diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 01e700ebba3..6aa65942445 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -26,8 +26,6 @@ #include #include #include -#include -#include #include #include #include @@ -59,6 +57,7 @@ #include #include #include +#include #if !defined(ARCADIA_BUILD) # include @@ -70,6 +69,7 @@ #endif #include +namespace fs = std::filesystem; DB::PipeFDs signal_pipe; @@ -437,11 +437,11 @@ static void sanitizerDeathCallback() static std::string createDirectory(const std::string & file) { - auto path = Poco::Path(file).makeParent(); - if (path.toString().empty()) + fs::path path = fs::path(file).parent_path(); + if (path.empty()) return ""; - Poco::File(path).createDirectories(); - return path.toString(); + fs::create_directories(path); + return path; }; @@ -449,7 +449,7 @@ static bool tryCreateDirectories(Poco::Logger * logger, const std::string & path { try { - Poco::File(path).createDirectories(); + fs::create_directories(path); return true; } catch (...) @@ -470,7 +470,7 @@ void BaseDaemon::reloadConfiguration() */ config_path = config().getString("config-file", getDefaultConfigFileName()); DB::ConfigProcessor config_processor(config_path, false, true); - config_processor.setConfigPath(Poco::Path(config_path).makeParent().toString()); + config_processor.setConfigPath(fs::path(config_path).parent_path()); loaded_config = config_processor.loadConfig(/* allow_zk_includes = */ true); if (last_configuration != nullptr) @@ -524,18 +524,20 @@ std::string BaseDaemon::getDefaultConfigFileName() const void BaseDaemon::closeFDs() { #if defined(OS_FREEBSD) || defined(OS_DARWIN) - Poco::File proc_path{"/dev/fd"}; + fs::path proc_path{"/dev/fd"}; #else - Poco::File proc_path{"/proc/self/fd"}; + fs::path proc_path{"/proc/self/fd"}; #endif - if (proc_path.isDirectory()) /// Hooray, proc exists + if (fs::is_directory(proc_path)) /// Hooray, proc exists { - std::vector fds; - /// in /proc/self/fd directory filenames are numeric file descriptors - proc_path.list(fds); - for (const auto & fd_str : fds) + /// in /proc/self/fd directory filenames are numeric file descriptors. + /// Iterate directory separately from closing fds to avoid closing iterated directory fd. + std::vector fds; + for (const auto & path : fs::directory_iterator(proc_path)) + fds.push_back(DB::parse(path.path().filename())); + + for (const auto & fd : fds) { - int fd = DB::parse(fd_str); if (fd > 2 && fd != signal_pipe.fds_rw[0] && fd != signal_pipe.fds_rw[1]) ::close(fd); } @@ -597,7 +599,7 @@ void BaseDaemon::initialize(Application & self) { /** When creating pid file and looking for config, will search for paths relative to the working path of the program when started. */ - std::string path = Poco::Path(config().getString("application.path")).setFileName("").toString(); + std::string path = fs::path(config().getString("application.path")).replace_filename(""); if (0 != chdir(path.c_str())) throw Poco::Exception("Cannot change directory to " + path); } @@ -645,7 +647,7 @@ void BaseDaemon::initialize(Application & self) std::string log_path = config().getString("logger.log", ""); if (!log_path.empty()) - log_path = Poco::Path(log_path).setFileName("").toString(); + log_path = fs::path(log_path).replace_filename(""); /** Redirect stdout, stderr to separate files in the log directory (or in the specified file). * Some libraries write to stderr in case of errors in debug mode, @@ -708,8 +710,7 @@ void BaseDaemon::initialize(Application & self) tryCreateDirectories(&logger(), core_path); - Poco::File cores = core_path; - if (!(cores.exists() && cores.isDirectory())) + if (!(fs::exists(core_path) && fs::is_directory(core_path))) { core_path = !log_path.empty() ? log_path : "/opt/"; tryCreateDirectories(&logger(), core_path); diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index 1028dc7d2dc..3571c64edd6 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -1,6 +1,5 @@ #include -#include #include #include @@ -25,6 +24,7 @@ # include # include +namespace fs = std::filesystem; namespace { @@ -53,8 +53,7 @@ void setExtras() sentry_set_extra("physical_cpu_cores", sentry_value_new_int32(getNumberOfPhysicalCPUCores())); if (!server_data_path.empty()) - sentry_set_extra("disk_free_space", sentry_value_new_string(formatReadableSizeWithBinarySuffix( - Poco::File(server_data_path).freeSpace()).c_str())); + sentry_set_extra("disk_free_space", sentry_value_new_string(formatReadableSizeWithBinarySuffix(fs::space(server_data_path).free).c_str())); } void sentry_logger(sentry_level_e level, const char * message, va_list args, void *) @@ -110,12 +109,12 @@ void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config) if (enabled) { server_data_path = config.getString("path", ""); - const std::filesystem::path & default_tmp_path = std::filesystem::path(config.getString("tmp_path", Poco::Path::temp())) / "sentry"; + const std::filesystem::path & default_tmp_path = fs::path(config.getString("tmp_path", fs::temp_directory_path())) / "sentry"; const std::string & endpoint = config.getString("send_crash_reports.endpoint"); const std::string & temp_folder_path = config.getString("send_crash_reports.tmp_path", default_tmp_path); - Poco::File(temp_folder_path).createDirectories(); + fs::create_directories(temp_folder_path); sentry_options_t * options = sentry_options_new(); /// will be freed by sentry_init or sentry_shutdown sentry_options_set_release(options, VERSION_STRING_SHORT); diff --git a/base/loggers/Loggers.cpp b/base/loggers/Loggers.cpp index 913deaf1eb8..80e62d0a6d6 100644 --- a/base/loggers/Loggers.cpp +++ b/base/loggers/Loggers.cpp @@ -6,10 +6,11 @@ #include "OwnFormattingChannel.h" #include "OwnPatternFormatter.h" #include -#include #include #include -#include +#include + +namespace fs = std::filesystem; namespace DB { @@ -20,11 +21,11 @@ namespace DB // TODO: move to libcommon static std::string createDirectory(const std::string & file) { - auto path = Poco::Path(file).makeParent(); - if (path.toString().empty()) + auto path = fs::path(file).parent_path(); + if (path.empty()) return ""; - Poco::File(path).createDirectories(); - return path.toString(); + fs::create_directories(path); + return path; }; void Loggers::setTextLog(std::shared_ptr log, int max_priority) @@ -70,7 +71,7 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log // Set up two channel chains. log_file = new Poco::FileChannel; - log_file->setProperty(Poco::FileChannel::PROP_PATH, Poco::Path(log_path).absolute().toString()); + log_file->setProperty(Poco::FileChannel::PROP_PATH, fs::weakly_canonical(log_path)); log_file->setProperty(Poco::FileChannel::PROP_ROTATION, config.getRawString("logger.size", "100M")); log_file->setProperty(Poco::FileChannel::PROP_ARCHIVE, "number"); log_file->setProperty(Poco::FileChannel::PROP_COMPRESS, config.getRawString("logger.compress", "true")); @@ -102,7 +103,7 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log std::cerr << "Logging errors to " << errorlog_path << std::endl; error_log_file = new Poco::FileChannel; - error_log_file->setProperty(Poco::FileChannel::PROP_PATH, Poco::Path(errorlog_path).absolute().toString()); + error_log_file->setProperty(Poco::FileChannel::PROP_PATH, fs::weakly_canonical(errorlog_path)); error_log_file->setProperty(Poco::FileChannel::PROP_ROTATION, config.getRawString("logger.size", "100M")); error_log_file->setProperty(Poco::FileChannel::PROP_ARCHIVE, "number"); error_log_file->setProperty(Poco::FileChannel::PROP_COMPRESS, config.getRawString("logger.compress", "true")); diff --git a/programs/benchmark/Benchmark.cpp b/programs/benchmark/Benchmark.cpp index 82c93eef9ff..2e48c5d20c5 100644 --- a/programs/benchmark/Benchmark.cpp +++ b/programs/benchmark/Benchmark.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -36,7 +35,9 @@ #include #include #include +#include +namespace fs = std::filesystem; /** A tool for evaluating ClickHouse performance. * The tool emulates a case with fixed amount of simultaneously executing queries. @@ -119,8 +120,8 @@ public: int main(const std::vector &) override { - if (!json_path.empty() && Poco::File(json_path).exists()) /// Clear file with previous results - Poco::File(json_path).remove(); + if (!json_path.empty() && fs::exists(json_path)) /// Clear file with previous results + fs::remove(json_path); readQueries(); runBenchmark(); diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index db2878d7d4c..2a6b9e5a79b 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -25,7 +25,6 @@ #include #include #include -#include #include #include #include @@ -86,6 +85,8 @@ #include #include #include +#include +#include #if !defined(ARCADIA_BUILD) # include @@ -95,6 +96,7 @@ #pragma GCC optimize("-fno-var-tracking-assignments") #endif +namespace fs = std::filesystem; namespace DB { @@ -275,7 +277,7 @@ private: /// Set path for format schema files if (config().has("format_schema_path")) - context->setFormatSchemaPath(Poco::Path(config().getString("format_schema_path")).toString()); + context->setFormatSchemaPath(fs::weakly_canonical(config().getString("format_schema_path"))); /// Initialize query_id_formats if any if (config().has("query_id_formats")) @@ -632,8 +634,8 @@ private: history_file = home_path + "/.clickhouse-client-history"; } - if (!history_file.empty() && !Poco::File(history_file).exists()) - Poco::File(history_file).createFile(); + if (!history_file.empty() && !fs::exists(history_file)) + FS::createFile(history_file); LineReader::Patterns query_extenders = {"\\"}; LineReader::Patterns query_delimiters = {";", "\\G"}; diff --git a/programs/copier/ClusterCopierApp.cpp b/programs/copier/ClusterCopierApp.cpp index d3fff616b65..8925ab63f99 100644 --- a/programs/copier/ClusterCopierApp.cpp +++ b/programs/copier/ClusterCopierApp.cpp @@ -5,7 +5,9 @@ #include #include #include +#include +namespace fs = std::filesystem; namespace DB { @@ -26,7 +28,7 @@ void ClusterCopierApp::initialize(Poco::Util::Application & self) copy_fault_probability = std::max(std::min(config().getDouble("copy-fault-probability"), 1.0), 0.0); if (config().has("move-fault-probability")) move_fault_probability = std::max(std::min(config().getDouble("move-fault-probability"), 1.0), 0.0); - base_dir = (config().has("base-dir")) ? config().getString("base-dir") : Poco::Path::current(); + base_dir = (config().has("base-dir")) ? config().getString("base-dir") : fs::current_path().string(); if (config().has("experimental-use-sample-offset")) @@ -38,18 +40,18 @@ void ClusterCopierApp::initialize(Poco::Util::Application & self) process_id = std::to_string(DateLUT::instance().toNumYYYYMMDDhhmmss(timestamp)) + "_" + std::to_string(curr_pid); host_id = escapeForFileName(getFQDNOrHostName()) + '#' + process_id; - process_path = Poco::Path(base_dir + "/clickhouse-copier_" + process_id).absolute().toString(); - Poco::File(process_path).createDirectories(); + process_path = fs::weakly_canonical(fs::path(base_dir) / ("clickhouse-copier_" + process_id)); + fs::create_directories(process_path); /// Override variables for BaseDaemon if (config().has("log-level")) config().setString("logger.level", config().getString("log-level")); if (config().has("base-dir") || !config().has("logger.log")) - config().setString("logger.log", process_path + "/log.log"); + config().setString("logger.log", fs::path(process_path) / "log.log"); if (config().has("base-dir") || !config().has("logger.errorlog")) - config().setString("logger.errorlog", process_path + "/log.err.log"); + config().setString("logger.errorlog", fs::path(process_path) / "log.err.log"); Base::initialize(self); } diff --git a/programs/copier/Internals.h b/programs/copier/Internals.h index 7e45c0ea2ee..9e40d7ebd7b 100644 --- a/programs/copier/Internals.h +++ b/programs/copier/Internals.h @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include diff --git a/programs/install/Install.cpp b/programs/install/Install.cpp index 8c187978106..a7f566a78b8 100644 --- a/programs/install/Install.cpp +++ b/programs/install/Install.cpp @@ -288,7 +288,7 @@ int mainEntryClickHouseInstall(int argc, char ** argv) bool is_symlink = fs::is_symlink(symlink_path); fs::path points_to; if (is_symlink) - points_to = fs::absolute(fs::read_symlink(symlink_path)); + points_to = fs::weakly_canonical(fs::read_symlink(symlink_path)); if (is_symlink && points_to == main_bin_path) { diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index aa10a9c2d14..2633f0e9426 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -42,9 +42,9 @@ #include #include #include - #include +namespace fs = std::filesystem; namespace DB { @@ -72,11 +72,11 @@ void LocalServer::initialize(Poco::Util::Application & self) Poco::Util::Application::initialize(self); /// Load config files if exists - if (config().has("config-file") || Poco::File("config.xml").exists()) + if (config().has("config-file") || fs::exists("config.xml")) { const auto config_path = config().getString("config-file", "config.xml"); ConfigProcessor config_processor(config_path, false, true); - config_processor.setConfigPath(Poco::Path(config_path).makeParent().toString()); + config_processor.setConfigPath(fs::path(config_path).parent_path()); auto loaded_config = config_processor.loadConfig(); config_processor.savePreprocessedConfig(loaded_config, loaded_config.configuration->getString("path", ".")); config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false); @@ -287,8 +287,8 @@ try status.emplace(path + "status", StatusFile::write_full_info); LOG_DEBUG(log, "Loading metadata from {}", path); - Poco::File(path + "data/").createDirectories(); - Poco::File(path + "metadata/").createDirectories(); + fs::create_directories(fs::path(path) / "data/"); + fs::create_directories(fs::path(path) / "metadata/"); loadMetadataSystem(global_context); attachSystemTables(global_context); loadMetadata(global_context); @@ -476,7 +476,7 @@ void LocalServer::setupUsers() { ConfigurationPtr users_config; - if (config().has("users_config") || config().has("config-file") || Poco::File("config.xml").exists()) + if (config().has("users_config") || config().has("config-file") || fs::exists("config.xml")) { const auto users_config_path = config().getString("users_config", config().getString("config-file", "config.xml")); ConfigProcessor config_processor(users_config_path); diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 6b3136dc200..f169d793ee9 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -74,6 +74,7 @@ #include #include #include +#include #if !defined(ARCADIA_BUILD) @@ -117,6 +118,8 @@ namespace CurrentMetrics extern const Metric MaxDDLEntryID; } +namespace fs = std::filesystem; + #if USE_JEMALLOC static bool jemallocOptionEnabled(const char *name) { @@ -183,19 +186,19 @@ void setupTmpPath(Poco::Logger * log, const std::string & path) { LOG_DEBUG(log, "Setting up {} to store temporary data in it", path); - Poco::File(path).createDirectories(); + fs::create_directories(path); /// Clearing old temporary files. - Poco::DirectoryIterator dir_end; - for (Poco::DirectoryIterator it(path); it != dir_end; ++it) + fs::directory_iterator dir_end; + for (fs::directory_iterator it(path); it != dir_end; ++it) { - if (it->isFile() && startsWith(it.name(), "tmp")) + if (it->is_regular_file() && startsWith(it->path().filename(), "tmp")) { - LOG_DEBUG(log, "Removing old temporary file {}", it->path()); - it->remove(); + LOG_DEBUG(log, "Removing old temporary file {}", it->path().string()); + fs::remove(it->path()); } else - LOG_DEBUG(log, "Skipped file in temporary path {}", it->path()); + LOG_DEBUG(log, "Skipped file in temporary path {}", it->path().string()); } } @@ -678,37 +681,38 @@ int Server::main(const std::vector & /*args*/) * Examples: do repair of local data; clone all replicated tables from replica. */ { - Poco::File(path + "flags/").createDirectories(); - global_context->setFlagsPath(path + "flags/"); + auto flags_path = fs::path(path) / "flags/"; + fs::create_directories(flags_path); + global_context->setFlagsPath(flags_path); } /** Directory with user provided files that are usable by 'file' table function. */ { - std::string user_files_path = config().getString("user_files_path", path + "user_files/"); + std::string user_files_path = config().getString("user_files_path", fs::path(path) / "user_files/"); global_context->setUserFilesPath(user_files_path); - Poco::File(user_files_path).createDirectories(); + fs::create_directories(user_files_path); } { - std::string dictionaries_lib_path = config().getString("dictionaries_lib_path", path + "dictionaries_lib/"); + std::string dictionaries_lib_path = config().getString("dictionaries_lib_path", fs::path(path) / "dictionaries_lib/"); global_context->setDictionariesLibPath(dictionaries_lib_path); - Poco::File(dictionaries_lib_path).createDirectories(); + fs::create_directories(dictionaries_lib_path); } /// top_level_domains_lists { - const std::string & top_level_domains_path = config().getString("top_level_domains_path", path + "top_level_domains/") + "/"; - TLDListsHolder::getInstance().parseConfig(top_level_domains_path, config()); + const std::string & top_level_domains_path = config().getString("top_level_domains_path", fs::path(path) / "top_level_domains/"); + TLDListsHolder::getInstance().parseConfig(fs::path(top_level_domains_path) / "", config()); } { - Poco::File(path + "data/").createDirectories(); - Poco::File(path + "metadata/").createDirectories(); + fs::create_directories(fs::path(path) / "data/"); + fs::create_directories(fs::path(path) / "metadata/"); /// Directory with metadata of tables, which was marked as dropped by Atomic database - Poco::File(path + "metadata_dropped/").createDirectories(); + fs::create_directories(fs::path(path) / "metadata_dropped/"); } if (config().has("interserver_http_port") && config().has("interserver_https_port")) @@ -891,9 +895,9 @@ int Server::main(const std::vector & /*args*/) #endif /// Set path for format schema files - auto format_schema_path = Poco::File(config().getString("format_schema_path", path + "format_schemas/")); - global_context->setFormatSchemaPath(format_schema_path.path()); - format_schema_path.createDirectories(); + fs::path format_schema_path(config().getString("format_schema_path", fs::path(path) / "format_schemas/")); + global_context->setFormatSchemaPath(format_schema_path); + fs::create_directories(format_schema_path); /// Check sanity of MergeTreeSettings on server startup global_context->getMergeTreeSettings().sanityCheck(settings); diff --git a/src/Access/AllowedClientHosts.h b/src/Access/AllowedClientHosts.h index a6895b120e0..7b21fd7e236 100644 --- a/src/Access/AllowedClientHosts.h +++ b/src/Access/AllowedClientHosts.h @@ -7,7 +7,9 @@ #include #include #include +#include +namespace fs = std::filesystem; namespace DB { @@ -198,9 +200,9 @@ inline String AllowedClientHosts::IPSubnet::toString() const if (isMaskAllBitsOne()) return prefix.toString(); else if (IPAddress{prefix_length, mask.family()} == mask) - return prefix.toString() + "/" + std::to_string(prefix_length); + return fs::path(prefix.toString()) / std::to_string(prefix_length); else - return prefix.toString() + "/" + mask.toString(); + return fs::path(prefix.toString()) / mask.toString(); } inline bool AllowedClientHosts::IPSubnet::isMaskAllBitsOne() const diff --git a/src/Bridge/IBridgeHelper.cpp b/src/Bridge/IBridgeHelper.cpp index 16886bae4fe..b6f3446d0a6 100644 --- a/src/Bridge/IBridgeHelper.cpp +++ b/src/Bridge/IBridgeHelper.cpp @@ -3,9 +3,10 @@ #include #include #include -#include #include +#include +namespace fs = std::filesystem; namespace DB { @@ -87,10 +88,10 @@ std::unique_ptr IBridgeHelper::startBridgeCommand() const const auto & config = getConfig(); /// Path to executable folder - Poco::Path path{config.getString("application.dir", "/usr/bin")}; + fs::path path(config.getString("application.dir", "/usr/bin")); std::vector cmd_args; - path.setFileName(serviceFileName()); + path /= serviceFileName(); cmd_args.push_back("--http-port"); cmd_args.push_back(std::to_string(config.getUInt(configPrefix() + ".port", getDefaultPort()))); @@ -126,7 +127,7 @@ std::unique_ptr IBridgeHelper::startBridgeCommand() const LOG_TRACE(getLog(), "Starting {}", serviceAlias()); - return ShellCommand::executeDirect(path.toString(), cmd_args, ShellCommandDestructorStrategy(true)); + return ShellCommand::executeDirect(path.string(), cmd_args, ShellCommandDestructorStrategy(true)); } } diff --git a/src/Bridge/LibraryBridgeHelper.cpp b/src/Bridge/LibraryBridgeHelper.cpp index 8234b5d755f..8967b191b73 100644 --- a/src/Bridge/LibraryBridgeHelper.cpp +++ b/src/Bridge/LibraryBridgeHelper.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include diff --git a/src/Bridge/XDBCBridgeHelper.h b/src/Bridge/XDBCBridgeHelper.h index ee6a178037d..97a6251a9a0 100644 --- a/src/Bridge/XDBCBridgeHelper.h +++ b/src/Bridge/XDBCBridgeHelper.h @@ -5,10 +5,8 @@ #include #include #include -#include #include #include -#include #include #include #include diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp index fa9e9b72087..7bbb4798d6f 100644 --- a/src/Common/Config/ConfigProcessor.cpp +++ b/src/Common/Config/ConfigProcessor.cpp @@ -62,7 +62,7 @@ static std::string numberFromHost(const std::string & s) bool ConfigProcessor::isPreprocessedFile(const std::string & path) { - return endsWith(Poco::Path(path).getBaseName(), PREPROCESSED_SUFFIX); + return endsWith(fs::path(path).stem(), PREPROCESSED_SUFFIX); } @@ -416,34 +416,32 @@ ConfigProcessor::Files ConfigProcessor::getConfigMergeFiles(const std::string & { Files files; - Poco::Path merge_dir_path(config_path); + fs::path merge_dir_path(config_path); std::set merge_dirs; /// Add path_to_config/config_name.d dir - merge_dir_path.setExtension("d"); - merge_dirs.insert(merge_dir_path.toString()); + merge_dir_path.replace_extension("d"); + merge_dirs.insert(merge_dir_path); /// Add path_to_config/conf.d dir - merge_dir_path.setBaseName("conf"); - merge_dirs.insert(merge_dir_path.toString()); + merge_dir_path.replace_filename("conf.d"); + merge_dirs.insert(merge_dir_path); for (const std::string & merge_dir_name : merge_dirs) { - Poco::File merge_dir(merge_dir_name); - if (!merge_dir.exists() || !merge_dir.isDirectory()) + if (!fs::exists(merge_dir_name) || !fs::is_directory(merge_dir_name)) continue; - for (Poco::DirectoryIterator it(merge_dir_name); it != Poco::DirectoryIterator(); ++it) + for (fs::directory_iterator it(merge_dir_name); it != fs::directory_iterator(); ++it) { - Poco::File & file = *it; - Poco::Path path(file.path()); - std::string extension = path.getExtension(); - std::string base_name = path.getBaseName(); + fs::path path(it->path()); + std::string extension = path.extension(); + std::string base_name = path.stem(); // Skip non-config and temporary files - if (file.isFile() && (extension == "xml" || extension == "conf" || extension == "yaml" || extension == "yml") && !startsWith(base_name, ".")) - { - files.push_back(file.path()); - } + if (fs::is_regular_file(path) + && (extension == ".xml" || extension == ".conf" || extension == ".yaml" || extension == ".yml") + && !startsWith(base_name, ".")) + files.push_back(it->path()); } } @@ -548,7 +546,7 @@ XMLDocumentPtr ConfigProcessor::processConfig( else { std::string default_path = "/etc/metrika.xml"; - if (Poco::File(default_path).exists()) + if (fs::exists(default_path)) include_from_path = default_path; } if (!include_from_path.empty()) @@ -660,11 +658,11 @@ void ConfigProcessor::savePreprocessedConfig(const LoadedConfig & loaded_config, if (!loaded_config.configuration->has("path")) { // Will use current directory - auto parent_path = Poco::Path(loaded_config.config_path).makeParent(); - preprocessed_dir = parent_path.toString(); - Poco::Path poco_new_path(new_path); - poco_new_path.setBaseName(poco_new_path.getBaseName() + PREPROCESSED_SUFFIX); - new_path = poco_new_path.toString(); + fs::path parent_path = fs::path(loaded_config.config_path).parent_path(); + preprocessed_dir = parent_path.string(); + fs::path fs_new_path(new_path); + fs_new_path.replace_filename(fs_new_path.stem().string() + PREPROCESSED_SUFFIX + fs_new_path.extension().string()); + new_path = fs_new_path.string(); } else { @@ -679,9 +677,9 @@ void ConfigProcessor::savePreprocessedConfig(const LoadedConfig & loaded_config, } preprocessed_path = (fs::path(preprocessed_dir) / fs::path(new_path)).string(); - auto preprocessed_path_parent = Poco::Path(preprocessed_path).makeParent(); - if (!preprocessed_path_parent.toString().empty()) - Poco::File(preprocessed_path_parent).createDirectories(); + auto preprocessed_path_parent = fs::path(preprocessed_path).parent_path(); + if (!preprocessed_path_parent.empty()) + fs::create_directories(preprocessed_path_parent); } DOMWriter().writeNode(preprocessed_path, loaded_config.preprocessed_xml); LOG_DEBUG(log, "Saved preprocessed configuration to '{}'.", preprocessed_path); diff --git a/src/Common/Config/ConfigProcessor.h b/src/Common/Config/ConfigProcessor.h index 5b16bc0cb1b..e13e66547a3 100644 --- a/src/Common/Config/ConfigProcessor.h +++ b/src/Common/Config/ConfigProcessor.h @@ -15,12 +15,9 @@ #include #include #include -#include -#include #include #include #include - #include diff --git a/src/Common/Config/ConfigReloader.cpp b/src/Common/Config/ConfigReloader.cpp index afff08e82bb..01bb4f556d6 100644 --- a/src/Common/Config/ConfigReloader.cpp +++ b/src/Common/Config/ConfigReloader.cpp @@ -1,12 +1,15 @@ #include "ConfigReloader.h" #include -#include #include #include #include "ConfigProcessor.h" +#include +#include +namespace fs = std::filesystem; + namespace DB { @@ -167,8 +170,8 @@ struct ConfigReloader::FileWithTimestamp void ConfigReloader::FilesChangesTracker::addIfExists(const std::string & path_to_add) { - if (!path_to_add.empty() && Poco::File(path_to_add).exists()) - files.emplace(path_to_add, Poco::File(path_to_add).getLastModified().epochTime()); + if (!path_to_add.empty() && fs::exists(path_to_add)) + files.emplace(path_to_add, FS::getModificationTime(path_to_add)); } bool ConfigReloader::FilesChangesTracker::isDifferOrNewerThan(const FilesChangesTracker & rhs) diff --git a/src/Common/Config/configReadClient.cpp b/src/Common/Config/configReadClient.cpp index 61d8e507c05..8ce2a8b03e9 100644 --- a/src/Common/Config/configReadClient.cpp +++ b/src/Common/Config/configReadClient.cpp @@ -1,8 +1,10 @@ #include "configReadClient.h" #include -#include #include "ConfigProcessor.h" +#include + +namespace fs = std::filesystem; namespace DB { @@ -11,11 +13,11 @@ bool configReadClient(Poco::Util::LayeredConfiguration & config, const std::stri std::string config_path; if (config.has("config-file")) config_path = config.getString("config-file"); - else if (Poco::File("./clickhouse-client.xml").exists()) + else if (fs::exists("./clickhouse-client.xml")) config_path = "./clickhouse-client.xml"; - else if (!home_path.empty() && Poco::File(home_path + "/.clickhouse-client/config.xml").exists()) + else if (!home_path.empty() && fs::exists(home_path + "/.clickhouse-client/config.xml")) config_path = home_path + "/.clickhouse-client/config.xml"; - else if (Poco::File("/etc/clickhouse-client/config.xml").exists()) + else if (fs::exists("/etc/clickhouse-client/config.xml")) config_path = "/etc/clickhouse-client/config.xml"; if (!config_path.empty()) diff --git a/src/Common/CounterInFile.h b/src/Common/CounterInFile.h index 8cd4534d413..6326261234d 100644 --- a/src/Common/CounterInFile.h +++ b/src/Common/CounterInFile.h @@ -7,7 +7,6 @@ #include #include -#include #include #include @@ -59,7 +58,7 @@ public: Int64 res = -1; - bool file_doesnt_exists = !Poco::File(path).exists(); + bool file_doesnt_exists = !fs::exists(path); if (file_doesnt_exists && !create_if_need) { throw Poco::Exception("File " + path + " does not exist. " @@ -138,7 +137,7 @@ public: // Not thread-safe and not synchronized between processes. void fixIfBroken(UInt64 value) { - bool file_exists = Poco::File(path).exists(); + bool file_exists = fs::exists(path); int fd = ::open(path.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0666); if (-1 == fd) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 330a193212b..d840830bf28 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -553,6 +553,7 @@ M(583, ILLEGAL_PROJECTION) \ M(584, PROJECTION_NOT_USED) \ M(585, CANNOT_PARSE_YAML) \ + M(586, CANNOT_CREATE_FILE) \ \ M(998, POSTGRESQL_CONNECTION_FAILURE) \ M(999, KEEPER_EXCEPTION) \ diff --git a/src/Common/Exception.cpp b/src/Common/Exception.cpp index ac1780a45a8..e98cd3c3046 100644 --- a/src/Common/Exception.cpp +++ b/src/Common/Exception.cpp @@ -21,6 +21,8 @@ # include #endif +namespace fs = std::filesystem; + namespace DB { @@ -177,7 +179,7 @@ void tryLogCurrentException(Poco::Logger * logger, const std::string & start_of_ tryLogCurrentExceptionImpl(logger, start_of_message); } -static void getNoSpaceLeftInfoMessage(std::filesystem::path path, std::string & msg) +static void getNoSpaceLeftInfoMessage(std::filesystem::path path, String & msg) { path = std::filesystem::absolute(path); /// It's possible to get ENOSPC for non existent file (e.g. if there are no free inodes and creat() fails) @@ -264,22 +266,12 @@ static std::string getExtraExceptionInfo(const std::exception & e) String msg; try { - if (const auto * file_exception = dynamic_cast(&e)) + if (const auto * file_exception = dynamic_cast(&e)) { - if (file_exception->code() == ENOSPC) - { - /// See Poco::FileImpl::handleLastErrorImpl(...) - constexpr const char * expected_error_message = "no space left on device: "; - if (startsWith(file_exception->message(), expected_error_message)) - { - String path = file_exception->message().substr(strlen(expected_error_message)); - getNoSpaceLeftInfoMessage(path, msg); - } - else - { - msg += "\nCannot print extra info for Poco::Exception"; - } - } + if (file_exception->code() == std::errc::no_space_on_device) + getNoSpaceLeftInfoMessage(file_exception->path1(), msg); + else + msg += "\nCannot print extra info for Poco::Exception"; } else if (const auto * errno_exception = dynamic_cast(&e)) { diff --git a/src/Common/FileUpdatesTracker.h b/src/Common/FileUpdatesTracker.h index 0914c75693c..9fe37c2a75d 100644 --- a/src/Common/FileUpdatesTracker.h +++ b/src/Common/FileUpdatesTracker.h @@ -1,10 +1,11 @@ #pragma once -#include #include - #include +#include +#include +namespace fs = std::filesystem; class FileUpdatesTracker { @@ -31,6 +32,6 @@ public: private: Poco::Timestamp getLastModificationTime() const { - return Poco::File(path).getLastModified(); + return FS::getModificationTimestamp(path); } }; diff --git a/src/Common/ProgressIndication.cpp b/src/Common/ProgressIndication.cpp index 3e1cf5978ee..e1a7c420c54 100644 --- a/src/Common/ProgressIndication.cpp +++ b/src/Common/ProgressIndication.cpp @@ -34,7 +34,7 @@ void ProgressIndication::resetProgress() write_progress_on_update = false; } -void ProgressIndication::setFileProgressCallback(ContextPtr context, bool write_progress_on_update_) +void ProgressIndication::setFileProgressCallback(ContextMutablePtr context, bool write_progress_on_update_) { write_progress_on_update = write_progress_on_update_; context->setFileProgressCallback([&](const FileProgress & file_progress) diff --git a/src/Common/ProgressIndication.h b/src/Common/ProgressIndication.h index df161387ec3..dd4d3e9d241 100644 --- a/src/Common/ProgressIndication.h +++ b/src/Common/ProgressIndication.h @@ -36,7 +36,7 @@ public: /// In some cases there is a need to update progress value, when there is no access to progress_inidcation object. /// In this case it is added via context. /// `write_progress_on_update` is needed to write progress for loading files data via pipe in non-interactive mode. - void setFileProgressCallback(ContextPtr context, bool write_progress_on_update = false); + void setFileProgressCallback(ContextMutablePtr context, bool write_progress_on_update = false); /// How much seconds passed since query execution start. UInt64 elapsedSeconds() const { return watch.elapsedSeconds(); } diff --git a/src/Common/StatusFile.cpp b/src/Common/StatusFile.cpp index b21454c9ed8..ceedf518608 100644 --- a/src/Common/StatusFile.cpp +++ b/src/Common/StatusFile.cpp @@ -4,7 +4,6 @@ #include #include -#include #include #include #include @@ -14,7 +13,9 @@ #include #include #include +#include +namespace fs = std::filesystem; namespace DB { @@ -45,7 +46,7 @@ StatusFile::StatusFile(std::string path_, FillFunction fill_) : path(std::move(path_)), fill(std::move(fill_)) { /// If file already exists. NOTE Minor race condition. - if (Poco::File(path).exists()) + if (fs::exists(path)) { std::string contents; { diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index 43092f719ec..f3aecd4e76b 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -4,6 +4,7 @@ #include "TestKeeper.h" #include +#include #include #include @@ -17,6 +18,7 @@ #define ZOOKEEPER_CONNECTION_TIMEOUT_MS 1000 +namespace fs = std::filesystem; namespace DB { @@ -612,7 +614,7 @@ void ZooKeeper::removeChildren(const std::string & path) Coordination::Requests ops; for (size_t i = 0; i < MULTI_BATCH_SIZE && !children.empty(); ++i) { - ops.emplace_back(makeRemoveRequest(path + "/" + children.back(), -1)); + ops.emplace_back(makeRemoveRequest(fs::path(path) / children.back(), -1)); children.pop_back(); } multi(ops); @@ -628,9 +630,9 @@ void ZooKeeper::removeChildrenRecursive(const std::string & path, const String & Coordination::Requests ops; for (size_t i = 0; i < MULTI_BATCH_SIZE && !children.empty(); ++i) { - removeChildrenRecursive(path + "/" + children.back()); + removeChildrenRecursive(fs::path(path) / children.back()); if (likely(keep_child_node.empty() || keep_child_node != children.back())) - ops.emplace_back(makeRemoveRequest(path + "/" + children.back(), -1)); + ops.emplace_back(makeRemoveRequest(fs::path(path) / children.back(), -1)); children.pop_back(); } multi(ops); @@ -648,7 +650,7 @@ void ZooKeeper::tryRemoveChildrenRecursive(const std::string & path, const Strin Strings batch; for (size_t i = 0; i < MULTI_BATCH_SIZE && !children.empty(); ++i) { - String child_path = path + "/" + children.back(); + String child_path = fs::path(path) / children.back(); tryRemoveChildrenRecursive(child_path); if (likely(keep_child_node.empty() || keep_child_node != children.back())) { diff --git a/src/Common/filesystemHelpers.cpp b/src/Common/filesystemHelpers.cpp index 6f0e3155cbe..4855500b776 100644 --- a/src/Common/filesystemHelpers.cpp +++ b/src/Common/filesystemHelpers.cpp @@ -6,10 +6,15 @@ # include #endif #include -#include -#include #include +#include +#include +#include +#include +#include +#include +namespace fs = std::filesystem; namespace DB { @@ -20,6 +25,8 @@ namespace ErrorCodes extern const int SYSTEM_ERROR; extern const int NOT_IMPLEMENTED; extern const int CANNOT_STATVFS; + extern const int PATH_ACCESS_DENIED; + extern const int CANNOT_CREATE_FILE; } @@ -38,17 +45,13 @@ struct statvfs getStatVFS(const String & path) bool enoughSpaceInDirectory(const std::string & path [[maybe_unused]], size_t data_size [[maybe_unused]]) { -#if POCO_VERSION >= 0x01090000 - auto free_space = Poco::File(path).freeSpace(); + auto free_space = fs::space(path).free; return data_size <= free_space; -#else - return true; -#endif } std::unique_ptr createTemporaryFile(const std::string & path) { - Poco::File(path).createDirectories(); + fs::create_directories(path); /// NOTE: std::make_shared cannot use protected constructors return std::make_unique(path); @@ -128,3 +131,73 @@ bool pathStartsWith(const String & path, const String & prefix_path) } } + + +/// Copied from Poco::File +namespace FS +{ + +bool createFile(const std::string & path) +{ + int n = open(path.c_str(), O_WRONLY | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); + if (n != -1) + { + close(n); + return true; + } + DB::throwFromErrnoWithPath("Cannot create file: " + path, path, DB::ErrorCodes::CANNOT_CREATE_FILE); +} + +bool canRead(const std::string & path) +{ + struct stat st; + if (stat(path.c_str(), &st) == 0) + { + if (st.st_uid == geteuid()) + return (st.st_mode & S_IRUSR) != 0; + else if (st.st_gid == getegid()) + return (st.st_mode & S_IRGRP) != 0; + else + return (st.st_mode & S_IROTH) != 0 || geteuid() == 0; + } + DB::throwFromErrnoWithPath("Cannot check read access to file: " + path, path, DB::ErrorCodes::PATH_ACCESS_DENIED); +} + + +bool canWrite(const std::string & path) +{ + struct stat st; + if (stat(path.c_str(), &st) == 0) + { + if (st.st_uid == geteuid()) + return (st.st_mode & S_IWUSR) != 0; + else if (st.st_gid == getegid()) + return (st.st_mode & S_IWGRP) != 0; + else + return (st.st_mode & S_IWOTH) != 0 || geteuid() == 0; + } + DB::throwFromErrnoWithPath("Cannot check write access to file: " + path, path, DB::ErrorCodes::PATH_ACCESS_DENIED); +} + +time_t getModificationTime(const std::string & path) +{ + struct stat st; + if (stat(path.c_str(), &st) == 0) + return st.st_mtime; + DB::throwFromErrnoWithPath("Cannot check modification time for file: " + path, path, DB::ErrorCodes::PATH_ACCESS_DENIED); +} + +Poco::Timestamp getModificationTimestamp(const std::string & path) +{ + return Poco::Timestamp::fromEpochTime(getModificationTime(path)); +} + +void setModificationTime(const std::string & path, time_t time) +{ + struct utimbuf tb; + tb.actime = time; + tb.modtime = time; + if (utime(path.c_str(), &tb) != 0) + DB::throwFromErrnoWithPath("Cannot set modification time for file: " + path, path, DB::ErrorCodes::PATH_ACCESS_DENIED); +} +} diff --git a/src/Common/filesystemHelpers.h b/src/Common/filesystemHelpers.h index 0ffbf19643d..b7525a64fae 100644 --- a/src/Common/filesystemHelpers.h +++ b/src/Common/filesystemHelpers.h @@ -36,3 +36,15 @@ bool pathStartsWith(const std::filesystem::path & path, const std::filesystem::p bool pathStartsWith(const String & path, const String & prefix_path); } + +namespace FS +{ +bool createFile(const std::string & path); + +bool canRead(const std::string & path); +bool canWrite(const std::string & path); + +time_t getModificationTime(const std::string & path); +Poco::Timestamp getModificationTimestamp(const std::string & path); +void setModificationTime(const std::string & path, time_t time); +} diff --git a/src/Common/renameat2.cpp b/src/Common/renameat2.cpp index a735a9d72d4..26d90427889 100644 --- a/src/Common/renameat2.cpp +++ b/src/Common/renameat2.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #if defined(linux) || defined(__linux) || defined(__linux__) #include @@ -10,6 +10,8 @@ #include #endif +namespace fs = std::filesystem; + namespace DB { @@ -93,9 +95,9 @@ static bool renameat2(const std::string &, const std::string &, int) static void renameNoReplaceFallback(const std::string & old_path, const std::string & new_path) { /// NOTE it's unsafe - if (Poco::File{new_path}.exists()) + if (fs::exists(new_path)) throw Exception("File " + new_path + " exists", ErrorCodes::FILE_ALREADY_EXISTS); - Poco::File{old_path}.renameTo(new_path); + fs::rename(old_path, new_path); } /// Do not use [[noreturn]] to avoid warnings like "code will never be executed" in other places diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index 88ead6c9b4c..6b8c470861d 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -1,7 +1,5 @@ #include #include -#include -#include #include #include #include @@ -13,6 +11,8 @@ #include #include +namespace fs = std::filesystem; + namespace DB { namespace ErrorCodes @@ -37,12 +37,12 @@ public: DatabaseAtomic::DatabaseAtomic(String name_, String metadata_path_, UUID uuid, const String & logger_name, ContextPtr context_) : DatabaseOrdinary(name_, std::move(metadata_path_), "store/", logger_name, context_) - , path_to_table_symlinks(getContext()->getPath() + "data/" + escapeForFileName(name_) + "/") - , path_to_metadata_symlink(getContext()->getPath() + "metadata/" + escapeForFileName(name_)) + , path_to_table_symlinks(fs::path(getContext()->getPath()) / "data" / escapeForFileName(name_) / "") + , path_to_metadata_symlink(fs::path(getContext()->getPath()) / "metadata" / escapeForFileName(name_)) , db_uuid(uuid) { assert(db_uuid != UUIDHelpers::Nil); - Poco::File(path_to_table_symlinks).createDirectories(); + fs::create_directories(path_to_table_symlinks); tryCreateMetadataSymlink(); } @@ -73,14 +73,14 @@ void DatabaseAtomic::drop(ContextPtr) assert(tables.empty()); try { - Poco::File(path_to_metadata_symlink).remove(); - Poco::File(path_to_table_symlinks).remove(true); + fs::remove(path_to_metadata_symlink); + fs::remove_all(path_to_table_symlinks); } catch (...) { LOG_WARNING(log, getCurrentExceptionMessage(true)); } - Poco::File(getMetadataPath()).remove(true); + fs::remove_all(getMetadataPath()); } void DatabaseAtomic::attachTable(const String & name, const StoragePtr & table, const String & relative_table_path) @@ -132,8 +132,8 @@ void DatabaseAtomic::dropTable(ContextPtr local_context, const String & table_na /// (it's more likely to lost connection, than to fail before applying local changes). /// TODO better detection and recovery - Poco::File(table_metadata_path).renameTo(table_metadata_path_drop); /// Mark table as dropped - DatabaseOrdinary::detachTableUnlocked(table_name, lock); /// Should never throw + fs::rename(table_metadata_path, table_metadata_path_drop); /// Mark table as dropped + DatabaseOrdinary::detachTableUnlocked(table_name, lock); /// Should never throw table_name_to_path.erase(table_name); } @@ -316,7 +316,7 @@ void DatabaseAtomic::commitCreateTable(const ASTCreateQuery & query, const Stora } catch (...) { - Poco::File(table_metadata_tmp_path).remove(); + fs::remove(table_metadata_tmp_path); if (locked_uuid) DatabaseCatalog::instance().removeUUIDMappingFinally(query.uuid); throw; @@ -420,7 +420,7 @@ void DatabaseAtomic::loadStoredObjects(ContextMutablePtr local_context, bool has { /// Recreate symlinks to table data dirs in case of force restore, because some of them may be broken if (has_force_restore_data_flag) - Poco::File(path_to_table_symlinks).remove(true); + fs::remove_all(path_to_table_symlinks); DatabaseOrdinary::loadStoredObjects(local_context, has_force_restore_data_flag, force_attach); @@ -432,7 +432,7 @@ void DatabaseAtomic::loadStoredObjects(ContextMutablePtr local_context, bool has table_names = table_name_to_path; } - Poco::File(path_to_table_symlinks).createDirectories(); + fs::create_directories(path_to_table_symlinks); for (const auto & table : table_names) tryCreateSymlink(table.first, table.second, true); } @@ -443,9 +443,9 @@ void DatabaseAtomic::tryCreateSymlink(const String & table_name, const String & try { String link = path_to_table_symlinks + escapeForFileName(table_name); - Poco::File data = Poco::Path(getContext()->getPath()).makeAbsolute().toString() + actual_data_path; - if (!if_data_path_exist || data.exists()) - data.linkTo(link, Poco::File::LINK_SYMBOLIC); + fs::path data = fs::canonical(getContext()->getPath()) / actual_data_path; + if (!if_data_path_exist || fs::exists(data)) + fs::create_directory_symlink(data, link); } catch (...) { @@ -458,7 +458,7 @@ void DatabaseAtomic::tryRemoveSymlink(const String & table_name) try { String path = path_to_table_symlinks + escapeForFileName(table_name); - Poco::File{path}.remove(); + fs::remove(path); } catch (...) { @@ -471,17 +471,17 @@ void DatabaseAtomic::tryCreateMetadataSymlink() /// Symlinks in data/db_name/ directory and metadata/db_name/ are not used by ClickHouse, /// it's needed only for convenient introspection. assert(path_to_metadata_symlink != metadata_path); - Poco::File metadata_symlink(path_to_metadata_symlink); - if (metadata_symlink.exists()) + fs::path metadata_symlink(path_to_metadata_symlink); + if (fs::exists(metadata_symlink)) { - if (!metadata_symlink.isLink()) + if (!fs::is_symlink(metadata_symlink)) throw Exception(ErrorCodes::FILE_ALREADY_EXISTS, "Directory {} exists", path_to_metadata_symlink); } else { try { - Poco::File{metadata_path}.linkTo(path_to_metadata_symlink, Poco::File::LINK_SYMBOLIC); + fs::create_directory_symlink(metadata_path, path_to_metadata_symlink); } catch (...) { @@ -495,7 +495,7 @@ void DatabaseAtomic::renameDatabase(const String & new_name) /// CREATE, ATTACH, DROP, DETACH and RENAME DATABASE must hold DDLGuard try { - Poco::File(path_to_metadata_symlink).remove(); + fs::remove(path_to_metadata_symlink); } catch (...) { @@ -526,7 +526,7 @@ void DatabaseAtomic::renameDatabase(const String & new_name) path_to_table_symlinks = getContext()->getPath() + "data/" + new_name_escaped + "/"; } - Poco::File(old_path_to_table_symlinks).renameTo(path_to_table_symlinks); + fs::rename(old_path_to_table_symlinks, path_to_table_symlinks); tryCreateMetadataSymlink(); } diff --git a/src/Databases/DatabaseFactory.cpp b/src/Databases/DatabaseFactory.cpp index 4626d4f47b6..e09860e79b6 100644 --- a/src/Databases/DatabaseFactory.cpp +++ b/src/Databases/DatabaseFactory.cpp @@ -11,10 +11,9 @@ #include #include #include -#include -#include #include #include +#include #if !defined(ARCADIA_BUILD) # include "config_core.h" @@ -40,6 +39,8 @@ #include #endif +namespace fs = std::filesystem; + namespace DB { @@ -58,11 +59,12 @@ DatabasePtr DatabaseFactory::get(const ASTCreateQuery & create, const String & m try { /// Creates store/xxx/ for Atomic - Poco::File(Poco::Path(metadata_path).makeParent()).createDirectories(); + fs::create_directories(fs::path(metadata_path).parent_path()); + /// Before 20.7 it's possible that .sql metadata file does not exist for some old database. /// In this case Ordinary database is created on server startup if the corresponding metadata directory exists. /// So we should remove metadata directory if database creation failed. - created = Poco::File(metadata_path).createDirectory(); + created = fs::create_directory(metadata_path); DatabasePtr impl = getImpl(create, metadata_path, context); @@ -74,11 +76,8 @@ DatabasePtr DatabaseFactory::get(const ASTCreateQuery & create, const String & m } catch (...) { - Poco::File metadata_dir(metadata_path); - - if (created && metadata_dir.exists()) - metadata_dir.remove(true); - + if (created && fs::exists(metadata_path)) + fs::remove_all(metadata_path); throw; } } diff --git a/src/Databases/DatabaseLazy.cpp b/src/Databases/DatabaseLazy.cpp index f77133f68e7..22afaa971a1 100644 --- a/src/Databases/DatabaseLazy.cpp +++ b/src/Databases/DatabaseLazy.cpp @@ -12,8 +12,9 @@ #include #include #include -#include +#include +namespace fs = std::filesystem; namespace DB { @@ -43,8 +44,8 @@ void DatabaseLazy::loadStoredObjects( { const std::string table_name = file_name.substr(0, file_name.size() - 4); - auto detached_permanently_flag = Poco::File(getMetadataPath() + "/" + file_name + detached_suffix); - if (detached_permanently_flag.exists()) + fs::path detached_permanently_flag = fs::path(getMetadataPath()) / (file_name + detached_suffix); + if (fs::exists(detached_permanently_flag)) { LOG_DEBUG(log, "Skipping permanently detached table {}.", backQuote(table_name)); return; @@ -228,7 +229,7 @@ StoragePtr DatabaseLazy::loadTable(const String & table_name) const LOG_DEBUG(log, "Load table {} to cache.", backQuote(table_name)); - const String table_metadata_path = getMetadataPath() + "/" + escapeForFileName(table_name) + ".sql"; + const String table_metadata_path = fs::path(getMetadataPath()) / (escapeForFileName(table_name) + ".sql"); try { diff --git a/src/Databases/DatabaseMemory.cpp b/src/Databases/DatabaseMemory.cpp index f21a145df55..c0af027e027 100644 --- a/src/Databases/DatabaseMemory.cpp +++ b/src/Databases/DatabaseMemory.cpp @@ -4,9 +4,9 @@ #include #include #include -#include #include +namespace fs = std::filesystem; namespace DB { @@ -42,9 +42,9 @@ void DatabaseMemory::dropTable( try { table->drop(); - Poco::File table_data_dir{getTableDataPath(table_name)}; - if (table_data_dir.exists()) - table_data_dir.remove(true); + fs::path table_data_dir{getTableDataPath(table_name)}; + if (fs::exists(table_data_dir)) + fs::remove_all(table_data_dir); } catch (...) { diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index af0ec35f5a2..620e560b64c 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -14,14 +14,14 @@ #include #include #include - #include -#include - #include #include #include +#include +#include +namespace fs = std::filesystem; namespace DB { @@ -201,8 +201,8 @@ DatabaseOnDisk::DatabaseOnDisk( , metadata_path(metadata_path_) , data_path(data_path_) { - Poco::File(local_context->getPath() + data_path).createDirectories(); - Poco::File(metadata_path).createDirectories(); + fs::create_directories(local_context->getPath() + data_path); + fs::create_directories(metadata_path); } @@ -245,7 +245,7 @@ void DatabaseOnDisk::createTable( if (!create.attach) checkMetadataFilenameAvailability(table_name); - if (create.attach && Poco::File(table_metadata_path).exists()) + if (create.attach && fs::exists(table_metadata_path)) { ASTPtr ast_detached = parseQueryFromMetadata(log, local_context, table_metadata_path); auto & create_detached = ast_detached->as(); @@ -285,10 +285,10 @@ void DatabaseOnDisk::removeDetachedPermanentlyFlag(ContextPtr, const String & ta { try { - auto detached_permanently_flag = Poco::File(table_metadata_path + detached_suffix); + fs::path detached_permanently_flag(table_metadata_path + detached_suffix); - if (detached_permanently_flag.exists()) - detached_permanently_flag.remove(); + if (fs::exists(detached_permanently_flag)) + fs::remove(detached_permanently_flag); } catch (Exception & e) { @@ -308,11 +308,11 @@ void DatabaseOnDisk::commitCreateTable(const ASTCreateQuery & query, const Stora /// If it was ATTACH query and file with table metadata already exist /// (so, ATTACH is done after DETACH), then rename atomically replaces old file with new one. - Poco::File(table_metadata_tmp_path).renameTo(table_metadata_path); + fs::rename(table_metadata_tmp_path, table_metadata_path); } catch (...) { - Poco::File(table_metadata_tmp_path).remove(); + fs::remove(table_metadata_tmp_path); throw; } } @@ -321,10 +321,10 @@ void DatabaseOnDisk::detachTablePermanently(ContextPtr, const String & table_nam { auto table = detachTable(table_name); - Poco::File detached_permanently_flag(getObjectMetadataPath(table_name) + detached_suffix); + fs::path detached_permanently_flag(getObjectMetadataPath(table_name) + detached_suffix); try { - detached_permanently_flag.createFile(); + FS::createFile(detached_permanently_flag); } catch (Exception & e) { @@ -350,25 +350,25 @@ void DatabaseOnDisk::dropTable(ContextPtr local_context, const String & table_na bool renamed = false; try { - Poco::File(table_metadata_path).renameTo(table_metadata_path_drop); + fs::rename(table_metadata_path, table_metadata_path_drop); renamed = true; table->drop(); table->is_dropped = true; - Poco::File table_data_dir{local_context->getPath() + table_data_path_relative}; - if (table_data_dir.exists()) - table_data_dir.remove(true); + fs::path table_data_dir(local_context->getPath() + table_data_path_relative); + if (fs::exists(table_data_dir)) + fs::remove_all(table_data_dir); } catch (...) { LOG_WARNING(log, getCurrentExceptionMessage(__PRETTY_FUNCTION__)); attachTable(table_name, table, table_data_path_relative); if (renamed) - Poco::File(table_metadata_path_drop).renameTo(table_metadata_path); + fs::rename(table_metadata_path_drop, table_metadata_path); throw; } - Poco::File(table_metadata_path_drop).remove(); + fs::remove(table_metadata_path_drop); } void DatabaseOnDisk::checkMetadataFilenameAvailability(const String & to_table_name) const @@ -381,11 +381,11 @@ void DatabaseOnDisk::checkMetadataFilenameAvailabilityUnlocked(const String & to { String table_metadata_path = getObjectMetadataPath(to_table_name); - if (Poco::File(table_metadata_path).exists()) + if (fs::exists(table_metadata_path)) { - auto detached_permanently_flag = Poco::File(table_metadata_path + detached_suffix); + fs::path detached_permanently_flag(table_metadata_path + detached_suffix); - if (detached_permanently_flag.exists()) + if (fs::exists(detached_permanently_flag)) throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Table {}.{} already exists (detached permanently)", backQuote(database_name), backQuote(to_table_name)); else throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Table {}.{} already exists (detached)", backQuote(database_name), backQuote(to_table_name)); @@ -463,7 +463,7 @@ void DatabaseOnDisk::renameTable( /// Now table data are moved to new database, so we must add metadata and attach table to new database to_database.createTable(local_context, to_table_name, table, attach_query); - Poco::File(table_metadata_path).remove(); + fs::remove(table_metadata_path); if (from_atomic_to_ordinary) { @@ -528,8 +528,8 @@ ASTPtr DatabaseOnDisk::getCreateDatabaseQuery() const void DatabaseOnDisk::drop(ContextPtr local_context) { assert(tables.empty()); - Poco::File(local_context->getPath() + getDataPath()).remove(false); - Poco::File(getMetadataPath()).remove(false); + fs::remove(local_context->getPath() + getDataPath()); + fs::remove(getMetadataPath()); } String DatabaseOnDisk::getObjectMetadataPath(const String & object_name) const @@ -540,10 +540,9 @@ String DatabaseOnDisk::getObjectMetadataPath(const String & object_name) const time_t DatabaseOnDisk::getObjectMetadataModificationTime(const String & object_name) const { String table_metadata_path = getObjectMetadataPath(object_name); - Poco::File meta_file(table_metadata_path); - if (meta_file.exists()) - return meta_file.getLastModified().epochTime(); + if (fs::exists(table_metadata_path)) + return FS::getModificationTime(table_metadata_path); else return static_cast(0); } @@ -555,56 +554,57 @@ void DatabaseOnDisk::iterateMetadataFiles(ContextPtr local_context, const Iterat assert(getUUID() == UUIDHelpers::Nil); static const char * tmp_drop_ext = ".sql.tmp_drop"; const std::string object_name = file_name.substr(0, file_name.size() - strlen(tmp_drop_ext)); - if (Poco::File(local_context->getPath() + getDataPath() + '/' + object_name).exists()) + + if (fs::exists(local_context->getPath() + getDataPath() + '/' + object_name)) { - Poco::File(getMetadataPath() + file_name).renameTo(getMetadataPath() + object_name + ".sql"); + fs::rename(getMetadataPath() + file_name, getMetadataPath() + object_name + ".sql"); LOG_WARNING(log, "Object {} was not dropped previously and will be restored", backQuote(object_name)); process_metadata_file(object_name + ".sql"); } else { LOG_INFO(log, "Removing file {}", getMetadataPath() + file_name); - Poco::File(getMetadataPath() + file_name).remove(); + fs::remove(getMetadataPath() + file_name); } }; /// Metadata files to load: name and flag for .tmp_drop files std::set> metadata_files; - Poco::DirectoryIterator dir_end; - for (Poco::DirectoryIterator dir_it(getMetadataPath()); dir_it != dir_end; ++dir_it) + fs::directory_iterator dir_end; + for (fs::directory_iterator dir_it(getMetadataPath()); dir_it != dir_end; ++dir_it) { + String file_name = dir_it->path().filename(); /// For '.svn', '.gitignore' directory and similar. - if (dir_it.name().at(0) == '.') + if (file_name.at(0) == '.') continue; /// There are .sql.bak files - skip them. - if (endsWith(dir_it.name(), ".sql.bak")) + if (endsWith(file_name, ".sql.bak")) continue; /// Permanently detached table flag - if (endsWith(dir_it.name(), ".sql.detached")) + if (endsWith(file_name, ".sql.detached")) continue; - if (endsWith(dir_it.name(), ".sql.tmp_drop")) + if (endsWith(file_name, ".sql.tmp_drop")) { /// There are files that we tried to delete previously - metadata_files.emplace(dir_it.name(), false); + metadata_files.emplace(file_name, false); } - else if (endsWith(dir_it.name(), ".sql.tmp")) + else if (endsWith(file_name, ".sql.tmp")) { /// There are files .sql.tmp - delete - LOG_INFO(log, "Removing file {}", dir_it->path()); - Poco::File(dir_it->path()).remove(); + LOG_INFO(log, "Removing file {}", dir_it->path().string()); + fs::remove(dir_it->path()); } - else if (endsWith(dir_it.name(), ".sql")) + else if (endsWith(file_name, ".sql")) { /// The required files have names like `table_name.sql` - metadata_files.emplace(dir_it.name(), true); + metadata_files.emplace(file_name, true); } else - throw Exception("Incorrect file extension: " + dir_it.name() + " in metadata directory " + getMetadataPath(), - ErrorCodes::INCORRECT_FILE_NAME); + throw Exception(ErrorCodes::INCORRECT_FILE_NAME, "Incorrect file extension: {} in metadata directory {}", file_name, getMetadataPath()); } /// Read and parse metadata in parallel @@ -651,7 +651,7 @@ ASTPtr DatabaseOnDisk::parseQueryFromMetadata( { if (logger) LOG_ERROR(logger, "File {} is empty. Removing.", metadata_file_path); - Poco::File(metadata_file_path).remove(); + fs::remove(metadata_file_path); return nullptr; } @@ -670,8 +670,7 @@ ASTPtr DatabaseOnDisk::parseQueryFromMetadata( auto & create = ast->as(); if (!create.table.empty() && create.uuid != UUIDHelpers::Nil) { - String table_name = Poco::Path(metadata_file_path).makeFile().getBaseName(); - table_name = unescapeForFileName(table_name); + String table_name = unescapeForFileName(fs::path(metadata_file_path).stem()); if (create.table != TABLE_WITH_UUID_NAME_PLACEHOLDER && logger) LOG_WARNING( diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index eccee68da40..7758841afc2 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include @@ -111,8 +110,7 @@ void DatabaseOrdinary::loadStoredObjects(ContextMutablePtr local_context, bool h auto * create_query = ast->as(); create_query->database = database_name; - auto detached_permanently_flag = Poco::File(full_path.string() + detached_suffix); - if (detached_permanently_flag.exists()) + if (fs::exists(full_path.string() + detached_suffix)) { /// FIXME: even if we don't load the table we can still mark the uuid of it as taken. /// if (create_query->uuid != UUIDHelpers::Nil) @@ -281,11 +279,11 @@ void DatabaseOrdinary::commitAlterTable(const StorageID &, const String & table_ try { /// rename atomically replaces the old file with the new one. - Poco::File(table_metadata_tmp_path).renameTo(table_metadata_path); + fs::rename(table_metadata_tmp_path, table_metadata_path); } catch (...) { - Poco::File(table_metadata_tmp_path).remove(); + fs::remove(table_metadata_tmp_path); throw; } } diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 8d27595338a..5d3c156f665 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -212,7 +212,7 @@ void DatabaseReplicated::tryConnectToZooKeeperAndInitDatabase(bool force_attach) createDatabaseNodesInZooKeeper(current_zookeeper); } - replica_path = zookeeper_path + "/replicas/" + getFullReplicaName(); + replica_path = fs::path(zookeeper_path) / "replicas" / getFullReplicaName(); String replica_host_id; if (current_zookeeper->tryGet(replica_path, replica_host_id)) diff --git a/src/Databases/DatabaseReplicatedWorker.cpp b/src/Databases/DatabaseReplicatedWorker.cpp index 9ae4d026bf0..760300d6750 100644 --- a/src/Databases/DatabaseReplicatedWorker.cpp +++ b/src/Databases/DatabaseReplicatedWorker.cpp @@ -1,6 +1,9 @@ #include #include #include +#include + +namespace fs = std::filesystem; namespace DB { @@ -156,7 +159,7 @@ DDLTaskPtr DatabaseReplicatedDDLWorker::initAndCheckTask(const String & entry_na } } - UInt32 our_log_ptr = parse(current_zookeeper->get(database->replica_path + "/log_ptr")); + UInt32 our_log_ptr = parse(current_zookeeper->get(fs::path(database->replica_path) / "log_ptr")); UInt32 entry_num = DatabaseReplicatedTask::getLogEntryNumber(entry_name); if (entry_num <= our_log_ptr) @@ -165,13 +168,13 @@ DDLTaskPtr DatabaseReplicatedDDLWorker::initAndCheckTask(const String & entry_na return {}; } - String entry_path = queue_dir + "/" + entry_name; + String entry_path = fs::path(queue_dir) / entry_name; auto task = std::make_unique(entry_name, entry_path, database); String initiator_name; zkutil::EventPtr wait_committed_or_failed = std::make_shared(); - String try_node_path = entry_path + "/try"; + String try_node_path = fs::path(entry_path) / "try"; if (zookeeper->tryGet(try_node_path, initiator_name, nullptr, wait_committed_or_failed)) { task->is_initial_query = initiator_name == task->host_id_str; @@ -203,7 +206,7 @@ DDLTaskPtr DatabaseReplicatedDDLWorker::initAndCheckTask(const String & entry_na if (code != Coordination::Error::ZOK && code != Coordination::Error::ZNONODE) throw Coordination::Exception(code, try_node_path); - if (!zookeeper->exists(entry_path + "/committed")) + if (!zookeeper->exists(fs::path(entry_path) / "committed")) { out_reason = fmt::format("Entry {} was forcefully cancelled due to timeout", entry_name); return {}; @@ -212,7 +215,7 @@ DDLTaskPtr DatabaseReplicatedDDLWorker::initAndCheckTask(const String & entry_na } } - if (!zookeeper->exists(entry_path + "/committed")) + if (!zookeeper->exists(fs::path(entry_path) / "committed")) { out_reason = fmt::format("Entry {} hasn't been committed", entry_name); return {}; @@ -220,8 +223,8 @@ DDLTaskPtr DatabaseReplicatedDDLWorker::initAndCheckTask(const String & entry_na if (task->is_initial_query) { - assert(!zookeeper->exists(entry_path + "/try")); - assert(zookeeper->exists(entry_path + "/committed") == (zookeeper->get(task->getFinishedNodePath()) == ExecutionStatus(0).serializeText())); + assert(!zookeeper->exists(fs::path(entry_path) / "try")); + assert(zookeeper->exists(fs::path(entry_path) / "committed") == (zookeeper->get(task->getFinishedNodePath()) == ExecutionStatus(0).serializeText())); out_reason = fmt::format("Entry {} has been executed as initial query", entry_name); return {}; } @@ -257,7 +260,7 @@ DDLTaskPtr DatabaseReplicatedDDLWorker::initAndCheckTask(const String & entry_na bool DatabaseReplicatedDDLWorker::canRemoveQueueEntry(const String & entry_name, const Coordination::Stat &) { UInt32 entry_number = DDLTaskBase::getLogEntryNumber(entry_name); - UInt32 max_log_ptr = parse(getAndSetZooKeeper()->get(database->zookeeper_path + "/max_log_ptr")); + UInt32 max_log_ptr = parse(getAndSetZooKeeper()->get(fs::path(database->zookeeper_path) / "max_log_ptr")); return entry_number + logs_to_keep < max_log_ptr; } diff --git a/src/Databases/MySQL/DatabaseConnectionMySQL.cpp b/src/Databases/MySQL/DatabaseConnectionMySQL.cpp index 50b1c77c87e..866b03e3bd6 100644 --- a/src/Databases/MySQL/DatabaseConnectionMySQL.cpp +++ b/src/Databases/MySQL/DatabaseConnectionMySQL.cpp @@ -24,10 +24,10 @@ # include # include # include +# include +# include -# include -# include - +namespace fs = std::filesystem; namespace DB { @@ -326,7 +326,7 @@ void DatabaseConnectionMySQL::shutdown() void DatabaseConnectionMySQL::drop(ContextPtr /*context*/) { - Poco::File(getMetadataPath()).remove(true); + fs::remove_all(getMetadataPath()); } void DatabaseConnectionMySQL::cleanOutdatedTables() @@ -372,10 +372,10 @@ void DatabaseConnectionMySQL::attachTable(const String & table_name, const Stora local_tables_cache[table_name].second = storage; remove_or_detach_tables.erase(table_name); - Poco::File remove_flag(getMetadataPath() + '/' + escapeForFileName(table_name) + suffix); + fs::path remove_flag = fs::path(getMetadataPath()) / (escapeForFileName(table_name) + suffix); - if (remove_flag.exists()) - remove_flag.remove(); + if (fs::exists(remove_flag)) + fs::remove(remove_flag); } StoragePtr DatabaseConnectionMySQL::detachTable(const String & table_name) @@ -403,13 +403,13 @@ void DatabaseConnectionMySQL::loadStoredObjects(ContextMutablePtr, bool, bool /* { std::lock_guard lock{mutex}; - Poco::DirectoryIterator iterator(getMetadataPath()); + fs::directory_iterator iter(getMetadataPath()); - for (Poco::DirectoryIterator end; iterator != end; ++iterator) + for (fs::directory_iterator end; iter != end; ++iter) { - if (iterator->isFile() && endsWith(iterator.name(), suffix)) + if (fs::is_regular_file(iter->path()) && endsWith(iter->path().filename(), suffix)) { - const auto & filename = iterator.name(); + const auto & filename = iter->path().filename().string(); const auto & table_name = unescapeForFileName(filename.substr(0, filename.size() - strlen(suffix))); remove_or_detach_tables.emplace(table_name); } @@ -420,27 +420,25 @@ void DatabaseConnectionMySQL::detachTablePermanently(ContextPtr, const String & { std::lock_guard lock{mutex}; - Poco::File remove_flag(getMetadataPath() + '/' + escapeForFileName(table_name) + suffix); + fs::path remove_flag = fs::path(getMetadataPath()) / (escapeForFileName(table_name) + suffix); if (remove_or_detach_tables.count(table_name)) - throw Exception("Table " + backQuoteIfNeed(database_name) + "." + backQuoteIfNeed(table_name) + " is dropped", - ErrorCodes::TABLE_IS_DROPPED); + throw Exception(ErrorCodes::TABLE_IS_DROPPED, "Table {}.{} is dropped", backQuoteIfNeed(database_name), backQuoteIfNeed(table_name)); - if (remove_flag.exists()) - throw Exception("The remove flag file already exists but the " + backQuoteIfNeed(database_name) + - "." + backQuoteIfNeed(table_name) + " does not exists remove tables, it is bug.", ErrorCodes::LOGICAL_ERROR); + if (fs::exists(remove_flag)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "The remove flag file already exists but the {}.{} does not exists remove tables, it is bug.", + backQuoteIfNeed(database_name), backQuoteIfNeed(table_name)); auto table_iter = local_tables_cache.find(table_name); if (table_iter == local_tables_cache.end()) - throw Exception("Table " + backQuoteIfNeed(database_name) + "." + backQuoteIfNeed(table_name) + " doesn't exist.", - ErrorCodes::UNKNOWN_TABLE); + throw Exception(ErrorCodes::UNKNOWN_TABLE, "Table {}.{} doesn't exist", backQuoteIfNeed(database_name), backQuoteIfNeed(table_name)); remove_or_detach_tables.emplace(table_name); try { table_iter->second.second->drop(); - remove_flag.createFile(); + FS::createFile(remove_flag); } catch (...) { diff --git a/src/Databases/MySQL/DatabaseMaterializeMySQL.cpp b/src/Databases/MySQL/DatabaseMaterializeMySQL.cpp index 6e82f88c574..bf9d6cdfbfa 100644 --- a/src/Databases/MySQL/DatabaseMaterializeMySQL.cpp +++ b/src/Databases/MySQL/DatabaseMaterializeMySQL.cpp @@ -13,9 +13,11 @@ # include # include # include -# include # include # include +# include + +namespace fs = std::filesystem; namespace DB { @@ -158,10 +160,10 @@ template void DatabaseMaterializeMySQL::drop(ContextPtr context_) { /// Remove metadata info - Poco::File metadata(Base::getMetadataPath() + "/.metadata"); + fs::path metadata(Base::getMetadataPath() + "/.metadata"); - if (metadata.exists()) - metadata.remove(false); + if (fs::exists(metadata)) + fs::remove(metadata); Base::drop(context_); } diff --git a/src/Databases/MySQL/MaterializeMetadata.cpp b/src/Databases/MySQL/MaterializeMetadata.cpp index c389ab5a1b0..4fd99ca3438 100644 --- a/src/Databases/MySQL/MaterializeMetadata.cpp +++ b/src/Databases/MySQL/MaterializeMetadata.cpp @@ -8,11 +8,13 @@ #include #include #include -#include #include #include #include #include +#include + +namespace fs = std::filesystem; namespace DB { @@ -193,12 +195,11 @@ void commitMetadata(const std::function & function, const String & persi try { function(); - - Poco::File(persistent_tmp_path).renameTo(persistent_path); + fs::rename(persistent_tmp_path, persistent_path); } catch (...) { - Poco::File(persistent_tmp_path).remove(); + fs::remove(persistent_tmp_path); throw; } } @@ -231,7 +232,7 @@ void MaterializeMetadata::transaction(const MySQLReplication::Position & positio MaterializeMetadata::MaterializeMetadata(const String & path_, const Settings & settings_) : persistent_path(path_), settings(settings_) { - if (Poco::File(persistent_path).exists()) + if (fs::exists(persistent_path)) { ReadBufferFromFile in(persistent_path, DBMS_DEFAULT_BUFFER_SIZE); assertString("Version:\t" + toString(meta_version), in); diff --git a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp index 1920c047b8e..243ab9166be 100644 --- a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp +++ b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp @@ -12,11 +12,12 @@ #include #include #include -#include -#include #include #include +#include +#include +namespace fs = std::filesystem; namespace DB { @@ -209,9 +210,9 @@ void DatabasePostgreSQL::attachTable(const String & table_name, const StoragePtr detached_or_dropped.erase(table_name); - Poco::File table_marked_as_removed(getMetadataPath() + '/' + escapeForFileName(table_name) + suffix); - if (table_marked_as_removed.exists()) - table_marked_as_removed.remove(); + fs::path table_marked_as_removed = fs::path(getMetadataPath()) / (escapeForFileName(table_name) + suffix); + if (fs::exists(table_marked_as_removed)) + fs::remove(table_marked_as_removed); } @@ -256,16 +257,8 @@ void DatabasePostgreSQL::dropTable(ContextPtr, const String & table_name, bool / if (detached_or_dropped.count(table_name)) throw Exception(fmt::format("Table {}.{} is already dropped/detached", database_name, table_name), ErrorCodes::TABLE_IS_DROPPED); - Poco::File mark_table_removed(getMetadataPath() + '/' + escapeForFileName(table_name) + suffix); - - try - { - mark_table_removed.createFile(); - } - catch (...) - { - throw; - } + fs::path mark_table_removed = fs::path(getMetadataPath()) / (escapeForFileName(table_name) + suffix); + FS::createFile(mark_table_removed); if (cache_tables) cached_tables.erase(table_name); @@ -276,7 +269,7 @@ void DatabasePostgreSQL::dropTable(ContextPtr, const String & table_name, bool / void DatabasePostgreSQL::drop(ContextPtr /*context*/) { - Poco::File(getMetadataPath()).remove(true); + fs::remove_all(getMetadataPath()); } @@ -284,14 +277,14 @@ void DatabasePostgreSQL::loadStoredObjects(ContextMutablePtr /* context */, bool { { std::lock_guard lock{mutex}; - Poco::DirectoryIterator iterator(getMetadataPath()); + fs::directory_iterator iter(getMetadataPath()); /// Check for previously dropped tables - for (Poco::DirectoryIterator end; iterator != end; ++iterator) + for (fs::directory_iterator end; iter != end; ++iter) { - if (iterator->isFile() && endsWith(iterator.name(), suffix)) + if (fs::is_regular_file(iter->path()) && endsWith(iter->path().filename(), suffix)) { - const auto & file_name = iterator.name(); + const auto & file_name = iter->path().filename().string(); const auto & table_name = unescapeForFileName(file_name.substr(0, file_name.size() - strlen(suffix))); detached_or_dropped.emplace(table_name); } @@ -325,9 +318,9 @@ void DatabasePostgreSQL::removeOutdatedTables() { auto table_name = *iter; iter = detached_or_dropped.erase(iter); - Poco::File table_marked_as_removed(getMetadataPath() + '/' + escapeForFileName(table_name) + suffix); - if (table_marked_as_removed.exists()) - table_marked_as_removed.remove(); + fs::path table_marked_as_removed = fs::path(getMetadataPath()) / (escapeForFileName(table_name) + suffix); + if (fs::exists(table_marked_as_removed)) + fs::remove(table_marked_as_removed); } else ++iter; diff --git a/src/Dictionaries/DictionarySourceHelpers.h b/src/Dictionaries/DictionarySourceHelpers.h index 2f39f951f8d..1066574f8e9 100644 --- a/src/Dictionaries/DictionarySourceHelpers.h +++ b/src/Dictionaries/DictionarySourceHelpers.h @@ -3,14 +3,13 @@ #include #include - -#include #include #include #include #include #include + namespace DB { diff --git a/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.cpp b/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.cpp index 3b8848ab19b..210459da0be 100644 --- a/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.cpp +++ b/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.cpp @@ -5,7 +5,9 @@ #include #include #include "HierarchyFormatReader.h" +#include +namespace fs = std::filesystem; bool RegionsHierarchyDataSource::isModified() const { @@ -27,14 +29,13 @@ RegionsHierarchiesDataProvider::RegionsHierarchiesDataProvider(const std::string void RegionsHierarchiesDataProvider::discoverFilesWithCustomHierarchies() { - std::string basename = Poco::Path(path).getBaseName(); + std::string basename = fs::path(path).stem(); + fs::path dir_path = fs::canonical(path).parent_path(); - Poco::Path dir_path = Poco::Path(path).absolute().parent(); - - Poco::DirectoryIterator dir_end; - for (Poco::DirectoryIterator dir_it(dir_path); dir_it != dir_end; ++dir_it) + fs::directory_iterator dir_end; + for (fs::directory_iterator dir_it(dir_path); dir_it != dir_end; ++dir_it) { - std::string candidate_basename = dir_it.path().getBaseName(); + std::string candidate_basename = dir_it->path().stem(); if (candidate_basename.starts_with(basename) && (candidate_basename.size() > basename.size() + 1) diff --git a/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.cpp b/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.cpp index f3e49545481..5f79fda070f 100644 --- a/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.cpp +++ b/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.cpp @@ -2,7 +2,9 @@ #include #include "NamesFormatReader.h" +#include +namespace fs = std::filesystem; bool LanguageRegionsNamesDataSource::isModified() const { @@ -11,7 +13,7 @@ bool LanguageRegionsNamesDataSource::isModified() const size_t LanguageRegionsNamesDataSource::estimateTotalSize() const { - return Poco::File(path).getSize(); + return fs::file_size(path); } ILanguageRegionsNamesReaderPtr LanguageRegionsNamesDataSource::createReader() @@ -39,7 +41,7 @@ RegionsNamesDataProvider::RegionsNamesDataProvider(const std::string & directory ILanguageRegionsNamesDataSourcePtr RegionsNamesDataProvider::getLanguageRegionsNamesSource(const std::string & language) const { const auto data_file = getDataFilePath(language); - if (Poco::File(data_file).exists()) + if (fs::exists(data_file)) return std::make_unique(data_file, language); else return {}; diff --git a/src/Dictionaries/FileDictionarySource.cpp b/src/Dictionaries/FileDictionarySource.cpp index cf722aba68f..49e87e24543 100644 --- a/src/Dictionaries/FileDictionarySource.cpp +++ b/src/Dictionaries/FileDictionarySource.cpp @@ -1,9 +1,4 @@ #include "FileDictionarySource.h" - -#include - -#include - #include #include #include @@ -15,6 +10,7 @@ #include "registerDictionaries.h" #include "DictionarySourceHelpers.h" + namespace DB { static const UInt64 max_block_size = 8192; @@ -68,9 +64,10 @@ std::string FileDictionarySource::toString() const Poco::Timestamp FileDictionarySource::getLastModification() const { - return Poco::File{filepath}.getLastModified(); + return FS::getModificationTimestamp(filepath); } + void registerDictionarySourceFile(DictionarySourceFactory & factory) { auto create_table_source = [=](const DictionaryStructure & dict_struct, diff --git a/src/Dictionaries/LibraryDictionarySource.cpp b/src/Dictionaries/LibraryDictionarySource.cpp index 3fa61408ba1..f52d774f279 100644 --- a/src/Dictionaries/LibraryDictionarySource.cpp +++ b/src/Dictionaries/LibraryDictionarySource.cpp @@ -1,19 +1,19 @@ #include "LibraryDictionarySource.h" -#include - +#include +#include #include #include #include #include -#include -#include +#include #include #include #include #include +namespace fs = std::filesystem; namespace DB { @@ -44,8 +44,8 @@ LibraryDictionarySource::LibraryDictionarySource( if (created_from_ddl && !pathStartsWith(path, context->getDictionariesLibPath())) throw Exception(ErrorCodes::PATH_ACCESS_DENIED, "File path {} is not inside {}", path, context->getDictionariesLibPath()); - if (!Poco::File(path).exists()) - throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "LibraryDictionarySource: Can't load library {}: file doesn't exist", Poco::File(path).path()); + if (!fs::exists(path)) + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "LibraryDictionarySource: Can't load library {}: file doesn't exist", path); description.init(sample_block); bridge_helper = std::make_shared(context, description.sample_block, dictionary_id); diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp index d0cf6a00344..768d1c2354a 100644 --- a/src/Disks/DiskLocal.cpp +++ b/src/Disks/DiskLocal.cpp @@ -6,9 +6,9 @@ #include #include #include - #include +#include #include @@ -60,27 +60,28 @@ class DiskLocalDirectoryIterator : public IDiskDirectoryIterator { public: explicit DiskLocalDirectoryIterator(const String & disk_path_, const String & dir_path_) - : dir_path(dir_path_), iter(disk_path_ + dir_path_) + : dir_path(dir_path_), entry(fs::path(disk_path_) / dir_path_) { } - void next() override { ++iter; } + void next() override { ++entry; } - bool isValid() const override { return iter != Poco::DirectoryIterator(); } + bool isValid() const override { return entry != fs::directory_iterator(); } String path() const override { - if (iter->isDirectory()) - return dir_path + iter.name() + '/'; + if (entry->is_directory()) + return dir_path / entry->path().filename() / ""; else - return dir_path + iter.name(); + return dir_path / entry->path().filename(); } - String name() const override { return iter.name(); } + + String name() const override { return entry->path().filename(); } private: - String dir_path; - Poco::DirectoryIterator iter; + fs::path dir_path; + fs::directory_iterator entry; }; @@ -118,7 +119,7 @@ UInt64 DiskLocal::getTotalSpace() const { struct statvfs fs; if (name == "default") /// for default disk we get space from path/data/ - fs = getStatVFS(disk_path + "data/"); + fs = getStatVFS((fs::path(disk_path) / "data/").string()); else fs = getStatVFS(disk_path); UInt64 total_size = fs.f_blocks * fs.f_bsize; @@ -133,7 +134,7 @@ UInt64 DiskLocal::getAvailableSpace() const /// available for superuser only and for system purposes struct statvfs fs; if (name == "default") /// for default disk we get space from path/data/ - fs = getStatVFS(disk_path + "data/"); + fs = getStatVFS((fs::path(disk_path) / "data/").string()); else fs = getStatVFS(disk_path); UInt64 total_size = fs.f_bavail * fs.f_bsize; @@ -152,45 +153,43 @@ UInt64 DiskLocal::getUnreservedSpace() const bool DiskLocal::exists(const String & path) const { - return Poco::File(disk_path + path).exists(); + return fs::exists(fs::path(disk_path) / path); } bool DiskLocal::isFile(const String & path) const { - return Poco::File(disk_path + path).isFile(); + return fs::is_regular_file(fs::path(disk_path) / path); } bool DiskLocal::isDirectory(const String & path) const { - return Poco::File(disk_path + path).isDirectory(); + return fs::is_directory(fs::path(disk_path) / path); } size_t DiskLocal::getFileSize(const String & path) const { - return Poco::File(disk_path + path).getSize(); + return fs::file_size(fs::path(disk_path) / path); } void DiskLocal::createDirectory(const String & path) { - Poco::File(disk_path + path).createDirectory(); + fs::create_directory(fs::path(disk_path) / path); } void DiskLocal::createDirectories(const String & path) { - Poco::File(disk_path + path).createDirectories(); + fs::create_directories(fs::path(disk_path) / path); } void DiskLocal::clearDirectory(const String & path) { - std::vector files; - Poco::File(disk_path + path).list(files); - for (auto & file : files) - file.remove(); + for (const auto & entry : fs::directory_iterator(fs::path(disk_path) / path)) + fs::remove(entry.path()); } void DiskLocal::moveDirectory(const String & from_path, const String & to_path) { - Poco::File(disk_path + from_path).renameTo(disk_path + to_path); + fs::rename(fs::path(disk_path) / from_path, fs::path(disk_path) / to_path); } DiskDirectoryIteratorPtr DiskLocal::iterateDirectory(const String & path) @@ -200,99 +199,95 @@ DiskDirectoryIteratorPtr DiskLocal::iterateDirectory(const String & path) void DiskLocal::moveFile(const String & from_path, const String & to_path) { - Poco::File(disk_path + from_path).renameTo(disk_path + to_path); + fs::rename(fs::path(disk_path) / from_path, fs::path(disk_path) / to_path); } void DiskLocal::replaceFile(const String & from_path, const String & to_path) { - Poco::File from_file(disk_path + from_path); - Poco::File to_file(disk_path + to_path); - if (to_file.exists()) - { - Poco::File tmp_file(disk_path + to_path + ".old"); - to_file.renameTo(tmp_file.path()); - from_file.renameTo(disk_path + to_path); - tmp_file.remove(); - } - else - from_file.renameTo(to_file.path()); + fs::path from_file = fs::path(disk_path) / from_path; + fs::path to_file = fs::path(disk_path) / to_path; + fs::rename(from_file, to_file); } std::unique_ptr DiskLocal::readFile( const String & path, size_t buf_size, size_t estimated_size, size_t aio_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache) const { - return createReadBufferFromFileBase(disk_path + path, estimated_size, aio_threshold, mmap_threshold, mmap_cache, buf_size); + return createReadBufferFromFileBase(fs::path(disk_path) / path, estimated_size, aio_threshold, mmap_threshold, mmap_cache, buf_size); } std::unique_ptr DiskLocal::writeFile(const String & path, size_t buf_size, WriteMode mode) { int flags = (mode == WriteMode::Append) ? (O_APPEND | O_CREAT | O_WRONLY) : -1; - return std::make_unique(disk_path + path, buf_size, flags); + return std::make_unique(fs::path(disk_path) / path, buf_size, flags); } void DiskLocal::removeFile(const String & path) { - auto fs_path = disk_path + path; + auto fs_path = fs::path(disk_path) / path; if (0 != unlink(fs_path.c_str())) - throwFromErrnoWithPath("Cannot unlink file " + fs_path, fs_path, ErrorCodes::CANNOT_UNLINK); + throwFromErrnoWithPath("Cannot unlink file " + fs_path.string(), fs_path, ErrorCodes::CANNOT_UNLINK); } void DiskLocal::removeFileIfExists(const String & path) { - auto fs_path = disk_path + path; + auto fs_path = fs::path(disk_path) / path; if (0 != unlink(fs_path.c_str()) && errno != ENOENT) - throwFromErrnoWithPath("Cannot unlink file " + fs_path, fs_path, ErrorCodes::CANNOT_UNLINK); + throwFromErrnoWithPath("Cannot unlink file " + fs_path.string(), fs_path, ErrorCodes::CANNOT_UNLINK); } void DiskLocal::removeDirectory(const String & path) { - auto fs_path = disk_path + path; + auto fs_path = fs::path(disk_path) / path; if (0 != rmdir(fs_path.c_str())) - throwFromErrnoWithPath("Cannot rmdir " + fs_path, fs_path, ErrorCodes::CANNOT_RMDIR); + throwFromErrnoWithPath("Cannot rmdir " + fs_path.string(), fs_path, ErrorCodes::CANNOT_RMDIR); } void DiskLocal::removeRecursive(const String & path) { - Poco::File(disk_path + path).remove(true); + fs::remove_all(fs::path(disk_path) / path); } void DiskLocal::listFiles(const String & path, std::vector & file_names) { - Poco::File(disk_path + path).list(file_names); + file_names.clear(); + for (const auto & entry : fs::directory_iterator(fs::path(disk_path) / path)) + file_names.emplace_back(entry.path().filename()); } void DiskLocal::setLastModified(const String & path, const Poco::Timestamp & timestamp) { - Poco::File(disk_path + path).setLastModified(timestamp); + FS::setModificationTime(fs::path(disk_path) / path, timestamp.epochTime()); } Poco::Timestamp DiskLocal::getLastModified(const String & path) { - return Poco::File(disk_path + path).getLastModified(); + return FS::getModificationTimestamp(fs::path(disk_path) / path); } void DiskLocal::createHardLink(const String & src_path, const String & dst_path) { - DB::createHardLink(disk_path + src_path, disk_path + dst_path); + DB::createHardLink(fs::path(disk_path) / src_path, fs::path(disk_path) / dst_path); } void DiskLocal::truncateFile(const String & path, size_t size) { - int res = truncate((disk_path + path).c_str(), size); + int res = truncate((fs::path(disk_path) / path).string().data(), size); if (-1 == res) throwFromErrnoWithPath("Cannot truncate file " + path, path, ErrorCodes::CANNOT_TRUNCATE_FILE); } void DiskLocal::createFile(const String & path) { - Poco::File(disk_path + path).createFile(); + FS::createFile(fs::path(disk_path) / path); } void DiskLocal::setReadOnly(const String & path) { - Poco::File(disk_path + path).setReadOnly(true); + fs::permissions(fs::path(disk_path) / path, + fs::perms::owner_write | fs::perms::group_write | fs::perms::others_write, + fs::perm_options::remove); } bool inline isSameDiskType(const IDisk & one, const IDisk & another) @@ -303,14 +298,23 @@ bool inline isSameDiskType(const IDisk & one, const IDisk & another) void DiskLocal::copy(const String & from_path, const std::shared_ptr & to_disk, const String & to_path) { if (isSameDiskType(*this, *to_disk)) - Poco::File(disk_path + from_path).copyTo(to_disk->getPath() + to_path); /// Use more optimal way. + { + fs::path to = fs::path(to_disk->getPath()) / to_path; + fs::path from = fs::path(disk_path) / from_path; + if (from_path.ends_with('/')) + from = from.parent_path(); + if (fs::is_directory(from)) + to /= from.filename(); + + fs::copy(from, to, fs::copy_options::recursive | fs::copy_options::overwrite_existing); /// Use more optimal way. + } else IDisk::copy(from_path, to_disk, to_path); /// Copy files through buffers. } SyncGuardPtr DiskLocal::getDirectorySyncGuard(const String & path) const { - return std::make_unique(disk_path + path); + return std::make_unique(fs::path(disk_path) / path); } DiskPtr DiskLocalReservation::getDisk(size_t i) const @@ -381,10 +385,8 @@ void registerDiskLocal(DiskFactory & factory) throw Exception("Disk path must end with /. Disk " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); } - if (Poco::File disk{path}; !disk.canRead() || !disk.canWrite()) - { + if (!FS::canRead(path) || !FS::canWrite(path)) throw Exception("There is no RW access to the disk " + name + " (" + path + ")", ErrorCodes::PATH_ACCESS_DENIED); - } bool has_space_ratio = config.has(config_prefix + ".keep_free_space_ratio"); diff --git a/src/Disks/DiskLocal.h b/src/Disks/DiskLocal.h index 567ca24eb50..47482ad8d67 100644 --- a/src/Disks/DiskLocal.h +++ b/src/Disks/DiskLocal.h @@ -6,8 +6,6 @@ #include #include -#include -#include namespace DB { @@ -27,7 +25,7 @@ public: : name(name_), disk_path(path_), keep_free_space_bytes(keep_free_space_bytes_) { if (disk_path.back() != '/') - throw Exception("Disk path must ends with '/', but '" + disk_path + "' doesn't.", ErrorCodes::LOGICAL_ERROR); + throw Exception("Disk path must end with '/', but '" + disk_path + "' doesn't.", ErrorCodes::LOGICAL_ERROR); } const String & getName() const override { return name; } diff --git a/src/Disks/DiskMemory.cpp b/src/Disks/DiskMemory.cpp index 68257ec4948..423dba6bed6 100644 --- a/src/Disks/DiskMemory.cpp +++ b/src/Disks/DiskMemory.cpp @@ -6,7 +6,6 @@ #include #include #include -#include namespace DB @@ -24,7 +23,7 @@ namespace ErrorCodes class DiskMemoryDirectoryIterator final : public IDiskDirectoryIterator { public: - explicit DiskMemoryDirectoryIterator(std::vector && dir_file_paths_) + explicit DiskMemoryDirectoryIterator(std::vector && dir_file_paths_) : dir_file_paths(std::move(dir_file_paths_)), iter(dir_file_paths.begin()) { } @@ -33,13 +32,13 @@ public: bool isValid() const override { return iter != dir_file_paths.end(); } - String path() const override { return (*iter).toString(); } + String path() const override { return iter->string(); } - String name() const override { return (*iter).getFileName(); } + String name() const override { return iter->filename(); } private: - std::vector dir_file_paths; - std::vector::iterator iter; + std::vector dir_file_paths; + std::vector::iterator iter; }; @@ -268,7 +267,7 @@ DiskDirectoryIteratorPtr DiskMemory::iterateDirectory(const String & path) if (!path.empty() && files.find(path) == files.end()) throw Exception("Directory '" + path + "' does not exist", ErrorCodes::DIRECTORY_DOESNT_EXIST); - std::vector dir_file_paths; + std::vector dir_file_paths; for (const auto & file : files) if (parentPath(file.first) == path) dir_file_paths.emplace_back(file.first); diff --git a/src/Disks/HDFS/DiskHDFS.cpp b/src/Disks/HDFS/DiskHDFS.cpp index 0648fd9f08c..da6ccb024c7 100644 --- a/src/Disks/HDFS/DiskHDFS.cpp +++ b/src/Disks/HDFS/DiskHDFS.cpp @@ -172,8 +172,8 @@ void registerDiskHDFS(DiskFactory & factory) const String & config_prefix, ContextConstPtr context_) -> DiskPtr { - Poco::File disk{context_->getPath() + "disks/" + name}; - disk.createDirectories(); + fs::path disk = fs::path(context_->getPath()) / "disks" / name; + fs::create_directories(disk); String uri{config.getString(config_prefix + ".endpoint")}; diff --git a/src/Disks/IDisk.cpp b/src/Disks/IDisk.cpp index ee7f57af771..82705b5dcc8 100644 --- a/src/Disks/IDisk.cpp +++ b/src/Disks/IDisk.cpp @@ -22,7 +22,8 @@ bool IDisk::isDirectoryEmpty(const String & path) void copyFile(IDisk & from_disk, const String & from_path, IDisk & to_disk, const String & to_path) { - LOG_DEBUG(&Poco::Logger::get("IDisk"), "Copying from {} {} to {} {}.", from_disk.getName(), from_path, to_disk.getName(), to_path); + LOG_DEBUG(&Poco::Logger::get("IDisk"), "Copying from {} (path: {}) {} to {} (path: {}) {}.", + from_disk.getName(), from_disk.getPath(), from_path, to_disk.getName(), to_disk.getPath(), to_path); auto in = from_disk.readFile(from_path); auto out = to_disk.writeFile(to_path); @@ -41,16 +42,15 @@ void asyncCopy(IDisk & from_disk, String from_path, IDisk & to_disk, String to_p [&from_disk, from_path, &to_disk, to_path]() { setThreadName("DiskCopier"); - DB::copyFile(from_disk, from_path, to_disk, to_path + fileName(from_path)); + DB::copyFile(from_disk, from_path, to_disk, fs::path(to_path) / fileName(from_path)); }); results.push_back(std::move(result)); } else { - Poco::Path path(from_path); - const String & dir_name = path.directory(path.depth() - 1); - const String dest = to_path + dir_name + "/"; + fs::path dir_name = fs::path(from_path).parent_path().filename(); + fs::path dest(fs::path(to_path) / dir_name); to_disk.createDirectories(dest); for (auto it = from_disk.iterateDirectory(from_path); it->isValid(); it->next()) diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index a42a60959c5..cb718605dd6 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -7,16 +7,16 @@ #include #include #include -#include "Disks/Executor.h" #include #include #include #include -#include #include +#include #include "Poco/Util/AbstractConfiguration.h" +namespace fs = std::filesystem; namespace CurrentMetrics { @@ -212,10 +212,10 @@ public: virtual DiskType::Type getType() const = 0; /// Invoked when Global Context is shutdown. - virtual void shutdown() { } + virtual void shutdown() {} /// Performs action on disk startup. - virtual void startup() { } + virtual void startup() {} /// Return some uniq string for file, overrode for S3 /// Required for distinguish different copies of the same part on S3 @@ -233,7 +233,7 @@ public: virtual SyncGuardPtr getDirectorySyncGuard(const String & path) const; /// Applies new settings for disk in runtime. - virtual void applyNewSettings(const Poco::Util::AbstractConfiguration &, ContextConstPtr) { } + virtual void applyNewSettings(const Poco::Util::AbstractConfiguration &, ContextConstPtr) {} protected: friend class DiskDecorator; @@ -294,25 +294,27 @@ public: /// Return full path to a file on disk. inline String fullPath(const DiskPtr & disk, const String & path) { - return disk->getPath() + path; + return fs::path(disk->getPath()) / path; } /// Return parent path for the specified path. inline String parentPath(const String & path) { - return Poco::Path(path).parent().toString(); + if (path.ends_with('/')) + return fs::path(path).parent_path().parent_path() / ""; + return fs::path(path).parent_path() / ""; } /// Return file name for the specified path. inline String fileName(const String & path) { - return Poco::Path(path).getFileName(); + return fs::path(path).filename(); } /// Return directory path for the specified path. inline String directoryPath(const String & path) { - return Poco::Path(path).setFileName("").toString(); + return fs::path(path).parent_path() / ""; } } diff --git a/src/Disks/IDiskRemote.cpp b/src/Disks/IDiskRemote.cpp index bcb399f5d07..b30e9613ed8 100644 --- a/src/Disks/IDiskRemote.cpp +++ b/src/Disks/IDiskRemote.cpp @@ -6,12 +6,12 @@ #include #include #include -#include #include #include #include #include #include +#include namespace DB @@ -179,9 +179,9 @@ void IDiskRemote::removeMeta(const String & path, RemoteFSPathKeeperPtr fs_paths { LOG_DEBUG(log, "Remove file by path: {}", backQuote(metadata_path + path)); - Poco::File file(metadata_path + path); + fs::path file(metadata_path + path); - if (!file.isFile()) + if (!fs::is_regular_file(file)) throw Exception(ErrorCodes::CANNOT_DELETE_DIRECTORY, "Path '{}' is a directory", path); try @@ -191,7 +191,7 @@ void IDiskRemote::removeMeta(const String & path, RemoteFSPathKeeperPtr fs_paths /// If there is no references - delete content from remote FS. if (metadata.ref_count == 0) { - file.remove(); + fs::remove(file); for (const auto & [remote_fs_object_path, _] : metadata.remote_fs_objects) fs_paths_keeper->addPath(remote_fs_root_path + remote_fs_object_path); } @@ -199,7 +199,7 @@ void IDiskRemote::removeMeta(const String & path, RemoteFSPathKeeperPtr fs_paths { --metadata.ref_count; metadata.save(); - file.remove(); + fs::remove(file); } } catch (const Exception & e) @@ -210,7 +210,7 @@ void IDiskRemote::removeMeta(const String & path, RemoteFSPathKeeperPtr fs_paths LOG_WARNING(log, "Metadata file {} can't be read by reason: {}. Removing it forcibly.", backQuote(path), e.nested() ? e.nested()->message() : e.message()); - file.remove(); + fs::remove(file); } else throw; @@ -222,8 +222,8 @@ void IDiskRemote::removeMetaRecursive(const String & path, RemoteFSPathKeeperPtr { checkStackSize(); /// This is needed to prevent stack overflow in case of cyclic symlinks. - Poco::File file(metadata_path + path); - if (file.isFile()) + fs::path file = fs::path(metadata_path) / path; + if (fs::is_regular_file(file)) { removeMeta(path, fs_paths_keeper); } @@ -231,7 +231,7 @@ void IDiskRemote::removeMetaRecursive(const String & path, RemoteFSPathKeeperPtr { for (auto it{iterateDirectory(path)}; it->isValid(); it->next()) removeMetaRecursive(it->path(), fs_paths_keeper); - file.remove(); + fs::remove(file); } } @@ -296,13 +296,13 @@ IDiskRemote::IDiskRemote( bool IDiskRemote::exists(const String & path) const { - return Poco::File(metadata_path + path).exists(); + return fs::exists(fs::path(metadata_path) / path); } bool IDiskRemote::isFile(const String & path) const { - return Poco::File(metadata_path + path).isFile(); + return fs::is_regular_file(fs::path(metadata_path) / path); } @@ -326,7 +326,7 @@ void IDiskRemote::moveFile(const String & from_path, const String & to_path) if (exists(to_path)) throw Exception("File already exists: " + to_path, ErrorCodes::FILE_ALREADY_EXISTS); - Poco::File(metadata_path + from_path).renameTo(metadata_path + to_path); + fs::rename(fs::path(metadata_path) / from_path, fs::path(metadata_path) / to_path); } @@ -347,7 +347,7 @@ void IDiskRemote::replaceFile(const String & from_path, const String & to_path) void IDiskRemote::removeFileIfExists(const String & path) { RemoteFSPathKeeperPtr fs_paths_keeper = createFSPathKeeper(); - if (Poco::File(metadata_path + path).exists()) + if (fs::exists(fs::path(metadata_path) / path)) { removeMeta(path, fs_paths_keeper); removeFromRemoteFS(fs_paths_keeper); @@ -385,19 +385,19 @@ void IDiskRemote::setReadOnly(const String & path) bool IDiskRemote::isDirectory(const String & path) const { - return Poco::File(metadata_path + path).isDirectory(); + return fs::is_directory(fs::path(metadata_path) / path); } void IDiskRemote::createDirectory(const String & path) { - Poco::File(metadata_path + path).createDirectory(); + fs::create_directory(fs::path(metadata_path) / path); } void IDiskRemote::createDirectories(const String & path) { - Poco::File(metadata_path + path).createDirectories(); + fs::create_directories(fs::path(metadata_path) / path); } @@ -411,7 +411,7 @@ void IDiskRemote::clearDirectory(const String & path) void IDiskRemote::removeDirectory(const String & path) { - Poco::File(metadata_path + path).remove(); + fs::remove(fs::path(metadata_path) / path); } @@ -430,13 +430,13 @@ void IDiskRemote::listFiles(const String & path, std::vector & file_name void IDiskRemote::setLastModified(const String & path, const Poco::Timestamp & timestamp) { - Poco::File(metadata_path + path).setLastModified(timestamp); + FS::setModificationTime(fs::path(metadata_path) / path, timestamp.epochTime()); } Poco::Timestamp IDiskRemote::getLastModified(const String & path) { - return Poco::File(metadata_path + path).getLastModified(); + return FS::getModificationTimestamp(fs::path(metadata_path) / path); } diff --git a/src/Disks/IDiskRemote.h b/src/Disks/IDiskRemote.h index b32258331a7..a90621443e4 100644 --- a/src/Disks/IDiskRemote.h +++ b/src/Disks/IDiskRemote.h @@ -4,11 +4,12 @@ #include #include "Disks/DiskFactory.h" #include "Disks/Executor.h" -#include #include #include #include +#include +namespace fs = std::filesystem; namespace DB { @@ -193,21 +194,21 @@ public: void next() override { ++iter; } - bool isValid() const override { return iter != Poco::DirectoryIterator(); } + bool isValid() const override { return iter != fs::directory_iterator(); } String path() const override { - if (iter->isDirectory()) - return folder_path + iter.name() + '/'; + if (fs::is_directory(iter->path())) + return folder_path / iter->path().filename().string() / ""; else - return folder_path + iter.name(); + return folder_path / iter->path().filename().string(); } - String name() const override { return iter.name(); } + String name() const override { return iter->path().filename(); } private: - Poco::DirectoryIterator iter; - String folder_path; + fs::directory_iterator iter; + fs::path folder_path; }; diff --git a/src/Disks/ReadIndirectBufferFromRemoteFS.cpp b/src/Disks/ReadIndirectBufferFromRemoteFS.cpp index 955986e5259..6d4764e4392 100644 --- a/src/Disks/ReadIndirectBufferFromRemoteFS.cpp +++ b/src/Disks/ReadIndirectBufferFromRemoteFS.cpp @@ -21,6 +21,7 @@ ReadIndirectBufferFromRemoteFS::ReadIndirectBufferFromRemoteFS( { } + template off_t ReadIndirectBufferFromRemoteFS::seek(off_t offset_, int whence) { diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index a3f5fe89870..89c2d20db9f 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -215,7 +214,7 @@ void DiskS3::moveFile(const String & from_path, const String & to_path, bool sen createFileOperationObject("rename", revision, object_metadata); } - Poco::File(metadata_path + from_path).renameTo(metadata_path + to_path); + fs::rename(fs::path(metadata_path) / from_path, fs::path(metadata_path) / to_path); } std::unique_ptr DiskS3::readFile(const String & path, size_t buf_size, size_t, size_t, size_t, MMappedFileCache *) const @@ -675,8 +674,8 @@ void DiskS3::restore() restoreFiles(information); restoreFileOperations(information); - Poco::File restore_file(metadata_path + RESTORE_FILE_NAME); - restore_file.remove(); + fs::path restore_file = fs::path(metadata_path) / RESTORE_FILE_NAME; + fs::remove(restore_file); saveSchemaVersion(RESTORABLE_SCHEMA_VERSION); @@ -863,8 +862,9 @@ void DiskS3::restoreFileOperations(const RestoreInformation & restore_informatio continue; /// Skip not finished parts. They shouldn't be in 'detached' directory, because CH wouldn't be able to finish processing them. - Poco::Path directory_path (path); - auto directory_name = directory_path.directory(directory_path.depth() - 1); + fs::path directory_path(path); + auto directory_name = directory_path.parent_path().filename().string(); + auto predicate = [&directory_name](String & prefix) { return directory_name.starts_with(prefix); }; if (std::any_of(not_finished_prefixes.begin(), not_finished_prefixes.end(), predicate)) continue; @@ -873,7 +873,14 @@ void DiskS3::restoreFileOperations(const RestoreInformation & restore_informatio LOG_DEBUG(log, "Move directory to 'detached' {} -> {}", path, detached_path); - Poco::File(metadata_path + path).moveTo(metadata_path + detached_path); + fs::path from_path = fs::path(metadata_path) / path; + fs::path to_path = fs::path(metadata_path) / detached_path; + if (path.ends_with('/')) + to_path /= from_path.parent_path().filename(); + else + to_path /= from_path.filename(); + fs::copy(from_path, to_path, fs::copy_options::recursive | fs::copy_options::overwrite_existing); + fs::remove_all(from_path); } } @@ -905,7 +912,9 @@ String DiskS3::revisionToString(UInt64 revision) String DiskS3::pathToDetached(const String & source_path) { - return Poco::Path(source_path).parent().append(Poco::Path("detached")).toString() + '/'; + if (source_path.ends_with('/')) + return fs::path(source_path).parent_path().parent_path() / "detached/"; + return fs::path(source_path).parent_path() / "detached/"; } void DiskS3::onFreeze(const String & path) diff --git a/src/Disks/S3/registerDiskS3.cpp b/src/Disks/S3/registerDiskS3.cpp index e02f413c65e..21eff5f3d31 100644 --- a/src/Disks/S3/registerDiskS3.cpp +++ b/src/Disks/S3/registerDiskS3.cpp @@ -174,7 +174,7 @@ void registerDiskS3(DiskFactory & factory) throw Exception("S3 path must ends with '/', but '" + uri.key + "' doesn't.", ErrorCodes::BAD_ARGUMENTS); String metadata_path = config.getString(config_prefix + ".metadata_path", context->getPath() + "disks/" + name + "/"); - Poco::File (metadata_path).createDirectories(); + fs::create_directories(metadata_path); std::shared_ptr s3disk = std::make_shared( name, diff --git a/src/Disks/StoragePolicy.cpp b/src/Disks/StoragePolicy.cpp index cff2685ca24..efedff9929f 100644 --- a/src/Disks/StoragePolicy.cpp +++ b/src/Disks/StoragePolicy.cpp @@ -8,8 +8,6 @@ #include -#include - namespace { diff --git a/src/Disks/WriteIndirectBufferFromRemoteFS.cpp b/src/Disks/WriteIndirectBufferFromRemoteFS.cpp index adc711608d7..6951b9fa92e 100644 --- a/src/Disks/WriteIndirectBufferFromRemoteFS.cpp +++ b/src/Disks/WriteIndirectBufferFromRemoteFS.cpp @@ -8,7 +8,6 @@ namespace DB { -/// Stores data in S3/HDFS and adds the object key (S3 path) and object size to metadata file on local FS. template WriteIndirectBufferFromRemoteFS::WriteIndirectBufferFromRemoteFS( std::unique_ptr impl_, diff --git a/src/Disks/WriteIndirectBufferFromRemoteFS.h b/src/Disks/WriteIndirectBufferFromRemoteFS.h index cda7523e19e..ece7b9d5871 100644 --- a/src/Disks/WriteIndirectBufferFromRemoteFS.h +++ b/src/Disks/WriteIndirectBufferFromRemoteFS.h @@ -10,7 +10,7 @@ namespace DB { -/// Stores data in S3/HDFS and adds the object key (S3 path) and object size to metadata file on local FS. +/// Stores data in S3/HDFS and adds the object path and object size to metadata file on local FS. template class WriteIndirectBufferFromRemoteFS final : public WriteBufferFromFileDecorator { diff --git a/src/Disks/tests/gtest_disk.cpp b/src/Disks/tests/gtest_disk.cpp index 714abf485ee..36f91249391 100644 --- a/src/Disks/tests/gtest_disk.cpp +++ b/src/Disks/tests/gtest_disk.cpp @@ -2,6 +2,9 @@ #include #include #include "gtest_disk.h" +#include + +namespace fs = std::filesystem; #if !defined(__clang__) @@ -22,7 +25,7 @@ DB::DiskPtr createDisk() template <> DB::DiskPtr createDisk() { - Poco::File("tmp/").createDirectory(); + fs::create_directory("tmp/"); return std::make_shared("local_disk", "tmp/", 0); } @@ -43,7 +46,7 @@ template <> void destroyDisk(DB::DiskPtr & disk) { disk.reset(); - Poco::File("tmp/").remove(true); + fs::remove_all("tmp/"); } diff --git a/src/Formats/FormatSchemaInfo.cpp b/src/Formats/FormatSchemaInfo.cpp index 707f9babe8d..2605c0bdf04 100644 --- a/src/Formats/FormatSchemaInfo.cpp +++ b/src/Formats/FormatSchemaInfo.cpp @@ -1,7 +1,7 @@ #include -#include #include #include +#include namespace DB @@ -11,6 +11,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } +namespace fs = std::filesystem; namespace { @@ -34,55 +35,66 @@ FormatSchemaInfo::FormatSchemaInfo(const String & format_schema, const String & String default_file_extension = getFormatSchemaDefaultFileExtension(format); - Poco::Path path; + fs::path path; if (require_message) { size_t colon_pos = format_schema.find(':'); - if ((colon_pos == String::npos) || (colon_pos == 0) || (colon_pos == format_schema.length() - 1) - || path.assign(format_schema.substr(0, colon_pos)).makeFile().getFileName().empty()) + if ((colon_pos == String::npos) || (colon_pos == 0) || (colon_pos == format_schema.length() - 1)) { throw Exception( "Format schema requires the 'format_schema' setting to have the 'schema_file:message_name' format" + (default_file_extension.empty() ? "" : ", e.g. 'schema." + default_file_extension + ":Message'") + - ". Got '" + format_schema - + "'", - ErrorCodes::BAD_ARGUMENTS); + ". Got '" + format_schema + "'", ErrorCodes::BAD_ARGUMENTS); + } + else + { + path = fs::path(format_schema.substr(0, colon_pos)); + String filename = path.has_filename() ? path.filename() : path.parent_path().filename(); + if (filename.empty()) + throw Exception( + "Format schema requires the 'format_schema' setting to have the 'schema_file:message_name' format" + + (default_file_extension.empty() ? "" : ", e.g. 'schema." + default_file_extension + ":Message'") + + ". Got '" + format_schema + "'", ErrorCodes::BAD_ARGUMENTS); } - message_name = format_schema.substr(colon_pos + 1); } else - path.assign(format_schema).makeFile().getFileName(); + { + path = fs::path(format_schema); + if (!path.has_filename()) + path = path.parent_path() / ""; + } auto default_schema_directory = [&format_schema_path]() { - static const String str = Poco::Path(format_schema_path).makeAbsolute().makeDirectory().toString(); + static const String str = fs::canonical(format_schema_path) / ""; return str; }; - if (path.getExtension().empty() && !default_file_extension.empty()) - path.setExtension(default_file_extension); + if (!path.has_extension() && !default_file_extension.empty()) + path = path.parent_path() / (path.stem().string() + '.' + default_file_extension); - if (path.isAbsolute()) + fs::path default_schema_directory_path(default_schema_directory()); + if (path.is_absolute()) { if (is_server) - throw Exception("Absolute path in the 'format_schema' setting is prohibited: " + path.toString(), ErrorCodes::BAD_ARGUMENTS); - schema_path = path.getFileName(); - schema_directory = path.makeParent().toString(); + throw Exception("Absolute path in the 'format_schema' setting is prohibited: " + path.string(), ErrorCodes::BAD_ARGUMENTS); + schema_path = path.filename(); + schema_directory = path.parent_path() / ""; } - else if (path.depth() >= 1 && path.directory(0) == "..") + else if (path.has_parent_path() && !fs::weakly_canonical(default_schema_directory_path / path).string().starts_with(fs::weakly_canonical(default_schema_directory_path).string())) { if (is_server) - throw Exception( - "Path in the 'format_schema' setting shouldn't go outside the 'format_schema_path' directory: " + path.toString(), - ErrorCodes::BAD_ARGUMENTS); - path = Poco::Path(default_schema_directory()).resolve(path).toString(); - schema_path = path.getFileName(); - schema_directory = path.makeParent().toString(); + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Path in the 'format_schema' setting shouldn't go outside the 'format_schema_path' directory: {} ({} not in {})", + path.string()); + path = default_schema_directory_path / path; + schema_path = path.filename(); + schema_directory = path.parent_path() / ""; } else { - schema_path = path.toString(); + schema_path = path; schema_directory = default_schema_directory(); } } diff --git a/src/Functions/FunctionFile.cpp b/src/Functions/FunctionFile.cpp index 4e81a4497b7..cd81f2b4e0b 100644 --- a/src/Functions/FunctionFile.cpp +++ b/src/Functions/FunctionFile.cpp @@ -3,10 +3,11 @@ #include #include #include -#include -#include #include #include +#include + +namespace fs = std::filesystem; namespace DB { @@ -68,21 +69,19 @@ public: { const char * filename = reinterpret_cast(&chars[source_offset]); - const String user_files_path = getContext()->getUserFilesPath(); - String user_files_absolute_path = Poco::Path(user_files_path).makeAbsolute().makeDirectory().toString(); - Poco::Path poco_filepath = Poco::Path(filename); - if (poco_filepath.isRelative()) - poco_filepath = Poco::Path(user_files_absolute_path, poco_filepath); - const String file_absolute_path = poco_filepath.absolute().toString(); - checkReadIsAllowedOrThrow(user_files_absolute_path, file_absolute_path); + fs::path user_files_absolute_path = fs::canonical(fs::path(getContext()->getUserFilesPath())); + fs::path file_path(filename); + if (file_path.is_relative()) + file_path = user_files_absolute_path / file_path; + fs::path file_absolute_path = fs::canonical(file_path); + checkReadIsAllowedOrThrow(user_files_absolute_path.string(), file_absolute_path); - checked_filenames[row] = file_absolute_path; - auto file = Poco::File(file_absolute_path); + checked_filenames[row] = file_absolute_path.string(); - if (!file.exists()) - throw Exception(fmt::format("File {} doesn't exist.", file_absolute_path), ErrorCodes::FILE_DOESNT_EXIST); + if (!fs::exists(file_absolute_path)) + throw Exception(fmt::format("File {} doesn't exist.", file_absolute_path.string()), ErrorCodes::FILE_DOESNT_EXIST); - const auto current_file_size = Poco::File(file_absolute_path).getSize(); + const auto current_file_size = fs::file_size(file_absolute_path); result_offset += current_file_size + 1; res_offsets[row] = result_offset; @@ -117,8 +116,8 @@ private: if (file_absolute_path.find(user_files_absolute_path) != 0) throw Exception("File is not inside " + user_files_absolute_path, ErrorCodes::DATABASE_ACCESS_DENIED); - Poco::File path_poco_file = Poco::File(file_absolute_path); - if (path_poco_file.exists() && path_poco_file.isDirectory()) + fs::path fs_path(file_absolute_path); + if (fs::exists(fs_path) && fs::is_directory(fs_path)) throw Exception("File can't be a directory", ErrorCodes::INCORRECT_FILE_NAME); } }; diff --git a/src/IO/tests/gtest_cascade_and_memory_write_buffer.cpp b/src/IO/tests/gtest_cascade_and_memory_write_buffer.cpp index 53248467418..656e40b2ad7 100644 --- a/src/IO/tests/gtest_cascade_and_memory_write_buffer.cpp +++ b/src/IO/tests/gtest_cascade_and_memory_write_buffer.cpp @@ -1,7 +1,6 @@ #include #include -#include #include #include #include @@ -9,7 +8,9 @@ #include #include #include +#include +namespace fs = std::filesystem; using namespace DB; @@ -235,7 +236,7 @@ try buf.reset(); reread_buf.reset(); - ASSERT_TRUE(!Poco::File(tmp_filename).exists()); + ASSERT_TRUE(!fs::exists(tmp_filename)); } } catch (...) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index b6b10906c6b..a920a26e4c4 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -74,8 +74,11 @@ #include #include #include +#include +namespace fs = std::filesystem; + namespace ProfileEvents { extern const Event ContextLock; @@ -2212,14 +2215,14 @@ void Context::checkCanBeDropped(const String & database, const String & table, c if (!max_size_to_drop || size <= max_size_to_drop) return; - Poco::File force_file(getFlagsPath() + "force_drop_table"); - bool force_file_exists = force_file.exists(); + fs::path force_file(getFlagsPath() + "force_drop_table"); + bool force_file_exists = fs::exists(force_file); if (force_file_exists) { try { - force_file.remove(); + fs::remove(force_file); return; } catch (...) @@ -2241,9 +2244,9 @@ void Context::checkCanBeDropped(const String & database, const String & table, c "Example:\nsudo touch '{}' && sudo chmod 666 '{}'", backQuoteIfNeed(database), backQuoteIfNeed(table), size_str, max_size_to_drop_str, - force_file.path(), force_file_exists ? "exists but not writeable (could not be removed)" : "doesn't exist", - force_file.path(), - force_file.path(), force_file.path()); + force_file.string(), force_file_exists ? "exists but not writeable (could not be removed)" : "doesn't exist", + force_file.string(), + force_file.string(), force_file.string()); } diff --git a/src/Interpreters/DDLTask.h b/src/Interpreters/DDLTask.h index 2dd9857b78a..703d691a358 100644 --- a/src/Interpreters/DDLTask.h +++ b/src/Interpreters/DDLTask.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace Poco { @@ -14,6 +15,8 @@ namespace zkutil class ZooKeeper; } +namespace fs = std::filesystem; + namespace DB { @@ -100,9 +103,9 @@ struct DDLTaskBase virtual ContextMutablePtr makeQueryContext(ContextPtr from_context, const ZooKeeperPtr & zookeeper); - inline String getActiveNodePath() const { return entry_path + "/active/" + host_id_str; } - inline String getFinishedNodePath() const { return entry_path + "/finished/" + host_id_str; } - inline String getShardNodePath() const { return entry_path + "/shards/" + getShardID(); } + inline String getActiveNodePath() const { return fs::path(entry_path) / "active" / host_id_str; } + inline String getFinishedNodePath() const { return fs::path(entry_path) / "finished" / host_id_str; } + inline String getShardNodePath() const { return fs::path(entry_path) / "shards" / getShardID(); } static String getLogEntryName(UInt32 log_entry_number); static UInt32 getLogEntryNumber(const String & log_entry_name); diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 7dfc7b93c96..a70cdb2e054 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -17,6 +16,8 @@ #include #include #include +#include +#include #if !defined(ARCADIA_BUILD) # include "config_core.h" @@ -27,8 +28,7 @@ # include #endif -#include - +namespace fs = std::filesystem; namespace CurrentMetrics { @@ -354,10 +354,9 @@ DatabasePtr DatabaseCatalog::detachDatabase(const String & database_name, bool d db->drop(getContext()); /// Old ClickHouse versions did not store database.sql files - Poco::File database_metadata_file( - getContext()->getPath() + "metadata/" + escapeForFileName(database_name) + ".sql"); - if (database_metadata_file.exists()) - database_metadata_file.remove(false); + fs::path database_metadata_file = fs::path(getContext()->getPath()) / "metadata" / (escapeForFileName(database_name) + ".sql"); + if (fs::exists(database_metadata_file)) + fs::remove(database_metadata_file); } return db; @@ -783,7 +782,7 @@ void DatabaseCatalog::enqueueDroppedTableCleanup(StorageID table_id, StoragePtr } addUUIDMapping(table_id.uuid); - drop_time = Poco::File(dropped_metadata_path).getLastModified().epochTime(); + drop_time = FS::getModificationTime(dropped_metadata_path); } std::lock_guard lock(tables_marked_dropped_mutex); @@ -892,16 +891,15 @@ void DatabaseCatalog::dropTableFinally(const TableMarkedAsDropped & table) /// Even if table is not loaded, try remove its data from disk. /// TODO remove data from all volumes - String data_path = getContext()->getPath() + "store/" + getPathForUUID(table.table_id.uuid); - Poco::File table_data_dir{data_path}; - if (table_data_dir.exists()) + fs::path data_path = fs::path(getContext()->getPath()) / "store" / getPathForUUID(table.table_id.uuid); + if (fs::exists(data_path)) { - LOG_INFO(log, "Removing data directory {} of dropped table {}", data_path, table.table_id.getNameForLogs()); - table_data_dir.remove(true); + LOG_INFO(log, "Removing data directory {} of dropped table {}", data_path.string(), table.table_id.getNameForLogs()); + fs::remove_all(data_path); } LOG_INFO(log, "Removing metadata {} of dropped table {}", table.metadata_path, table.table_id.getNameForLogs()); - Poco::File(table.metadata_path).remove(); + fs::remove(fs::path(table.metadata_path)); removeUUIDMappingFinally(table.table_id.uuid); CurrentMetrics::sub(CurrentMetrics::TablesToDropQueueSize, 1); diff --git a/src/Interpreters/ExternalLoaderXMLConfigRepository.cpp b/src/Interpreters/ExternalLoaderXMLConfigRepository.cpp index 63755ee1839..00c31110ea4 100644 --- a/src/Interpreters/ExternalLoaderXMLConfigRepository.cpp +++ b/src/Interpreters/ExternalLoaderXMLConfigRepository.cpp @@ -3,12 +3,13 @@ #include #include #include - #include -#include -#include +#include +#include +namespace fs = std::filesystem; + namespace DB { ExternalLoaderXMLConfigRepository::ExternalLoaderXMLConfigRepository( @@ -19,7 +20,7 @@ ExternalLoaderXMLConfigRepository::ExternalLoaderXMLConfigRepository( Poco::Timestamp ExternalLoaderXMLConfigRepository::getUpdateTime(const std::string & definition_entity_name) { - return Poco::File(definition_entity_name).getLastModified(); + return FS::getModificationTimestamp(definition_entity_name); } std::set ExternalLoaderXMLConfigRepository::getAllLoadablesDefinitionNames() @@ -36,8 +37,8 @@ std::set ExternalLoaderXMLConfigRepository::getAllLoadablesDefiniti if (pattern[0] != '/') { const auto app_config_path = main_config.getString("config-file", "config.xml"); - const auto config_dir = Poco::Path{app_config_path}.parent().toString(); - const auto absolute_path = config_dir + pattern; + const String config_dir = fs::path(app_config_path).parent_path(); + const String absolute_path = fs::path(config_dir) / pattern; Poco::Glob::glob(absolute_path, files, 0); if (!files.empty()) continue; @@ -59,7 +60,7 @@ std::set ExternalLoaderXMLConfigRepository::getAllLoadablesDefiniti bool ExternalLoaderXMLConfigRepository::exists(const std::string & definition_entity_name) { - return Poco::File(definition_entity_name).exists(); + return fs::exists(fs::path(definition_entity_name)); } Poco::AutoPtr ExternalLoaderXMLConfigRepository::load( diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index 332a52a1d1f..3c515d58e0c 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -1,5 +1,3 @@ -#include - #include #include #include diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 7dab12eb803..f97001883bd 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -542,7 +542,7 @@ void InterpreterSystemQuery::dropReplica(ASTSystemQuery & query) else if (!query.replica_zk_path.empty()) { getContext()->checkAccess(AccessType::SYSTEM_DROP_REPLICA); - auto remote_replica_path = query.replica_zk_path + "/replicas/" + query.replica; + String remote_replica_path = fs::path(query.replica_zk_path) / "replicas" / query.replica; /// This check is actually redundant, but it may prevent from some user mistakes for (auto & elem : DatabaseCatalog::instance().getDatabases()) diff --git a/src/Interpreters/loadMetadata.cpp b/src/Interpreters/loadMetadata.cpp index 9f1934e2e2f..43f9727c355 100644 --- a/src/Interpreters/loadMetadata.cpp +++ b/src/Interpreters/loadMetadata.cpp @@ -18,7 +18,9 @@ #include #include +#include +namespace fs = std::filesystem; namespace DB { @@ -54,13 +56,13 @@ static void loadDatabase( String database_attach_query; String database_metadata_file = database_path + ".sql"; - if (Poco::File(database_metadata_file).exists()) + if (fs::exists(fs::path(database_metadata_file))) { /// There is .sql file with database creation statement. ReadBufferFromFile in(database_metadata_file, 1024); readStringUntilEOF(database_attach_query, in); } - else if (Poco::File(database_path).exists()) + else if (fs::exists(fs::path(database_path))) { /// Database exists, but .sql file is absent. It's old-style Ordinary database (e.g. system or default) database_attach_query = "ATTACH DATABASE " + backQuoteIfNeed(database) + " ENGINE = Ordinary"; @@ -95,34 +97,35 @@ void loadMetadata(ContextMutablePtr context, const String & default_database_nam * This file is deleted after successful loading of tables. * (flag is "one-shot") */ - Poco::File force_restore_data_flag_file(context->getFlagsPath() + "force_restore_data"); - bool has_force_restore_data_flag = force_restore_data_flag_file.exists(); + auto force_restore_data_flag_file = fs::path(context->getFlagsPath()) / "force_restore_data"; + bool has_force_restore_data_flag = fs::exists(force_restore_data_flag_file); /// Loop over databases. std::map databases; - Poco::DirectoryIterator dir_end; - for (Poco::DirectoryIterator it(path); it != dir_end; ++it) + fs::directory_iterator dir_end; + for (fs::directory_iterator it(path); it != dir_end; ++it) { - if (it->isLink()) + if (it->is_symlink()) continue; - if (!it->isDirectory()) + const auto current_file = it->path().filename().string(); + if (!it->is_directory()) { /// TODO: DETACH DATABASE PERMANENTLY ? - if (endsWith(it.name(), ".sql")) + if (fs::path(current_file).extension() == ".sql") { - String db_name = it.name().substr(0, it.name().size() - 4); + String db_name = fs::path(current_file).stem(); if (db_name != DatabaseCatalog::SYSTEM_DATABASE) - databases.emplace(unescapeForFileName(db_name), path + "/" + db_name); + databases.emplace(unescapeForFileName(db_name), fs::path(path) / db_name); } /// Temporary fails may be left from previous server runs. - if (endsWith(it.name(), ".tmp")) + if (fs::path(current_file).extension() == ".tmp") { - LOG_WARNING(log, "Removing temporary file {}", it->path()); + LOG_WARNING(log, "Removing temporary file {}", it->path().string()); try { - it->remove(); + fs::remove(it->path()); } catch (...) { @@ -135,13 +138,13 @@ void loadMetadata(ContextMutablePtr context, const String & default_database_nam } /// For '.svn', '.gitignore' directory and similar. - if (it.name().at(0) == '.') + if (current_file.at(0) == '.') continue; - if (it.name() == DatabaseCatalog::SYSTEM_DATABASE) + if (current_file == DatabaseCatalog::SYSTEM_DATABASE) continue; - databases.emplace(unescapeForFileName(it.name()), it.path().toString()); + databases.emplace(unescapeForFileName(current_file), it->path().string()); } /// clickhouse-local creates DatabaseMemory as default database by itself @@ -158,7 +161,7 @@ void loadMetadata(ContextMutablePtr context, const String & default_database_nam { try { - force_restore_data_flag_file.remove(); + fs::remove(force_restore_data_flag_file); } catch (...) { @@ -172,7 +175,7 @@ void loadMetadataSystem(ContextMutablePtr context) { String path = context->getPath() + "metadata/" + DatabaseCatalog::SYSTEM_DATABASE; String metadata_file = path + ".sql"; - if (Poco::File(path).exists() || Poco::File(metadata_file).exists()) + if (fs::exists(fs::path(path)) || fs::exists(fs::path(metadata_file))) { /// 'has_force_restore_data_flag' is true, to not fail on loading query_log table, if it is corrupted. loadDatabase(context, DatabaseCatalog::SYSTEM_DATABASE, path, true); diff --git a/src/Server/HTTP/HTTPServerResponse.cpp b/src/Server/HTTP/HTTPServerResponse.cpp index db5cfb132e3..25e7604a515 100644 --- a/src/Server/HTTP/HTTPServerResponse.cpp +++ b/src/Server/HTTP/HTTPServerResponse.cpp @@ -1,11 +1,8 @@ #include - #include - #include #include #include -#include #include #include #include @@ -13,6 +10,7 @@ #include #include + namespace DB { diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index 39dc764b958..6eab6cdda5e 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -39,7 +39,6 @@ #include #include -#include #include #include #include diff --git a/src/Server/StaticRequestHandler.cpp b/src/Server/StaticRequestHandler.cpp index 169d6859b43..19b91ae9c42 100644 --- a/src/Server/StaticRequestHandler.cpp +++ b/src/Server/StaticRequestHandler.cpp @@ -14,14 +14,15 @@ #include -#include -#include #include #include #include #include +#include +namespace fs = std::filesystem; + namespace DB { @@ -137,11 +138,14 @@ void StaticRequestHandler::writeResponse(WriteBuffer & out) if (startsWith(response_expression, file_prefix)) { - const auto & user_files_absolute_path = Poco::Path(server.context()->getUserFilesPath()).makeAbsolute().makeDirectory().toString(); - const auto & file_name = response_expression.substr(file_prefix.size(), response_expression.size() - file_prefix.size()); + auto file_name = response_expression.substr(file_prefix.size(), response_expression.size() - file_prefix.size()); + if (file_name.starts_with('/')) + file_name = file_name.substr(1); - const auto & file_path = Poco::Path(user_files_absolute_path, file_name).makeAbsolute().toString(); - if (!Poco::File(file_path).exists()) + fs::path user_files_absolute_path = fs::canonical(fs::path(server.context()->getUserFilesPath())); + String file_path = fs::weakly_canonical(user_files_absolute_path / file_name); + + if (!fs::exists(file_path)) throw Exception("Invalid file name " + file_path + " for static HTTPHandler. ", ErrorCodes::INCORRECT_FILE_NAME); ReadBufferFromFile in(file_path); diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index 55ee884646e..e8835132f8f 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -25,11 +25,9 @@ #include #include #include - #include #include - -#include +#include namespace CurrentMetrics @@ -39,6 +37,8 @@ namespace CurrentMetrics extern const Metric BrokenDistributedFilesToInsert; } +namespace fs = std::filesystem; + namespace DB { @@ -293,7 +293,7 @@ StorageDistributedDirectoryMonitor::StorageDistributedDirectoryMonitor( , pool(std::move(pool_)) , disk(disk_) , relative_path(relative_path_) - , path(disk->getPath() + relative_path + '/') + , path(fs::path(disk->getPath()) / relative_path / "") , should_batch_inserts(storage.getContext()->getSettingsRef().distributed_directory_monitor_batch_inserts) , dir_fsync(storage.getDistributedSettingsRef().fsync_directories) , min_batched_block_size_rows(storage.getContext()->getSettingsRef().min_insert_block_size_rows) @@ -347,7 +347,7 @@ void StorageDistributedDirectoryMonitor::shutdownAndDropAllData() } auto dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, relative_path); - Poco::File(path).remove(true); + fs::remove_all(path); } @@ -490,16 +490,14 @@ std::map StorageDistributedDirectoryMonitor::getFiles() std::map files; size_t new_bytes_count = 0; - Poco::DirectoryIterator end; - for (Poco::DirectoryIterator it{path}; it != end; ++it) + fs::directory_iterator end; + for (fs::directory_iterator it{path}; it != end; ++it) { const auto & file_path_str = it->path(); - Poco::Path file_path{file_path_str}; - - if (!it->isDirectory() && startsWith(file_path.getExtension(), "bin")) + if (!it->is_directory() && startsWith(fs::path(file_path_str).extension(), ".bin")) { - files[parse(file_path.getBaseName())] = file_path_str; - new_bytes_count += Poco::File(file_path).getSize(); + files[parse(fs::path(file_path_str).stem())] = file_path_str; + new_bytes_count += fs::file_size(fs::path(file_path_str)); } } @@ -663,8 +661,7 @@ struct StorageDistributedDirectoryMonitor::Batch String tmp_file{parent.current_batch_file_path + ".tmp"}; auto dir_sync_guard = getDirectorySyncGuard(dir_fsync, parent.disk, parent.relative_path); - - if (Poco::File{tmp_file}.exists()) + if (fs::exists(tmp_file)) LOG_ERROR(parent.log, "Temporary file {} exists. Unclean shutdown?", backQuote(tmp_file)); { @@ -676,7 +673,7 @@ struct StorageDistributedDirectoryMonitor::Batch out.sync(); } - Poco::File{tmp_file}.renameTo(parent.current_batch_file_path); + fs::rename(tmp_file, parent.current_batch_file_path); } auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(parent.storage.getContext()->getSettingsRef()); auto connection = parent.pool->get(timeouts); @@ -757,7 +754,7 @@ struct StorageDistributedDirectoryMonitor::Batch total_bytes = 0; recovered = false; - Poco::File{parent.current_batch_file_path}.setSize(0); + fs::resize_file(parent.current_batch_file_path, 0); } void writeText(WriteBuffer & out) @@ -850,7 +847,7 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map { std::unordered_set file_indices_to_skip; - if (Poco::File{current_batch_file_path}.exists()) + if (fs::exists(current_batch_file_path)) { /// Possibly, we failed to send a batch on the previous iteration. Try to send exactly the same batch. Batch batch(*this, files); @@ -951,8 +948,8 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map /// current_batch.txt will not exist if there was no send /// (this is the case when all batches that was pending has been marked as pending) - if (Poco::File{current_batch_file_path}.exists()) - Poco::File{current_batch_file_path}.remove(); + if (fs::exists(current_batch_file_path)) + fs::remove(current_batch_file_path); } } @@ -961,20 +958,18 @@ void StorageDistributedDirectoryMonitor::markAsBroken(const std::string & file_p const auto last_path_separator_pos = file_path.rfind('/'); const auto & base_path = file_path.substr(0, last_path_separator_pos + 1); const auto & file_name = file_path.substr(last_path_separator_pos + 1); - const auto & broken_path = base_path + "broken/"; - const auto & broken_file_path = broken_path + file_name; + const String & broken_path = fs::path(base_path) / "broken/"; + const String & broken_file_path = fs::path(broken_path) / file_name; - Poco::File{broken_path}.createDirectory(); + fs::create_directory(broken_path); auto dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, relative_path); - auto broken_dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, relative_path + "/broken/"); - - Poco::File file(file_path); + auto broken_dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, fs::path(relative_path) / "broken/"); { std::lock_guard status_lock(status_mutex); - size_t file_size = file.getSize(); + size_t file_size = fs::file_size(file_path); --status.files_count; status.bytes_count -= file_size; @@ -985,15 +980,13 @@ void StorageDistributedDirectoryMonitor::markAsBroken(const std::string & file_p metric_broken_files.add(); } - file.renameTo(broken_file_path); - + fs::rename(file_path, broken_file_path); LOG_ERROR(log, "Renamed `{}` to `{}`", file_path, broken_file_path); } + void StorageDistributedDirectoryMonitor::markAsSend(const std::string & file_path) { - Poco::File file(file_path); - - size_t file_size = file.getSize(); + size_t file_size = fs::file_size(file_path); { std::lock_guard status_lock(status_mutex); @@ -1002,7 +995,7 @@ void StorageDistributedDirectoryMonitor::markAsSend(const std::string & file_pat status.bytes_count -= file_size; } - file.remove(); + fs::remove(file_path); } bool StorageDistributedDirectoryMonitor::maybeMarkAsBroken(const std::string & file_path, const Exception & e) @@ -1030,7 +1023,7 @@ void StorageDistributedDirectoryMonitor::updatePath(const std::string & new_rela { std::lock_guard status_lock(status_mutex); relative_path = new_relative_path; - path = disk->getPath() + relative_path + '/'; + path = fs::path(disk->getPath()) / relative_path / ""; } current_batch_file_path = path + "current_batch.txt"; diff --git a/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/src/Storages/Distributed/DistributedBlockOutputStream.cpp index a4aa2779771..ea694fb0cfe 100644 --- a/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -33,11 +33,10 @@ #include #include -#include - #include #include #include +#include namespace CurrentMetrics @@ -50,10 +49,11 @@ namespace ProfileEvents extern const Event DistributedSyncInsertionTimeoutExceeded; } +namespace fs = std::filesystem; + namespace DB { - namespace ErrorCodes { extern const int LOGICAL_ERROR; @@ -660,10 +660,10 @@ void DistributedBlockOutputStream::writeToShard(const Block & block, const std:: /// hardlinking to ensure the inode is not freed until we're done { const std::string path(disk_path + data_path + *it); - Poco::File(path).createDirectory(); - const std::string tmp_path(path + "/tmp/"); - Poco::File(tmp_path).createDirectory(); + + fs::create_directory(path); + fs::create_directory(tmp_path); const std::string file_name(toString(storage.file_names_increment.get()) + ".bin"); @@ -717,7 +717,7 @@ void DistributedBlockOutputStream::writeToShard(const Block & block, const std:: } // Create hardlink here to reuse increment number - const std::string block_file_path(path + '/' + file_name); + const std::string block_file_path(fs::path(path) / file_name); createHardLink(first_file_tmp_path, block_file_path); auto dir_sync_guard = make_directory_sync_guard(*it); } @@ -726,18 +726,18 @@ void DistributedBlockOutputStream::writeToShard(const Block & block, const std:: /// Make hardlinks for (; it != dir_names.end(); ++it) { - const std::string path(disk_path + data_path + *it); - Poco::File(path).createDirectory(); + const std::string path(fs::path(disk_path) / (data_path + *it)); + fs::create_directory(path); - const std::string block_file_path(path + '/' + toString(storage.file_names_increment.get()) + ".bin"); + const std::string block_file_path(fs::path(path) / (toString(storage.file_names_increment.get()) + ".bin")); createHardLink(first_file_tmp_path, block_file_path); auto dir_sync_guard = make_directory_sync_guard(*it); } - auto file_size = Poco::File(first_file_tmp_path).getSize(); + auto file_size = fs::file_size(first_file_tmp_path); /// remove the temporary file, enabling the OS to reclaim inode after all threads /// have removed their corresponding files - Poco::File(first_file_tmp_path).remove(); + fs::remove(first_file_tmp_path); /// Notify auto sleep_ms = context->getSettingsRef().distributed_directory_monitor_sleep_time_ms; diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index 1428e80f9af..e3fd287bad8 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -18,7 +18,6 @@ #include #include #include - #include #include #include @@ -26,7 +25,9 @@ #include #include #include +#include +namespace fs = std::filesystem; namespace DB { @@ -257,7 +258,7 @@ Strings LSWithRegexpMatching(const String & path_for_ls, const HDFSFSPtr & fs, c { if (re2::RE2::FullMatch(file_name, matcher)) { - Strings result_part = LSWithRegexpMatching(full_path + "/", fs, suffix_with_globs.substr(next_slash)); + Strings result_part = LSWithRegexpMatching(fs::path(full_path) / "", fs, suffix_with_globs.substr(next_slash)); /// Recursion depth is limited by pattern. '*' works only for depth = 1, for depth = 2 pattern path is '*/*'. So we do not need additional check. std::move(result_part.begin(), result_part.end(), std::back_inserter(result)); } diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 5ccb2ca8c27..529fc9f54b8 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -13,11 +13,11 @@ #include #include #include - -#include #include +namespace fs = std::filesystem; + namespace CurrentMetrics { extern const Metric ReplicatedSend; @@ -289,7 +289,7 @@ MergeTreeData::DataPart::Checksums Service::sendPartFromDisk( { String file_name = it.first; - String path = part->getFullRelativePath() + file_name; + String path = fs::path(part->getFullRelativePath()) / file_name; UInt64 size = disk->getFileSize(path); @@ -339,15 +339,15 @@ void Service::sendPartS3Metadata(const MergeTreeData::DataPartPtr & part, WriteB { String file_name = it.first; - String metadata_file = disk->getPath() + part->getFullRelativePath() + file_name; + String metadata_file = fs::path(disk->getPath()) / part->getFullRelativePath() / file_name; - Poco::File metadata(metadata_file); + fs::path metadata(metadata_file); - if (!metadata.exists()) + if (!fs::exists(metadata)) throw Exception("S3 metadata '" + file_name + "' is not exists", ErrorCodes::CORRUPTED_DATA); - if (!metadata.isFile()) + if (!fs::is_regular_file(metadata)) throw Exception("S3 metadata '" + file_name + "' is not a file", ErrorCodes::CORRUPTED_DATA); - UInt64 file_size = metadata.getSize(); + UInt64 file_size = fs::file_size(metadata); writeStringBinary(it.first, out); writeBinary(file_size, out); @@ -570,7 +570,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( readUUIDText(part_uuid, in); auto storage_id = data.getStorageID(); - String new_part_path = part_type == "InMemory" ? "memory" : data.getFullPathOnDisk(reservation->getDisk()) + part_name + "/"; + String new_part_path = part_type == "InMemory" ? "memory" : fs::path(data.getFullPathOnDisk(reservation->getDisk())) / part_name / ""; auto entry = data.getContext()->getReplicatedFetchList().insert( storage_id.getDatabaseName(), storage_id.getTableName(), part_info.partition_id, part_name, new_part_path, @@ -681,13 +681,13 @@ void Fetcher::downloadBaseOrProjectionPartToDisk( /// File must be inside "absolute_part_path" directory. /// Otherwise malicious ClickHouse replica may force us to write to arbitrary path. - String absolute_file_path = Poco::Path(part_download_path + file_name).absolute().toString(); - if (!startsWith(absolute_file_path, Poco::Path(part_download_path).absolute().toString())) + String absolute_file_path = fs::weakly_canonical(fs::path(part_download_path) / file_name); + if (!startsWith(absolute_file_path, fs::weakly_canonical(part_download_path).string())) throw Exception("File path (" + absolute_file_path + ") doesn't appear to be inside part path (" + part_download_path + ")." " This may happen if we are trying to download part from malicious replica or logical error.", ErrorCodes::INSECURE_PATH); - auto file_out = disk->writeFile(part_download_path + file_name); + auto file_out = disk->writeFile(fs::path(part_download_path) / file_name); HashingWriteBuffer hashing_out(*file_out); copyData(in, hashing_out, file_size, blocker.getCounter()); @@ -704,7 +704,7 @@ void Fetcher::downloadBaseOrProjectionPartToDisk( readPODBinary(expected_hash, in); if (expected_hash != hashing_out.getHash()) - throw Exception("Checksum mismatch for file " + fullPath(disk, part_download_path + file_name) + " transferred from " + replica_path, + throw Exception("Checksum mismatch for file " + fullPath(disk, (fs::path(part_download_path) / file_name).string()) + " transferred from " + replica_path, ErrorCodes::CHECKSUM_DOESNT_MATCH); if (file_name != "checksums.txt" && @@ -811,7 +811,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToS3( String tmp_prefix = tmp_prefix_.empty() ? TMP_PREFIX : tmp_prefix_; String part_relative_path = String(to_detached ? "detached/" : "") + tmp_prefix + part_name; - String part_download_path = data.getRelativeDataPath() + part_relative_path + "/"; + String part_download_path = fs::path(data.getRelativeDataPath()) / part_relative_path / ""; if (disk->exists(part_download_path)) throw Exception("Directory " + fullPath(disk, part_download_path) + " already exists.", ErrorCodes::DIRECTORY_ALREADY_EXISTS); @@ -833,7 +833,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToS3( readStringBinary(file_name, in); readBinary(file_size, in); - String data_path = part_download_path + file_name; + String data_path = fs::path(part_download_path) / file_name; String metadata_file = fullPath(disk, data_path); { diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 8a35ce3dcb8..703cf32f743 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -69,7 +69,7 @@ void IMergeTreeDataPart::MinMaxIndex::load(const MergeTreeData & data, const Dis hyperrectangle.reserve(minmax_idx_size); for (size_t i = 0; i < minmax_idx_size; ++i) { - String file_name = part_path + "minmax_" + escapeForFileName(minmax_column_names[i]) + ".idx"; + String file_name = fs::path(part_path) / ("minmax_" + escapeForFileName(minmax_column_names[i]) + ".idx"); auto file = openForReading(disk_, file_name); auto serialization = minmax_column_types[i]->getDefaultSerialization(); @@ -111,7 +111,7 @@ void IMergeTreeDataPart::MinMaxIndex::store( String file_name = "minmax_" + escapeForFileName(column_names[i]) + ".idx"; auto serialization = data_types.at(i)->getDefaultSerialization(); - auto out = disk_->writeFile(part_path + file_name); + auto out = disk_->writeFile(fs::path(part_path) / file_name); HashingWriteBuffer out_hashing(*out); serialization->serializeBinary(hyperrectangle[i].left, out_hashing); serialization->serializeBinary(hyperrectangle[i].right, out_hashing); @@ -560,7 +560,7 @@ String IMergeTreeDataPart::getFullPath() const if (relative_path.empty()) throw Exception("Part relative_path cannot be empty. It's bug.", ErrorCodes::LOGICAL_ERROR); - return storage.getFullPathOnDisk(volume->getDisk()) + (parent_part ? parent_part->relative_path + "/" : "") + relative_path + "/"; + return fs::path(storage.getFullPathOnDisk(volume->getDisk())) / (parent_part ? parent_part->relative_path : "") / relative_path / ""; } String IMergeTreeDataPart::getFullRelativePath() const @@ -568,7 +568,7 @@ String IMergeTreeDataPart::getFullRelativePath() const if (relative_path.empty()) throw Exception("Part relative_path cannot be empty. It's bug.", ErrorCodes::LOGICAL_ERROR); - return storage.relative_data_path + (parent_part ? parent_part->relative_path + "/" : "") + relative_path + "/"; + return fs::path(storage.relative_data_path) / (parent_part ? parent_part->relative_path : "") / relative_path / ""; } void IMergeTreeDataPart::loadColumnsChecksumsIndexes(bool require_columns_checksums, bool check_consistency) @@ -643,7 +643,7 @@ void IMergeTreeDataPart::loadIndex() loaded_index[i]->reserve(index_granularity.getMarksCount()); } - String index_path = getFullRelativePath() + "primary.idx"; + String index_path = fs::path(getFullRelativePath()) / "primary.idx"; auto index_file = openForReading(volume->getDisk(), index_path); size_t marks_count = index_granularity.getMarksCount(); @@ -678,7 +678,7 @@ NameSet IMergeTreeDataPart::getFileNamesWithoutChecksums() const return {}; NameSet result = {"checksums.txt", "columns.txt"}; - String default_codec_path = getFullRelativePath() + DEFAULT_COMPRESSION_CODEC_FILE_NAME; + String default_codec_path = fs::path(getFullRelativePath()) / DEFAULT_COMPRESSION_CODEC_FILE_NAME; if (volume->getDisk()->exists(default_codec_path)) result.emplace(DEFAULT_COMPRESSION_CODEC_FILE_NAME); @@ -695,7 +695,7 @@ void IMergeTreeDataPart::loadDefaultCompressionCodec() return; } - String path = getFullRelativePath() + DEFAULT_COMPRESSION_CODEC_FILE_NAME; + String path = fs::path(getFullRelativePath()) / DEFAULT_COMPRESSION_CODEC_FILE_NAME; if (!volume->getDisk()->exists(path)) { default_codec = detectDefaultCompressionCodec(); @@ -756,7 +756,7 @@ CompressionCodecPtr IMergeTreeDataPart::detectDefaultCompressionCodec() const { if (path_to_data_file.empty()) { - String candidate_path = getFullRelativePath() + ISerialization::getFileNameForStream(part_column, substream_path) + ".bin"; + String candidate_path = fs::path(getFullRelativePath()) / (ISerialization::getFileNameForStream(part_column, substream_path) + ".bin"); /// We can have existing, but empty .bin files. Example: LowCardinality(Nullable(...)) columns and column_name.dict.null.bin file. if (volume->getDisk()->exists(candidate_path) && volume->getDisk()->getFileSize(candidate_path) != 0) @@ -822,7 +822,7 @@ void IMergeTreeDataPart::loadPartitionAndMinMaxIndex() void IMergeTreeDataPart::loadChecksums(bool require) { - const String path = getFullRelativePath() + "checksums.txt"; + const String path = fs::path(getFullRelativePath()) / "checksums.txt"; if (volume->getDisk()->exists(path)) { @@ -847,11 +847,11 @@ void IMergeTreeDataPart::loadChecksums(bool require) checksums = checkDataPart(shared_from_this(), false); { - auto out = volume->getDisk()->writeFile(getFullRelativePath() + "checksums.txt.tmp", 4096); + auto out = volume->getDisk()->writeFile(fs::path(getFullRelativePath()) / "checksums.txt.tmp", 4096); checksums.write(*out); } - volume->getDisk()->moveFile(getFullRelativePath() + "checksums.txt.tmp", getFullRelativePath() + "checksums.txt"); + volume->getDisk()->moveFile(fs::path(getFullRelativePath()) / "checksums.txt.tmp", fs::path(getFullRelativePath()) / "checksums.txt"); bytes_on_disk = checksums.getTotalSizeOnDisk(); } @@ -859,7 +859,7 @@ void IMergeTreeDataPart::loadChecksums(bool require) void IMergeTreeDataPart::loadRowsCount() { - String path = getFullRelativePath() + "count.txt"; + String path = fs::path(getFullRelativePath()) / "count.txt"; if (index_granularity.empty()) { rows_count = 0; @@ -960,7 +960,7 @@ void IMergeTreeDataPart::loadRowsCount() void IMergeTreeDataPart::loadTTLInfos() { - String path = getFullRelativePath() + "ttl.txt"; + String path = fs::path(getFullRelativePath()) / "ttl.txt"; if (volume->getDisk()->exists(path)) { auto in = openForReading(volume->getDisk(), path); @@ -987,7 +987,7 @@ void IMergeTreeDataPart::loadTTLInfos() void IMergeTreeDataPart::loadUUID() { - String path = getFullRelativePath() + UUID_FILE_NAME; + String path = fs::path(getFullRelativePath()) / UUID_FILE_NAME; if (volume->getDisk()->exists(path)) { @@ -1000,7 +1000,7 @@ void IMergeTreeDataPart::loadUUID() void IMergeTreeDataPart::loadColumns(bool require) { - String path = getFullRelativePath() + "columns.txt"; + String path = fs::path(getFullRelativePath()) / "columns.txt"; auto metadata_snapshot = storage.getInMemoryMetadataPtr(); if (parent_part) metadata_snapshot = metadata_snapshot->projections.get(name).metadata; @@ -1015,7 +1015,7 @@ void IMergeTreeDataPart::loadColumns(bool require) /// If there is no file with a list of columns, write it down. for (const NameAndTypePair & column : metadata_snapshot->getColumns().getAllPhysical()) - if (volume->getDisk()->exists(getFullRelativePath() + getFileNameForColumn(column) + ".bin")) + if (volume->getDisk()->exists(fs::path(getFullRelativePath()) / (getFileNameForColumn(column) + ".bin"))) loaded_columns.push_back(column); if (columns.empty()) @@ -1053,7 +1053,7 @@ UInt64 IMergeTreeDataPart::calculateTotalSizeOnDisk(const DiskPtr & disk_, const disk_->listFiles(from, files); UInt64 res = 0; for (const auto & file : files) - res += calculateTotalSizeOnDisk(disk_, from + "/" + file); + res += calculateTotalSizeOnDisk(disk_, fs::path(from) / file); return res; } @@ -1063,7 +1063,7 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_ assertOnDisk(); String from = getFullRelativePath(); - String to = storage.relative_data_path + (parent_part ? parent_part->relative_path + "/" : "") + new_relative_path + "/"; + String to = fs::path(storage.relative_data_path) / (parent_part ? parent_part->relative_path : "") / new_relative_path / ""; if (!volume->getDisk()->exists(from)) throw Exception("Part directory " + fullPath(volume->getDisk(), from) + " doesn't exist. Most likely it is a logical error.", ErrorCodes::FILE_DOESNT_EXIST); @@ -1124,8 +1124,8 @@ void IMergeTreeDataPart::remove(bool keep_s3) const * And a race condition can happen that will lead to "File not found" error here. */ - String from = storage.relative_data_path + relative_path; - String to = storage.relative_data_path + "delete_tmp_" + name; + fs::path from = fs::path(storage.relative_data_path) / relative_path; + fs::path to = fs::path(storage.relative_data_path) / ("delete_tmp_" + name); // TODO directory delete_tmp_ is never removed if server crashes before returning from this function auto disk = volume->getDisk(); @@ -1134,7 +1134,7 @@ void IMergeTreeDataPart::remove(bool keep_s3) const LOG_WARNING(storage.log, "Directory {} (to which part must be renamed before removing) already exists. Most likely this is due to unclean restart. Removing it.", fullPath(disk, to)); try { - disk->removeSharedRecursive(to + "/", keep_s3); + disk->removeSharedRecursive(fs::path(to) / "", keep_s3); } catch (...) { @@ -1147,11 +1147,14 @@ void IMergeTreeDataPart::remove(bool keep_s3) const { disk->moveDirectory(from, to); } - catch (const Poco::FileNotFoundException &) + catch (const fs::filesystem_error & e) { - LOG_ERROR(storage.log, "Directory {} (part to remove) doesn't exist or one of nested files has gone. Most likely this is due to manual removing. This should be discouraged. Ignoring.", fullPath(disk, to)); - - return; + if (e.code() == std::errc::no_such_file_or_directory) + { + LOG_ERROR(storage.log, "Directory {} (part to remove) doesn't exist or one of nested files has gone. Most likely this is due to manual removing. This should be discouraged. Ignoring.", fullPath(disk, to)); + return; + } + throw; } // Record existing projection directories so we don't remove them twice @@ -1166,7 +1169,7 @@ void IMergeTreeDataPart::remove(bool keep_s3) const if (checksums.empty()) { /// If the part is not completely written, we cannot use fast path by listing files. - disk->removeSharedRecursive(to + "/", keep_s3); + disk->removeSharedRecursive(fs::path(to) / "", keep_s3); } else { @@ -1181,17 +1184,17 @@ void IMergeTreeDataPart::remove(bool keep_s3) const for (const auto & [file, _] : checksums.files) { if (projection_directories.find(file) == projection_directories.end()) - disk->removeSharedFile(to + "/" + file, keep_s3); + disk->removeSharedFile(fs::path(to) / file, keep_s3); } #if !defined(__clang__) # pragma GCC diagnostic pop #endif for (const auto & file : {"checksums.txt", "columns.txt"}) - disk->removeSharedFile(to + "/" + file, keep_s3); + disk->removeSharedFile(fs::path(to) / file, keep_s3); - disk->removeSharedFileIfExists(to + "/" + DEFAULT_COMPRESSION_CODEC_FILE_NAME, keep_s3); - disk->removeSharedFileIfExists(to + "/" + DELETE_ON_DESTROY_MARKER_FILE_NAME, keep_s3); + disk->removeSharedFileIfExists(fs::path(to) / DEFAULT_COMPRESSION_CODEC_FILE_NAME, keep_s3); + disk->removeSharedFileIfExists(fs::path(to) / DELETE_ON_DESTROY_MARKER_FILE_NAME, keep_s3); disk->removeDirectory(to); } @@ -1201,7 +1204,7 @@ void IMergeTreeDataPart::remove(bool keep_s3) const LOG_ERROR(storage.log, "Cannot quickly remove directory {} by removing files; fallback to recursive removal. Reason: {}", fullPath(disk, to), getCurrentExceptionMessage(false)); - disk->removeSharedRecursive(to + "/", keep_s3); + disk->removeSharedRecursive(fs::path(to) / "", keep_s3); } } } @@ -1268,7 +1271,7 @@ String IMergeTreeDataPart::getRelativePathForPrefix(const String & prefix) const { res = (prefix.empty() ? "" : prefix + "_") + name + (try_no ? "_try" + DB::toString(try_no) : ""); - if (!volume->getDisk()->exists(getFullRelativePath() + res)) + if (!volume->getDisk()->exists(fs::path(getFullRelativePath()) / res)) return res; LOG_WARNING(storage.log, "Directory {} (to detach to) already exists. Will detach to directory with '_tryN' suffix.", res); @@ -1291,11 +1294,11 @@ void IMergeTreeDataPart::renameToDetached(const String & prefix) const void IMergeTreeDataPart::makeCloneInDetached(const String & prefix, const StorageMetadataPtr & /*metadata_snapshot*/) const { - String destination_path = storage.relative_data_path + getRelativePathForDetachedPart(prefix); + String destination_path = fs::path(storage.relative_data_path) / getRelativePathForDetachedPart(prefix); /// Backup is not recursive (max_level is 0), so do not copy inner directories localBackup(volume->getDisk(), getFullRelativePath(), destination_path, 0); - volume->getDisk()->removeFileIfExists(destination_path + "/" + DELETE_ON_DESTROY_MARKER_FILE_NAME); + volume->getDisk()->removeFileIfExists(fs::path(destination_path) / DELETE_ON_DESTROY_MARKER_FILE_NAME); } void IMergeTreeDataPart::makeCloneOnDisk(const DiskPtr & disk, const String & directory_name) const @@ -1307,16 +1310,16 @@ void IMergeTreeDataPart::makeCloneOnDisk(const DiskPtr & disk, const String & di if (directory_name.empty()) throw Exception("Can not clone data part " + name + " to empty directory.", ErrorCodes::LOGICAL_ERROR); - String path_to_clone = storage.relative_data_path + directory_name + '/'; + String path_to_clone = fs::path(storage.relative_data_path) / directory_name / ""; - if (disk->exists(path_to_clone + relative_path)) + if (disk->exists(fs::path(path_to_clone) / relative_path)) { LOG_WARNING(storage.log, "Path " + fullPath(disk, path_to_clone + relative_path) + " already exists. Will remove it and clone again."); - disk->removeRecursive(path_to_clone + relative_path + '/'); + disk->removeRecursive(fs::path(path_to_clone) / relative_path / ""); } disk->createDirectories(path_to_clone); volume->getDisk()->copy(getFullRelativePath(), disk, path_to_clone); - volume->getDisk()->removeFileIfExists(path_to_clone + '/' + DELETE_ON_DESTROY_MARKER_FILE_NAME); + volume->getDisk()->removeFileIfExists(fs::path(path_to_clone) / DELETE_ON_DESTROY_MARKER_FILE_NAME); } void IMergeTreeDataPart::checkConsistencyBase() const @@ -1370,19 +1373,19 @@ void IMergeTreeDataPart::checkConsistencyBase() const /// Check that the primary key index is not empty. if (!pk.column_names.empty()) - check_file_not_empty(volume->getDisk(), path + "primary.idx"); + check_file_not_empty(volume->getDisk(), fs::path(path) / "primary.idx"); if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) { - check_file_not_empty(volume->getDisk(), path + "count.txt"); + check_file_not_empty(volume->getDisk(), fs::path(path) / "count.txt"); if (metadata_snapshot->hasPartitionKey()) - check_file_not_empty(volume->getDisk(), path + "partition.dat"); + check_file_not_empty(volume->getDisk(), fs::path(path) / "partition.dat"); if (!parent_part) { for (const String & col_name : storage.getMinMaxColumnsNames(partition_key)) - check_file_not_empty(volume->getDisk(), path + "minmax_" + escapeForFileName(col_name) + ".idx"); + check_file_not_empty(volume->getDisk(), fs::path(path) / ("minmax_" + escapeForFileName(col_name) + ".idx")); } } } @@ -1477,7 +1480,7 @@ String IMergeTreeDataPart::getUniqueId() const auto disk = volume->getDisk(); if (disk->getType() == DB::DiskType::Type::S3) - id = disk->getUniqueId(getFullRelativePath() + "checksums.txt"); + id = disk->getUniqueId(fs::path(getFullRelativePath()) / "checksums.txt"); if (id.empty()) throw Exception("Can't get unique S3 object", ErrorCodes::LOGICAL_ERROR); diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index a8a49680dd7..53640b41507 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -17,8 +17,6 @@ #include #include -#include - #include namespace zkutil diff --git a/src/Storages/MergeTree/IMergeTreeReader.cpp b/src/Storages/MergeTree/IMergeTreeReader.cpp index 52d3e7ca9ab..14187564536 100644 --- a/src/Storages/MergeTree/IMergeTreeReader.cpp +++ b/src/Storages/MergeTree/IMergeTreeReader.cpp @@ -6,7 +6,6 @@ #include #include #include -#include namespace DB diff --git a/src/Storages/MergeTree/LeaderElection.h b/src/Storages/MergeTree/LeaderElection.h index 7cdfd8e566d..2810385e9e5 100644 --- a/src/Storages/MergeTree/LeaderElection.h +++ b/src/Storages/MergeTree/LeaderElection.h @@ -83,7 +83,7 @@ private: void createNode() { shutdown_called = false; - node = EphemeralNodeHolder::createSequential(path + "/leader_election-", zookeeper, identifier); + node = EphemeralNodeHolder::createSequential(fs::path(path) / "leader_election-", zookeeper, identifier); std::string node_path = node->getPath(); node_name = node_path.substr(node_path.find_last_of('/') + 1); diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 90540b71543..5c9d75d1fbb 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -52,8 +52,6 @@ #include #include -#include - #include #include @@ -67,8 +65,11 @@ #include #include #include +#include +namespace fs = std::filesystem; + namespace ProfileEvents { extern const Event RejectedInserts; @@ -210,8 +211,8 @@ MergeTreeData::MergeTreeData( for (const auto & [path, disk] : getRelativeDataPathsWithDisks()) { disk->createDirectories(path); - disk->createDirectories(path + MergeTreeData::DETACHED_DIR_NAME); - auto current_version_file_path = path + MergeTreeData::FORMAT_VERSION_FILE_NAME; + disk->createDirectories(fs::path(path) / MergeTreeData::DETACHED_DIR_NAME); + String current_version_file_path = fs::path(path) / MergeTreeData::FORMAT_VERSION_FILE_NAME; if (disk->exists(current_version_file_path)) { if (!version_file.first.empty()) @@ -225,7 +226,7 @@ MergeTreeData::MergeTreeData( /// If not choose any if (version_file.first.empty()) - version_file = {relative_data_path + MergeTreeData::FORMAT_VERSION_FILE_NAME, getStoragePolicy()->getAnyDisk()}; + version_file = {fs::path(relative_data_path) / MergeTreeData::FORMAT_VERSION_FILE_NAME, getStoragePolicy()->getAnyDisk()}; bool version_file_exists = version_file.second->exists(version_file.first); @@ -927,8 +928,8 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) auto part = createPart(part_name, part_info, single_disk_volume, part_name); bool broken = false; - String part_path = relative_data_path + "/" + part_name; - String marker_path = part_path + "/" + IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME; + String part_path = fs::path(relative_data_path) / part_name; + String marker_path = fs::path(part_path) / IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME; if (part_disk_ptr->exists(marker_path)) { LOG_WARNING(log, "Detaching stale part {}{}, which should have been deleted after a move. That can only happen after unclean restart of ClickHouse after move of a part having an operation blocking that stale copy of part.", getFullPathOnDisk(part_disk_ptr), part_name); @@ -1016,7 +1017,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) else has_adaptive_parts.store(true, std::memory_order_relaxed); - part->modification_time = part_disk_ptr->getLastModified(relative_data_path + part_name).epochTime(); + part->modification_time = part_disk_ptr->getLastModified(fs::path(relative_data_path) / part_name).epochTime(); /// Assume that all parts are Committed, covered parts will be detected and marked as Outdated later part->setState(DataPartState::Committed); @@ -1160,9 +1161,14 @@ void MergeTreeData::clearOldTemporaryDirectories(ssize_t custom_directories_life disk->removeRecursive(it->path()); } } - catch (const Poco::FileNotFoundException &) + catch (const fs::filesystem_error & e) { - /// If the file is already deleted, do nothing. + if (e.code() == std::errc::no_such_file_or_directory) + { + /// If the file is already deleted, do nothing. + } + else + throw; } } } @@ -1448,10 +1454,15 @@ void MergeTreeData::dropAllData() { disk->removeRecursive(path); } - catch (const Poco::FileNotFoundException &) + catch (const fs::filesystem_error & e) { - /// If the file is already deleted, log the error message and do nothing. - tryLogCurrentException(__PRETTY_FUNCTION__); + if (e.code() == std::errc::no_such_file_or_directory) + { + /// If the file is already deleted, log the error message and do nothing. + tryLogCurrentException(__PRETTY_FUNCTION__); + } + else + throw; } } @@ -1474,8 +1485,8 @@ void MergeTreeData::dropIfEmpty() for (const auto & [path, disk] : getRelativeDataPathsWithDisks()) { /// Non recursive, exception is thrown if there are more files. - disk->removeFileIfExists(path + MergeTreeData::FORMAT_VERSION_FILE_NAME); - disk->removeDirectory(path + MergeTreeData::DETACHED_DIR_NAME); + disk->removeFileIfExists(fs::path(path) / MergeTreeData::FORMAT_VERSION_FILE_NAME); + disk->removeDirectory(fs::path(path) / MergeTreeData::DETACHED_DIR_NAME); disk->removeDirectory(path); } } @@ -1925,7 +1936,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::createPart( const VolumePtr & volume, const String & relative_path, const IMergeTreeDataPart * parent_part) const { MergeTreeDataPartType type; - auto full_path = relative_data_path + (parent_part ? parent_part->relative_path + "/" : "") + relative_path + "/"; + auto full_path = fs::path(relative_data_path) / (parent_part ? parent_part->relative_path : "") / relative_path / ""; auto mrk_ext = MergeTreeIndexGranularityInfo::getMarksExtensionFromFilesystem(volume->getDisk(), full_path); if (mrk_ext) @@ -1978,7 +1989,7 @@ void MergeTreeData::changeSettings( { auto disk = new_storage_policy->getDiskByName(disk_name); disk->createDirectories(relative_data_path); - disk->createDirectories(relative_data_path + MergeTreeData::DETACHED_DIR_NAME); + disk->createDirectories(fs::path(relative_data_path) / MergeTreeData::DETACHED_DIR_NAME); } /// FIXME how would that be done while reloading configuration??? @@ -2007,7 +2018,7 @@ void MergeTreeData::PartsTemporaryRename::addPart(const String & old_name, const old_and_new_names.push_back({old_name, new_name}); for (const auto & [path, disk] : storage.getRelativeDataPathsWithDisks()) { - for (auto it = disk->iterateDirectory(path + source_dir); it->isValid(); it->next()) + for (auto it = disk->iterateDirectory(fs::path(path) / source_dir); it->isValid(); it->next()) { if (it->name() == old_name) { @@ -2029,8 +2040,8 @@ void MergeTreeData::PartsTemporaryRename::tryRenameAll() if (old_name.empty() || new_name.empty()) throw DB::Exception("Empty part name. Most likely it's a bug.", ErrorCodes::INCORRECT_FILE_NAME); const auto & [path, disk] = old_part_name_to_path_and_disk[old_name]; - const auto full_path = path + source_dir; /// for old_name - disk->moveFile(full_path + old_name, full_path + new_name); + const auto full_path = fs::path(path) / source_dir; /// for old_name + disk->moveFile(fs::path(full_path) / old_name, fs::path(full_path) / new_name); } catch (...) { @@ -2054,8 +2065,8 @@ MergeTreeData::PartsTemporaryRename::~PartsTemporaryRename() try { const auto & [path, disk] = old_part_name_to_path_and_disk[old_name]; - const auto full_path = path + source_dir; /// for old_name - disk->moveFile(full_path + new_name, full_path + old_name); + const String full_path = fs::path(path) / source_dir; /// for old_name + disk->moveFile(fs::path(full_path) / new_name, fs::path(full_path) / old_name); } catch (...) { @@ -2752,7 +2763,7 @@ void MergeTreeData::swapActivePart(MergeTreeData::DataPartPtr part_copy) addPartContributionToDataVolume(part_copy); auto disk = original_active_part->volume->getDisk(); - String marker_path = original_active_part->getFullRelativePath() + IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME; + String marker_path = fs::path(original_active_part->getFullRelativePath()) / IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME; try { disk->createFile(marker_path); @@ -3315,7 +3326,7 @@ MergeTreeData::getDetachedParts() const for (const auto & [path, disk] : getRelativeDataPathsWithDisks()) { - for (auto it = disk->iterateDirectory(path + MergeTreeData::DETACHED_DIR_NAME); it->isValid(); it->next()) + for (auto it = disk->iterateDirectory(fs::path(path) / MergeTreeData::DETACHED_DIR_NAME); it->isValid(); it->next()) { res.emplace_back(); auto & part = res.back(); @@ -3370,7 +3381,7 @@ void MergeTreeData::dropDetached(const ASTPtr & partition, bool part, ContextPtr for (auto & [old_name, new_name] : renamed_parts.old_and_new_names) { const auto & [path, disk] = renamed_parts.old_part_name_to_path_and_disk[old_name]; - disk->removeRecursive(path + "detached/" + new_name + "/"); + disk->removeRecursive(fs::path(path) / "detached" / new_name / ""); LOG_DEBUG(log, "Dropped detached part {}", old_name); old_name.clear(); } @@ -3431,8 +3442,8 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const if (!containing_part.empty() && containing_part != name) // TODO maybe use PartsTemporaryRename here? - disk->moveDirectory(relative_data_path + source_dir + name, - relative_data_path + source_dir + "inactive_" + name); + disk->moveDirectory(fs::path(relative_data_path) / source_dir / name, + fs::path(relative_data_path) / source_dir / ("inactive_" + name)); else renamed_parts.addPart(name, "attaching_" + name); } @@ -4294,12 +4305,12 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::cloneAndLoadDataPartOnSameDisk( const auto & src_relative_data_path = src_part_in_memory->storage.relative_data_path; auto flushed_part_path = src_part_in_memory->getRelativePathForPrefix(tmp_part_prefix); src_part_in_memory->flushToDisk(src_relative_data_path, flushed_part_path, metadata_snapshot); - src_part_path = src_relative_data_path + flushed_part_path + "/"; + src_part_path = fs::path(src_relative_data_path) / flushed_part_path / ""; } LOG_DEBUG(log, "Cloning part {} to {}", fullPath(disk, src_part_path), fullPath(disk, dst_part_path)); localBackup(disk, src_part_path, dst_part_path); - disk->removeFileIfExists(dst_part_path + "/" + IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME); + disk->removeFileIfExists(fs::path(dst_part_path) / IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME); auto single_disk_volume = std::make_shared(disk->getName(), disk, 0); auto dst_data_part = createPart(dst_part_name, dst_part_info, single_disk_volume, tmp_dst_part_name); @@ -4411,10 +4422,10 @@ PartitionCommandsResultInfo MergeTreeData::freezePartitionsByMatcher( const String & with_name, ContextPtr local_context) { - String clickhouse_path = Poco::Path(local_context->getPath()).makeAbsolute().toString(); - String default_shadow_path = clickhouse_path + "shadow/"; - Poco::File(default_shadow_path).createDirectories(); - auto increment = Increment(default_shadow_path + "increment.txt").get(true); + String clickhouse_path = fs::canonical(local_context->getPath()); + String default_shadow_path = fs::path(clickhouse_path) / "shadow/"; + fs::create_directories(default_shadow_path); + auto increment = Increment(fs::path(default_shadow_path) / "increment.txt").get(true); const String shadow_path = "shadow/"; @@ -4422,7 +4433,7 @@ PartitionCommandsResultInfo MergeTreeData::freezePartitionsByMatcher( const auto data_parts = getDataParts(); String backup_name = (!with_name.empty() ? escapeForFileName(with_name) : toString(increment)); - String backup_path = shadow_path + backup_name + "/"; + String backup_path = fs::path(shadow_path) / backup_name / ""; for (const auto & disk : getStoragePolicy()->getDisks()) disk->onFreeze(backup_path); @@ -4439,20 +4450,20 @@ PartitionCommandsResultInfo MergeTreeData::freezePartitionsByMatcher( part->volume->getDisk()->createDirectories(backup_path); - String backup_part_path = backup_path + relative_data_path + part->relative_path; + String backup_part_path = fs::path(backup_path) / relative_data_path / part->relative_path; if (auto part_in_memory = asInMemoryPart(part)) - part_in_memory->flushToDisk(backup_path + relative_data_path, part->relative_path, metadata_snapshot); + part_in_memory->flushToDisk(fs::path(backup_path) / relative_data_path, part->relative_path, metadata_snapshot); else localBackup(part->volume->getDisk(), part->getFullRelativePath(), backup_part_path); - part->volume->getDisk()->removeFileIfExists(backup_part_path + "/" + IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME); + part->volume->getDisk()->removeFileIfExists(fs::path(backup_part_path) / IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME); part->is_frozen.store(true, std::memory_order_relaxed); result.push_back(PartitionCommandResultInfo{ .partition_id = part->info.partition_id, .part_name = part->name, - .backup_path = part->volume->getDisk()->getPath() + backup_path, - .part_backup_path = part->volume->getDisk()->getPath() + backup_part_path, + .backup_path = fs::path(part->volume->getDisk()->getPath()) / backup_path, + .part_backup_path = fs::path(part->volume->getDisk()->getPath()) / backup_part_path, .backup_name = backup_name, }); ++parts_processed; @@ -4481,7 +4492,7 @@ PartitionCommandsResultInfo MergeTreeData::unfreezeAll( PartitionCommandsResultInfo MergeTreeData::unfreezePartitionsByMatcher(MatcherFn matcher, const String & backup_name, ContextPtr) { - auto backup_path = std::filesystem::path("shadow") / escapeForFileName(backup_name) / relative_data_path; + auto backup_path = fs::path("shadow") / escapeForFileName(backup_name) / relative_data_path; LOG_DEBUG(log, "Unfreezing parts by path {}", backup_path.generic_string()); diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index f5edde01478..846ad7b026d 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -2268,7 +2268,7 @@ void MergeTreeDataMergerMutator::finalizeMutatedPart( if (need_remove_expired_values) { /// Write a file with ttl infos in json format. - auto out_ttl = disk->writeFile(new_data_part->getFullRelativePath() + "ttl.txt", 4096); + auto out_ttl = disk->writeFile(fs::path(new_data_part->getFullRelativePath()) / "ttl.txt", 4096); HashingWriteBuffer out_hashing(*out_ttl); new_data_part->ttl_infos.write(out_hashing); new_data_part->checksums.files["ttl.txt"].file_size = out_hashing.count(); @@ -2277,7 +2277,7 @@ void MergeTreeDataMergerMutator::finalizeMutatedPart( { /// Write file with checksums. - auto out_checksums = disk->writeFile(new_data_part->getFullRelativePath() + "checksums.txt", 4096); + auto out_checksums = disk->writeFile(fs::path(new_data_part->getFullRelativePath()) / "checksums.txt", 4096); new_data_part->checksums.write(*out_checksums); } /// close fd @@ -2288,7 +2288,7 @@ void MergeTreeDataMergerMutator::finalizeMutatedPart( { /// Write a file with a description of columns. - auto out_columns = disk->writeFile(new_data_part->getFullRelativePath() + "columns.txt", 4096); + auto out_columns = disk->writeFile(fs::path(new_data_part->getFullRelativePath()) / "columns.txt", 4096); new_data_part->getColumns().writeText(*out_columns); } /// close fd diff --git a/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp b/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp index 8f5c4b6a848..1df97dc9241 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp @@ -7,7 +7,6 @@ #include #include #include -#include namespace DB diff --git a/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 5c1a3b01804..7b1641a0537 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -2,7 +2,6 @@ #include #include #include -#include namespace DB diff --git a/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp b/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp index 473f2e598a9..e929bfc6862 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #include #include diff --git a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index 84378faa513..4eebd729dd8 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -1,5 +1,4 @@ #include "MergeTreeDataPartWide.h" -#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index b1d2f33c659..00bc4306e4b 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -3,8 +3,6 @@ #include #include -#include - #include #include #include diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 3bf33d45802..0b05650b42c 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp b/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp index ed0e7e55fc8..6da0a822f7f 100644 --- a/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp @@ -1,6 +1,8 @@ #include #include -#include + + +namespace fs = std::filesystem; namespace DB { @@ -17,8 +19,7 @@ std::optional MergeTreeIndexGranularityInfo::getMarksExtensionFromF { for (DiskDirectoryIteratorPtr it = disk->iterateDirectory(path_to_part); it->isValid(); it->next()) { - Poco::Path path(it->path()); - const auto & ext = "." + path.getExtension(); + const auto & ext = fs::path(it->path()).extension(); if (ext == getNonAdaptiveMrkExtension() || ext == getAdaptiveMrkExtension(MergeTreeDataPartType::WIDE) || ext == getAdaptiveMrkExtension(MergeTreeDataPartType::COMPACT)) diff --git a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp index c5a99b128e9..717179e5f26 100644 --- a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp +++ b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp @@ -1,7 +1,6 @@ #include #include #include -#include #include diff --git a/src/Storages/MergeTree/MergeTreeMutationEntry.cpp b/src/Storages/MergeTree/MergeTreeMutationEntry.cpp index 49c4e93eb1d..2aefb3df2be 100644 --- a/src/Storages/MergeTree/MergeTreeMutationEntry.cpp +++ b/src/Storages/MergeTree/MergeTreeMutationEntry.cpp @@ -4,9 +4,6 @@ #include #include -#include -#include - #include diff --git a/src/Storages/MergeTree/MergeTreePartsMover.cpp b/src/Storages/MergeTree/MergeTreePartsMover.cpp index f9e3883d5e2..5b77ac9ec4a 100644 --- a/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -206,18 +206,18 @@ MergeTreeData::DataPartPtr MergeTreePartsMover::clonePart(const MergeTreeMoveEnt /// Try to fetch part from S3 without copy and fallback to default copy /// if it's not possible moving_part.part->assertOnDisk(); - String path_to_clone = data->getRelativeDataPath() + directory_to_move + "/"; + String path_to_clone = fs::path(data->getRelativeDataPath()) / directory_to_move / ""; String relative_path = part->relative_path; if (disk->exists(path_to_clone + relative_path)) { LOG_WARNING(log, "Path " + fullPath(disk, path_to_clone + relative_path) + " already exists. Will remove it and clone again."); - disk->removeRecursive(path_to_clone + relative_path + "/"); + disk->removeRecursive(fs::path(path_to_clone) / relative_path / ""); } disk->createDirectories(path_to_clone); - bool is_fetched = data->tryToFetchIfShared(*part, disk, path_to_clone + "/" + part->name); + bool is_fetched = data->tryToFetchIfShared(*part, disk, fs::path(path_to_clone) / part->name); if (!is_fetched) - part->volume->getDisk()->copy(data->getRelativeDataPath() + relative_path + "/", disk, path_to_clone); - part->volume->getDisk()->removeFileIfExists(path_to_clone + "/" + IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME); + part->volume->getDisk()->copy(fs::path(data->getRelativeDataPath()) / relative_path / "", disk, path_to_clone); + part->volume->getDisk()->removeFileIfExists(fs::path(path_to_clone) / IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME); } else { @@ -226,7 +226,7 @@ MergeTreeData::DataPartPtr MergeTreePartsMover::clonePart(const MergeTreeMoveEnt auto single_disk_volume = std::make_shared("volume_" + part->name, moving_part.reserved_space->getDisk(), 0); MergeTreeData::MutableDataPartPtr cloned_part = - data->createPart(part->name, single_disk_volume, directory_to_move + '/' + part->name); + data->createPart(part->name, single_disk_volume, fs::path(directory_to_move) / part->name); LOG_TRACE(log, "Part {} was cloned to {}", part->name, cloned_part->getFullPath()); cloned_part->loadColumnsChecksumsIndexes(true, true); diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index da28f75b57f..783a37cce60 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -2,7 +2,6 @@ #include #include #include -#include namespace DB { diff --git a/src/Storages/MergeTree/MergeTreeReaderInMemory.cpp b/src/Storages/MergeTree/MergeTreeReaderInMemory.cpp index 5ee4aa555e6..cf5c850daa1 100644 --- a/src/Storages/MergeTree/MergeTreeReaderInMemory.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderInMemory.cpp @@ -3,7 +3,6 @@ #include #include #include -#include namespace DB { diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 3655743d41a..6de01caa833 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -1,6 +1,5 @@ #include #include -#include #include @@ -123,7 +122,7 @@ void MergedBlockOutputStream::finalizePartOnDisk( { if (new_part->uuid != UUIDHelpers::Nil) { - auto out = volume->getDisk()->writeFile(part_path + IMergeTreeDataPart::UUID_FILE_NAME, 4096); + auto out = volume->getDisk()->writeFile(fs::path(part_path) / IMergeTreeDataPart::UUID_FILE_NAME, 4096); HashingWriteBuffer out_hashing(*out); writeUUIDText(new_part->uuid, out_hashing); checksums.files[IMergeTreeDataPart::UUID_FILE_NAME].file_size = out_hashing.count(); @@ -142,7 +141,7 @@ void MergedBlockOutputStream::finalizePartOnDisk( throw Exception("MinMax index was not initialized for new non-empty part " + new_part->name + ". It is a bug.", ErrorCodes::LOGICAL_ERROR); - auto count_out = volume->getDisk()->writeFile(part_path + "count.txt", 4096); + auto count_out = volume->getDisk()->writeFile(fs::path(part_path) / "count.txt", 4096); HashingWriteBuffer count_out_hashing(*count_out); writeIntText(rows_count, count_out_hashing); count_out_hashing.next(); @@ -157,7 +156,7 @@ void MergedBlockOutputStream::finalizePartOnDisk( if (!new_part->ttl_infos.empty()) { /// Write a file with ttl infos in json format. - auto out = volume->getDisk()->writeFile(part_path + "ttl.txt", 4096); + auto out = volume->getDisk()->writeFile(fs::path(part_path) / "ttl.txt", 4096); HashingWriteBuffer out_hashing(*out); new_part->ttl_infos.write(out_hashing); checksums.files["ttl.txt"].file_size = out_hashing.count(); @@ -171,7 +170,7 @@ void MergedBlockOutputStream::finalizePartOnDisk( { /// Write a file with a description of columns. - auto out = volume->getDisk()->writeFile(part_path + "columns.txt", 4096); + auto out = volume->getDisk()->writeFile(fs::path(part_path) / "columns.txt", 4096); part_columns.writeText(*out); out->finalize(); if (sync) @@ -192,7 +191,7 @@ void MergedBlockOutputStream::finalizePartOnDisk( { /// Write file with checksums. - auto out = volume->getDisk()->writeFile(part_path + "checksums.txt", 4096); + auto out = volume->getDisk()->writeFile(fs::path(part_path) / "checksums.txt", 4096); checksums.write(*out); out->finalize(); if (sync) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 30569e53f64..439d4970aa8 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -62,7 +62,7 @@ bool ReplicatedMergeTreeQueue::isVirtualPart(const MergeTreeData::DataPartPtr & bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper) { - auto queue_path = replica_path + "/queue"; + String queue_path = fs::path(replica_path) / "queue"; LOG_DEBUG(log, "Loading queue from {}", queue_path); bool updated = false; @@ -74,7 +74,7 @@ bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper) /// Reset batch size on initialization to recover from possible errors of too large batch size. current_multi_batch_size = 1; - String log_pointer_str = zookeeper->get(replica_path + "/log_pointer"); + String log_pointer_str = zookeeper->get(fs::path(replica_path) / "log_pointer"); log_pointer = log_pointer_str.empty() ? 0 : parse(log_pointer_str); std::unordered_set already_loaded_paths; @@ -101,7 +101,7 @@ bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper) futures.reserve(children.size()); for (const String & child : children) - futures.emplace_back(child, zookeeper->asyncGet(queue_path + "/" + child)); + futures.emplace_back(child, zookeeper->asyncGet(fs::path(queue_path) / child)); for (auto & future : futures) { @@ -116,7 +116,7 @@ bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper) updated = true; } - zookeeper->tryGet(replica_path + "/mutation_pointer", mutation_pointer); + zookeeper->tryGet(fs::path(replica_path) / "mutation_pointer", mutation_pointer); } updateTimesInZooKeeper(zookeeper, min_unprocessed_insert_time_changed, {}); @@ -399,7 +399,7 @@ void ReplicatedMergeTreeQueue::removeProcessedEntry(zkutil::ZooKeeperPtr zookeep if (!need_remove_from_zk) return; - auto code = zookeeper->tryRemove(replica_path + "/queue/" + entry->znode_name); + auto code = zookeeper->tryRemove(fs::path(replica_path) / "queue" / entry->znode_name); if (code != Coordination::Error::ZOK) LOG_ERROR(log, "Couldn't remove {}/queue/{}: {}. This shouldn't happen often.", replica_path, entry->znode_name, Coordination::errorMessage(code)); @@ -455,7 +455,7 @@ bool ReplicatedMergeTreeQueue::remove(zkutil::ZooKeeperPtr zookeeper, const Stri notifySubscribers(queue_size); - zookeeper->tryRemove(replica_path + "/queue/" + found->znode_name); + zookeeper->tryRemove(fs::path(replica_path) / "queue" / found->znode_name); updateTimesInZooKeeper(zookeeper, min_unprocessed_insert_time_changed, max_processed_insert_time_changed); return true; @@ -474,14 +474,14 @@ int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper if (pull_log_blocker.isCancelled()) throw Exception("Log pulling is cancelled", ErrorCodes::ABORTED); - String index_str = zookeeper->get(replica_path + "/log_pointer"); + String index_str = zookeeper->get(fs::path(replica_path) / "log_pointer"); UInt64 index; /// The version of "/log" is modified when new entries to merge/mutate/drop appear. Coordination::Stat stat; - zookeeper->get(zookeeper_path + "/log", &stat); + zookeeper->get(fs::path(zookeeper_path) / "log", &stat); - Strings log_entries = zookeeper->getChildrenWatch(zookeeper_path + "/log", nullptr, watch_callback); + Strings log_entries = zookeeper->getChildrenWatch(fs::path(zookeeper_path) / "log", nullptr, watch_callback); /// We update mutations after we have loaded the list of log entries, but before we insert them /// in the queue. @@ -494,7 +494,7 @@ int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper /// If we do not already have a pointer to the log, put a pointer to the first entry in it. index = log_entries.empty() ? 0 : parse(std::min_element(log_entries.begin(), log_entries.end())->substr(strlen("log-"))); - zookeeper->set(replica_path + "/log_pointer", toString(index)); + zookeeper->set(fs::path(replica_path) / "log_pointer", toString(index)); } else { @@ -541,7 +541,7 @@ int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper futures.reserve(end - begin); for (auto it = begin; it != end; ++it) - futures.emplace_back(*it, zookeeper->asyncGet(zookeeper_path + "/log/" + *it)); + futures.emplace_back(*it, zookeeper->asyncGet(fs::path(zookeeper_path) / "log" / *it)); /// Simultaneously add all new entries to the queue and move the pointer to the log. @@ -558,7 +558,7 @@ int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper copied_entries.emplace_back(LogEntry::parse(res.data, res.stat)); ops.emplace_back(zkutil::makeCreateRequest( - replica_path + "/queue/queue-", res.data, zkutil::CreateMode::PersistentSequential)); + fs::path(replica_path) / "queue/queue-", res.data, zkutil::CreateMode::PersistentSequential)); const auto & entry = *copied_entries.back(); if (entry.type == LogEntry::GET_PART || entry.type == LogEntry::ATTACH_PART) @@ -573,11 +573,11 @@ int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper } ops.emplace_back(zkutil::makeSetRequest( - replica_path + "/log_pointer", toString(last_entry_index + 1), -1)); + fs::path(replica_path) / "log_pointer", toString(last_entry_index + 1), -1)); if (min_unprocessed_insert_time_changed) ops.emplace_back(zkutil::makeSetRequest( - replica_path + "/min_unprocessed_insert_time", toString(*min_unprocessed_insert_time_changed), -1)); + fs::path(replica_path) / "min_unprocessed_insert_time", toString(*min_unprocessed_insert_time_changed), -1)); auto responses = zookeeper->multi(ops); @@ -655,7 +655,7 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C { std::lock_guard lock(update_mutations_mutex); - Strings entries_in_zk = zookeeper->getChildrenWatch(zookeeper_path + "/mutations", nullptr, watch_callback); + Strings entries_in_zk = zookeeper->getChildrenWatch(fs::path(zookeeper_path) / "mutations", nullptr, watch_callback); StringSet entries_in_zk_set(entries_in_zk.begin(), entries_in_zk.end()); /// Compare with the local state, delete obsolete entries and determine which new entries to load. @@ -712,7 +712,7 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C std::vector> futures; for (const String & entry : entries_to_load) - futures.emplace_back(zookeeper->asyncGet(zookeeper_path + "/mutations/" + entry)); + futures.emplace_back(zookeeper->asyncGet(fs::path(zookeeper_path) / "mutations" / entry)); std::vector new_mutations; for (size_t i = 0; i < entries_to_load.size(); ++i) @@ -796,7 +796,7 @@ ReplicatedMergeTreeMutationEntryPtr ReplicatedMergeTreeQueue::removeMutation( { std::lock_guard lock(update_mutations_mutex); - auto rc = zookeeper->tryRemove(zookeeper_path + "/mutations/" + mutation_id); + auto rc = zookeeper->tryRemove(fs::path(zookeeper_path) / "mutations" / mutation_id); if (rc == Coordination::Error::ZOK) LOG_DEBUG(log, "Removed mutation {} from ZooKeeper.", mutation_id); @@ -940,12 +940,12 @@ void ReplicatedMergeTreeQueue::removePartProducingOpsInRange( { if ((*it)->currently_executing) to_wait.push_back(*it); - auto code = zookeeper->tryRemove(replica_path + "/queue/" + (*it)->znode_name); + auto code = zookeeper->tryRemove(fs::path(replica_path) / "queue" / (*it)->znode_name); /// FIXME it's probably unsafe to remove entries non-atomically /// when this method called directly from alter query (not from replication queue task), /// because entries will be lost if ALTER fails. if (code != Coordination::Error::ZOK) - LOG_INFO(log, "Couldn't remove {}: {}", replica_path + "/queue/" + (*it)->znode_name, Coordination::errorMessage(code)); + LOG_INFO(log, "Couldn't remove {}: {}", (fs::path(replica_path) / "queue" / (*it)->znode_name).string(), Coordination::errorMessage(code)); updateStateOnQueueEntryRemoval( *it, /* is_successful = */ false, @@ -1554,7 +1554,7 @@ bool ReplicatedMergeTreeQueue::tryFinalizeMutations(zkutil::ZooKeeperPtr zookeep if (!finished.empty()) { - zookeeper->set(replica_path + "/mutation_pointer", finished.back()->znode_name); + zookeeper->set(fs::path(replica_path) / "mutation_pointer", finished.back()->znode_name); std::lock_guard lock(state_mutex); @@ -1750,22 +1750,22 @@ ReplicatedMergeTreeMergePredicate::ReplicatedMergeTreeMergePredicate( } /// Load current quorum status. - auto quorum_status_future = zookeeper->asyncTryGet(queue.zookeeper_path + "/quorum/status"); + auto quorum_status_future = zookeeper->asyncTryGet(fs::path(queue.zookeeper_path) / "quorum" / "status"); /// Load current inserts std::unordered_set lock_holder_paths; - for (const String & entry : zookeeper->getChildren(queue.zookeeper_path + "/temp")) + for (const String & entry : zookeeper->getChildren(fs::path(queue.zookeeper_path) / "temp")) { if (startsWith(entry, "abandonable_lock-")) - lock_holder_paths.insert(queue.zookeeper_path + "/temp/" + entry); + lock_holder_paths.insert(fs::path(queue.zookeeper_path) / "temp" / entry); } if (!lock_holder_paths.empty()) { - Strings partitions = zookeeper->getChildren(queue.zookeeper_path + "/block_numbers"); + Strings partitions = zookeeper->getChildren(fs::path(queue.zookeeper_path) / "block_numbers"); std::vector> lock_futures; for (const String & partition : partitions) - lock_futures.push_back(zookeeper->asyncGetChildren(queue.zookeeper_path + "/block_numbers/" + partition)); + lock_futures.push_back(zookeeper->asyncGetChildren(fs::path(queue.zookeeper_path) / "block_numbers" / partition)); struct BlockInfoInZooKeeper { @@ -1786,7 +1786,7 @@ ReplicatedMergeTreeMergePredicate::ReplicatedMergeTreeMergePredicate( if (startsWith(entry, "block-")) { Int64 block_number = parse(entry.substr(strlen("block-"))); - String zk_path = queue.zookeeper_path + "/block_numbers/" + partitions[i] + "/" + entry; + String zk_path = fs::path(queue.zookeeper_path) / "block_numbers" / partitions[i] / entry; block_infos.emplace_back( BlockInfoInZooKeeper{partitions[i], block_number, zk_path, zookeeper->asyncTryGet(zk_path)}); } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index 6b7fb3bf17f..7288396b58b 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -238,7 +238,7 @@ void ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart() auto zookeeper = storage.getZooKeeper(); String quorum_str; - if (zookeeper->tryGet(storage.zookeeper_path + "/quorum/status", quorum_str)) + if (zookeeper->tryGet(fs::path(storage.zookeeper_path) / "quorum" / "status", quorum_str)) { ReplicatedMergeTreeQuorumEntry quorum_entry(quorum_str); @@ -251,12 +251,12 @@ void ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart() } Strings part_names; - String parallel_quorum_parts_path = storage.zookeeper_path + "/quorum/parallel"; + String parallel_quorum_parts_path = fs::path(storage.zookeeper_path) / "quorum" / "parallel"; if (zookeeper->tryGetChildren(parallel_quorum_parts_path, part_names) == Coordination::Error::ZOK) { for (auto & part_name : part_names) { - if (zookeeper->tryGet(parallel_quorum_parts_path + "/" + part_name, quorum_str)) + if (zookeeper->tryGet(fs::path(parallel_quorum_parts_path) / part_name, quorum_str)) { ReplicatedMergeTreeQuorumEntry quorum_entry(quorum_str); if (!quorum_entry.replicas.count(storage.replica_name) @@ -278,7 +278,7 @@ void ReplicatedMergeTreeRestartingThread::activateReplica() /// How other replicas can access this one. ReplicatedMergeTreeAddress address = storage.getReplicatedMergeTreeAddress(); - String is_active_path = storage.replica_path + "/is_active"; + String is_active_path = fs::path(storage.replica_path) / "is_active"; /** If the node is marked as active, but the mark is made in the same instance, delete it. * This is possible only when session in ZooKeeper expires. @@ -302,7 +302,7 @@ void ReplicatedMergeTreeRestartingThread::activateReplica() /// Simultaneously declare that this replica is active, and update the host. Coordination::Requests ops; ops.emplace_back(zkutil::makeCreateRequest(is_active_path, active_node_identifier, zkutil::CreateMode::Ephemeral)); - ops.emplace_back(zkutil::makeSetRequest(storage.replica_path + "/host", address.toString(), -1)); + ops.emplace_back(zkutil::makeSetRequest(fs::path(storage.replica_path) / "host", address.toString(), -1)); try { @@ -311,7 +311,7 @@ void ReplicatedMergeTreeRestartingThread::activateReplica() catch (const Coordination::Exception & e) { String existing_replica_host; - zookeeper->tryGet(storage.replica_path + "/host", existing_replica_host); + zookeeper->tryGet(fs::path(storage.replica_path) / "host", existing_replica_host); if (existing_replica_host.empty()) existing_replica_host = "without host node"; diff --git a/src/Storages/MergeTree/checkDataPart.cpp b/src/Storages/MergeTree/checkDataPart.cpp index 8f0847fc106..8a234833da7 100644 --- a/src/Storages/MergeTree/checkDataPart.cpp +++ b/src/Storages/MergeTree/checkDataPart.cpp @@ -1,7 +1,6 @@ #include #include -#include #include #include @@ -70,7 +69,7 @@ IMergeTreeDataPart::Checksums checkDataPart( NamesAndTypesList columns_txt; { - auto buf = disk->readFile(path + "columns.txt"); + auto buf = disk->readFile(fs::path(path) / "columns.txt"); columns_txt.readText(*buf); assertEOF(*buf); } @@ -231,9 +230,9 @@ IMergeTreeDataPart::Checksums checkDataPart( /// Checksums from the rest files listed in checksums.txt. May be absent. If present, they are subsequently compared with the actual data checksums. IMergeTreeDataPart::Checksums checksums_txt; - if (require_checksums || disk->exists(path + "checksums.txt")) + if (require_checksums || disk->exists(fs::path(path) / "checksums.txt")) { - auto buf = disk->readFile(path + "checksums.txt"); + auto buf = disk->readFile(fs::path(path) / "checksums.txt"); checksums_txt.read(*buf); assertEOF(*buf); } diff --git a/src/Storages/MergeTree/localBackup.cpp b/src/Storages/MergeTree/localBackup.cpp index 7d7dacaeaf1..2e02481e065 100644 --- a/src/Storages/MergeTree/localBackup.cpp +++ b/src/Storages/MergeTree/localBackup.cpp @@ -29,7 +29,7 @@ static void localBackupImpl(const DiskPtr & disk, const String & source_path, co for (auto it = disk->iterateDirectory(source_path); it->isValid(); it->next()) { auto source = it->path(); - auto destination = destination_path + "/" + it->name(); + auto destination = fs::path(destination_path) / it->name(); if (!disk->isDirectory(source)) { @@ -74,13 +74,16 @@ void localBackup(const DiskPtr & disk, const String & source_path, const String continue; } - catch (const Poco::FileNotFoundException &) + catch (const fs::filesystem_error & e) { - ++try_no; - if (try_no == max_tries) - throw; - - continue; + if (e.code() == std::errc::no_such_file_or_directory) + { + ++try_no; + if (try_no == max_tries) + throw; + continue; + } + throw; } break; diff --git a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp index af6f01a0ab5..70cc173e38a 100644 --- a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp +++ b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp @@ -27,14 +27,16 @@ #include #include -#include -#include #include #include #include #include +#include + + +namespace fs = std::filesystem; namespace DB { @@ -254,7 +256,7 @@ StorageEmbeddedRocksDB::StorageEmbeddedRocksDB(const StorageID & table_id_, rocksdb_dir = context_->getPath() + relative_data_path_; if (!attach) { - Poco::File(rocksdb_dir).createDirectories(); + fs::create_directories(rocksdb_dir); } initDb(); } @@ -262,8 +264,8 @@ StorageEmbeddedRocksDB::StorageEmbeddedRocksDB(const StorageID & table_id_, void StorageEmbeddedRocksDB::truncate(const ASTPtr &, const StorageMetadataPtr & , ContextPtr, TableExclusiveLockHolder &) { rocksdb_ptr->Close(); - Poco::File(rocksdb_dir).remove(true); - Poco::File(rocksdb_dir).createDirectories(); + fs::remove_all(rocksdb_dir); + fs::create_directories(rocksdb_dir); initDb(); } diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 3a3291c6c48..256875454db 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -76,6 +76,8 @@ #include +namespace fs = std::filesystem; + namespace { const UInt64 FORCE_OPTIMIZE_SKIP_UNUSED_SHARDS_HAS_SHARDING_KEY = 1; @@ -864,7 +866,7 @@ StoragePolicyPtr StorageDistributed::getStoragePolicy() const void StorageDistributed::createDirectoryMonitors(const DiskPtr & disk) { const std::string path(disk->getPath() + relative_data_path); - Poco::File{path}.createDirectories(); + fs::create_directories(path); std::filesystem::directory_iterator begin(path); std::filesystem::directory_iterator end; diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index afd0d1f47f9..b67cd0a0af7 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -28,9 +28,6 @@ #include #include -#include -#include - #include #include #include @@ -79,10 +76,9 @@ std::vector listFilesWithRegexpMatching(const std::string & path_fo std::vector result; const std::string prefix_without_globs = path_for_ls + for_match.substr(1, end_of_path_without_globs); - if (!fs::exists(fs::path(prefix_without_globs))) - { + if (!fs::exists(prefix_without_globs)) return result; - } + const fs::directory_iterator end; for (fs::directory_iterator it(prefix_without_globs); it != end; ++it) { @@ -104,7 +100,7 @@ std::vector listFilesWithRegexpMatching(const std::string & path_fo if (re2::RE2::FullMatch(file_name, matcher)) { /// Recursion depth is limited by pattern. '*' works only for depth = 1, for depth = 2 pattern path is '*/*'. So we do not need additional check. - Strings result_part = listFilesWithRegexpMatching(full_path + "/", suffix_with_globs.substr(next_slash), total_bytes_to_read); + Strings result_part = listFilesWithRegexpMatching(fs::path(full_path) / "", suffix_with_globs.substr(next_slash), total_bytes_to_read); std::move(result_part.begin(), result_part.end(), std::back_inserter(result)); } } @@ -127,21 +123,20 @@ void checkCreationIsAllowed(ContextPtr context_global, const std::string & db_di if (!startsWith(table_path, db_dir_path) && table_path != "/dev/null") throw Exception("File is not inside " + db_dir_path, ErrorCodes::DATABASE_ACCESS_DENIED); - Poco::File table_path_poco_file = Poco::File(table_path); - if (table_path_poco_file.exists() && table_path_poco_file.isDirectory()) + if (fs::exists(table_path) && fs::is_directory(table_path)) throw Exception("File must not be a directory", ErrorCodes::INCORRECT_FILE_NAME); } } Strings StorageFile::getPathsList(const String & table_path, const String & user_files_path, ContextPtr context, size_t & total_bytes_to_read) { - String user_files_absolute_path = Poco::Path(user_files_path).makeAbsolute().makeDirectory().toString(); - Poco::Path poco_path = Poco::Path(table_path); - if (poco_path.isRelative()) - poco_path = Poco::Path(user_files_absolute_path, poco_path); + fs::path user_files_absolute_path = fs::weakly_canonical(user_files_path); + fs::path fs_table_path(table_path); + if (fs_table_path.is_relative()) + fs_table_path = user_files_absolute_path / fs_table_path; Strings paths; - const String path = poco_path.absolute().toString(); + const String path = fs::weakly_canonical(fs_table_path); if (path.find_first_of("*?{") == std::string::npos) { std::error_code error; @@ -211,8 +206,8 @@ StorageFile::StorageFile(const std::string & relative_table_dir_path, CommonArgu if (args.format_name == "Distributed") throw Exception("Distributed format is allowed only with explicit file path", ErrorCodes::INCORRECT_FILE_NAME); - String table_dir_path = base_path + relative_table_dir_path + "/"; - Poco::File(table_dir_path).createDirectories(); + String table_dir_path = fs::path(base_path) / relative_table_dir_path / ""; + fs::create_directories(table_dir_path); paths = {getTablePath(table_dir_path, format_name)}; } @@ -471,7 +466,7 @@ Pipe StorageFile::read( if (use_table_fd) /// need to call ctr BlockInputStream paths = {""}; /// when use fd, paths are empty else - if (paths.size() == 1 && !Poco::File(paths[0]).exists()) + if (paths.size() == 1 && !fs::exists(paths[0])) { if (context->getSettingsRef().engine_file_empty_if_not_exists) return Pipe(std::make_shared(metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID()))); @@ -619,7 +614,7 @@ BlockOutputStreamPtr StorageFile::write( if (!paths.empty()) { path = paths[0]; - Poco::File(Poco::Path(path).makeParent()).createDirectories(); + fs::create_directories(fs::path(path).parent_path()); } return std::make_shared( @@ -656,8 +651,8 @@ void StorageFile::rename(const String & new_path_to_table_data, const StorageID if (path_new == paths[0]) return; - Poco::File(Poco::Path(path_new).parent()).createDirectories(); - Poco::File(paths[0]).renameTo(path_new); + fs::create_directories(fs::path(path_new).parent_path()); + fs::rename(paths[0], path_new); paths[0] = std::move(path_new); renameInMemory(new_table_id); @@ -679,7 +674,7 @@ void StorageFile::truncate( } else { - if (!Poco::File(paths[0]).exists()) + if (!fs::exists(paths[0])) return; if (0 != ::truncate(paths[0].c_str(), 0)) diff --git a/src/Storages/StorageFile.h b/src/Storages/StorageFile.h index 9eabbfed241..74ee818cc9c 100644 --- a/src/Storages/StorageFile.h +++ b/src/Storages/StorageFile.h @@ -1,10 +1,6 @@ #pragma once #include - -#include -#include - #include #include diff --git a/src/Storages/StorageJoin.cpp b/src/Storages/StorageJoin.cpp index f0129e95495..7a448e1dde9 100644 --- a/src/Storages/StorageJoin.cpp +++ b/src/Storages/StorageJoin.cpp @@ -14,9 +14,7 @@ #include #include #include - #include /// toLower -#include #include #include diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index c0ec041d7c6..b33daf67269 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1509,8 +1509,8 @@ CheckResults StorageMergeTree::checkData(const ASTPtr & query, ContextPtr local_ auto disk = part->volume->getDisk(); String part_path = part->getFullRelativePath(); /// If the checksums file is not present, calculate the checksums and write them to disk. - String checksums_path = part_path + "checksums.txt"; - String tmp_checksums_path = part_path + "checksums.txt.tmp"; + String checksums_path = fs::path(part_path) / "checksums.txt"; + String tmp_checksums_path = fs::path(part_path) / "checksums.txt.tmp"; if (part->isStoredOnDisk() && !disk->exists(checksums_path)) { try diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index debed255c9f..57e7229de16 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -66,9 +66,12 @@ #include #include #include +#include #include +namespace fs = std::filesystem; + namespace ProfileEvents { extern const Event ReplicatedPartMerges; @@ -272,7 +275,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( , zookeeper_name(extractZooKeeperName(zookeeper_path_)) , zookeeper_path(extractZooKeeperPath(zookeeper_path_)) , replica_name(replica_name_) - , replica_path(zookeeper_path + "/replicas/" + replica_name_) + , replica_path(fs::path(zookeeper_path) / "replicas" / replica_name_) , reader(*this) , writer(*this) , merger_mutator(*this, getContext()->getSettingsRef().background_pool_size) @@ -493,14 +496,14 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( /// Mutation maybe killed or whole replica was deleted. /// Wait event will unblock at this moment. Coordination::Stat exists_stat; - if (!getZooKeeper()->exists(zookeeper_path + "/mutations/" + mutation_id, &exists_stat, wait_event)) + if (!getZooKeeper()->exists(fs::path(zookeeper_path) / "mutations" / mutation_id, &exists_stat, wait_event)) { throw Exception(ErrorCodes::UNFINISHED, "Mutation {} was killed, manually removed or table was dropped", mutation_id); } auto zookeeper = getZooKeeper(); /// Replica could be inactive. - if (!zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active")) + if (!zookeeper->exists(fs::path(zookeeper_path) / "replicas" / replica / "is_active")) { LOG_WARNING(log, "Replica {} is not active during mutation. Mutation will be done asynchronously when replica becomes active.", replica); @@ -508,7 +511,7 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( break; } - String mutation_pointer = zookeeper_path + "/replicas/" + replica + "/mutation_pointer"; + String mutation_pointer = fs::path(zookeeper_path) / "replicas" / replica / "mutation_pointer"; std::string mutation_pointer_value; /// Replica could be removed if (!zookeeper->tryGet(mutation_pointer, mutation_pointer_value, nullptr, wait_event)) @@ -523,16 +526,17 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( if (wait_event->tryWait(1000)) continue; - /// Here we check mutation for errors or kill on local replica. If they happen on this replica + /// Here we check mutation for errors on local replica. If they happen on this replica /// they will happen on each replica, so we can check only in-memory info. auto mutation_status = queue.getIncompleteMutationsStatus(mutation_id); - if (!mutation_status || !mutation_status->latest_fail_reason.empty()) + /// If mutation status is empty, than local replica may just not loaded it into memory. + if (mutation_status && !mutation_status->latest_fail_reason.empty()) break; } /// It maybe already removed from zk, but local in-memory mutations /// state was not updated. - if (!getZooKeeper()->exists(zookeeper_path + "/mutations/" + mutation_id)) + if (!getZooKeeper()->exists(fs::path(zookeeper_path) / "mutations" / mutation_id)) { throw Exception(ErrorCodes::UNFINISHED, "Mutation {} was killed, manually removed or table was dropped", mutation_id); } @@ -877,12 +881,12 @@ bool StorageReplicatedMergeTree::removeTableNodesFromZooKeeper(zkutil::ZooKeeper for (const auto & child : children) if (child != "dropped") - zookeeper->tryRemoveRecursive(zookeeper_path + "/" + child); + zookeeper->tryRemoveRecursive(fs::path(zookeeper_path) / child); Coordination::Requests ops; Coordination::Responses responses; ops.emplace_back(zkutil::makeRemoveRequest(metadata_drop_lock->getPath(), -1)); - ops.emplace_back(zkutil::makeRemoveRequest(zookeeper_path + "/dropped", -1)); + ops.emplace_back(zkutil::makeRemoveRequest(fs::path(zookeeper_path) / "dropped", -1)); ops.emplace_back(zkutil::makeRemoveRequest(zookeeper_path, -1)); code = zookeeper->tryMulti(ops, responses); @@ -921,12 +925,12 @@ void StorageReplicatedMergeTree::checkTableStructure(const String & zookeeper_pr ReplicatedMergeTreeTableMetadata old_metadata(*this, metadata_snapshot); Coordination::Stat metadata_stat; - String metadata_str = zookeeper->get(zookeeper_prefix + "/metadata", &metadata_stat); + String metadata_str = zookeeper->get(fs::path(zookeeper_prefix) / "metadata", &metadata_stat); auto metadata_from_zk = ReplicatedMergeTreeTableMetadata::parse(metadata_str); old_metadata.checkEquals(metadata_from_zk, metadata_snapshot->getColumns(), getContext()); Coordination::Stat columns_stat; - auto columns_from_zk = ColumnsDescription::parse(zookeeper->get(zookeeper_prefix + "/columns", &columns_stat)); + auto columns_from_zk = ColumnsDescription::parse(zookeeper->get(fs::path(zookeeper_prefix) / "columns", &columns_stat)); const ColumnsDescription & old_columns = metadata_snapshot->getColumns(); if (columns_from_zk != old_columns) @@ -1071,7 +1075,7 @@ static time_t tryGetPartCreateTime(zkutil::ZooKeeperPtr & zookeeper, const Strin /// We get creation time of part, if it still exists (was not merged, for example). Coordination::Stat stat; String unused; - if (zookeeper->tryGet(replica_path + "/parts/" + part_name, unused, &stat)) + if (zookeeper->tryGet(fs::path(replica_path) / "parts" / part_name, unused, &stat)) res = stat.ctime / 1000; return res; @@ -1082,7 +1086,7 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) { auto zookeeper = getZooKeeper(); - Strings expected_parts_vec = zookeeper->getChildren(replica_path + "/parts"); + Strings expected_parts_vec = zookeeper->getChildren(fs::path(replica_path) / "parts"); /// Parts in ZK. NameSet expected_parts(expected_parts_vec.begin(), expected_parts_vec.end()); @@ -1185,7 +1189,7 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) exists_futures.reserve(parts_to_fetch.size()); for (const String & part_name : parts_to_fetch) { - String part_path = replica_path + "/parts/" + part_name; + String part_path = fs::path(replica_path) / "parts" / part_name; exists_futures.emplace_back(zookeeper->asyncExists(part_path)); } @@ -1214,7 +1218,7 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) /// We assume that this occurs before the queue is loaded (queue.initialize). ops.emplace_back(zkutil::makeCreateRequest( - replica_path + "/queue/queue-", log_entry.toString(), zkutil::CreateMode::PersistentSequential)); + fs::path(replica_path) / "queue/queue-", log_entry.toString(), zkutil::CreateMode::PersistentSequential)); enqueue_futures.emplace_back(zookeeper->asyncMulti(ops)); } @@ -1261,13 +1265,13 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: auto local_part_header = ReplicatedMergeTreePartHeader::fromColumnsAndChecksums( part->getColumns(), part->checksums); - Strings replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); + Strings replicas = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas"); std::shuffle(replicas.begin(), replicas.end(), thread_local_rng); bool has_been_already_added = false; for (const String & replica : replicas) { - String current_part_path = zookeeper_path + "/replicas/" + replica + "/parts/" + part_name; + String current_part_path = fs::path(zookeeper_path) / "replicas" / replica / "parts" / part_name; String part_zk_str; if (!zookeeper->tryGet(current_part_path, part_zk_str)) @@ -1288,9 +1292,9 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: String checksums_str; /// Let's check that the node's version with the columns did not change while we were reading the checksums. /// This ensures that the columns and the checksum refer to the same - if (!zookeeper->tryGet(current_part_path + "/columns", columns_str, &columns_stat_before) || - !zookeeper->tryGet(current_part_path + "/checksums", checksums_str) || - !zookeeper->exists(current_part_path + "/columns", &columns_stat_after) || + if (!zookeeper->tryGet(fs::path(current_part_path) / "columns", columns_str, &columns_stat_before) || + !zookeeper->tryGet(fs::path(current_part_path) / "checksums", checksums_str) || + !zookeeper->exists(fs::path(current_part_path) / "columns", &columns_stat_after) || columns_stat_before.version != columns_stat_after.version) { LOG_INFO(log, "Not checking checksums of part {} with replica {} because part changed while we were reading its checksums", part_name, replica); @@ -1324,7 +1328,7 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: if (!has_been_already_added) { const auto storage_settings_ptr = getSettings(); - String part_path = replica_path + "/parts/" + part_name; + String part_path = fs::path(replica_path) / "parts" / part_name; //ops.emplace_back(zkutil::makeCheckRequest( // zookeeper_path + "/columns", expected_columns_version)); @@ -1339,14 +1343,15 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: ops.emplace_back(zkutil::makeCreateRequest( part_path, "", zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest( - part_path + "/columns", part->getColumns().toString(), zkutil::CreateMode::Persistent)); + fs::path(part_path) / "columns", part->getColumns().toString(), zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest( - part_path + "/checksums", getChecksumsForZooKeeper(part->checksums), zkutil::CreateMode::Persistent)); + fs::path(part_path) / "checksums", getChecksumsForZooKeeper(part->checksums), zkutil::CreateMode::Persistent)); } } else { - LOG_WARNING(log, "checkPartAndAddToZooKeeper: node {} already exists. Will not commit any nodes.", replica_path + "/parts/" + part_name); + LOG_WARNING(log, "checkPartAndAddToZooKeeper: node {} already exists. Will not commit any nodes.", + (fs::path(replica_path) / "parts" / part_name).string()); } } @@ -1414,7 +1419,7 @@ MergeTreeData::MutableDataPartPtr StorageReplicatedMergeTree::attachPartHelperFo const String part_new_name = actual_part_info.getPartName(); for (const DiskPtr & disk : getStoragePolicy()->getDisks()) - for (const auto it = disk->iterateDirectory(relative_data_path + "detached/"); it->isValid(); it->next()) + for (const auto it = disk->iterateDirectory(fs::path(relative_data_path) / "detached/"); it->isValid(); it->next()) { MergeTreePartInfo part_info; @@ -1423,7 +1428,7 @@ MergeTreeData::MutableDataPartPtr StorageReplicatedMergeTree::attachPartHelperFo continue; const String part_old_name = part_info.getPartName(); - const String part_path = "detached/" + part_old_name; + const String part_path = fs::path("detached") / part_old_name; const VolumePtr volume = std::make_shared("volume_" + part_old_name, disk); @@ -1478,7 +1483,7 @@ bool StorageReplicatedMergeTree::executeLogEntry(LogEntry & entry) existing_part = getActiveContainingPart(entry.new_part_name); /// Even if the part is local, it (in exceptional cases) may not be in ZooKeeper. Let's check that it is there. - if (existing_part && getZooKeeper()->exists(replica_path + "/parts/" + existing_part->name)) + if (existing_part && getZooKeeper()->exists(fs::path(replica_path) / "parts" / existing_part->name)) { if (!is_get_or_attach || entry.source_replica != replica_name) LOG_DEBUG(log, "Skipping action for part {} because part {} already exists.", @@ -1513,7 +1518,7 @@ bool StorageReplicatedMergeTree::executeLogEntry(LogEntry & entry) /// Perhaps we don't need this part, because during write with quorum, the quorum has failed /// (see below about `/quorum/failed_parts`). - if (entry.quorum && getZooKeeper()->exists(zookeeper_path + "/quorum/failed_parts/" + entry.new_part_name)) + if (entry.quorum && getZooKeeper()->exists(fs::path(zookeeper_path) / "quorum" / "failed_parts" / entry.new_part_name)) { LOG_DEBUG(log, "Skipping action for part {} because quorum for that part was failed.", entry.new_part_name); return true; /// NOTE Deletion from `virtual_parts` is not done, but it is only necessary for merge. @@ -1983,14 +1988,14 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) auto zookeeper = getZooKeeper(); - Strings replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); + Strings replicas = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas"); Coordination::Requests ops; for (const auto & path_part : replicas) { Coordination::Stat stat; - String path = zookeeper_path + "/replicas/" + path_part + "/host"; + String path = fs::path(zookeeper_path) / "replicas" / path_part / "host"; zookeeper->get(path, &stat); ops.emplace_back(zkutil::makeCheckRequest(path, stat.version)); } @@ -2004,8 +2009,8 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) if (replica.empty()) { Coordination::Stat quorum_stat; - const String quorum_unparallel_path = zookeeper_path + "/quorum/status"; - const String quorum_parallel_path = zookeeper_path + "/quorum/parallel/" + entry.new_part_name; + const String quorum_unparallel_path = fs::path(zookeeper_path) / "quorum" / "status"; + const String quorum_parallel_path = fs::path(zookeeper_path) / "quorum" / "parallel" / entry.new_part_name; String quorum_str, quorum_path; ReplicatedMergeTreeQuorumEntry quorum_entry; @@ -2029,13 +2034,13 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) ErrorCodes::LOGICAL_ERROR); ops.emplace_back(zkutil::makeCreateRequest( - zookeeper_path + "/quorum/failed_parts/" + entry.new_part_name, + fs::path(zookeeper_path) / "quorum" / "failed_parts" / entry.new_part_name, "", zkutil::CreateMode::Persistent)); /// Deleting from `blocks`. - if (!entry.block_id.empty() && zookeeper->exists(zookeeper_path + "/blocks/" + entry.block_id)) - ops.emplace_back(zkutil::makeRemoveRequest(zookeeper_path + "/blocks/" + entry.block_id, -1)); + if (!entry.block_id.empty() && zookeeper->exists(fs::path(zookeeper_path) / "blocks" / entry.block_id)) + ops.emplace_back(zkutil::makeRemoveRequest(fs::path(zookeeper_path) / "blocks" / entry.block_id, -1)); Coordination::Responses responses; auto code = zookeeper->tryMulti(ops, responses); @@ -2048,7 +2053,8 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) } else if (code == Coordination::Error::ZBADVERSION || code == Coordination::Error::ZNONODE || code == Coordination::Error::ZNODEEXISTS) { - LOG_DEBUG(log, "State was changed or isn't expected when trying to mark quorum for part {} as failed. Code: {}", entry.new_part_name, Coordination::errorMessage(code)); + LOG_DEBUG(log, "State was changed or isn't expected when trying to mark quorum for part {} as failed. Code: {}", + entry.new_part_name, Coordination::errorMessage(code)); } else throw Coordination::Exception(code); @@ -2070,7 +2076,7 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) try { String part_name = entry.actual_new_part_name.empty() ? entry.new_part_name : entry.actual_new_part_name; - if (!fetchPart(part_name, metadata_snapshot, zookeeper_path + "/replicas/" + replica, false, entry.quorum)) + if (!fetchPart(part_name, metadata_snapshot, fs::path(zookeeper_path) / "replicas" / replica, false, entry.quorum)) return false; } catch (Exception & e) @@ -2154,7 +2160,7 @@ bool StorageReplicatedMergeTree::executeFetchShared( try { - if (!fetchExistsPart(new_part_name, metadata_snapshot, zookeeper_path + "/replicas/" + source_replica, disk, path)) + if (!fetchExistsPart(new_part_name, metadata_snapshot, fs::path(zookeeper_path) / "replicas" / source_replica, disk, path)) return false; } catch (Exception & e) @@ -2489,8 +2495,8 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) } else if (!part_desc->replica.empty()) { - String source_replica_path = zookeeper_path + "/replicas/" + part_desc->replica; - ReplicatedMergeTreeAddress address(getZooKeeper()->get(source_replica_path + "/host")); + String source_replica_path = fs::path(zookeeper_path) / "replicas" / part_desc->replica; + ReplicatedMergeTreeAddress address(getZooKeeper()->get(fs::path(source_replica_path) / "host")); auto timeouts = getFetchPartHTTPTimeouts(getContext()); auto credentials = getContext()->getInterserverCredentials(); @@ -2632,18 +2638,18 @@ void StorageReplicatedMergeTree::executeClonePartFromShard(const LogEntry & entr void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coordination::Stat source_is_lost_stat, zkutil::ZooKeeperPtr & zookeeper) { - String source_path = zookeeper_path + "/replicas/" + source_replica; + String source_path = fs::path(zookeeper_path) / "replicas" / source_replica; /** TODO: it will be deleted! (It is only to support old version of CH server). * In current code, the replica is created in single transaction. * If the reference/master replica is not yet fully created, let's wait. */ - while (!zookeeper->exists(source_path + "/columns")) + while (!zookeeper->exists(fs::path(source_path) / "columns")) { LOG_INFO(log, "Waiting for replica {} to be fully created", source_path); zkutil::EventPtr event = std::make_shared(); - if (zookeeper->exists(source_path + "/columns", nullptr, event)) + if (zookeeper->exists(fs::path(source_path) / "columns", nullptr, event)) { LOG_WARNING(log, "Oops, a watch has leaked"); break; @@ -2660,29 +2666,29 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo while (true) { Coordination::Stat log_pointer_stat; - String raw_log_pointer = zookeeper->get(source_path + "/log_pointer", &log_pointer_stat); + String raw_log_pointer = zookeeper->get(fs::path(source_path) / "log_pointer", &log_pointer_stat); Coordination::Requests ops; - ops.push_back(zkutil::makeSetRequest(replica_path + "/log_pointer", raw_log_pointer, -1)); + ops.push_back(zkutil::makeSetRequest(fs::path(replica_path) / "log_pointer", raw_log_pointer, -1)); /// For support old versions CH. if (source_is_lost_stat.version == -1) { /// We check that it was not suddenly upgraded to new version. /// Otherwise it can be upgraded and instantly become lost, but we cannot notice that. - ops.push_back(zkutil::makeCreateRequest(source_path + "/is_lost", "0", zkutil::CreateMode::Persistent)); - ops.push_back(zkutil::makeRemoveRequest(source_path + "/is_lost", -1)); + ops.push_back(zkutil::makeCreateRequest(fs::path(source_path) / "is_lost", "0", zkutil::CreateMode::Persistent)); + ops.push_back(zkutil::makeRemoveRequest(fs::path(source_path) / "is_lost", -1)); } else /// The replica we clone should not suddenly become lost. - ops.push_back(zkutil::makeCheckRequest(source_path + "/is_lost", source_is_lost_stat.version)); + ops.push_back(zkutil::makeCheckRequest(fs::path(source_path) / "is_lost", source_is_lost_stat.version)); Coordination::Responses responses; /// Let's remember the queue of the reference/master replica. - source_queue_names = zookeeper->getChildren(source_path + "/queue"); + source_queue_names = zookeeper->getChildren(fs::path(source_path) / "queue"); /// Check that log pointer of source replica didn't changed while we read queue entries - ops.push_back(zkutil::makeCheckRequest(source_path + "/log_pointer", log_pointer_stat.version)); + ops.push_back(zkutil::makeCheckRequest(fs::path(source_path) / "log_pointer", log_pointer_stat.version)); auto rc = zookeeper->tryMulti(ops, responses); @@ -2722,7 +2728,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo for (const String & entry_name : source_queue_names) { String entry; - if (!zookeeper->tryGet(source_path + "/queue/" + entry_name, entry)) + if (!zookeeper->tryGet(fs::path(source_path) / "queue" / entry_name, entry)) continue; source_queue.push_back(entry); } @@ -2731,13 +2737,13 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo cloneMetadataIfNeeded(source_replica, source_path, zookeeper); /// Add to the queue jobs to receive all the active parts that the reference/master replica has. - Strings source_replica_parts = zookeeper->getChildren(source_path + "/parts"); + Strings source_replica_parts = zookeeper->getChildren(fs::path(source_path) / "parts"); ActiveDataPartSet active_parts_set(format_version, source_replica_parts); Strings active_parts = active_parts_set.getParts(); /// Remove local parts if source replica does not have them, because such parts will never be fetched by other replicas. - Strings local_parts_in_zk = zookeeper->getChildren(replica_path + "/parts"); + Strings local_parts_in_zk = zookeeper->getChildren(fs::path(replica_path) / "parts"); Strings parts_to_remove_from_zk; for (const auto & part : local_parts_in_zk) { @@ -2781,7 +2787,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo log_entry.new_part_name = name; log_entry.create_time = tryGetPartCreateTime(zookeeper, source_path, name); - zookeeper->create(replica_path + "/queue/queue-", log_entry.toString(), zkutil::CreateMode::PersistentSequential); + zookeeper->create(fs::path(replica_path) / "queue/queue-", log_entry.toString(), zkutil::CreateMode::PersistentSequential); } LOG_DEBUG(log, "Queued {} parts to be fetched", active_parts.size()); @@ -2789,7 +2795,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo /// Add content of the reference/master replica queue to the queue. for (const String & entry : source_queue) { - zookeeper->create(replica_path + "/queue/queue-", entry, zkutil::CreateMode::PersistentSequential); + zookeeper->create(fs::path(replica_path) / "queue/queue-", entry, zkutil::CreateMode::PersistentSequential); } LOG_DEBUG(log, "Copied {} queue entries", source_queue.size()); @@ -2876,7 +2882,7 @@ void StorageReplicatedMergeTree::cloneReplicaIfNeeded(zkutil::ZooKeeperPtr zooke Coordination::Stat is_lost_stat; bool is_new_replica = true; String res; - if (zookeeper->tryGet(replica_path + "/is_lost", res, &is_lost_stat)) + if (zookeeper->tryGet(fs::path(replica_path) / "is_lost", res, &is_lost_stat)) { if (res == "0") return; @@ -2888,14 +2894,14 @@ void StorageReplicatedMergeTree::cloneReplicaIfNeeded(zkutil::ZooKeeperPtr zooke /// Replica was created by old version of CH, so me must create "/is_lost". /// Note that in old version of CH there was no "lost" replicas possible. /// TODO is_lost node should always exist since v18.12, maybe we can replace `tryGet` with `get` and remove old code? - zookeeper->create(replica_path + "/is_lost", "0", zkutil::CreateMode::Persistent); + zookeeper->create(fs::path(replica_path) / "is_lost", "0", zkutil::CreateMode::Persistent); return; } /// is_lost is "1": it means that we are in repair mode. /// Try choose source replica to clone. /// Source replica must not be lost and should have minimal queue size and maximal log pointer. - Strings replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); + Strings replicas = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas"); std::vector futures; for (const String & source_replica_name : replicas) { @@ -2903,20 +2909,20 @@ void StorageReplicatedMergeTree::cloneReplicaIfNeeded(zkutil::ZooKeeperPtr zooke if (source_replica_name == replica_name) continue; - String source_replica_path = zookeeper_path + "/replicas/" + source_replica_name; + String source_replica_path = fs::path(zookeeper_path) / "replicas" / source_replica_name; /// Obviously the following get operations are not atomic, but it's ok to choose good enough replica, not the best one. /// NOTE: We may count some entries twice if log_pointer is moved. - futures.emplace_back(zookeeper->asyncTryGet(source_replica_path + "/is_lost")); - futures.emplace_back(zookeeper->asyncTryGet(source_replica_path + "/log_pointer")); - futures.emplace_back(zookeeper->asyncTryGet(source_replica_path + "/queue")); + futures.emplace_back(zookeeper->asyncTryGet(fs::path(source_replica_path) / "is_lost")); + futures.emplace_back(zookeeper->asyncTryGet(fs::path(source_replica_path) / "log_pointer")); + futures.emplace_back(zookeeper->asyncTryGet(fs::path(source_replica_path) / "queue")); } /// Wait for results before getting log entries for (auto & future : futures) future.wait(); - Strings log_entries = zookeeper->getChildren(zookeeper_path + "/log"); + Strings log_entries = zookeeper->getChildren(fs::path(zookeeper_path) / "log"); size_t max_log_entry = 0; if (!log_entries.empty()) { @@ -2987,14 +2993,14 @@ void StorageReplicatedMergeTree::cloneReplicaIfNeeded(zkutil::ZooKeeperPtr zooke LOG_WARNING(log, "Will mimic {}", source_replica); /// Clear obsolete queue that we no longer need. - zookeeper->removeChildren(replica_path + "/queue"); + zookeeper->removeChildren(fs::path(replica_path) / "queue"); /// Will do repair from the selected replica. cloneReplica(source_replica, source_is_lost_stat, zookeeper); /// If repair fails to whatever reason, the exception is thrown, is_lost will remain "1" and the replica will be repaired later. /// If replica is repaired successfully, we remove is_lost flag. - zookeeper->set(replica_path + "/is_lost", "0"); + zookeeper->set(fs::path(replica_path) / "is_lost", "0"); } @@ -3321,7 +3327,7 @@ StorageReplicatedMergeTree::CreateMergeEntryResult StorageReplicatedMergeTree::c std::vector> exists_futures; exists_futures.reserve(parts.size()); for (const auto & part : parts) - exists_futures.emplace_back(zookeeper->asyncExists(replica_path + "/parts/" + part->name)); + exists_futures.emplace_back(zookeeper->asyncExists(fs::path(replica_path) / "parts" / part->name)); bool all_in_zk = true; for (size_t i = 0; i < parts.size(); ++i) @@ -3361,11 +3367,11 @@ StorageReplicatedMergeTree::CreateMergeEntryResult StorageReplicatedMergeTree::c Coordination::Responses responses; ops.emplace_back(zkutil::makeCreateRequest( - zookeeper_path + "/log/log-", entry.toString(), + fs::path(zookeeper_path) / "log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); ops.emplace_back(zkutil::makeSetRequest( - zookeeper_path + "/log", "", log_version)); /// Check and update version. + fs::path(zookeeper_path) / "log", "", log_version)); /// Check and update version. Coordination::Error code = zookeeper->tryMulti(ops, responses); @@ -3401,7 +3407,7 @@ StorageReplicatedMergeTree::CreateMergeEntryResult StorageReplicatedMergeTree::c auto zookeeper = getZooKeeper(); /// If there is no information about part in ZK, we will not mutate it. - if (!zookeeper->exists(replica_path + "/parts/" + part.name)) + if (!zookeeper->exists(fs::path(replica_path) / "parts" / part.name)) { if (part.modification_time + MAX_AGE_OF_LOCAL_PART_THAT_WASNT_ADDED_TO_ZOOKEEPER < time(nullptr)) { @@ -3431,11 +3437,11 @@ StorageReplicatedMergeTree::CreateMergeEntryResult StorageReplicatedMergeTree::c Coordination::Responses responses; ops.emplace_back(zkutil::makeCreateRequest( - zookeeper_path + "/log/log-", entry.toString(), + fs::path(zookeeper_path) / "log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); ops.emplace_back(zkutil::makeSetRequest( - zookeeper_path + "/log", "", log_version)); /// Check and update version. + fs::path(zookeeper_path) / "log", "", log_version)); /// Check and update version. Coordination::Error code = zookeeper->tryMulti(ops, responses); @@ -3456,12 +3462,12 @@ StorageReplicatedMergeTree::CreateMergeEntryResult StorageReplicatedMergeTree::c void StorageReplicatedMergeTree::removePartFromZooKeeper(const String & part_name, Coordination::Requests & ops, bool has_children) { - String part_path = replica_path + "/parts/" + part_name; + String part_path = fs::path(replica_path) / "parts" / part_name; if (has_children) { - ops.emplace_back(zkutil::makeRemoveRequest(part_path + "/checksums", -1)); - ops.emplace_back(zkutil::makeRemoveRequest(part_path + "/columns", -1)); + ops.emplace_back(zkutil::makeRemoveRequest(fs::path(part_path) / "checksums", -1)); + ops.emplace_back(zkutil::makeRemoveRequest(fs::path(part_path) / "columns", -1)); } ops.emplace_back(zkutil::makeRemoveRequest(part_path, -1)); } @@ -3469,7 +3475,7 @@ void StorageReplicatedMergeTree::removePartFromZooKeeper(const String & part_nam void StorageReplicatedMergeTree::removePartFromZooKeeper(const String & part_name) { auto zookeeper = getZooKeeper(); - String part_path = replica_path + "/parts/" + part_name; + String part_path = fs::path(replica_path) / "parts" / part_name; Coordination::Stat stat; /// Part doesn't exist, nothing to remove @@ -3486,7 +3492,7 @@ void StorageReplicatedMergeTree::removePartAndEnqueueFetch(const String & part_n { auto zookeeper = getZooKeeper(); - String part_path = replica_path + "/parts/" + part_name; + String part_path = fs::path(replica_path) / "parts" / part_name; Coordination::Requests ops; @@ -3505,7 +3511,7 @@ void StorageReplicatedMergeTree::removePartAndEnqueueFetch(const String & part_n log_entry->new_part_name = part_name; ops.emplace_back(zkutil::makeCreateRequest( - replica_path + "/queue/queue-", log_entry->toString(), + fs::path(replica_path) / "queue/queue-", log_entry->toString(), zkutil::CreateMode::PersistentSequential)); auto results = zookeeper->multi(ops); @@ -3530,7 +3536,7 @@ void StorageReplicatedMergeTree::enterLeaderElection() { leader_election = std::make_shared( getContext()->getSchedulePool(), - zookeeper_path + "/leader_election", + fs::path(zookeeper_path) / "leader_election", *current_zookeeper, /// current_zookeeper lives for the lifetime of leader_election, /// since before changing `current_zookeeper`, `leader_election` object is destroyed in `partialShutdown` method. callback, @@ -3585,13 +3591,13 @@ ConnectionTimeouts StorageReplicatedMergeTree::getFetchPartHTTPTimeouts(ContextP bool StorageReplicatedMergeTree::checkReplicaHavePart(const String & replica, const String & part_name) { auto zookeeper = getZooKeeper(); - return zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/parts/" + part_name); + return zookeeper->exists(fs::path(zookeeper_path) / "replicas" / replica / "parts" / part_name); } String StorageReplicatedMergeTree::findReplicaHavingPart(const String & part_name, bool active) { auto zookeeper = getZooKeeper(); - Strings replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); + Strings replicas = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas"); /// Select replicas in uniformly random order. std::shuffle(replicas.begin(), replicas.end(), thread_local_rng); @@ -3607,7 +3613,7 @@ String StorageReplicatedMergeTree::findReplicaHavingPart(const String & part_nam LOG_TRACE(log, "Candidate replica: {}", replica); if (checkReplicaHavePart(replica, part_name) && - (!active || zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active"))) + (!active || zookeeper->exists(fs::path(zookeeper_path) / "replicas" / replica / "is_active"))) return replica; /// Obviously, replica could become inactive or even vanish after return from this method. @@ -3619,7 +3625,7 @@ String StorageReplicatedMergeTree::findReplicaHavingPart(const String & part_nam String StorageReplicatedMergeTree::findReplicaHavingCoveringPart(LogEntry & entry, bool active) { auto zookeeper = getZooKeeper(); - Strings replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); + Strings replicas = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas"); /// Select replicas in uniformly random order. std::shuffle(replicas.begin(), replicas.end(), thread_local_rng); @@ -3629,11 +3635,11 @@ String StorageReplicatedMergeTree::findReplicaHavingCoveringPart(LogEntry & entr if (replica == replica_name) continue; - if (active && !zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active")) + if (active && !zookeeper->exists(fs::path(zookeeper_path) / "replicas" / replica / "is_active")) continue; String largest_part_found; - Strings parts = zookeeper->getChildren(zookeeper_path + "/replicas/" + replica + "/parts"); + Strings parts = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas" / replica / "parts"); for (const String & part_on_replica : parts) { if (part_on_replica == entry.new_part_name @@ -3674,7 +3680,7 @@ String StorageReplicatedMergeTree::findReplicaHavingCoveringPart( const String & part_name, bool active, String & found_part_name) { auto zookeeper = getZooKeeper(); - Strings replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); + Strings replicas = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas"); /// Select replicas in uniformly random order. std::shuffle(replicas.begin(), replicas.end(), thread_local_rng); @@ -3687,10 +3693,10 @@ String StorageReplicatedMergeTree::findReplicaHavingCoveringPart( if (replica == replica_name) continue; - if (active && !zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active")) + if (active && !zookeeper->exists(fs::path(zookeeper_path) / "replicas" / replica / "is_active")) continue; - Strings parts = zookeeper->getChildren(zookeeper_path + "/replicas/" + replica + "/parts"); + Strings parts = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas" / replica / "parts"); for (const String & part_on_replica : parts) { if (part_on_replica == part_name @@ -3718,11 +3724,11 @@ void StorageReplicatedMergeTree::updateQuorum(const String & part_name, bool is_ auto zookeeper = getZooKeeper(); /// Information on which replicas a part has been added, if the quorum has not yet been reached. - String quorum_status_path = zookeeper_path + "/quorum/status"; + String quorum_status_path = fs::path(zookeeper_path) / "quorum" / "status"; if (is_parallel) - quorum_status_path = zookeeper_path + "/quorum/parallel/" + part_name; + quorum_status_path = fs::path(zookeeper_path) / "quorum" / "parallel" / part_name; /// The name of the previous part for which the quorum was reached. - const String quorum_last_part_path = zookeeper_path + "/quorum/last_part"; + const String quorum_last_part_path = fs::path(zookeeper_path) / "quorum" / "last_part"; String value; Coordination::Stat stat; @@ -3824,7 +3830,7 @@ void StorageReplicatedMergeTree::cleanLastPartNode(const String & partition_id) auto zookeeper = getZooKeeper(); /// The name of the previous part for which the quorum was reached. - const String quorum_last_part_path = zookeeper_path + "/quorum/last_part"; + const String quorum_last_part_path = fs::path(zookeeper_path) / "quorum" / "last_part"; /// Delete information from "last_part" node. @@ -3874,7 +3880,7 @@ void StorageReplicatedMergeTree::cleanLastPartNode(const String & partition_id) bool StorageReplicatedMergeTree::partIsInsertingWithParallelQuorum(const MergeTreePartInfo & part_info) const { auto zookeeper = getZooKeeper(); - return zookeeper->exists(zookeeper_path + "/quorum/parallel/" + part_info.getPartName()); + return zookeeper->exists(fs::path(zookeeper_path) / "quorum" / "parallel" / part_info.getPartName()); } @@ -3882,7 +3888,7 @@ bool StorageReplicatedMergeTree::partIsLastQuorumPart(const MergeTreePartInfo & { auto zookeeper = getZooKeeper(); - const String parts_with_quorum_path = zookeeper_path + "/quorum/last_part"; + const String parts_with_quorum_path = fs::path(zookeeper_path) / "quorum" / "last_part"; String parts_with_quorum_str = zookeeper->get(parts_with_quorum_path); @@ -3966,13 +3972,13 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora source_part_checksums.computeTotalChecksums(source_part->checksums); MinimalisticDataPartChecksums desired_checksums; - String part_path = source_replica_path + "/parts/" + part_name; + String part_path = fs::path(source_replica_path) / "parts" / part_name; String part_znode = zookeeper->get(part_path); if (!part_znode.empty()) desired_checksums = ReplicatedMergeTreePartHeader::fromString(part_znode).getChecksums(); else { - String desired_checksums_str = zookeeper->get(part_path + "/checksums"); + String desired_checksums_str = zookeeper->get(fs::path(part_path) / "checksums"); desired_checksums = MinimalisticDataPartChecksums::deserializeFrom(desired_checksums_str); } @@ -4001,7 +4007,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora } else { - address.fromString(zookeeper->get(source_replica_path + "/host")); + address.fromString(zookeeper->get(fs::path(source_replica_path) / "host")); timeouts = getFetchPartHTTPTimeouts(getContext()); credentials = getContext()->getInterserverCredentials(); @@ -4049,16 +4055,16 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora if (quorum) { /// Check if this quorum insert is parallel or not - if (zookeeper->exists(zookeeper_path + "/quorum/parallel/" + part_name)) + if (zookeeper->exists(fs::path(zookeeper_path) / "quorum" / "parallel" / part_name)) updateQuorum(part_name, true); - else if (zookeeper->exists(zookeeper_path + "/quorum/status")) + else if (zookeeper->exists(fs::path(zookeeper_path) / "quorum" / "status")) updateQuorum(part_name, false); } /// merged parts that are still inserted with quorum. if it only contains one block, it hasn't been merged before if (part_info.level != 0 || part_info.mutation != 0) { - Strings quorum_parts = zookeeper->getChildren(zookeeper_path + "/quorum/parallel"); + Strings quorum_parts = zookeeper->getChildren(fs::path(zookeeper_path) / "quorum" / "parallel"); for (const String & quorum_part : quorum_parts) { auto quorum_part_info = MergeTreePartInfo::fromPartName(quorum_part, format_version); @@ -4081,7 +4087,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora { // The fetched part is valuable and should not be cleaned like a temp part. part->is_temp = false; - part->renameTo("detached/" + part_name, true); + part->renameTo(fs::path("detached") / part_name, true); } } catch (const Exception & e) @@ -4159,7 +4165,7 @@ bool StorageReplicatedMergeTree::fetchExistsPart(const String & part_name, const std::function get_part; - ReplicatedMergeTreeAddress address(zookeeper->get(source_replica_path + "/host")); + ReplicatedMergeTreeAddress address(zookeeper->get(fs::path(source_replica_path) / "host")); auto timeouts = ConnectionTimeouts::getHTTPTimeouts(getContext()); auto credentials = getContext()->getInterserverCredentials(); String interserver_scheme = getContext()->getInterserverScheme(); @@ -4320,7 +4326,7 @@ ReplicatedMergeTreeQuorumAddedParts::PartitionIdToMaxBlock StorageReplicatedMerg auto zookeeper = getZooKeeper(); - const String quorum_status_path = zookeeper_path + "/quorum/status"; + const String quorum_status_path = fs::path(zookeeper_path) / "quorum" / "status"; String value; Coordination::Stat stat; @@ -4336,7 +4342,7 @@ ReplicatedMergeTreeQuorumAddedParts::PartitionIdToMaxBlock StorageReplicatedMerg } String added_parts_str; - if (zookeeper->tryGet(zookeeper_path + "/quorum/last_part", added_parts_str)) + if (zookeeper->tryGet(fs::path(zookeeper_path) / "quorum" / "last_part", added_parts_str)) { if (!added_parts_str.empty()) { @@ -4663,8 +4669,8 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer /// If metadata nodes have changed, we will update table structure locally. Coordination::Requests requests; - requests.emplace_back(zkutil::makeSetRequest(replica_path + "/columns", entry.columns_str, -1)); - requests.emplace_back(zkutil::makeSetRequest(replica_path + "/metadata", entry.metadata_str, -1)); + requests.emplace_back(zkutil::makeSetRequest(fs::path(replica_path) / "columns", entry.columns_str, -1)); + requests.emplace_back(zkutil::makeSetRequest(fs::path(replica_path) / "metadata", entry.metadata_str, -1)); zookeeper->multi(requests); @@ -4680,7 +4686,7 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer } /// This transaction may not happen, but it's OK, because on the next retry we will eventually create/update this node - zookeeper->createOrUpdate(replica_path + "/metadata_version", std::to_string(metadata_version), zkutil::CreateMode::Persistent); + zookeeper->createOrUpdate(fs::path(replica_path) / "metadata_version", std::to_string(metadata_version), zkutil::CreateMode::Persistent); return true; } @@ -4726,7 +4732,7 @@ PartitionBlockNumbersHolder StorageReplicatedMergeTree::allocateBlockNumbersInAf { /// TODO: Implement optimal block number aqcuisition algorithm in multiple (but not all) partitions EphemeralLocksInAllPartitions lock_holder( - zookeeper_path + "/block_numbers", "block-", zookeeper_path + "/temp", *zookeeper); + fs::path(zookeeper_path) / "block_numbers", "block-", fs::path(zookeeper_path) / "temp", *zookeeper); PartitionBlockNumbersHolder::BlockNumbersType block_numbers; for (const auto & lock : lock_holder.getLocks()) @@ -4826,10 +4832,10 @@ void StorageReplicatedMergeTree::alter( size_t mutation_path_idx = std::numeric_limits::max(); String new_metadata_str = future_metadata_in_zk.toString(); - ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/metadata", new_metadata_str, metadata_version)); + ops.emplace_back(zkutil::makeSetRequest(fs::path(zookeeper_path) / "metadata", new_metadata_str, metadata_version)); String new_columns_str = future_metadata.columns.toString(); - ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/columns", new_columns_str, -1)); + ops.emplace_back(zkutil::makeSetRequest(fs::path(zookeeper_path) / "columns", new_columns_str, -1)); if (ast_to_str(current_metadata->settings_changes) != ast_to_str(future_metadata.settings_changes)) { @@ -4858,12 +4864,12 @@ void StorageReplicatedMergeTree::alter( alter_path_idx = ops.size(); ops.emplace_back(zkutil::makeCreateRequest( - zookeeper_path + "/log/log-", alter_entry->toString(), zkutil::CreateMode::PersistentSequential)); + fs::path(zookeeper_path) / "log/log-", alter_entry->toString(), zkutil::CreateMode::PersistentSequential)); PartitionBlockNumbersHolder partition_block_numbers_holder; if (have_mutation) { - const String mutations_path(zookeeper_path + "/mutations"); + const String mutations_path(fs::path(zookeeper_path) / "mutations"); ReplicatedMergeTreeMutationEntry mutation_entry; mutation_entry.alter_version = new_metadata_version; @@ -4882,7 +4888,7 @@ void StorageReplicatedMergeTree::alter( ops.emplace_back(zkutil::makeSetRequest(mutations_path, String(), mutations_stat.version)); mutation_path_idx = ops.size(); ops.emplace_back( - zkutil::makeCreateRequest(mutations_path + "/", mutation_entry.toString(), zkutil::CreateMode::PersistentSequential)); + zkutil::makeCreateRequest(fs::path(mutations_path) / "", mutation_entry.toString(), zkutil::CreateMode::PersistentSequential)); } if (auto txn = query_context->getZooKeeperMetadataTransaction()) @@ -4890,7 +4896,7 @@ void StorageReplicatedMergeTree::alter( txn->moveOpsTo(ops); /// NOTE: IDatabase::alterTable(...) is called when executing ALTER_METADATA queue entry without query context, /// so we have to update metadata of DatabaseReplicated here. - String metadata_zk_path = txn->getDatabaseZooKeeperPath() + "/metadata/" + escapeForFileName(table_id.table_name); + String metadata_zk_path = fs::path(txn->getDatabaseZooKeeperPath()) / "metadata" / escapeForFileName(table_id.table_name); auto ast = DatabaseCatalog::instance().getDatabase(table_id.database_name)->getCreateTableQuery(table_id.table_name, query_context); applyMetadataChangesToCreateQuery(ast, future_metadata); ops.emplace_back(zkutil::makeSetRequest(metadata_zk_path, getObjectDefinitionFromCreateQuery(ast), -1)); @@ -5082,7 +5088,7 @@ void StorageReplicatedMergeTree::truncate( zkutil::ZooKeeperPtr zookeeper = getZooKeeper(); - Strings partitions = zookeeper->getChildren(zookeeper_path + "/block_numbers"); + Strings partitions = zookeeper->getChildren(fs::path(zookeeper_path) / "block_numbers"); for (String & partition_id : partitions) { @@ -5157,7 +5163,7 @@ void StorageReplicatedMergeTree::rename(const String & new_path_to_table_data, c try { auto zookeeper = getZooKeeper(); - zookeeper->set(replica_path + "/host", getReplicatedMergeTreeAddress().toString()); + zookeeper->set(fs::path(replica_path) / "host", getReplicatedMergeTreeAddress().toString()); } catch (Coordination::Exception & e) { @@ -5207,8 +5213,8 @@ StorageReplicatedMergeTree::allocateBlockNumber( deduplication_check_ops.emplace_back(zkutil::makeRemoveRequest(zookeeper_block_id_path, -1)); } - String block_numbers_path = zookeeper_table_path + "/block_numbers"; - String partition_path = block_numbers_path + "/" + partition_id; + String block_numbers_path = fs::path(zookeeper_table_path) / "block_numbers"; + String partition_path = fs::path(block_numbers_path) / partition_id; if (!existsNodeCached(partition_path)) { @@ -5230,7 +5236,7 @@ StorageReplicatedMergeTree::allocateBlockNumber( try { lock = EphemeralLockInZooKeeper( - partition_path + "/block-", zookeeper_table_path + "/temp", *zookeeper, &deduplication_check_ops); + fs::path(partition_path) / "block-", fs::path(zookeeper_table_path) / "temp", *zookeeper, &deduplication_check_ops); } catch (const zkutil::KeeperMultiException & e) { @@ -5254,11 +5260,11 @@ Strings StorageReplicatedMergeTree::waitForAllTableReplicasToProcessLogEntry( LOG_DEBUG(log, "Waiting for all replicas to process {}", entry.znode_name); auto zookeeper = getZooKeeper(); - Strings replicas = zookeeper->getChildren(table_zookeeper_path + "/replicas"); + Strings replicas = zookeeper->getChildren(fs::path(table_zookeeper_path) / "replicas"); Strings unwaited; for (const String & replica : replicas) { - if (wait_for_non_active || zookeeper->exists(table_zookeeper_path + "/replicas/" + replica + "/is_active")) + if (wait_for_non_active || zookeeper->exists(fs::path(table_zookeeper_path) / "replicas" / replica / "is_active")) { if (!waitForTableReplicaToProcessLogEntry(table_zookeeper_path, replica, entry, wait_for_non_active)) unwaited.push_back(replica); @@ -5308,7 +5314,7 @@ bool StorageReplicatedMergeTree::waitForTableReplicaToProcessLogEntry( const auto & stop_waiting = [&]() { bool stop_waiting_itself = waiting_itself && (partial_shutdown_called || is_dropped); - bool stop_waiting_non_active = !wait_for_non_active && !getZooKeeper()->exists(table_zookeeper_path + "/replicas/" + replica + "/is_active"); + bool stop_waiting_non_active = !wait_for_non_active && !getZooKeeper()->exists(fs::path(table_zookeeper_path) / "replicas" / replica / "is_active"); return stop_waiting_itself || stop_waiting_non_active; }; @@ -5330,7 +5336,7 @@ bool StorageReplicatedMergeTree::waitForTableReplicaToProcessLogEntry( { zkutil::EventPtr event = std::make_shared(); - String log_pointer = getZooKeeper()->get(table_zookeeper_path + "/replicas/" + replica + "/log_pointer", nullptr, event); + String log_pointer = getZooKeeper()->get(fs::path(table_zookeeper_path) / "replicas" / replica / "log_pointer", nullptr, event); if (!log_pointer.empty() && parse(log_pointer) > log_index) break; @@ -5347,9 +5353,9 @@ bool StorageReplicatedMergeTree::waitForTableReplicaToProcessLogEntry( * looking for a node with the same content. And if we do not find it - then the replica has already taken this entry in its queue. */ - String log_pointer = getZooKeeper()->get(table_zookeeper_path + "/replicas/" + replica + "/log_pointer"); + String log_pointer = getZooKeeper()->get(fs::path(table_zookeeper_path) / "replicas" / replica / "log_pointer"); - Strings log_entries = getZooKeeper()->getChildren(table_zookeeper_path + "/log"); + Strings log_entries = getZooKeeper()->getChildren(fs::path(table_zookeeper_path) / "log"); UInt64 log_index = 0; bool found = false; @@ -5361,7 +5367,7 @@ bool StorageReplicatedMergeTree::waitForTableReplicaToProcessLogEntry( continue; String log_entry_str; - bool exists = getZooKeeper()->tryGet(table_zookeeper_path + "/log/" + log_entry_name, log_entry_str); + bool exists = getZooKeeper()->tryGet(fs::path(table_zookeeper_path) / "log" / log_entry_name, log_entry_str); if (exists && entry_str == log_entry_str) { found = true; @@ -5379,7 +5385,7 @@ bool StorageReplicatedMergeTree::waitForTableReplicaToProcessLogEntry( { zkutil::EventPtr event = std::make_shared(); - String log_pointer_new = getZooKeeper()->get(table_zookeeper_path + "/replicas/" + replica + "/log_pointer", nullptr, event); + String log_pointer_new = getZooKeeper()->get(fs::path(table_zookeeper_path) / "replicas" / replica / "log_pointer", nullptr, event); if (!log_pointer_new.empty() && parse(log_pointer_new) > log_index) break; @@ -5404,13 +5410,13 @@ bool StorageReplicatedMergeTree::waitForTableReplicaToProcessLogEntry( * Therefore, we search by comparing the content. */ - Strings queue_entries = getZooKeeper()->getChildren(table_zookeeper_path + "/replicas/" + replica + "/queue"); + Strings queue_entries = getZooKeeper()->getChildren(fs::path(table_zookeeper_path) / "replicas" / replica / "queue"); String queue_entry_to_wait_for; for (const String & entry_name : queue_entries) { String queue_entry_str; - bool exists = getZooKeeper()->tryGet(table_zookeeper_path + "/replicas/" + replica + "/queue/" + entry_name, queue_entry_str); + bool exists = getZooKeeper()->tryGet(fs::path(table_zookeeper_path) / "replicas" / replica / "queue" / entry_name, queue_entry_str); if (exists && queue_entry_str == entry_str) { queue_entry_to_wait_for = entry_name; @@ -5428,7 +5434,7 @@ bool StorageReplicatedMergeTree::waitForTableReplicaToProcessLogEntry( LOG_DEBUG(log, "Waiting for {} to disappear from {} queue", queue_entry_to_wait_for, replica); /// Third - wait until the entry disappears from the replica queue or replica become inactive. - String path_to_wait_on = table_zookeeper_path + "/replicas/" + replica + "/queue/" + queue_entry_to_wait_for; + String path_to_wait_on = fs::path(table_zookeeper_path) / "replicas" / replica / "queue" / queue_entry_to_wait_for; return getZooKeeper()->waitForDisappear(path_to_wait_on, stop_waiting); } @@ -5470,7 +5476,7 @@ void StorageReplicatedMergeTree::getStatus(Status & res, bool with_zk_fields) { try { - auto log_entries = zookeeper->getChildren(zookeeper_path + "/log"); + auto log_entries = zookeeper->getChildren(fs::path(zookeeper_path) / "log"); if (!log_entries.empty()) { @@ -5478,14 +5484,14 @@ void StorageReplicatedMergeTree::getStatus(Status & res, bool with_zk_fields) res.log_max_index = parse(last_log_entry.substr(strlen("log-"))); } - String log_pointer_str = zookeeper->get(replica_path + "/log_pointer"); + String log_pointer_str = zookeeper->get(fs::path(replica_path) / "log_pointer"); res.log_pointer = log_pointer_str.empty() ? 0 : parse(log_pointer_str); - auto all_replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); + auto all_replicas = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas"); res.total_replicas = all_replicas.size(); for (const String & replica : all_replicas) - if (zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active")) + if (zookeeper->exists(fs::path(zookeeper_path) / "replicas" / replica / "is_active")) ++res.active_replicas; } catch (const Coordination::Exception &) @@ -5568,7 +5574,7 @@ void StorageReplicatedMergeTree::getReplicaDelays(time_t & out_absolute_delay, t time_t max_replicas_unprocessed_insert_time = 0; bool have_replica_with_nothing_unprocessed = false; - Strings replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); + Strings replicas = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas"); for (const auto & replica : replicas) { @@ -5576,11 +5582,11 @@ void StorageReplicatedMergeTree::getReplicaDelays(time_t & out_absolute_delay, t continue; /// Skip dead replicas. - if (!zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active")) + if (!zookeeper->exists(fs::path(zookeeper_path) / "replicas" / replica / "is_active")) continue; String value; - if (!zookeeper->tryGet(zookeeper_path + "/replicas/" + replica + "/min_unprocessed_insert_time", value)) + if (!zookeeper->tryGet(fs::path(zookeeper_path) / "replicas" / replica / "min_unprocessed_insert_time", value)) continue; time_t replica_time = value.empty() ? 0 : parse(value); @@ -5687,13 +5693,13 @@ void StorageReplicatedMergeTree::fetchPartition( { /// List of replicas of source shard. - replicas = zookeeper->getChildren(from + "/replicas"); + replicas = zookeeper->getChildren(fs::path(from) / "replicas"); /// Leave only active replicas. active_replicas.reserve(replicas.size()); for (const String & replica : replicas) - if (zookeeper->exists(from + "/replicas/" + replica + "/is_active")) + if (zookeeper->exists(fs::path(from) / "replicas" / replica / "is_active")) active_replicas.push_back(replica); if (active_replicas.empty()) @@ -5710,13 +5716,13 @@ void StorageReplicatedMergeTree::fetchPartition( for (const String & replica : active_replicas) { - String current_replica_path = from + "/replicas/" + replica; + String current_replica_path = fs::path(from) / "replicas" / replica; - String log_pointer_str = zookeeper->get(current_replica_path + "/log_pointer"); + String log_pointer_str = zookeeper->get(fs::path(current_replica_path) / "log_pointer"); Int64 log_pointer = log_pointer_str.empty() ? 0 : parse(log_pointer_str); Coordination::Stat stat; - zookeeper->get(current_replica_path + "/queue", &stat); + zookeeper->get(fs::path(current_replica_path) / "queue", &stat); size_t queue_size = stat.numChildren; if (log_pointer > max_log_pointer @@ -5734,7 +5740,7 @@ void StorageReplicatedMergeTree::fetchPartition( LOG_INFO(log, "Found {} replicas, {} of them are active. Selected {} to fetch from.", replicas.size(), active_replicas.size(), best_replica); - String best_replica_path = from + "/replicas/" + best_replica; + String best_replica_path = fs::path(from) / "replicas" / best_replica; /// Let's find out which parts are on the best replica. @@ -5753,7 +5759,7 @@ void StorageReplicatedMergeTree::fetchPartition( if (try_no >= query_context->getSettings().max_fetch_partition_retries_count) throw Exception("Too many retries to fetch parts from " + best_replica_path, ErrorCodes::TOO_MANY_RETRIES_TO_FETCH_PARTS); - Strings parts = zookeeper->getChildren(best_replica_path + "/parts"); + Strings parts = zookeeper->getChildren(fs::path(best_replica_path) / "parts"); ActiveDataPartSet active_parts_set(format_version, parts); Strings parts_to_fetch; @@ -5874,7 +5880,7 @@ void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, Conte mutation_entry.source_replica = replica_name; mutation_entry.commands = commands; - const String mutations_path = zookeeper_path + "/mutations"; + const String mutations_path = fs::path(zookeeper_path) / "mutations"; const auto zookeeper = getZooKeeper(); /// Update the mutations_path node when creating the mutation and check its version to ensure that @@ -5897,7 +5903,7 @@ void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, Conte Coordination::Requests requests; requests.emplace_back(zkutil::makeSetRequest(mutations_path, String(), mutations_stat.version)); requests.emplace_back(zkutil::makeCreateRequest( - mutations_path + "/", mutation_entry.toString(), zkutil::CreateMode::PersistentSequential)); + fs::path(mutations_path) / "", mutation_entry.toString(), zkutil::CreateMode::PersistentSequential)); if (auto txn = query_context->getZooKeeperMetadataTransaction()) txn->moveOpsTo(requests); @@ -5936,7 +5942,7 @@ void StorageReplicatedMergeTree::waitMutation(const String & znode_name, size_t auto zookeeper = getZooKeeper(); Strings replicas; if (mutations_sync == 2) /// wait for all replicas - replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); + replicas = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas"); else if (mutations_sync == 1) /// just wait for ourself replicas.push_back(replica_name); @@ -6131,7 +6137,7 @@ bool StorageReplicatedMergeTree::tryRemovePartsFromZooKeeperWithRetries(const St exists_futures.reserve(part_names.size()); for (const String & part_name : part_names) { - String part_path = replica_path + "/parts/" + part_name; + String part_path = fs::path(replica_path) / "parts" / part_name; exists_futures.emplace_back(zookeeper->asyncExists(part_path)); } @@ -6194,7 +6200,7 @@ void StorageReplicatedMergeTree::removePartsFromZooKeeper( /// if zk session will be dropped for (const String & part_name : part_names) { - String part_path = replica_path + "/parts/" + part_name; + String part_path = fs::path(replica_path) / "parts" / part_name; exists_futures.emplace_back(zookeeper->asyncExists(part_path)); } @@ -6253,7 +6259,7 @@ void StorageReplicatedMergeTree::getClearBlocksInPartitionOps( Coordination::Requests & ops, zkutil::ZooKeeper & zookeeper, const String & partition_id, Int64 min_block_num, Int64 max_block_num) { Strings blocks; - if (Coordination::Error::ZOK != zookeeper.tryGetChildren(zookeeper_path + "/blocks", blocks)) + if (Coordination::Error::ZOK != zookeeper.tryGetChildren(fs::path(zookeeper_path) / "blocks", blocks)) throw Exception(zookeeper_path + "/blocks doesn't exist", ErrorCodes::NOT_FOUND_NODE); String partition_prefix = partition_id + "_"; @@ -6262,7 +6268,7 @@ void StorageReplicatedMergeTree::getClearBlocksInPartitionOps( { if (startsWith(block_id, partition_prefix)) { - String path = zookeeper_path + "/blocks/" + block_id; + String path = fs::path(zookeeper_path) / "blocks" / block_id; get_futures.emplace_back(path, zookeeper.asyncTryGet(path)); } } @@ -6372,7 +6378,7 @@ void StorageReplicatedMergeTree::replacePartitionFrom( else LOG_INFO(log, "Trying to attach {} with hash_hex {}", src_part->name, hash_hex); - String block_id_path = replace ? "" : (zookeeper_path + "/blocks/" + partition_id + "_replace_from_" + hash_hex); + String block_id_path = replace ? "" : (fs::path(zookeeper_path) / "blocks" / (partition_id + "_replace_from_" + hash_hex)); auto lock = allocateBlockNumber(partition_id, zookeeper, block_id_path); if (!lock) @@ -6444,8 +6450,8 @@ void StorageReplicatedMergeTree::replacePartitionFrom( ops.emplace_back(zkutil::makeCheckRequest(alter_partition_version_path, alter_partition_version_stat.version)); ops.emplace_back(zkutil::makeSetRequest(alter_partition_version_path, "", -1)); /// Just update version, because merges assignment relies on it - ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/log", "", -1)); - ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); + ops.emplace_back(zkutil::makeSetRequest(fs::path(zookeeper_path) / "log", "", -1)); + ops.emplace_back(zkutil::makeCreateRequest(fs::path(zookeeper_path) / "log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); Transaction transaction(*this); { @@ -6632,8 +6638,8 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta ops.emplace_back(zkutil::makeCheckRequest(alter_partition_version_path, alter_partition_version_stat.version)); ops.emplace_back(zkutil::makeSetRequest(alter_partition_version_path, "", -1)); /// Just update version, because merges assignment relies on it - ops.emplace_back(zkutil::makeSetRequest(dest_table_storage->zookeeper_path + "/log", "", -1)); - ops.emplace_back(zkutil::makeCreateRequest(dest_table_storage->zookeeper_path + "/log/log-", + ops.emplace_back(zkutil::makeSetRequest(fs::path(dest_table_storage->zookeeper_path) / "log", "", -1)); + ops.emplace_back(zkutil::makeCreateRequest(fs::path(dest_table_storage->zookeeper_path) / "log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); { @@ -6689,12 +6695,12 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta Coordination::Requests ops_src; ops_src.emplace_back(zkutil::makeCreateRequest( - zookeeper_path + "/log/log-", entry_delete.toString(), zkutil::CreateMode::PersistentSequential)); + fs::path(zookeeper_path) / "log/log-", entry_delete.toString(), zkutil::CreateMode::PersistentSequential)); /// Check and update version to avoid race with REPLACE_RANGE ops_src.emplace_back(zkutil::makeCheckRequest(alter_partition_version_path, alter_partition_version_stat.version)); ops_src.emplace_back(zkutil::makeSetRequest(alter_partition_version_path, "", -1)); /// Just update version, because merges assignment relies on it - ops_src.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/log", "", -1)); + ops_src.emplace_back(zkutil::makeSetRequest(fs::path(zookeeper_path) / "log", "", -1)); delimiting_block_lock->getUnlockOps(ops_src); Coordination::Error code = zookeeper->tryMulti(ops_src, op_results); @@ -6834,22 +6840,22 @@ void StorageReplicatedMergeTree::getCommitPartOps( if (storage_settings_ptr->use_minimalistic_part_header_in_zookeeper) { ops.emplace_back(zkutil::makeCreateRequest( - replica_path + "/parts/" + part->name, + fs::path(replica_path) / "parts" / part->name, ReplicatedMergeTreePartHeader::fromColumnsAndChecksums(part->getColumns(), part->checksums).toString(), zkutil::CreateMode::Persistent)); } else { ops.emplace_back(zkutil::makeCreateRequest( - replica_path + "/parts/" + part->name, + fs::path(replica_path) / "parts" / part->name, "", zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest( - replica_path + "/parts/" + part->name + "/columns", + fs::path(replica_path) / "parts" / part->name / "columns", part->getColumns().toString(), zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest( - replica_path + "/parts/" + part->name + "/checksums", + fs::path(replica_path) / "parts" / part->name / "checksums", getChecksumsForZooKeeper(part->checksums), zkutil::CreateMode::Persistent)); } @@ -7000,10 +7006,10 @@ bool StorageReplicatedMergeTree::dropPart( entry.detach = detach; entry.create_time = time(nullptr); - ops.emplace_back(zkutil::makeCheckRequest(zookeeper_path + "/log", merge_pred.getVersion())); /// Make sure no new events were added to the log. - ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); + ops.emplace_back(zkutil::makeCheckRequest(fs::path(zookeeper_path) / "log", merge_pred.getVersion())); /// Make sure no new events were added to the log. + ops.emplace_back(zkutil::makeCreateRequest(fs::path(zookeeper_path) / "log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); /// Just update version, because merges assignment relies on it - ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/log", "", -1)); + ops.emplace_back(zkutil::makeSetRequest(fs::path(zookeeper_path) / "log", "", -1)); Coordination::Responses responses; Coordination::Error rc = zookeeper->tryMulti(ops, responses); @@ -7057,14 +7063,14 @@ bool StorageReplicatedMergeTree::dropAllPartsInPartition( entry.create_time = time(nullptr); Coordination::Requests ops; - ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); + ops.emplace_back(zkutil::makeCreateRequest(fs::path(zookeeper_path) / "log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); /// Check and update version to avoid race with REPLACE_RANGE. /// Otherwise new parts covered by drop_range_info may appear after execution of current DROP_RANGE entry /// as a result of execution of concurrently created REPLACE_RANGE entry. ops.emplace_back(zkutil::makeCheckRequest(alter_partition_version_path, alter_partition_version_stat.version)); ops.emplace_back(zkutil::makeSetRequest(alter_partition_version_path, "", -1)); /// Just update version, because merges assignment relies on it - ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/log", "", -1)); + ops.emplace_back(zkutil::makeSetRequest(fs::path(zookeeper_path) / "log", "", -1)); delimiting_block_lock->getUnlockOps(ops); if (auto txn = query_context->getZooKeeperMetadataTransaction()) txn->moveOpsTo(ops); @@ -7150,7 +7156,7 @@ void StorageReplicatedMergeTree::lockSharedData(const IMergeTreeDataPart & part) String id = part.getUniqueId(); boost::replace_all(id, "/", "_"); - String zookeeper_node = zookeeper_path + "/zero_copy_s3/shared/" + part.name + "/" + id + "/" + replica_name; + String zookeeper_node = fs::path(zookeeper_path) / "zero_copy_s3" / "shared" / part.name / id / replica_name; LOG_TRACE(log, "Set zookeeper lock {}", zookeeper_node); @@ -7191,9 +7197,9 @@ bool StorageReplicatedMergeTree::unlockSharedData(const IMergeTreeDataPart & par String id = part.getUniqueId(); boost::replace_all(id, "/", "_"); - String zookeeper_part_node = zookeeper_path + "/zero_copy_s3/shared/" + part.name; - String zookeeper_part_uniq_node = zookeeper_part_node + "/" + id; - String zookeeper_node = zookeeper_part_uniq_node + "/" + replica_name; + String zookeeper_part_node = fs::path(zookeeper_path) / "zero_copy_s3" / "shared" / part.name; + String zookeeper_part_uniq_node = fs::path(zookeeper_part_node) / id; + String zookeeper_node = fs::path(zookeeper_part_uniq_node) / replica_name; LOG_TRACE(log, "Remove zookeeper lock {}", zookeeper_node); @@ -7252,7 +7258,7 @@ String StorageReplicatedMergeTree::getSharedDataReplica( if (!zookeeper) return best_replica; - String zookeeper_part_node = zookeeper_path + "/zero_copy_s3/shared/" + part.name; + String zookeeper_part_node = fs::path(zookeeper_path) / "zero_copy_s3" / "shared" / part.name; Strings ids; zookeeper->tryGetChildren(zookeeper_part_node, ids); @@ -7260,7 +7266,7 @@ String StorageReplicatedMergeTree::getSharedDataReplica( Strings replicas; for (const auto & id : ids) { - String zookeeper_part_uniq_node = zookeeper_part_node + "/" + id; + String zookeeper_part_uniq_node = fs::path(zookeeper_part_node) / id; Strings id_replicas; zookeeper->tryGetChildren(zookeeper_part_uniq_node, id_replicas); LOG_TRACE(log, "Found zookeper replicas for {}: {}", zookeeper_part_uniq_node, id_replicas.size()); @@ -7277,7 +7283,7 @@ String StorageReplicatedMergeTree::getSharedDataReplica( active_replicas.reserve(replicas.size()); for (const String & replica : replicas) - if ((replica != replica_name) && (zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active"))) + if ((replica != replica_name) && (zookeeper->exists(fs::path(zookeeper_path) / "replicas" / replica / "is_active"))) active_replicas.push_back(replica); LOG_TRACE(log, "Found zookeper active replicas for part {}: {}", part.name, active_replicas.size()); @@ -7296,13 +7302,13 @@ String StorageReplicatedMergeTree::getSharedDataReplica( for (const String & replica : active_replicas) { - String current_replica_path = zookeeper_path + "/replicas/" + replica; + String current_replica_path = fs::path(zookeeper_path) / "replicas" / replica; - String log_pointer_str = zookeeper->get(current_replica_path + "/log_pointer"); + String log_pointer_str = zookeeper->get(fs::path(current_replica_path) / "log_pointer"); Int64 log_pointer = log_pointer_str.empty() ? 0 : parse(log_pointer_str); Coordination::Stat stat; - zookeeper->get(current_replica_path + "/queue", &stat); + zookeeper->get(fs::path(current_replica_path) / "queue", &stat); size_t queue_size = stat.numChildren; if (log_pointer > max_log_pointer @@ -7320,16 +7326,16 @@ String StorageReplicatedMergeTree::getSharedDataReplica( String StorageReplicatedMergeTree::findReplicaHavingPart( const String & part_name, const String & zookeeper_path_, zkutil::ZooKeeper::Ptr zookeeper_) { - Strings replicas = zookeeper_->getChildren(zookeeper_path_ + "/replicas"); + Strings replicas = zookeeper_->getChildren(fs::path(zookeeper_path_) / "replicas"); /// Select replicas in uniformly random order. std::shuffle(replicas.begin(), replicas.end(), thread_local_rng); for (const String & replica : replicas) { - if (zookeeper_->exists(zookeeper_path_ + "/replicas/" + replica + "/parts/" + part_name) - && zookeeper_->exists(zookeeper_path_ + "/replicas/" + replica + "/is_active")) - return zookeeper_path_ + "/replicas/" + replica; + if (zookeeper_->exists(fs::path(zookeeper_path_) / "replicas" / replica / "parts" / part_name) + && zookeeper_->exists(fs::path(zookeeper_path_) / "replicas" / replica / "is_active")) + return fs::path(zookeeper_path_) / "replicas" / replica; } return {}; @@ -7337,23 +7343,23 @@ String StorageReplicatedMergeTree::findReplicaHavingPart( bool StorageReplicatedMergeTree::checkIfDetachedPartExists(const String & part_name) { - Poco::DirectoryIterator dir_end; + fs::directory_iterator dir_end; for (const std::string & path : getDataPaths()) - for (Poco::DirectoryIterator dir_it{path + "detached/"}; dir_it != dir_end; ++dir_it) - if (dir_it.name() == part_name) + for (fs::directory_iterator dir_it{fs::path(path) / "detached/"}; dir_it != dir_end; ++dir_it) + if (dir_it->path().filename().string() == part_name) return true; return false; } bool StorageReplicatedMergeTree::checkIfDetachedPartitionExists(const String & partition_name) { - Poco::DirectoryIterator dir_end; + fs::directory_iterator dir_end; for (const std::string & path : getDataPaths()) { - for (Poco::DirectoryIterator dir_it{path + "detached/"}; dir_it != dir_end; ++dir_it) + for (fs::directory_iterator dir_it{fs::path(path) / "detached/"}; dir_it != dir_end; ++dir_it) { MergeTreePartInfo part_info; - if (MergeTreePartInfo::tryParsePartName(dir_it.name(), &part_info, format_version) && part_info.partition_id == partition_name) + if (MergeTreePartInfo::tryParsePartName(dir_it->path().filename(), &part_info, format_version) && part_info.partition_id == partition_name) return true; } } diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 36d078e7bf3..fd9ff596329 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -35,8 +35,10 @@ #include #include #include - #include +#include + +namespace fs = std::filesystem; namespace DB { @@ -197,7 +199,7 @@ bool StorageS3Source::initialize() if (current_key.empty()) return false; - file_path = bucket + "/" + current_key; + file_path = fs::path(bucket) / current_key; read_buf = wrapReadBufferWithCompressionMethod( std::make_unique(client, bucket, current_key, s3_max_single_read_retries), chooseCompressionMethod(current_key, compression_hint)); diff --git a/src/Storages/StorageXDBC.cpp b/src/Storages/StorageXDBC.cpp index 88d54669889..9cffc32fda1 100644 --- a/src/Storages/StorageXDBC.cpp +++ b/src/Storages/StorageXDBC.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include diff --git a/src/Storages/examples/get_current_inserts_in_replicated.cpp b/src/Storages/examples/get_current_inserts_in_replicated.cpp index fa998d20f66..56364997643 100644 --- a/src/Storages/examples/get_current_inserts_in_replicated.cpp +++ b/src/Storages/examples/get_current_inserts_in_replicated.cpp @@ -4,11 +4,12 @@ #include #include #include - #include +#include #include +namespace fs = std::filesystem; using namespace DB; @@ -36,10 +37,10 @@ try Stopwatch stage; /// Load current inserts std::unordered_set lock_holder_paths; - for (const String & entry : zookeeper->getChildren(zookeeper_path + "/temp")) + for (const String & entry : zookeeper->getChildren(fs::path(zookeeper_path) / "temp")) { if (startsWith(entry, "abandonable_lock-")) - lock_holder_paths.insert(zookeeper_path + "/temp/" + entry); + lock_holder_paths.insert(fs::path(zookeeper_path) / "temp" / entry); } std::cerr << "Stage 1 (get lock holders): " << lock_holder_paths.size() << " lock holders, elapsed: " << stage.elapsedSeconds() << "s." << std::endl; @@ -47,14 +48,14 @@ try if (!lock_holder_paths.empty()) { - Strings partitions = zookeeper->getChildren(zookeeper_path + "/block_numbers"); + Strings partitions = zookeeper->getChildren(fs::path(zookeeper_path) / "block_numbers"); std::cerr << "Stage 2 (get partitions): " << partitions.size() << " partitions, elapsed: " << stage.elapsedSeconds() << "s." << std::endl; stage.restart(); std::vector> lock_futures; for (const String & partition : partitions) - lock_futures.push_back(zookeeper->asyncGetChildren(zookeeper_path + "/block_numbers/" + partition)); + lock_futures.push_back(zookeeper->asyncGetChildren(fs::path(zookeeper_path) / "block_numbers" / partition)); struct BlockInfo { @@ -71,7 +72,7 @@ try for (const String & entry : partition_block_numbers) { Int64 block_number = parse(entry.substr(strlen("block-"))); - String zk_path = zookeeper_path + "/block_numbers/" + partitions[i] + "/" + entry; + String zk_path = fs::path(zookeeper_path) / "block_numbers" / partitions[i] / entry; block_infos.push_back( BlockInfo{partitions[i], block_number, zk_path, zookeeper->asyncTryGet(zk_path)}); } diff --git a/src/Storages/examples/remove_symlink_directory.cpp b/src/Storages/examples/remove_symlink_directory.cpp index ae5fa72fa66..db436c0a608 100644 --- a/src/Storages/examples/remove_symlink_directory.cpp +++ b/src/Storages/examples/remove_symlink_directory.cpp @@ -1,9 +1,10 @@ #include #include -#include -#include #include +#include +#include +namespace fs = std::filesystem; namespace DB { @@ -16,18 +17,15 @@ namespace DB int main(int, char **) try { - Poco::File dir("./test_dir/"); - dir.createDirectories(); - - Poco::File("./test_dir/file").createFile(); + fs::path dir("./test_dir/"); + fs::create_directories(dir); + FS::createFile("./test_dir/file"); if (0 != symlink("./test_dir", "./test_link")) DB::throwFromErrnoWithPath("Cannot create symlink", "./test_link", DB::ErrorCodes::SYSTEM_ERROR); - Poco::File link("./test_link"); - link.renameTo("./test_link2"); - - Poco::File("./test_link2").remove(true); + fs::rename("./test_link", "./test_link2"); + fs::remove_all("./test_link2"); return 0; } catch (...) diff --git a/tests/queries/0_stateless/01601_detach_permanently.reference b/tests/queries/0_stateless/01601_detach_permanently.reference index 6683603c972..0232c1694e4 100644 --- a/tests/queries/0_stateless/01601_detach_permanently.reference +++ b/tests/queries/0_stateless/01601_detach_permanently.reference @@ -50,7 +50,7 @@ After database reattachement the table is still absent (it was detached permamen And we can not detach it permanently But we can attach it back And detach permanently again to check how database drop will behave -DROP database - Directory not empty error, but database deteched +DROP database - Directory not empty error, but database detached DROP database - now success ----------------------- database lazy tests diff --git a/tests/queries/0_stateless/01601_detach_permanently.sql b/tests/queries/0_stateless/01601_detach_permanently.sql index 0e21fd8f463..3af8ed573ef 100644 --- a/tests/queries/0_stateless/01601_detach_permanently.sql +++ b/tests/queries/0_stateless/01601_detach_permanently.sql @@ -128,8 +128,8 @@ ATTACH TABLE test1601_detach_permanently_ordinary.test_name_reuse; SELECT 'And detach permanently again to check how database drop will behave'; DETACH table test1601_detach_permanently_ordinary.test_name_reuse PERMANENTLY; -SELECT 'DROP database - Directory not empty error, but database deteched'; -DROP DATABASE test1601_detach_permanently_ordinary; -- { serverError 1000 } +SELECT 'DROP database - Directory not empty error, but database detached'; +DROP DATABASE test1601_detach_permanently_ordinary; -- { serverError 1001 } ATTACH DATABASE test1601_detach_permanently_ordinary; @@ -203,7 +203,7 @@ SELECT 'And detach permanently again to check how database drop will behave'; DETACH table test1601_detach_permanently_lazy.test_name_reuse PERMANENTLY; SELECT 'DROP database - Directory not empty error, but database deteched'; -DROP DATABASE test1601_detach_permanently_lazy; -- { serverError 1000 } +DROP DATABASE test1601_detach_permanently_lazy; -- { serverError 1001 } ATTACH DATABASE test1601_detach_permanently_lazy; diff --git a/tests/queries/0_stateless/01658_read_file_to_stringcolumn.reference b/tests/queries/0_stateless/01658_read_file_to_stringcolumn.reference index 87659c32e39..1d0901cf9f6 100644 --- a/tests/queries/0_stateless/01658_read_file_to_stringcolumn.reference +++ b/tests/queries/0_stateless/01658_read_file_to_stringcolumn.reference @@ -8,11 +8,11 @@ ccccccccc aaaaaaaaa bbbbbbbbb aaaaaaaaa bbbbbbbbb ccccccccc -:107 +:233 :79 :35 :35 -:35 +:233 699415 aaaaaaaaa bbbbbbbbb ccccccccc aaaaaaaaa bbbbbbbbb @@ -21,5 +21,5 @@ ccccccccc aaaaaaaaa bbbbbbbbb ccccccccc aaaaaaaaa bbbbbbbbb 699415 0 :0 -:107 +:70 :79