thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/ClickHouse/ClickHouse into progress-bar

Browse Source
This commit is contained in:
kssenii 2021-06-01 07:56:20 +00:00
commit de277f8ac4
115 changed files with 1143 additions and 1034 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
base/daemon/BaseDaemon.cpp
View File

@ -26,8 +26,6 @@
#include <Poco/Observer.h>
#include <Poco/AutoPtr.h>
#include <Poco/PatternFormatter.h>
#include <Poco/File.h>
#include <Poco/Path.h>
#include <Poco/Message.h>
#include <Poco/Util/Application.h>
#include <Poco/Exception.h>
@ -59,6 +57,7 @@
#include <Common/getExecutablePath.h>
#include <Common/getHashOfLoadedBinary.h>
#include <Common/Elf.h>
#include <filesystem>
#if !defined(ARCADIA_BUILD)
# include <Common/config_version.h>
@ -70,6 +69,7 @@
#endif
#include <ucontext.h>
namespace fs = std::filesystem;
DB::PipeFDs signal_pipe;
@ -437,11 +437,11 @@ static void sanitizerDeathCallback()
static std::string createDirectory(const std::string & file)
{
auto path = Poco::Path(file).makeParent();
if (path.toString().empty())
fs::path path = fs::path(file).parent_path();
if (path.empty())
return "";
Poco::File(path).createDirectories();
return path.toString();
fs::create_directories(path);
return path;
};
@ -449,7 +449,7 @@ static bool tryCreateDirectories(Poco::Logger * logger, const std::string & path
{
try
{
Poco::File(path).createDirectories();
fs::create_directories(path);
return true;
}
catch (...)
@ -470,7 +470,7 @@ void BaseDaemon::reloadConfiguration()
*/
config_path = config().getString("config-file", getDefaultConfigFileName());
DB::ConfigProcessor config_processor(config_path, false, true);
config_processor.setConfigPath(Poco::Path(config_path).makeParent().toString());
config_processor.setConfigPath(fs::path(config_path).parent_path());
loaded_config = config_processor.loadConfig(/* allow_zk_includes = */ true);
if (last_configuration != nullptr)
@ -524,18 +524,20 @@ std::string BaseDaemon::getDefaultConfigFileName() const
void BaseDaemon::closeFDs()
{
#if defined(OS_FREEBSD) || defined(OS_DARWIN)
Poco::File proc_path{"/dev/fd"};
fs::path proc_path{"/dev/fd"};
#else
Poco::File proc_path{"/proc/self/fd"};
fs::path proc_path{"/proc/self/fd"};
#endif
if (proc_path.isDirectory()) /// Hooray, proc exists
if (fs::is_directory(proc_path)) /// Hooray, proc exists
{
std::vector<std::string> fds;
/// in /proc/self/fd directory filenames are numeric file descriptors
proc_path.list(fds);
for (const auto & fd_str : fds)
/// in /proc/self/fd directory filenames are numeric file descriptors.
/// Iterate directory separately from closing fds to avoid closing iterated directory fd.
std::vector<int> fds;
for (const auto & path : fs::directory_iterator(proc_path))
fds.push_back(DB::parse<int>(path.path().filename()));
for (const auto & fd : fds)
{
int fd = DB::parse<int>(fd_str);
if (fd > 2 && fd != signal_pipe.fds_rw[0] && fd != signal_pipe.fds_rw[1])
::close(fd);
}
@ -597,7 +599,7 @@ void BaseDaemon::initialize(Application & self)
{
/** When creating pid file and looking for config, will search for paths relative to the working path of the program when started.
*/
std::string path = Poco::Path(config().getString("application.path")).setFileName("").toString();
std::string path = fs::path(config().getString("application.path")).replace_filename("");
if (0 != chdir(path.c_str()))
throw Poco::Exception("Cannot change directory to " + path);
}
@ -645,7 +647,7 @@ void BaseDaemon::initialize(Application & self)
std::string log_path = config().getString("logger.log", "");
if (!log_path.empty())
log_path = Poco::Path(log_path).setFileName("").toString();
log_path = fs::path(log_path).replace_filename("");
/** Redirect stdout, stderr to separate files in the log directory (or in the specified file).
* Some libraries write to stderr in case of errors in debug mode,
@ -708,8 +710,7 @@ void BaseDaemon::initialize(Application & self)
tryCreateDirectories(&logger(), core_path);
Poco::File cores = core_path;
if (!(cores.exists() && cores.isDirectory()))
if (!(fs::exists(core_path) && fs::is_directory(core_path)))
{
core_path = !log_path.empty() ? log_path : "/opt/";
tryCreateDirectories(&logger(), core_path);

9
base/daemon/SentryWriter.cpp
View File

@ -1,6 +1,5 @@
#include <daemon/SentryWriter.h>
#include <Poco/File.h>
#include <Poco/Util/Application.h>
#include <Poco/Util/LayeredConfiguration.h>
@ -25,6 +24,7 @@
# include <stdio.h>
# include <filesystem>
namespace fs = std::filesystem;
namespace
{
@ -53,8 +53,7 @@ void setExtras()
sentry_set_extra("physical_cpu_cores", sentry_value_new_int32(getNumberOfPhysicalCPUCores()));
if (!server_data_path.empty())
sentry_set_extra("disk_free_space", sentry_value_new_string(formatReadableSizeWithBinarySuffix(
Poco::File(server_data_path).freeSpace()).c_str()));
sentry_set_extra("disk_free_space", sentry_value_new_string(formatReadableSizeWithBinarySuffix(fs::space(server_data_path).free).c_str()));
}
void sentry_logger(sentry_level_e level, const char * message, va_list args, void *)
@ -110,12 +109,12 @@ void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config)
if (enabled)
{
server_data_path = config.getString("path", "");
const std::filesystem::path & default_tmp_path = std::filesystem::path(config.getString("tmp_path", Poco::Path::temp())) / "sentry";
const std::filesystem::path & default_tmp_path = fs::path(config.getString("tmp_path", fs::temp_directory_path())) / "sentry";
const std::string & endpoint
= config.getString("send_crash_reports.endpoint");
const std::string & temp_folder_path
= config.getString("send_crash_reports.tmp_path", default_tmp_path);
Poco::File(temp_folder_path).createDirectories();
fs::create_directories(temp_folder_path);
sentry_options_t * options = sentry_options_new(); /// will be freed by sentry_init or sentry_shutdown
sentry_options_set_release(options, VERSION_STRING_SHORT);

17
base/loggers/Loggers.cpp
View File

@ -6,10 +6,11 @@
#include "OwnFormattingChannel.h"
#include "OwnPatternFormatter.h"
#include <Poco/ConsoleChannel.h>
#include <Poco/File.h>
#include <Poco/Logger.h>
#include <Poco/Net/RemoteSyslogChannel.h>
#include <Poco/Path.h>
#include <filesystem>
namespace fs = std::filesystem;
namespace DB
{
@ -20,11 +21,11 @@ namespace DB
// TODO: move to libcommon
static std::string createDirectory(const std::string & file)
{
auto path = Poco::Path(file).makeParent();
if (path.toString().empty())
auto path = fs::path(file).parent_path();
if (path.empty())
return "";
Poco::File(path).createDirectories();
return path.toString();
fs::create_directories(path);
return path;
};
void Loggers::setTextLog(std::shared_ptr<DB::TextLog> log, int max_priority)
@ -70,7 +71,7 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
// Set up two channel chains.
log_file = new Poco::FileChannel;
log_file->setProperty(Poco::FileChannel::PROP_PATH, Poco::Path(log_path).absolute().toString());
log_file->setProperty(Poco::FileChannel::PROP_PATH, fs::weakly_canonical(log_path));
log_file->setProperty(Poco::FileChannel::PROP_ROTATION, config.getRawString("logger.size", "100M"));
log_file->setProperty(Poco::FileChannel::PROP_ARCHIVE, "number");
log_file->setProperty(Poco::FileChannel::PROP_COMPRESS, config.getRawString("logger.compress", "true"));
@ -102,7 +103,7 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
std::cerr << "Logging errors to " << errorlog_path << std::endl;
error_log_file = new Poco::FileChannel;
error_log_file->setProperty(Poco::FileChannel::PROP_PATH, Poco::Path(errorlog_path).absolute().toString());
error_log_file->setProperty(Poco::FileChannel::PROP_PATH, fs::weakly_canonical(errorlog_path));
error_log_file->setProperty(Poco::FileChannel::PROP_ROTATION, config.getRawString("logger.size", "100M"));
error_log_file->setProperty(Poco::FileChannel::PROP_ARCHIVE, "number");
error_log_file->setProperty(Poco::FileChannel::PROP_COMPRESS, config.getRawString("logger.compress", "true"));

7
programs/benchmark/Benchmark.cpp
View File

@ -8,7 +8,6 @@
#include <iomanip>
#include <random>
#include <pcg_random.hpp>
#include <Poco/File.h>
#include <Poco/Util/Application.h>
#include <Common/Stopwatch.h>
#include <Common/ThreadPool.h>
@ -36,7 +35,9 @@
#include <Common/Config/configReadClient.h>
#include <Common/TerminalSize.h>
#include <Common/StudentTTest.h>
#include <filesystem>
namespace fs = std::filesystem;
/** A tool for evaluating ClickHouse performance.
* The tool emulates a case with fixed amount of simultaneously executing queries.
@ -119,8 +120,8 @@ public:
int main(const std::vector<std::string> &) override
{
if (!json_path.empty() && Poco::File(json_path).exists()) /// Clear file with previous results
Poco::File(json_path).remove();
if (!json_path.empty() && fs::exists(json_path)) /// Clear file with previous results
fs::remove(json_path);
readQueries();
runBenchmark();

10
programs/client/Client.cpp
View File

@ -25,7 +25,6 @@
#include <boost/program_options.hpp>
#include <boost/algorithm/string/replace.hpp>
#include <Poco/String.h>
#include <Poco/File.h>
#include <Poco/Util/Application.h>
#include <common/find_symbols.h>
#include <common/LineReader.h>
@ -86,6 +85,8 @@
#include <Common/TerminalSize.h>
#include <Common/UTF8Helpers.h>
#include <Common/ProgressIndication.h>
#include <filesystem>
#include <Common/filesystemHelpers.h>
#if !defined(ARCADIA_BUILD)
# include <Common/config_version.h>
@ -95,6 +96,7 @@
#pragma GCC optimize("-fno-var-tracking-assignments")
#endif
namespace fs = std::filesystem;
namespace DB
{
@ -275,7 +277,7 @@ private:
/// Set path for format schema files
if (config().has("format_schema_path"))
context->setFormatSchemaPath(Poco::Path(config().getString("format_schema_path")).toString());
context->setFormatSchemaPath(fs::weakly_canonical(config().getString("format_schema_path")));
/// Initialize query_id_formats if any
if (config().has("query_id_formats"))
@ -632,8 +634,8 @@ private:
history_file = home_path + "/.clickhouse-client-history";
}
if (!history_file.empty() && !Poco::File(history_file).exists())
Poco::File(history_file).createFile();
if (!history_file.empty() && !fs::exists(history_file))
FS::createFile(history_file);
LineReader::Patterns query_extenders = {"\\"};
LineReader::Patterns query_delimiters = {";", "\\G"};

12
programs/copier/ClusterCopierApp.cpp
View File

@ -5,7 +5,9 @@
#include <Formats/registerFormats.h>
#include <ext/scope_guard_safe.h>
#include <unistd.h>
#include <filesystem>
namespace fs = std::filesystem;
namespace DB
{
@ -26,7 +28,7 @@ void ClusterCopierApp::initialize(Poco::Util::Application & self)
copy_fault_probability = std::max(std::min(config().getDouble("copy-fault-probability"), 1.0), 0.0);
if (config().has("move-fault-probability"))
move_fault_probability = std::max(std::min(config().getDouble("move-fault-probability"), 1.0), 0.0);
base_dir = (config().has("base-dir")) ? config().getString("base-dir") : Poco::Path::current();
base_dir = (config().has("base-dir")) ? config().getString("base-dir") : fs::current_path().string();
if (config().has("experimental-use-sample-offset"))
@ -38,18 +40,18 @@ void ClusterCopierApp::initialize(Poco::Util::Application & self)
process_id = std::to_string(DateLUT::instance().toNumYYYYMMDDhhmmss(timestamp)) + "_" + std::to_string(curr_pid);
host_id = escapeForFileName(getFQDNOrHostName()) + '#' + process_id;
process_path = Poco::Path(base_dir + "/clickhouse-copier_" + process_id).absolute().toString();
Poco::File(process_path).createDirectories();
process_path = fs::weakly_canonical(fs::path(base_dir) / ("clickhouse-copier_" + process_id));
fs::create_directories(process_path);
/// Override variables for BaseDaemon
if (config().has("log-level"))
config().setString("logger.level", config().getString("log-level"));
if (config().has("base-dir") || !config().has("logger.log"))
config().setString("logger.log", process_path + "/log.log");
config().setString("logger.log", fs::path(process_path) / "log.log");
if (config().has("base-dir") || !config().has("logger.errorlog"))
config().setString("logger.errorlog", process_path + "/log.err.log");
config().setString("logger.errorlog", fs::path(process_path) / "log.err.log");
Base::initialize(self);
}

1
programs/copier/Internals.h
View File

@ -8,7 +8,6 @@
#include <Poco/FormattingChannel.h>
#include <Poco/PatternFormatter.h>
#include <Poco/UUIDGenerator.h>
#include <Poco/File.h>
#include <Poco/Process.h>
#include <Poco/FileChannel.h>
#include <Poco/SplitterChannel.h>

2
programs/install/Install.cpp
View File

@ -288,7 +288,7 @@ int mainEntryClickHouseInstall(int argc, char ** argv)
bool is_symlink = fs::is_symlink(symlink_path);
fs::path points_to;
if (is_symlink)
points_to = fs::absolute(fs::read_symlink(symlink_path));
points_to = fs::weakly_canonical(fs::read_symlink(symlink_path));
if (is_symlink && points_to == main_bin_path)
{

12
programs/local/LocalServer.cpp
View File

@ -42,9 +42,9 @@
#include <common/argsToConfig.h>
#include <Common/TerminalSize.h>
#include <Common/randomSeed.h>
#include <filesystem>
namespace fs = std::filesystem;
namespace DB
{
@ -72,11 +72,11 @@ void LocalServer::initialize(Poco::Util::Application & self)
Poco::Util::Application::initialize(self);
/// Load config files if exists
if (config().has("config-file") || Poco::File("config.xml").exists())
if (config().has("config-file") || fs::exists("config.xml"))
{
const auto config_path = config().getString("config-file", "config.xml");
ConfigProcessor config_processor(config_path, false, true);
config_processor.setConfigPath(Poco::Path(config_path).makeParent().toString());
config_processor.setConfigPath(fs::path(config_path).parent_path());
auto loaded_config = config_processor.loadConfig();
config_processor.savePreprocessedConfig(loaded_config, loaded_config.configuration->getString("path", "."));
config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false);
@ -287,8 +287,8 @@ try
status.emplace(path + "status", StatusFile::write_full_info);
LOG_DEBUG(log, "Loading metadata from {}", path);
Poco::File(path + "data/").createDirectories();
Poco::File(path + "metadata/").createDirectories();
fs::create_directories(fs::path(path) / "data/");
fs::create_directories(fs::path(path) / "metadata/");
loadMetadataSystem(global_context);
attachSystemTables(global_context);
loadMetadata(global_context);
@ -476,7 +476,7 @@ void LocalServer::setupUsers()
{
ConfigurationPtr users_config;
if (config().has("users_config") || config().has("config-file") || Poco::File("config.xml").exists())
if (config().has("users_config") || config().has("config-file") || fs::exists("config.xml"))
{
const auto users_config_path = config().getString("users_config", config().getString("config-file", "config.xml"));
ConfigProcessor config_processor(users_config_path);

46
programs/server/Server.cpp
View File

@ -74,6 +74,7 @@
#include <Server/PostgreSQLHandlerFactory.h>
#include <Server/ProtocolServerAdapter.h>
#include <Server/HTTP/HTTPServer.h>
#include <filesystem>
#if !defined(ARCADIA_BUILD)
@ -117,6 +118,8 @@ namespace CurrentMetrics
extern const Metric MaxDDLEntryID;
}
namespace fs = std::filesystem;
#if USE_JEMALLOC
static bool jemallocOptionEnabled(const char *name)
{
@ -183,19 +186,19 @@ void setupTmpPath(Poco::Logger * log, const std::string & path)
{
LOG_DEBUG(log, "Setting up {} to store temporary data in it", path);
Poco::File(path).createDirectories();
fs::create_directories(path);
/// Clearing old temporary files.
Poco::DirectoryIterator dir_end;
for (Poco::DirectoryIterator it(path); it != dir_end; ++it)
fs::directory_iterator dir_end;
for (fs::directory_iterator it(path); it != dir_end; ++it)
{
if (it->isFile() && startsWith(it.name(), "tmp"))
if (it->is_regular_file() && startsWith(it->path().filename(), "tmp"))
{
LOG_DEBUG(log, "Removing old temporary file {}", it->path());
it->remove();
LOG_DEBUG(log, "Removing old temporary file {}", it->path().string());
fs::remove(it->path());
}
else
LOG_DEBUG(log, "Skipped file in temporary path {}", it->path());
LOG_DEBUG(log, "Skipped file in temporary path {}", it->path().string());
}
}
@ -678,37 +681,38 @@ int Server::main(const std::vector<std::string> & /*args*/)
* Examples: do repair of local data; clone all replicated tables from replica.
*/
{
Poco::File(path + "flags/").createDirectories();
global_context->setFlagsPath(path + "flags/");
auto flags_path = fs::path(path) / "flags/";
fs::create_directories(flags_path);
global_context->setFlagsPath(flags_path);
}
/** Directory with user provided files that are usable by 'file' table function.
*/
{
std::string user_files_path = config().getString("user_files_path", path + "user_files/");
std::string user_files_path = config().getString("user_files_path", fs::path(path) / "user_files/");
global_context->setUserFilesPath(user_files_path);
Poco::File(user_files_path).createDirectories();
fs::create_directories(user_files_path);
}
{
std::string dictionaries_lib_path = config().getString("dictionaries_lib_path", path + "dictionaries_lib/");
std::string dictionaries_lib_path = config().getString("dictionaries_lib_path", fs::path(path) / "dictionaries_lib/");
global_context->setDictionariesLibPath(dictionaries_lib_path);
Poco::File(dictionaries_lib_path).createDirectories();
fs::create_directories(dictionaries_lib_path);
}
/// top_level_domains_lists
{
const std::string & top_level_domains_path = config().getString("top_level_domains_path", path + "top_level_domains/") + "/";
TLDListsHolder::getInstance().parseConfig(top_level_domains_path, config());
const std::string & top_level_domains_path = config().getString("top_level_domains_path", fs::path(path) / "top_level_domains/");
TLDListsHolder::getInstance().parseConfig(fs::path(top_level_domains_path) / "", config());
}
{
Poco::File(path + "data/").createDirectories();
Poco::File(path + "metadata/").createDirectories();
fs::create_directories(fs::path(path) / "data/");
fs::create_directories(fs::path(path) / "metadata/");
/// Directory with metadata of tables, which was marked as dropped by Atomic database
Poco::File(path + "metadata_dropped/").createDirectories();
fs::create_directories(fs::path(path) / "metadata_dropped/");
}
if (config().has("interserver_http_port") && config().has("interserver_https_port"))
@ -891,9 +895,9 @@ int Server::main(const std::vector<std::string> & /*args*/)
#endif
/// Set path for format schema files
auto format_schema_path = Poco::File(config().getString("format_schema_path", path + "format_schemas/"));
global_context->setFormatSchemaPath(format_schema_path.path());
format_schema_path.createDirectories();
fs::path format_schema_path(config().getString("format_schema_path", fs::path(path) / "format_schemas/"));
global_context->setFormatSchemaPath(format_schema_path);
fs::create_directories(format_schema_path);
/// Check sanity of MergeTreeSettings on server startup
global_context->getMergeTreeSettings().sanityCheck(settings);

6
src/Access/AllowedClientHosts.h
View File

@ -7,7 +7,9 @@
#include <boost/range/algorithm/find.hpp>
#include <boost/range/algorithm_ext/erase.hpp>
#include <boost/algorithm/string/predicate.hpp>
#include <filesystem>
namespace fs = std::filesystem;
namespace DB
{
@ -198,9 +200,9 @@ inline String AllowedClientHosts::IPSubnet::toString() const
if (isMaskAllBitsOne())
return prefix.toString();
else if (IPAddress{prefix_length, mask.family()} == mask)
return prefix.toString() + "/" + std::to_string(prefix_length);
return fs::path(prefix.toString()) / std::to_string(prefix_length);
else
return prefix.toString() + "/" + mask.toString();
return fs::path(prefix.toString()) / mask.toString();
}
inline bool AllowedClientHosts::IPSubnet::isMaskAllBitsOne() const

9
src/Bridge/IBridgeHelper.cpp
View File

@ -3,9 +3,10 @@
#include <IO/ReadWriteBufferFromHTTP.h>
#include <IO/ReadHelpers.h>
#include <Poco/Net/HTTPRequest.h>
#include <Poco/Path.h>
#include <Poco/URI.h>
#include <filesystem>
namespace fs = std::filesystem;
namespace DB
{
@ -87,10 +88,10 @@ std::unique_ptr<ShellCommand> IBridgeHelper::startBridgeCommand() const
const auto & config = getConfig();
/// Path to executable folder
Poco::Path path{config.getString("application.dir", "/usr/bin")};
fs::path path(config.getString("application.dir", "/usr/bin"));
std::vector<std::string> cmd_args;
path.setFileName(serviceFileName());
path /= serviceFileName();
cmd_args.push_back("--http-port");
cmd_args.push_back(std::to_string(config.getUInt(configPrefix() + ".port", getDefaultPort())));
@ -126,7 +127,7 @@ std::unique_ptr<ShellCommand> IBridgeHelper::startBridgeCommand() const
LOG_TRACE(getLog(), "Starting {}", serviceAlias());
return ShellCommand::executeDirect(path.toString(), cmd_args, ShellCommandDestructorStrategy(true));
return ShellCommand::executeDirect(path.string(), cmd_args, ShellCommandDestructorStrategy(true));
}
}

1
src/Bridge/LibraryBridgeHelper.cpp
View File

@ -8,7 +8,6 @@
#include <IO/WriteBufferFromOStream.h>
#include <IO/WriteBufferFromString.h>
#include <Formats/FormatFactory.h>
#include <Poco/Path.h>
#include <Poco/Util/AbstractConfiguration.h>
#include <Common/ShellCommand.h>
#include <common/logger_useful.h>

2
src/Bridge/XDBCBridgeHelper.h
View File

@ -5,10 +5,8 @@
#include <Interpreters/Context.h>
#include <Access/AccessType.h>
#include <Parsers/IdentifierQuotingStyle.h>
#include <Poco/File.h>
#include <Poco/Logger.h>
#include <Poco/Net/HTTPRequest.h>
#include <Poco/Path.h>
#include <Poco/URI.h>
#include <Poco/Util/AbstractConfiguration.h>
#include <Common/ShellCommand.h>

50
src/Common/Config/ConfigProcessor.cpp
View File

@ -62,7 +62,7 @@ static std::string numberFromHost(const std::string & s)
bool ConfigProcessor::isPreprocessedFile(const std::string & path)
{
return endsWith(Poco::Path(path).getBaseName(), PREPROCESSED_SUFFIX);
return endsWith(fs::path(path).stem(), PREPROCESSED_SUFFIX);
}
@ -416,34 +416,32 @@ ConfigProcessor::Files ConfigProcessor::getConfigMergeFiles(const std::string &
{
Files files;
Poco::Path merge_dir_path(config_path);
fs::path merge_dir_path(config_path);
std::set<std::string> merge_dirs;
/// Add path_to_config/config_name.d dir
merge_dir_path.setExtension("d");
merge_dirs.insert(merge_dir_path.toString());
merge_dir_path.replace_extension("d");
merge_dirs.insert(merge_dir_path);
/// Add path_to_config/conf.d dir
merge_dir_path.setBaseName("conf");
merge_dirs.insert(merge_dir_path.toString());
merge_dir_path.replace_filename("conf.d");
merge_dirs.insert(merge_dir_path);
for (const std::string & merge_dir_name : merge_dirs)
{
Poco::File merge_dir(merge_dir_name);
if (!merge_dir.exists() || !merge_dir.isDirectory())
if (!fs::exists(merge_dir_name) || !fs::is_directory(merge_dir_name))
continue;
for (Poco::DirectoryIterator it(merge_dir_name); it != Poco::DirectoryIterator(); ++it)
for (fs::directory_iterator it(merge_dir_name); it != fs::directory_iterator(); ++it)
{
Poco::File & file = *it;
Poco::Path path(file.path());
std::string extension = path.getExtension();
std::string base_name = path.getBaseName();
fs::path path(it->path());
std::string extension = path.extension();
std::string base_name = path.stem();
// Skip non-config and temporary files
if (file.isFile() && (extension == "xml" || extension == "conf" || extension == "yaml" || extension == "yml") && !startsWith(base_name, "."))
{
files.push_back(file.path());
}
if (fs::is_regular_file(path)
&& (extension == ".xml" || extension == ".conf" || extension == ".yaml" || extension == ".yml")
&& !startsWith(base_name, "."))
files.push_back(it->path());
}
}
@ -548,7 +546,7 @@ XMLDocumentPtr ConfigProcessor::processConfig(
else
{
std::string default_path = "/etc/metrika.xml";
if (Poco::File(default_path).exists())
if (fs::exists(default_path))
include_from_path = default_path;
}
if (!include_from_path.empty())
@ -660,11 +658,11 @@ void ConfigProcessor::savePreprocessedConfig(const LoadedConfig & loaded_config,
if (!loaded_config.configuration->has("path"))
{
// Will use current directory
auto parent_path = Poco::Path(loaded_config.config_path).makeParent();
preprocessed_dir = parent_path.toString();
Poco::Path poco_new_path(new_path);
poco_new_path.setBaseName(poco_new_path.getBaseName() + PREPROCESSED_SUFFIX);
new_path = poco_new_path.toString();
fs::path parent_path = fs::path(loaded_config.config_path).parent_path();
preprocessed_dir = parent_path.string();
fs::path fs_new_path(new_path);
fs_new_path.replace_filename(fs_new_path.stem().string() + PREPROCESSED_SUFFIX + fs_new_path.extension().string());
new_path = fs_new_path.string();
}
else
{
@ -679,9 +677,9 @@ void ConfigProcessor::savePreprocessedConfig(const LoadedConfig & loaded_config,
}
preprocessed_path = (fs::path(preprocessed_dir) / fs::path(new_path)).string();
auto preprocessed_path_parent = Poco::Path(preprocessed_path).makeParent();
if (!preprocessed_path_parent.toString().empty())
Poco::File(preprocessed_path_parent).createDirectories();
auto preprocessed_path_parent = fs::path(preprocessed_path).parent_path();
if (!preprocessed_path_parent.empty())
fs::create_directories(preprocessed_path_parent);
}
DOMWriter().writeNode(preprocessed_path, loaded_config.preprocessed_xml);
LOG_DEBUG(log, "Saved preprocessed configuration to '{}'.", preprocessed_path);

3
src/Common/Config/ConfigProcessor.h
View File

@ -15,12 +15,9 @@
#include <Poco/DOM/NodeList.h>
#include <Poco/DOM/NamedNodeMap.h>
#include <Poco/AutoPtr.h>
#include <Poco/File.h>
#include <Poco/Path.h>
#include <Poco/DirectoryIterator.h>
#include <Poco/ConsoleChannel.h>
#include <Poco/Util/AbstractConfiguration.h>
#include <common/logger_useful.h>

9
src/Common/Config/ConfigReloader.cpp
View File

@ -1,12 +1,15 @@
#include "ConfigReloader.h"
#include <Poco/Util/Application.h>
#include <Poco/File.h>
#include <common/logger_useful.h>
#include <Common/setThreadName.h>
#include "ConfigProcessor.h"
#include <filesystem>
#include <Common/filesystemHelpers.h>
namespace fs = std::filesystem;
namespace DB
{
@ -167,8 +170,8 @@ struct ConfigReloader::FileWithTimestamp
void ConfigReloader::FilesChangesTracker::addIfExists(const std::string & path_to_add)
{
if (!path_to_add.empty() && Poco::File(path_to_add).exists())
files.emplace(path_to_add, Poco::File(path_to_add).getLastModified().epochTime());
if (!path_to_add.empty() && fs::exists(path_to_add))
files.emplace(path_to_add, FS::getModificationTime(path_to_add));
}
bool ConfigReloader::FilesChangesTracker::isDifferOrNewerThan(const FilesChangesTracker & rhs)

10
src/Common/Config/configReadClient.cpp
View File

@ -1,8 +1,10 @@
#include "configReadClient.h"
#include <Poco/Util/LayeredConfiguration.h>
#include <Poco/File.h>
#include "ConfigProcessor.h"
#include <filesystem>
namespace fs = std::filesystem;
namespace DB
{
@ -11,11 +13,11 @@ bool configReadClient(Poco::Util::LayeredConfiguration & config, const std::stri
std::string config_path;
if (config.has("config-file"))
config_path = config.getString("config-file");
else if (Poco::File("./clickhouse-client.xml").exists())
else if (fs::exists("./clickhouse-client.xml"))
config_path = "./clickhouse-client.xml";
else if (!home_path.empty() && Poco::File(home_path + "/.clickhouse-client/config.xml").exists())
else if (!home_path.empty() && fs::exists(home_path + "/.clickhouse-client/config.xml"))
config_path = home_path + "/.clickhouse-client/config.xml";
else if (Poco::File("/etc/clickhouse-client/config.xml").exists())
else if (fs::exists("/etc/clickhouse-client/config.xml"))
config_path = "/etc/clickhouse-client/config.xml";
if (!config_path.empty())

5
src/Common/CounterInFile.h
View File

@ -7,7 +7,6 @@
#include <iostream>
#include <mutex>
#include <Poco/File.h>
#include <Poco/Exception.h>
#include <IO/ReadBufferFromFileDescriptor.h>
@ -59,7 +58,7 @@ public:
Int64 res = -1;
bool file_doesnt_exists = !Poco::File(path).exists();
bool file_doesnt_exists = !fs::exists(path);
if (file_doesnt_exists && !create_if_need)
{
throw Poco::Exception("File " + path + " does not exist. "
@ -138,7 +137,7 @@ public:
// Not thread-safe and not synchronized between processes.
void fixIfBroken(UInt64 value)
{
bool file_exists = Poco::File(path).exists();
bool file_exists = fs::exists(path);
int fd = ::open(path.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0666);
if (-1 == fd)

1
src/Common/ErrorCodes.cpp
View File

@ -553,6 +553,7 @@
M(583, ILLEGAL_PROJECTION) \
M(584, PROJECTION_NOT_USED) \
M(585, CANNOT_PARSE_YAML) \
M(586, CANNOT_CREATE_FILE) \
\
M(998, POSTGRESQL_CONNECTION_FAILURE) \
M(999, KEEPER_EXCEPTION) \

20
src/Common/Exception.cpp
View File

@ -21,6 +21,8 @@
# include <Common/config_version.h>
#endif
namespace fs = std::filesystem;
namespace DB
{
@ -177,7 +179,7 @@ void tryLogCurrentException(Poco::Logger * logger, const std::string & start_of_
tryLogCurrentExceptionImpl(logger, start_of_message);
}
static void getNoSpaceLeftInfoMessage(std::filesystem::path path, std::string & msg)
static void getNoSpaceLeftInfoMessage(std::filesystem::path path, String & msg)
{
path = std::filesystem::absolute(path);
/// It's possible to get ENOSPC for non existent file (e.g. if there are no free inodes and creat() fails)
@ -264,23 +266,13 @@ static std::string getExtraExceptionInfo(const std::exception & e)
String msg;
try
{
if (const auto * file_exception = dynamic_cast<const Poco::FileException *>(&e))
if (const auto * file_exception = dynamic_cast<const fs::filesystem_error *>(&e))
{
if (file_exception->code() == ENOSPC)
{
/// See Poco::FileImpl::handleLastErrorImpl(...)
constexpr const char * expected_error_message = "no space left on device: ";
if (startsWith(file_exception->message(), expected_error_message))
{
String path = file_exception->message().substr(strlen(expected_error_message));
getNoSpaceLeftInfoMessage(path, msg);
}
if (file_exception->code() == std::errc::no_space_on_device)
getNoSpaceLeftInfoMessage(file_exception->path1(), msg);
else
{
msg += "\nCannot print extra info for Poco::Exception";
}
}
}
else if (const auto * errno_exception = dynamic_cast<const DB::ErrnoException *>(&e))
{
if (errno_exception->getErrno() == ENOSPC && errno_exception->getPath())

7
src/Common/FileUpdatesTracker.h
View File

@ -1,10 +1,11 @@
#pragma once
#include <Poco/File.h>
#include <Poco/Timestamp.h>
#include <string>
#include <filesystem>
#include <Common/filesystemHelpers.h>
namespace fs = std::filesystem;
class FileUpdatesTracker
{
@ -31,6 +32,6 @@ public:
private:
Poco::Timestamp getLastModificationTime() const
{
return Poco::File(path).getLastModified();
return FS::getModificationTimestamp(path);
}
};

2
src/Common/ProgressIndication.cpp
View File

@ -34,7 +34,7 @@ void ProgressIndication::resetProgress()
write_progress_on_update = false;
}
void ProgressIndication::setFileProgressCallback(ContextPtr context, bool write_progress_on_update_)
void ProgressIndication::setFileProgressCallback(ContextMutablePtr context, bool write_progress_on_update_)
{
write_progress_on_update = write_progress_on_update_;
context->setFileProgressCallback([&](const FileProgress & file_progress)

2
src/Common/ProgressIndication.h
View File

@ -36,7 +36,7 @@ public:
/// In some cases there is a need to update progress value, when there is no access to progress_inidcation object.
/// In this case it is added via context.
/// `write_progress_on_update` is needed to write progress for loading files data via pipe in non-interactive mode.
void setFileProgressCallback(ContextPtr context, bool write_progress_on_update = false);
void setFileProgressCallback(ContextMutablePtr context, bool write_progress_on_update = false);
/// How much seconds passed since query execution start.
UInt64 elapsedSeconds() const { return watch.elapsedSeconds(); }

5
src/Common/StatusFile.cpp
View File

@ -4,7 +4,6 @@
#include <fcntl.h>
#include <errno.h>
#include <Poco/File.h>
#include <common/logger_useful.h>
#include <common/errnoToString.h>
#include <Common/ClickHouseRevision.h>
@ -14,7 +13,9 @@
#include <IO/LimitReadBuffer.h>
#include <IO/WriteBufferFromFileDescriptor.h>
#include <IO/Operators.h>
#include <filesystem>
namespace fs = std::filesystem;
namespace DB
{
@ -45,7 +46,7 @@ StatusFile::StatusFile(std::string path_, FillFunction fill_)
: path(std::move(path_)), fill(std::move(fill_))
{
/// If file already exists. NOTE Minor race condition.
if (Poco::File(path).exists())
if (fs::exists(path))
{
std::string contents;
{

10
src/Common/ZooKeeper/ZooKeeper.cpp
View File

@ -4,6 +4,7 @@
#include "TestKeeper.h"
#include <functional>
#include <filesystem>
#include <pcg-random/pcg_random.hpp>
#include <common/logger_useful.h>
@ -17,6 +18,7 @@
#define ZOOKEEPER_CONNECTION_TIMEOUT_MS 1000
namespace fs = std::filesystem;
namespace DB
{
@ -612,7 +614,7 @@ void ZooKeeper::removeChildren(const std::string & path)
Coordination::Requests ops;
for (size_t i = 0; i < MULTI_BATCH_SIZE && !children.empty(); ++i)
{
ops.emplace_back(makeRemoveRequest(path + "/" + children.back(), -1));
ops.emplace_back(makeRemoveRequest(fs::path(path) / children.back(), -1));
children.pop_back();
}
multi(ops);
@ -628,9 +630,9 @@ void ZooKeeper::removeChildrenRecursive(const std::string & path, const String &
Coordination::Requests ops;
for (size_t i = 0; i < MULTI_BATCH_SIZE && !children.empty(); ++i)
{
removeChildrenRecursive(path + "/" + children.back());
removeChildrenRecursive(fs::path(path) / children.back());
if (likely(keep_child_node.empty() || keep_child_node != children.back()))
ops.emplace_back(makeRemoveRequest(path + "/" + children.back(), -1));
ops.emplace_back(makeRemoveRequest(fs::path(path) / children.back(), -1));
children.pop_back();
}
multi(ops);
@ -648,7 +650,7 @@ void ZooKeeper::tryRemoveChildrenRecursive(const std::string & path, const Strin
Strings batch;
for (size_t i = 0; i < MULTI_BATCH_SIZE && !children.empty(); ++i)
{
String child_path = path + "/" + children.back();
String child_path = fs::path(path) / children.back();
tryRemoveChildrenRecursive(child_path);
if (likely(keep_child_node.empty() || keep_child_node != children.back()))
{

89
src/Common/filesystemHelpers.cpp
View File

@ -6,10 +6,15 @@
# include <mntent.h>
#endif
#include <cerrno>
#include <Poco/File.h>
#include <Poco/Path.h>
#include <Poco/Version.h>
#include <Poco/Timestamp.h>
#include <filesystem>
#include <fcntl.h>
#include <unistd.h>
#include <sys/types.h>
#include <utime.h>
namespace fs = std::filesystem;
namespace DB
{
@ -20,6 +25,8 @@ namespace ErrorCodes
extern const int SYSTEM_ERROR;
extern const int NOT_IMPLEMENTED;
extern const int CANNOT_STATVFS;
extern const int PATH_ACCESS_DENIED;
extern const int CANNOT_CREATE_FILE;
}
@ -38,17 +45,13 @@ struct statvfs getStatVFS(const String & path)
bool enoughSpaceInDirectory(const std::string & path [[maybe_unused]], size_t data_size [[maybe_unused]])
{
#if POCO_VERSION >= 0x01090000
auto free_space = Poco::File(path).freeSpace();
auto free_space = fs::space(path).free;
return data_size <= free_space;
#else
return true;
#endif
}
std::unique_ptr<TemporaryFile> createTemporaryFile(const std::string & path)
{
Poco::File(path).createDirectories();
fs::create_directories(path);
/// NOTE: std::make_shared cannot use protected constructors
return std::make_unique<TemporaryFile>(path);
@ -128,3 +131,73 @@ bool pathStartsWith(const String & path, const String & prefix_path)
}
}
/// Copied from Poco::File
namespace FS
{
bool createFile(const std::string & path)
{
int n = open(path.c_str(), O_WRONLY | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
if (n != -1)
{
close(n);
return true;
}
DB::throwFromErrnoWithPath("Cannot create file: " + path, path, DB::ErrorCodes::CANNOT_CREATE_FILE);
}
bool canRead(const std::string & path)
{
struct stat st;
if (stat(path.c_str(), &st) == 0)
{
if (st.st_uid == geteuid())
return (st.st_mode & S_IRUSR) != 0;
else if (st.st_gid == getegid())
return (st.st_mode & S_IRGRP) != 0;
else
return (st.st_mode & S_IROTH) != 0 || geteuid() == 0;
}
DB::throwFromErrnoWithPath("Cannot check read access to file: " + path, path, DB::ErrorCodes::PATH_ACCESS_DENIED);
}
bool canWrite(const std::string & path)
{
struct stat st;
if (stat(path.c_str(), &st) == 0)
{
if (st.st_uid == geteuid())
return (st.st_mode & S_IWUSR) != 0;
else if (st.st_gid == getegid())
return (st.st_mode & S_IWGRP) != 0;
else
return (st.st_mode & S_IWOTH) != 0 || geteuid() == 0;
}
DB::throwFromErrnoWithPath("Cannot check write access to file: " + path, path, DB::ErrorCodes::PATH_ACCESS_DENIED);
}
time_t getModificationTime(const std::string & path)
{
struct stat st;
if (stat(path.c_str(), &st) == 0)
return st.st_mtime;
DB::throwFromErrnoWithPath("Cannot check modification time for file: " + path, path, DB::ErrorCodes::PATH_ACCESS_DENIED);
}
Poco::Timestamp getModificationTimestamp(const std::string & path)
{
return Poco::Timestamp::fromEpochTime(getModificationTime(path));
}
void setModificationTime(const std::string & path, time_t time)
{
struct utimbuf tb;
tb.actime = time;
tb.modtime = time;
if (utime(path.c_str(), &tb) != 0)
DB::throwFromErrnoWithPath("Cannot set modification time for file: " + path, path, DB::ErrorCodes::PATH_ACCESS_DENIED);
}
}

12
src/Common/filesystemHelpers.h
View File

@ -36,3 +36,15 @@ bool pathStartsWith(const std::filesystem::path & path, const std::filesystem::p
bool pathStartsWith(const String & path, const String & prefix_path);
}
namespace FS
{
bool createFile(const std::string & path);
bool canRead(const std::string & path);
bool canWrite(const std::string & path);
time_t getModificationTime(const std::string & path);
Poco::Timestamp getModificationTimestamp(const std::string & path);
void setModificationTime(const std::string & path, time_t time);
}

8
src/Common/renameat2.cpp
View File

@ -1,6 +1,6 @@
#include <Common/renameat2.h>
#include <Common/Exception.h>
#include <Poco/File.h>
#include <filesystem>
#if defined(linux) || defined(__linux) || defined(__linux__)
#include <unistd.h>
@ -10,6 +10,8 @@
#include <sys/utsname.h>
#endif
namespace fs = std::filesystem;
namespace DB
{
@ -93,9 +95,9 @@ static bool renameat2(const std::string &, const std::string &, int)
static void renameNoReplaceFallback(const std::string & old_path, const std::string & new_path)
{
/// NOTE it's unsafe
if (Poco::File{new_path}.exists())
if (fs::exists(new_path))
throw Exception("File " + new_path + " exists", ErrorCodes::FILE_ALREADY_EXISTS);
Poco::File{old_path}.renameTo(new_path);
fs::rename(old_path, new_path);
}
/// Do not use [[noreturn]] to avoid warnings like "code will never be executed" in other places

44
src/Databases/DatabaseAtomic.cpp
View File

@ -1,7 +1,5 @@
#include <Databases/DatabaseAtomic.h>
#include <Databases/DatabaseOnDisk.h>
#include <Poco/File.h>
#include <Poco/Path.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#include <IO/ReadBufferFromFile.h>
@ -13,6 +11,8 @@
#include <filesystem>
#include <Interpreters/DDLTask.h>
namespace fs = std::filesystem;
namespace DB
{
namespace ErrorCodes
@ -37,12 +37,12 @@ public:
DatabaseAtomic::DatabaseAtomic(String name_, String metadata_path_, UUID uuid, const String & logger_name, ContextPtr context_)
: DatabaseOrdinary(name_, std::move(metadata_path_), "store/", logger_name, context_)
, path_to_table_symlinks(getContext()->getPath() + "data/" + escapeForFileName(name_) + "/")
, path_to_metadata_symlink(getContext()->getPath() + "metadata/" + escapeForFileName(name_))
, path_to_table_symlinks(fs::path(getContext()->getPath()) / "data" / escapeForFileName(name_) / "")
, path_to_metadata_symlink(fs::path(getContext()->getPath()) / "metadata" / escapeForFileName(name_))
, db_uuid(uuid)
{
assert(db_uuid != UUIDHelpers::Nil);
Poco::File(path_to_table_symlinks).createDirectories();
fs::create_directories(path_to_table_symlinks);
tryCreateMetadataSymlink();
}
@ -73,14 +73,14 @@ void DatabaseAtomic::drop(ContextPtr)
assert(tables.empty());
try
{
Poco::File(path_to_metadata_symlink).remove();
Poco::File(path_to_table_symlinks).remove(true);
fs::remove(path_to_metadata_symlink);
fs::remove_all(path_to_table_symlinks);
}
catch (...)
{
LOG_WARNING(log, getCurrentExceptionMessage(true));
}
Poco::File(getMetadataPath()).remove(true);
fs::remove_all(getMetadataPath());
}
void DatabaseAtomic::attachTable(const String & name, const StoragePtr & table, const String & relative_table_path)
@ -132,7 +132,7 @@ void DatabaseAtomic::dropTable(ContextPtr local_context, const String & table_na
/// (it's more likely to lost connection, than to fail before applying local changes).
/// TODO better detection and recovery
Poco::File(table_metadata_path).renameTo(table_metadata_path_drop); /// Mark table as dropped
fs::rename(table_metadata_path, table_metadata_path_drop); /// Mark table as dropped
DatabaseOrdinary::detachTableUnlocked(table_name, lock); /// Should never throw
table_name_to_path.erase(table_name);
}
@ -316,7 +316,7 @@ void DatabaseAtomic::commitCreateTable(const ASTCreateQuery & query, const Stora
}
catch (...)
{
Poco::File(table_metadata_tmp_path).remove();
fs::remove(table_metadata_tmp_path);
if (locked_uuid)
DatabaseCatalog::instance().removeUUIDMappingFinally(query.uuid);
throw;
@ -420,7 +420,7 @@ void DatabaseAtomic::loadStoredObjects(ContextMutablePtr local_context, bool has
{
/// Recreate symlinks to table data dirs in case of force restore, because some of them may be broken
if (has_force_restore_data_flag)
Poco::File(path_to_table_symlinks).remove(true);
fs::remove_all(path_to_table_symlinks);
DatabaseOrdinary::loadStoredObjects(local_context, has_force_restore_data_flag, force_attach);
@ -432,7 +432,7 @@ void DatabaseAtomic::loadStoredObjects(ContextMutablePtr local_context, bool has
table_names = table_name_to_path;
}
Poco::File(path_to_table_symlinks).createDirectories();
fs::create_directories(path_to_table_symlinks);
for (const auto & table : table_names)
tryCreateSymlink(table.first, table.second, true);
}
@ -443,9 +443,9 @@ void DatabaseAtomic::tryCreateSymlink(const String & table_name, const String &
try
{
String link = path_to_table_symlinks + escapeForFileName(table_name);
Poco::File data = Poco::Path(getContext()->getPath()).makeAbsolute().toString() + actual_data_path;
if (!if_data_path_exist || data.exists())
data.linkTo(link, Poco::File::LINK_SYMBOLIC);
fs::path data = fs::canonical(getContext()->getPath()) / actual_data_path;
if (!if_data_path_exist || fs::exists(data))
fs::create_directory_symlink(data, link);
}
catch (...)
{
@ -458,7 +458,7 @@ void DatabaseAtomic::tryRemoveSymlink(const String & table_name)
try
{
String path = path_to_table_symlinks + escapeForFileName(table_name);
Poco::File{path}.remove();
fs::remove(path);
}
catch (...)
{
@ -471,17 +471,17 @@ void DatabaseAtomic::tryCreateMetadataSymlink()
/// Symlinks in data/db_name/ directory and metadata/db_name/ are not used by ClickHouse,
/// it's needed only for convenient introspection.
assert(path_to_metadata_symlink != metadata_path);
Poco::File metadata_symlink(path_to_metadata_symlink);
if (metadata_symlink.exists())
fs::path metadata_symlink(path_to_metadata_symlink);
if (fs::exists(metadata_symlink))
{
if (!metadata_symlink.isLink())
if (!fs::is_symlink(metadata_symlink))
throw Exception(ErrorCodes::FILE_ALREADY_EXISTS, "Directory {} exists", path_to_metadata_symlink);
}
else
{
try
{
Poco::File{metadata_path}.linkTo(path_to_metadata_symlink, Poco::File::LINK_SYMBOLIC);
fs::create_directory_symlink(metadata_path, path_to_metadata_symlink);
}
catch (...)
{
@ -495,7 +495,7 @@ void DatabaseAtomic::renameDatabase(const String & new_name)
/// CREATE, ATTACH, DROP, DETACH and RENAME DATABASE must hold DDLGuard
try
{
Poco::File(path_to_metadata_symlink).remove();
fs::remove(path_to_metadata_symlink);
}
catch (...)
{
@ -526,7 +526,7 @@ void DatabaseAtomic::renameDatabase(const String & new_name)
path_to_table_symlinks = getContext()->getPath() + "data/" + new_name_escaped + "/";
}
Poco::File(old_path_to_table_symlinks).renameTo(path_to_table_symlinks);
fs::rename(old_path_to_table_symlinks, path_to_table_symlinks);
tryCreateMetadataSymlink();
}

17
src/Databases/DatabaseFactory.cpp
View File

@ -11,10 +11,9 @@
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/formatAST.h>
#include <Poco/File.h>
#include <Poco/Path.h>
#include <Interpreters/Context.h>
#include <Common/Macros.h>
#include <filesystem>
#if !defined(ARCADIA_BUILD)
# include "config_core.h"
@ -40,6 +39,8 @@
#include <Storages/PostgreSQL/PoolWithFailover.h>
#endif
namespace fs = std::filesystem;
namespace DB
{
@ -58,11 +59,12 @@ DatabasePtr DatabaseFactory::get(const ASTCreateQuery & create, const String & m
try
{
/// Creates store/xxx/ for Atomic
Poco::File(Poco::Path(metadata_path).makeParent()).createDirectories();
fs::create_directories(fs::path(metadata_path).parent_path());
/// Before 20.7 it's possible that .sql metadata file does not exist for some old database.
/// In this case Ordinary database is created on server startup if the corresponding metadata directory exists.
/// So we should remove metadata directory if database creation failed.
created = Poco::File(metadata_path).createDirectory();
created = fs::create_directory(metadata_path);
DatabasePtr impl = getImpl(create, metadata_path, context);
@ -74,11 +76,8 @@ DatabasePtr DatabaseFactory::get(const ASTCreateQuery & create, const String & m
}
catch (...)
{
Poco::File metadata_dir(metadata_path);
if (created && metadata_dir.exists())
metadata_dir.remove(true);
if (created && fs::exists(metadata_path))
fs::remove_all(metadata_path);
throw;
}
}

9
src/Databases/DatabaseLazy.cpp
View File

@ -12,8 +12,9 @@
#include <common/logger_useful.h>
#include <ext/scope_guard_safe.h>
#include <iomanip>
#include <Poco/File.h>
#include <filesystem>
namespace fs = std::filesystem;
namespace DB
{
@ -43,8 +44,8 @@ void DatabaseLazy::loadStoredObjects(
{
const std::string table_name = file_name.substr(0, file_name.size() - 4);
auto detached_permanently_flag = Poco::File(getMetadataPath() + "/" + file_name + detached_suffix);
if (detached_permanently_flag.exists())
fs::path detached_permanently_flag = fs::path(getMetadataPath()) / (file_name + detached_suffix);
if (fs::exists(detached_permanently_flag))
{
LOG_DEBUG(log, "Skipping permanently detached table {}.", backQuote(table_name));
return;
@ -228,7 +229,7 @@ StoragePtr DatabaseLazy::loadTable(const String & table_name) const
LOG_DEBUG(log, "Load table {} to cache.", backQuote(table_name));
const String table_metadata_path = getMetadataPath() + "/" + escapeForFileName(table_name) + ".sql";
const String table_metadata_path = fs::path(getMetadataPath()) / (escapeForFileName(table_name) + ".sql");
try
{

8
src/Databases/DatabaseMemory.cpp
View File

@ -4,9 +4,9 @@
#include <Interpreters/Context.h>
#include <Parsers/ASTCreateQuery.h>
#include <Storages/IStorage.h>
#include <Poco/File.h>
#include <filesystem>
namespace fs = std::filesystem;
namespace DB
{
@ -42,9 +42,9 @@ void DatabaseMemory::dropTable(
try
{
table->drop();
Poco::File table_data_dir{getTableDataPath(table_name)};
if (table_data_dir.exists())
table_data_dir.remove(true);
fs::path table_data_dir{getTableDataPath(table_name)};
if (fs::exists(table_data_dir))
fs::remove_all(table_data_dir);
}
catch (...)
{

95
src/Databases/DatabaseOnDisk.cpp
View File

@ -14,14 +14,14 @@
#include <Storages/StorageFactory.h>
#include <TableFunctions/TableFunctionFactory.h>
#include <Common/escapeForFileName.h>
#include <common/logger_useful.h>
#include <Poco/DirectoryIterator.h>
#include <Databases/DatabaseOrdinary.h>
#include <Databases/DatabaseAtomic.h>
#include <Common/assert_cast.h>
#include <filesystem>
#include <Common/filesystemHelpers.h>
namespace fs = std::filesystem;
namespace DB
{
@ -201,8 +201,8 @@ DatabaseOnDisk::DatabaseOnDisk(
, metadata_path(metadata_path_)
, data_path(data_path_)
{
Poco::File(local_context->getPath() + data_path).createDirectories();
Poco::File(metadata_path).createDirectories();
fs::create_directories(local_context->getPath() + data_path);
fs::create_directories(metadata_path);
}
@ -245,7 +245,7 @@ void DatabaseOnDisk::createTable(
if (!create.attach)
checkMetadataFilenameAvailability(table_name);
if (create.attach && Poco::File(table_metadata_path).exists())
if (create.attach && fs::exists(table_metadata_path))
{
ASTPtr ast_detached = parseQueryFromMetadata(log, local_context, table_metadata_path);
auto & create_detached = ast_detached->as<ASTCreateQuery &>();
@ -285,10 +285,10 @@ void DatabaseOnDisk::removeDetachedPermanentlyFlag(ContextPtr, const String & ta
{
try
{
auto detached_permanently_flag = Poco::File(table_metadata_path + detached_suffix);
fs::path detached_permanently_flag(table_metadata_path + detached_suffix);
if (detached_permanently_flag.exists())
detached_permanently_flag.remove();
if (fs::exists(detached_permanently_flag))
fs::remove(detached_permanently_flag);
}
catch (Exception & e)
{
@ -308,11 +308,11 @@ void DatabaseOnDisk::commitCreateTable(const ASTCreateQuery & query, const Stora
/// If it was ATTACH query and file with table metadata already exist
/// (so, ATTACH is done after DETACH), then rename atomically replaces old file with new one.
Poco::File(table_metadata_tmp_path).renameTo(table_metadata_path);
fs::rename(table_metadata_tmp_path, table_metadata_path);
}
catch (...)
{
Poco::File(table_metadata_tmp_path).remove();
fs::remove(table_metadata_tmp_path);
throw;
}
}
@ -321,10 +321,10 @@ void DatabaseOnDisk::detachTablePermanently(ContextPtr, const String & table_nam
{
auto table = detachTable(table_name);
Poco::File detached_permanently_flag(getObjectMetadataPath(table_name) + detached_suffix);
fs::path detached_permanently_flag(getObjectMetadataPath(table_name) + detached_suffix);
try
{
detached_permanently_flag.createFile();
FS::createFile(detached_permanently_flag);
}
catch (Exception & e)
{
@ -350,25 +350,25 @@ void DatabaseOnDisk::dropTable(ContextPtr local_context, const String & table_na
bool renamed = false;
try
{
Poco::File(table_metadata_path).renameTo(table_metadata_path_drop);
fs::rename(table_metadata_path, table_metadata_path_drop);
renamed = true;
table->drop();
table->is_dropped = true;
Poco::File table_data_dir{local_context->getPath() + table_data_path_relative};
if (table_data_dir.exists())
table_data_dir.remove(true);
fs::path table_data_dir(local_context->getPath() + table_data_path_relative);
if (fs::exists(table_data_dir))
fs::remove_all(table_data_dir);
}
catch (...)
{
LOG_WARNING(log, getCurrentExceptionMessage(__PRETTY_FUNCTION__));
attachTable(table_name, table, table_data_path_relative);
if (renamed)
Poco::File(table_metadata_path_drop).renameTo(table_metadata_path);
fs::rename(table_metadata_path_drop, table_metadata_path);
throw;
}
Poco::File(table_metadata_path_drop).remove();
fs::remove(table_metadata_path_drop);
}
void DatabaseOnDisk::checkMetadataFilenameAvailability(const String & to_table_name) const
@ -381,11 +381,11 @@ void DatabaseOnDisk::checkMetadataFilenameAvailabilityUnlocked(const String & to
{
String table_metadata_path = getObjectMetadataPath(to_table_name);
if (Poco::File(table_metadata_path).exists())
if (fs::exists(table_metadata_path))
{
auto detached_permanently_flag = Poco::File(table_metadata_path + detached_suffix);
fs::path detached_permanently_flag(table_metadata_path + detached_suffix);
if (detached_permanently_flag.exists())
if (fs::exists(detached_permanently_flag))
throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Table {}.{} already exists (detached permanently)", backQuote(database_name), backQuote(to_table_name));
else
throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Table {}.{} already exists (detached)", backQuote(database_name), backQuote(to_table_name));
@ -463,7 +463,7 @@ void DatabaseOnDisk::renameTable(
/// Now table data are moved to new database, so we must add metadata and attach table to new database
to_database.createTable(local_context, to_table_name, table, attach_query);
Poco::File(table_metadata_path).remove();
fs::remove(table_metadata_path);
if (from_atomic_to_ordinary)
{
@ -528,8 +528,8 @@ ASTPtr DatabaseOnDisk::getCreateDatabaseQuery() const
void DatabaseOnDisk::drop(ContextPtr local_context)
{
assert(tables.empty());
Poco::File(local_context->getPath() + getDataPath()).remove(false);
Poco::File(getMetadataPath()).remove(false);
fs::remove(local_context->getPath() + getDataPath());
fs::remove(getMetadataPath());
}
String DatabaseOnDisk::getObjectMetadataPath(const String & object_name) const
@ -540,10 +540,9 @@ String DatabaseOnDisk::getObjectMetadataPath(const String & object_name) const
time_t DatabaseOnDisk::getObjectMetadataModificationTime(const String & object_name) const
{
String table_metadata_path = getObjectMetadataPath(object_name);
Poco::File meta_file(table_metadata_path);
if (meta_file.exists())
return meta_file.getLastModified().epochTime();
if (fs::exists(table_metadata_path))
return FS::getModificationTime(table_metadata_path);
else
return static_cast<time_t>(0);
}
@ -555,56 +554,57 @@ void DatabaseOnDisk::iterateMetadataFiles(ContextPtr local_context, const Iterat
assert(getUUID() == UUIDHelpers::Nil);
static const char * tmp_drop_ext = ".sql.tmp_drop";
const std::string object_name = file_name.substr(0, file_name.size() - strlen(tmp_drop_ext));
if (Poco::File(local_context->getPath() + getDataPath() + '/' + object_name).exists())
if (fs::exists(local_context->getPath() + getDataPath() + '/' + object_name))
{
Poco::File(getMetadataPath() + file_name).renameTo(getMetadataPath() + object_name + ".sql");
fs::rename(getMetadataPath() + file_name, getMetadataPath() + object_name + ".sql");
LOG_WARNING(log, "Object {} was not dropped previously and will be restored", backQuote(object_name));
process_metadata_file(object_name + ".sql");
}
else
{
LOG_INFO(log, "Removing file {}", getMetadataPath() + file_name);
Poco::File(getMetadataPath() + file_name).remove();
fs::remove(getMetadataPath() + file_name);
}
};
/// Metadata files to load: name and flag for .tmp_drop files
std::set<std::pair<String, bool>> metadata_files;
Poco::DirectoryIterator dir_end;
for (Poco::DirectoryIterator dir_it(getMetadataPath()); dir_it != dir_end; ++dir_it)
fs::directory_iterator dir_end;
for (fs::directory_iterator dir_it(getMetadataPath()); dir_it != dir_end; ++dir_it)
{
String file_name = dir_it->path().filename();
/// For '.svn', '.gitignore' directory and similar.
if (dir_it.name().at(0) == '.')
if (file_name.at(0) == '.')
continue;
/// There are .sql.bak files - skip them.
if (endsWith(dir_it.name(), ".sql.bak"))
if (endsWith(file_name, ".sql.bak"))
continue;
/// Permanently detached table flag
if (endsWith(dir_it.name(), ".sql.detached"))
if (endsWith(file_name, ".sql.detached"))
continue;
if (endsWith(dir_it.name(), ".sql.tmp_drop"))
if (endsWith(file_name, ".sql.tmp_drop"))
{
/// There are files that we tried to delete previously
metadata_files.emplace(dir_it.name(), false);
metadata_files.emplace(file_name, false);
}
else if (endsWith(dir_it.name(), ".sql.tmp"))
else if (endsWith(file_name, ".sql.tmp"))
{
/// There are files .sql.tmp - delete
LOG_INFO(log, "Removing file {}", dir_it->path());
Poco::File(dir_it->path()).remove();
LOG_INFO(log, "Removing file {}", dir_it->path().string());
fs::remove(dir_it->path());
}
else if (endsWith(dir_it.name(), ".sql"))
else if (endsWith(file_name, ".sql"))
{
/// The required files have names like `table_name.sql`
metadata_files.emplace(dir_it.name(), true);
metadata_files.emplace(file_name, true);
}
else
throw Exception("Incorrect file extension: " + dir_it.name() + " in metadata directory " + getMetadataPath(),
ErrorCodes::INCORRECT_FILE_NAME);
throw Exception(ErrorCodes::INCORRECT_FILE_NAME, "Incorrect file extension: {} in metadata directory {}", file_name, getMetadataPath());
}
/// Read and parse metadata in parallel
@ -651,7 +651,7 @@ ASTPtr DatabaseOnDisk::parseQueryFromMetadata(
{
if (logger)
LOG_ERROR(logger, "File {} is empty. Removing.", metadata_file_path);
Poco::File(metadata_file_path).remove();
fs::remove(metadata_file_path);
return nullptr;
}
@ -670,8 +670,7 @@ ASTPtr DatabaseOnDisk::parseQueryFromMetadata(
auto & create = ast->as<ASTCreateQuery &>();
if (!create.table.empty() && create.uuid != UUIDHelpers::Nil)
{
String table_name = Poco::Path(metadata_file_path).makeFile().getBaseName();
table_name = unescapeForFileName(table_name);
String table_name = unescapeForFileName(fs::path(metadata_file_path).stem());
if (create.table != TABLE_WITH_UUID_NAME_PLACEHOLDER && logger)
LOG_WARNING(

8
src/Databases/DatabaseOrdinary.cpp
View File

@ -16,7 +16,6 @@
#include <Parsers/formatAST.h>
#include <Parsers/parseQuery.h>
#include <Parsers/queryToString.h>
#include <Poco/DirectoryIterator.h>
#include <Common/Stopwatch.h>
#include <Common/ThreadPool.h>
#include <Common/escapeForFileName.h>
@ -111,8 +110,7 @@ void DatabaseOrdinary::loadStoredObjects(ContextMutablePtr local_context, bool h
auto * create_query = ast->as<ASTCreateQuery>();
create_query->database = database_name;
auto detached_permanently_flag = Poco::File(full_path.string() + detached_suffix);
if (detached_permanently_flag.exists())
if (fs::exists(full_path.string() + detached_suffix))
{
/// FIXME: even if we don't load the table we can still mark the uuid of it as taken.
/// if (create_query->uuid != UUIDHelpers::Nil)
@ -281,11 +279,11 @@ void DatabaseOrdinary::commitAlterTable(const StorageID &, const String & table_
try
{
/// rename atomically replaces the old file with the new one.
Poco::File(table_metadata_tmp_path).renameTo(table_metadata_path);
fs::rename(table_metadata_tmp_path, table_metadata_path);
}
catch (...)
{
Poco::File(table_metadata_tmp_path).remove();
fs::remove(table_metadata_tmp_path);
throw;
}
}

2
src/Databases/DatabaseReplicated.cpp
View File

@ -212,7 +212,7 @@ void DatabaseReplicated::tryConnectToZooKeeperAndInitDatabase(bool force_attach)
createDatabaseNodesInZooKeeper(current_zookeeper);
}
replica_path = zookeeper_path + "/replicas/" + getFullReplicaName();
replica_path = fs::path(zookeeper_path) / "replicas" / getFullReplicaName();
String replica_host_id;
if (current_zookeeper->tryGet(replica_path, replica_host_id))

19
src/Databases/DatabaseReplicatedWorker.cpp
View File

@ -1,6 +1,9 @@
#include <Databases/DatabaseReplicatedWorker.h>
#include <Databases/DatabaseReplicated.h>
#include <Interpreters/DDLTask.h>
#include <filesystem>
namespace fs = std::filesystem;
namespace DB
{
@ -156,7 +159,7 @@ DDLTaskPtr DatabaseReplicatedDDLWorker::initAndCheckTask(const String & entry_na
}
}
UInt32 our_log_ptr = parse<UInt32>(current_zookeeper->get(database->replica_path + "/log_ptr"));
UInt32 our_log_ptr = parse<UInt32>(current_zookeeper->get(fs::path(database->replica_path) / "log_ptr"));
UInt32 entry_num = DatabaseReplicatedTask::getLogEntryNumber(entry_name);
if (entry_num <= our_log_ptr)
@ -165,13 +168,13 @@ DDLTaskPtr DatabaseReplicatedDDLWorker::initAndCheckTask(const String & entry_na
return {};
}
String entry_path = queue_dir + "/" + entry_name;
String entry_path = fs::path(queue_dir) / entry_name;
auto task = std::make_unique<DatabaseReplicatedTask>(entry_name, entry_path, database);
String initiator_name;
zkutil::EventPtr wait_committed_or_failed = std::make_shared<Poco::Event>();
String try_node_path = entry_path + "/try";
String try_node_path = fs::path(entry_path) / "try";
if (zookeeper->tryGet(try_node_path, initiator_name, nullptr, wait_committed_or_failed))
{
task->is_initial_query = initiator_name == task->host_id_str;
@ -203,7 +206,7 @@ DDLTaskPtr DatabaseReplicatedDDLWorker::initAndCheckTask(const String & entry_na
if (code != Coordination::Error::ZOK && code != Coordination::Error::ZNONODE)
throw Coordination::Exception(code, try_node_path);
if (!zookeeper->exists(entry_path + "/committed"))
if (!zookeeper->exists(fs::path(entry_path) / "committed"))
{
out_reason = fmt::format("Entry {} was forcefully cancelled due to timeout", entry_name);
return {};
@ -212,7 +215,7 @@ DDLTaskPtr DatabaseReplicatedDDLWorker::initAndCheckTask(const String & entry_na
}
}
if (!zookeeper->exists(entry_path + "/committed"))
if (!zookeeper->exists(fs::path(entry_path) / "committed"))
{
out_reason = fmt::format("Entry {} hasn't been committed", entry_name);
return {};
@ -220,8 +223,8 @@ DDLTaskPtr DatabaseReplicatedDDLWorker::initAndCheckTask(const String & entry_na
if (task->is_initial_query)
{
assert(!zookeeper->exists(entry_path + "/try"));
assert(zookeeper->exists(entry_path + "/committed") == (zookeeper->get(task->getFinishedNodePath()) == ExecutionStatus(0).serializeText()));
assert(!zookeeper->exists(fs::path(entry_path) / "try"));
assert(zookeeper->exists(fs::path(entry_path) / "committed") == (zookeeper->get(task->getFinishedNodePath()) == ExecutionStatus(0).serializeText()));
out_reason = fmt::format("Entry {} has been executed as initial query", entry_name);
return {};
}
@ -257,7 +260,7 @@ DDLTaskPtr DatabaseReplicatedDDLWorker::initAndCheckTask(const String & entry_na
bool DatabaseReplicatedDDLWorker::canRemoveQueueEntry(const String & entry_name, const Coordination::Stat &)
{
UInt32 entry_number = DDLTaskBase::getLogEntryNumber(entry_name);
UInt32 max_log_ptr = parse<UInt32>(getAndSetZooKeeper()->get(database->zookeeper_path + "/max_log_ptr"));
UInt32 max_log_ptr = parse<UInt32>(getAndSetZooKeeper()->get(fs::path(database->zookeeper_path) / "max_log_ptr"));
return entry_number + logs_to_keep < max_log_ptr;
}

38
src/Databases/MySQL/DatabaseConnectionMySQL.cpp
View File

@ -24,10 +24,10 @@
# include <Common/escapeForFileName.h>
# include <Common/parseAddress.h>
# include <Common/setThreadName.h>
# include <filesystem>
# include <Common/filesystemHelpers.h>
# include <Poco/DirectoryIterator.h>
# include <Poco/File.h>
namespace fs = std::filesystem;
namespace DB
{
@ -326,7 +326,7 @@ void DatabaseConnectionMySQL::shutdown()
void DatabaseConnectionMySQL::drop(ContextPtr /*context*/)
{
Poco::File(getMetadataPath()).remove(true);
fs::remove_all(getMetadataPath());
}
void DatabaseConnectionMySQL::cleanOutdatedTables()
@ -372,10 +372,10 @@ void DatabaseConnectionMySQL::attachTable(const String & table_name, const Stora
local_tables_cache[table_name].second = storage;
remove_or_detach_tables.erase(table_name);
Poco::File remove_flag(getMetadataPath() + '/' + escapeForFileName(table_name) + suffix);
fs::path remove_flag = fs::path(getMetadataPath()) / (escapeForFileName(table_name) + suffix);
if (remove_flag.exists())
remove_flag.remove();
if (fs::exists(remove_flag))
fs::remove(remove_flag);
}
StoragePtr DatabaseConnectionMySQL::detachTable(const String & table_name)
@ -403,13 +403,13 @@ void DatabaseConnectionMySQL::loadStoredObjects(ContextMutablePtr, bool, bool /*
{
std::lock_guard<std::mutex> lock{mutex};
Poco::DirectoryIterator iterator(getMetadataPath());
fs::directory_iterator iter(getMetadataPath());
for (Poco::DirectoryIterator end; iterator != end; ++iterator)
for (fs::directory_iterator end; iter != end; ++iter)
{
if (iterator->isFile() && endsWith(iterator.name(), suffix))
if (fs::is_regular_file(iter->path()) && endsWith(iter->path().filename(), suffix))
{
const auto & filename = iterator.name();
const auto & filename = iter->path().filename().string();
const auto & table_name = unescapeForFileName(filename.substr(0, filename.size() - strlen(suffix)));
remove_or_detach_tables.emplace(table_name);
}
@ -420,27 +420,25 @@ void DatabaseConnectionMySQL::detachTablePermanently(ContextPtr, const String &
{
std::lock_guard<std::mutex> lock{mutex};
Poco::File remove_flag(getMetadataPath() + '/' + escapeForFileName(table_name) + suffix);
fs::path remove_flag = fs::path(getMetadataPath()) / (escapeForFileName(table_name) + suffix);
if (remove_or_detach_tables.count(table_name))
throw Exception("Table " + backQuoteIfNeed(database_name) + "." + backQuoteIfNeed(table_name) + " is dropped",
ErrorCodes::TABLE_IS_DROPPED);
throw Exception(ErrorCodes::TABLE_IS_DROPPED, "Table {}.{} is dropped", backQuoteIfNeed(database_name), backQuoteIfNeed(table_name));
if (remove_flag.exists())
throw Exception("The remove flag file already exists but the " + backQuoteIfNeed(database_name) +
"." + backQuoteIfNeed(table_name) + " does not exists remove tables, it is bug.", ErrorCodes::LOGICAL_ERROR);
if (fs::exists(remove_flag))
throw Exception(ErrorCodes::LOGICAL_ERROR, "The remove flag file already exists but the {}.{} does not exists remove tables, it is bug.",
backQuoteIfNeed(database_name), backQuoteIfNeed(table_name));
auto table_iter = local_tables_cache.find(table_name);
if (table_iter == local_tables_cache.end())
throw Exception("Table " + backQuoteIfNeed(database_name) + "." + backQuoteIfNeed(table_name) + " doesn't exist.",
ErrorCodes::UNKNOWN_TABLE);
throw Exception(ErrorCodes::UNKNOWN_TABLE, "Table {}.{} doesn't exist", backQuoteIfNeed(database_name), backQuoteIfNeed(table_name));
remove_or_detach_tables.emplace(table_name);
try
{
table_iter->second.second->drop();
remove_flag.createFile();
FS::createFile(remove_flag);
}
catch (...)
{

10
src/Databases/MySQL/DatabaseMaterializeMySQL.cpp
View File

@ -13,9 +13,11 @@
# include <Databases/MySQL/MaterializeMySQLSyncThread.h>
# include <Parsers/ASTCreateQuery.h>
# include <Storages/StorageMaterializeMySQL.h>
# include <Poco/File.h>
# include <Poco/Logger.h>
# include <Common/setThreadName.h>
# include <filesystem>
namespace fs = std::filesystem;
namespace DB
{
@ -158,10 +160,10 @@ template<typename Base>
void DatabaseMaterializeMySQL<Base>::drop(ContextPtr context_)
{
/// Remove metadata info
Poco::File metadata(Base::getMetadataPath() + "/.metadata");
fs::path metadata(Base::getMetadataPath() + "/.metadata");
if (metadata.exists())
metadata.remove(false);
if (fs::exists(metadata))
fs::remove(metadata);
Base::drop(context_);
}

11
src/Databases/MySQL/MaterializeMetadata.cpp
View File

@ -8,11 +8,13 @@
#include <Formats/MySQLBlockInputStream.h>
#include <IO/ReadBufferFromFile.h>
#include <IO/WriteBufferFromFile.h>
#include <Poco/File.h>
#include <Common/quoteString.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#include <IO/Operators.h>
#include <filesystem>
namespace fs = std::filesystem;
namespace DB
{
@ -193,12 +195,11 @@ void commitMetadata(const std::function<void()> & function, const String & persi
try
{
function();
Poco::File(persistent_tmp_path).renameTo(persistent_path);
fs::rename(persistent_tmp_path, persistent_path);
}
catch (...)
{
Poco::File(persistent_tmp_path).remove();
fs::remove(persistent_tmp_path);
throw;
}
}
@ -231,7 +232,7 @@ void MaterializeMetadata::transaction(const MySQLReplication::Position & positio
MaterializeMetadata::MaterializeMetadata(const String & path_, const Settings & settings_) : persistent_path(path_), settings(settings_)
{
if (Poco::File(persistent_path).exists())
if (fs::exists(persistent_path))
{
ReadBufferFromFile in(persistent_path, DBMS_DEFAULT_BUFFER_SIZE);
assertString("Version:\t" + toString(meta_version), in);

39
src/Databases/PostgreSQL/DatabasePostgreSQL.cpp
View File

@ -12,11 +12,12 @@
#include <Parsers/parseQuery.h>
#include <Parsers/queryToString.h>
#include <Common/escapeForFileName.h>
#include <Poco/DirectoryIterator.h>
#include <Poco/File.h>
#include <Databases/PostgreSQL/fetchPostgreSQLTableStructure.h>
#include <Common/quoteString.h>
#include <Common/filesystemHelpers.h>
#include <filesystem>
namespace fs = std::filesystem;
namespace DB
{
@ -209,9 +210,9 @@ void DatabasePostgreSQL::attachTable(const String & table_name, const StoragePtr
detached_or_dropped.erase(table_name);
Poco::File table_marked_as_removed(getMetadataPath() + '/' + escapeForFileName(table_name) + suffix);
if (table_marked_as_removed.exists())
table_marked_as_removed.remove();
fs::path table_marked_as_removed = fs::path(getMetadataPath()) / (escapeForFileName(table_name) + suffix);
if (fs::exists(table_marked_as_removed))
fs::remove(table_marked_as_removed);
}
@ -256,16 +257,8 @@ void DatabasePostgreSQL::dropTable(ContextPtr, const String & table_name, bool /
if (detached_or_dropped.count(table_name))
throw Exception(fmt::format("Table {}.{} is already dropped/detached", database_name, table_name), ErrorCodes::TABLE_IS_DROPPED);
Poco::File mark_table_removed(getMetadataPath() + '/' + escapeForFileName(table_name) + suffix);
try
{
mark_table_removed.createFile();
}
catch (...)
{
throw;
}
fs::path mark_table_removed = fs::path(getMetadataPath()) / (escapeForFileName(table_name) + suffix);
FS::createFile(mark_table_removed);
if (cache_tables)
cached_tables.erase(table_name);
@ -276,7 +269,7 @@ void DatabasePostgreSQL::dropTable(ContextPtr, const String & table_name, bool /
void DatabasePostgreSQL::drop(ContextPtr /*context*/)
{
Poco::File(getMetadataPath()).remove(true);
fs::remove_all(getMetadataPath());
}
@ -284,14 +277,14 @@ void DatabasePostgreSQL::loadStoredObjects(ContextMutablePtr /* context */, bool
{
{
std::lock_guard<std::mutex> lock{mutex};
Poco::DirectoryIterator iterator(getMetadataPath());
fs::directory_iterator iter(getMetadataPath());
/// Check for previously dropped tables
for (Poco::DirectoryIterator end; iterator != end; ++iterator)
for (fs::directory_iterator end; iter != end; ++iter)
{
if (iterator->isFile() && endsWith(iterator.name(), suffix))
if (fs::is_regular_file(iter->path()) && endsWith(iter->path().filename(), suffix))
{
const auto & file_name = iterator.name();
const auto & file_name = iter->path().filename().string();
const auto & table_name = unescapeForFileName(file_name.substr(0, file_name.size() - strlen(suffix)));
detached_or_dropped.emplace(table_name);
}
@ -325,9 +318,9 @@ void DatabasePostgreSQL::removeOutdatedTables()
{
auto table_name = *iter;
iter = detached_or_dropped.erase(iter);
Poco::File table_marked_as_removed(getMetadataPath() + '/' + escapeForFileName(table_name) + suffix);
if (table_marked_as_removed.exists())
table_marked_as_removed.remove();
fs::path table_marked_as_removed = fs::path(getMetadataPath()) / (escapeForFileName(table_name) + suffix);
if (fs::exists(table_marked_as_removed))
fs::remove(table_marked_as_removed);
}
else
++iter;

3
src/Dictionaries/DictionarySourceHelpers.h
View File

@ -3,14 +3,13 @@
#include <vector>
#include <common/types.h>
#include <Poco/File.h>
#include <Poco/Util/AbstractConfiguration.h>
#include <DataStreams/IBlockInputStream.h>
#include <Columns/IColumn.h>
#include <Core/Block.h>
#include <Interpreters/Context_fwd.h>
namespace DB
{

13
src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.cpp
View File

@ -5,7 +5,9 @@
#include <Poco/Exception.h>
#include <Poco/Util/Application.h>
#include "HierarchyFormatReader.h"
#include <filesystem>
namespace fs = std::filesystem;
bool RegionsHierarchyDataSource::isModified() const
{
@ -27,14 +29,13 @@ RegionsHierarchiesDataProvider::RegionsHierarchiesDataProvider(const std::string
void RegionsHierarchiesDataProvider::discoverFilesWithCustomHierarchies()
{
std::string basename = Poco::Path(path).getBaseName();
std::string basename = fs::path(path).stem();
fs::path dir_path = fs::canonical(path).parent_path();
Poco::Path dir_path = Poco::Path(path).absolute().parent();
Poco::DirectoryIterator dir_end;
for (Poco::DirectoryIterator dir_it(dir_path); dir_it != dir_end; ++dir_it)
fs::directory_iterator dir_end;
for (fs::directory_iterator dir_it(dir_path); dir_it != dir_end; ++dir_it)
{
std::string candidate_basename = dir_it.path().getBaseName();
std::string candidate_basename = dir_it->path().stem();
if (candidate_basename.starts_with(basename)
&& (candidate_basename.size() > basename.size() + 1)

6
src/Dictionaries/Embedded/GeodataProviders/NamesProvider.cpp
View File

@ -2,7 +2,9 @@
#include <IO/ReadBufferFromFile.h>
#include "NamesFormatReader.h"
#include <filesystem>
namespace fs = std::filesystem;
bool LanguageRegionsNamesDataSource::isModified() const
{
@ -11,7 +13,7 @@ bool LanguageRegionsNamesDataSource::isModified() const
size_t LanguageRegionsNamesDataSource::estimateTotalSize() const
{
return Poco::File(path).getSize();
return fs::file_size(path);
}
ILanguageRegionsNamesReaderPtr LanguageRegionsNamesDataSource::createReader()
@ -39,7 +41,7 @@ RegionsNamesDataProvider::RegionsNamesDataProvider(const std::string & directory
ILanguageRegionsNamesDataSourcePtr RegionsNamesDataProvider::getLanguageRegionsNamesSource(const std::string & language) const
{
const auto data_file = getDataFilePath(language);
if (Poco::File(data_file).exists())
if (fs::exists(data_file))
return std::make_unique<LanguageRegionsNamesDataSource>(data_file, language);
else
return {};

9
src/Dictionaries/FileDictionarySource.cpp
View File

@ -1,9 +1,4 @@
#include "FileDictionarySource.h"
#include <filesystem>
#include <Poco/File.h>
#include <common/logger_useful.h>
#include <Common/StringUtils/StringUtils.h>
#include <Common/filesystemHelpers.h>
@ -15,6 +10,7 @@
#include "registerDictionaries.h"
#include "DictionarySourceHelpers.h"
namespace DB
{
static const UInt64 max_block_size = 8192;
@ -68,9 +64,10 @@ std::string FileDictionarySource::toString() const
Poco::Timestamp FileDictionarySource::getLastModification() const
{
return Poco::File{filepath}.getLastModified();
return FS::getModificationTimestamp(filepath);
}
void registerDictionarySourceFile(DictionarySourceFactory & factory)
{
auto create_table_source = [=](const DictionaryStructure & dict_struct,

12
src/Dictionaries/LibraryDictionarySource.cpp
View File

@ -1,19 +1,19 @@
#include "LibraryDictionarySource.h"
#include <Poco/File.h>
#include <DataStreams/OneBlockInputStream.h>
#include <Interpreters/Context.h>
#include <common/logger_useful.h>
#include <Common/filesystemHelpers.h>
#include <IO/WriteBufferFromString.h>
#include <IO/WriteHelpers.h>
#include <DataStreams/OneBlockInputStream.h>
#include <Interpreters/Context.h>
#include <filesystem>
#include <Dictionaries/DictionarySourceFactory.h>
#include <Dictionaries/DictionarySourceHelpers.h>
#include <Dictionaries/DictionaryStructure.h>
#include <Dictionaries/registerDictionaries.h>
namespace fs = std::filesystem;
namespace DB
{
@ -44,8 +44,8 @@ LibraryDictionarySource::LibraryDictionarySource(
if (created_from_ddl && !pathStartsWith(path, context->getDictionariesLibPath()))
throw Exception(ErrorCodes::PATH_ACCESS_DENIED, "File path {} is not inside {}", path, context->getDictionariesLibPath());
if (!Poco::File(path).exists())
throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "LibraryDictionarySource: Can't load library {}: file doesn't exist", Poco::File(path).path());
if (!fs::exists(path))
throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "LibraryDictionarySource: Can't load library {}: file doesn't exist", path);
description.init(sample_block);
bridge_helper = std::make_shared<LibraryBridgeHelper>(context, description.sample_block, dictionary_id);

114
src/Disks/DiskLocal.cpp
View File

@ -6,9 +6,9 @@
#include <Interpreters/Context.h>
#include <Common/filesystemHelpers.h>
#include <Common/quoteString.h>
#include <IO/createReadBufferFromFileBase.h>
#include <fstream>
#include <unistd.h>
@ -60,27 +60,28 @@ class DiskLocalDirectoryIterator : public IDiskDirectoryIterator
{
public:
explicit DiskLocalDirectoryIterator(const String & disk_path_, const String & dir_path_)
: dir_path(dir_path_), iter(disk_path_ + dir_path_)
: dir_path(dir_path_), entry(fs::path(disk_path_) / dir_path_)
{
}
void next() override { ++iter; }
void next() override { ++entry; }
bool isValid() const override { return iter != Poco::DirectoryIterator(); }
bool isValid() const override { return entry != fs::directory_iterator(); }
String path() const override
{
if (iter->isDirectory())
return dir_path + iter.name() + '/';
if (entry->is_directory())
return dir_path / entry->path().filename() / "";
else
return dir_path + iter.name();
return dir_path / entry->path().filename();
}
String name() const override { return iter.name(); }
String name() const override { return entry->path().filename(); }
private:
String dir_path;
Poco::DirectoryIterator iter;
fs::path dir_path;
fs::directory_iterator entry;
};
@ -118,7 +119,7 @@ UInt64 DiskLocal::getTotalSpace() const
{
struct statvfs fs;
if (name == "default") /// for default disk we get space from path/data/
fs = getStatVFS(disk_path + "data/");
fs = getStatVFS((fs::path(disk_path) / "data/").string());
else
fs = getStatVFS(disk_path);
UInt64 total_size = fs.f_blocks * fs.f_bsize;
@ -133,7 +134,7 @@ UInt64 DiskLocal::getAvailableSpace() const
/// available for superuser only and for system purposes
struct statvfs fs;
if (name == "default") /// for default disk we get space from path/data/
fs = getStatVFS(disk_path + "data/");
fs = getStatVFS((fs::path(disk_path) / "data/").string());
else
fs = getStatVFS(disk_path);
UInt64 total_size = fs.f_bavail * fs.f_bsize;
@ -152,45 +153,43 @@ UInt64 DiskLocal::getUnreservedSpace() const
bool DiskLocal::exists(const String & path) const
{
return Poco::File(disk_path + path).exists();
return fs::exists(fs::path(disk_path) / path);
}
bool DiskLocal::isFile(const String & path) const
{
return Poco::File(disk_path + path).isFile();
return fs::is_regular_file(fs::path(disk_path) / path);
}
bool DiskLocal::isDirectory(const String & path) const
{
return Poco::File(disk_path + path).isDirectory();
return fs::is_directory(fs::path(disk_path) / path);
}
size_t DiskLocal::getFileSize(const String & path) const
{
return Poco::File(disk_path + path).getSize();
return fs::file_size(fs::path(disk_path) / path);
}
void DiskLocal::createDirectory(const String & path)
{
Poco::File(disk_path + path).createDirectory();
fs::create_directory(fs::path(disk_path) / path);
}
void DiskLocal::createDirectories(const String & path)
{
Poco::File(disk_path + path).createDirectories();
fs::create_directories(fs::path(disk_path) / path);
}
void DiskLocal::clearDirectory(const String & path)
{
std::vector<Poco::File> files;
Poco::File(disk_path + path).list(files);
for (auto & file : files)
file.remove();
for (const auto & entry : fs::directory_iterator(fs::path(disk_path) / path))
fs::remove(entry.path());
}
void DiskLocal::moveDirectory(const String & from_path, const String & to_path)
{
Poco::File(disk_path + from_path).renameTo(disk_path + to_path);
fs::rename(fs::path(disk_path) / from_path, fs::path(disk_path) / to_path);
}
DiskDirectoryIteratorPtr DiskLocal::iterateDirectory(const String & path)
@ -200,99 +199,95 @@ DiskDirectoryIteratorPtr DiskLocal::iterateDirectory(const String & path)
void DiskLocal::moveFile(const String & from_path, const String & to_path)
{
Poco::File(disk_path + from_path).renameTo(disk_path + to_path);
fs::rename(fs::path(disk_path) / from_path, fs::path(disk_path) / to_path);
}
void DiskLocal::replaceFile(const String & from_path, const String & to_path)
{
Poco::File from_file(disk_path + from_path);
Poco::File to_file(disk_path + to_path);
if (to_file.exists())
{
Poco::File tmp_file(disk_path + to_path + ".old");
to_file.renameTo(tmp_file.path());
from_file.renameTo(disk_path + to_path);
tmp_file.remove();
}
else
from_file.renameTo(to_file.path());
fs::path from_file = fs::path(disk_path) / from_path;
fs::path to_file = fs::path(disk_path) / to_path;
fs::rename(from_file, to_file);
}
std::unique_ptr<ReadBufferFromFileBase>
DiskLocal::readFile(
const String & path, size_t buf_size, size_t estimated_size, size_t aio_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache) const
{
return createReadBufferFromFileBase(disk_path + path, estimated_size, aio_threshold, mmap_threshold, mmap_cache, buf_size);
return createReadBufferFromFileBase(fs::path(disk_path) / path, estimated_size, aio_threshold, mmap_threshold, mmap_cache, buf_size);
}
std::unique_ptr<WriteBufferFromFileBase>
DiskLocal::writeFile(const String & path, size_t buf_size, WriteMode mode)
{
int flags = (mode == WriteMode::Append) ? (O_APPEND | O_CREAT | O_WRONLY) : -1;
return std::make_unique<WriteBufferFromFile>(disk_path + path, buf_size, flags);
return std::make_unique<WriteBufferFromFile>(fs::path(disk_path) / path, buf_size, flags);
}
void DiskLocal::removeFile(const String & path)
{
auto fs_path = disk_path + path;
auto fs_path = fs::path(disk_path) / path;
if (0 != unlink(fs_path.c_str()))
throwFromErrnoWithPath("Cannot unlink file " + fs_path, fs_path, ErrorCodes::CANNOT_UNLINK);
throwFromErrnoWithPath("Cannot unlink file " + fs_path.string(), fs_path, ErrorCodes::CANNOT_UNLINK);
}
void DiskLocal::removeFileIfExists(const String & path)
{
auto fs_path = disk_path + path;
auto fs_path = fs::path(disk_path) / path;
if (0 != unlink(fs_path.c_str()) && errno != ENOENT)
throwFromErrnoWithPath("Cannot unlink file " + fs_path, fs_path, ErrorCodes::CANNOT_UNLINK);
throwFromErrnoWithPath("Cannot unlink file " + fs_path.string(), fs_path, ErrorCodes::CANNOT_UNLINK);
}
void DiskLocal::removeDirectory(const String & path)
{
auto fs_path = disk_path + path;
auto fs_path = fs::path(disk_path) / path;
if (0 != rmdir(fs_path.c_str()))
throwFromErrnoWithPath("Cannot rmdir " + fs_path, fs_path, ErrorCodes::CANNOT_RMDIR);
throwFromErrnoWithPath("Cannot rmdir " + fs_path.string(), fs_path, ErrorCodes::CANNOT_RMDIR);
}
void DiskLocal::removeRecursive(const String & path)
{
Poco::File(disk_path + path).remove(true);
fs::remove_all(fs::path(disk_path) / path);
}
void DiskLocal::listFiles(const String & path, std::vector<String> & file_names)
{
Poco::File(disk_path + path).list(file_names);
file_names.clear();
for (const auto & entry : fs::directory_iterator(fs::path(disk_path) / path))
file_names.emplace_back(entry.path().filename());
}
void DiskLocal::setLastModified(const String & path, const Poco::Timestamp & timestamp)
{
Poco::File(disk_path + path).setLastModified(timestamp);
FS::setModificationTime(fs::path(disk_path) / path, timestamp.epochTime());
}
Poco::Timestamp DiskLocal::getLastModified(const String & path)
{
return Poco::File(disk_path + path).getLastModified();
return FS::getModificationTimestamp(fs::path(disk_path) / path);
}
void DiskLocal::createHardLink(const String & src_path, const String & dst_path)
{
DB::createHardLink(disk_path + src_path, disk_path + dst_path);
DB::createHardLink(fs::path(disk_path) / src_path, fs::path(disk_path) / dst_path);
}
void DiskLocal::truncateFile(const String & path, size_t size)
{
int res = truncate((disk_path + path).c_str(), size);
int res = truncate((fs::path(disk_path) / path).string().data(), size);
if (-1 == res)
throwFromErrnoWithPath("Cannot truncate file " + path, path, ErrorCodes::CANNOT_TRUNCATE_FILE);
}
void DiskLocal::createFile(const String & path)
{
Poco::File(disk_path + path).createFile();
FS::createFile(fs::path(disk_path) / path);
}
void DiskLocal::setReadOnly(const String & path)
{
Poco::File(disk_path + path).setReadOnly(true);
fs::permissions(fs::path(disk_path) / path,
fs::perms::owner_write | fs::perms::group_write | fs::perms::others_write,
fs::perm_options::remove);
}
bool inline isSameDiskType(const IDisk & one, const IDisk & another)
@ -303,14 +298,23 @@ bool inline isSameDiskType(const IDisk & one, const IDisk & another)
void DiskLocal::copy(const String & from_path, const std::shared_ptr<IDisk> & to_disk, const String & to_path)
{
if (isSameDiskType(*this, *to_disk))
Poco::File(disk_path + from_path).copyTo(to_disk->getPath() + to_path); /// Use more optimal way.
{
fs::path to = fs::path(to_disk->getPath()) / to_path;
fs::path from = fs::path(disk_path) / from_path;
if (from_path.ends_with('/'))
from = from.parent_path();
if (fs::is_directory(from))
to /= from.filename();
fs::copy(from, to, fs::copy_options::recursive | fs::copy_options::overwrite_existing); /// Use more optimal way.
}
else
IDisk::copy(from_path, to_disk, to_path); /// Copy files through buffers.
}
SyncGuardPtr DiskLocal::getDirectorySyncGuard(const String & path) const
{
return std::make_unique<LocalDirectorySyncGuard>(disk_path + path);
return std::make_unique<LocalDirectorySyncGuard>(fs::path(disk_path) / path);
}
DiskPtr DiskLocalReservation::getDisk(size_t i) const
@ -381,10 +385,8 @@ void registerDiskLocal(DiskFactory & factory)
throw Exception("Disk path must end with /. Disk " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG);
}
if (Poco::File disk{path}; !disk.canRead() || !disk.canWrite())
{
if (!FS::canRead(path) || !FS::canWrite(path))
throw Exception("There is no RW access to the disk " + name + " (" + path + ")", ErrorCodes::PATH_ACCESS_DENIED);
}
bool has_space_ratio = config.has(config_prefix + ".keep_free_space_ratio");

4
src/Disks/DiskLocal.h
View File

@ -6,8 +6,6 @@
#include <IO/ReadBufferFromFileBase.h>
#include <IO/WriteBufferFromFile.h>
#include <Poco/DirectoryIterator.h>
#include <Poco/File.h>
namespace DB
{
@ -27,7 +25,7 @@ public:
: name(name_), disk_path(path_), keep_free_space_bytes(keep_free_space_bytes_)
{
if (disk_path.back() != '/')
throw Exception("Disk path must ends with '/', but '" + disk_path + "' doesn't.", ErrorCodes::LOGICAL_ERROR);
throw Exception("Disk path must end with '/', but '" + disk_path + "' doesn't.", ErrorCodes::LOGICAL_ERROR);
}
const String & getName() const override { return name; }

13
src/Disks/DiskMemory.cpp
View File

@ -6,7 +6,6 @@
#include <IO/WriteBufferFromFileBase.h>
#include <IO/WriteBufferFromString.h>
#include <Interpreters/Context.h>
#include <Poco/Path.h>
namespace DB
@ -24,7 +23,7 @@ namespace ErrorCodes
class DiskMemoryDirectoryIterator final : public IDiskDirectoryIterator
{
public:
explicit DiskMemoryDirectoryIterator(std::vector<Poco::Path> && dir_file_paths_)
explicit DiskMemoryDirectoryIterator(std::vector<fs::path> && dir_file_paths_)
: dir_file_paths(std::move(dir_file_paths_)), iter(dir_file_paths.begin())
{
}
@ -33,13 +32,13 @@ public:
bool isValid() const override { return iter != dir_file_paths.end(); }
String path() const override { return (*iter).toString(); }
String path() const override { return iter->string(); }
String name() const override { return (*iter).getFileName(); }
String name() const override { return iter->filename(); }
private:
std::vector<Poco::Path> dir_file_paths;
std::vector<Poco::Path>::iterator iter;
std::vector<fs::path> dir_file_paths;
std::vector<fs::path>::iterator iter;
};
@ -268,7 +267,7 @@ DiskDirectoryIteratorPtr DiskMemory::iterateDirectory(const String & path)
if (!path.empty() && files.find(path) == files.end())
throw Exception("Directory '" + path + "' does not exist", ErrorCodes::DIRECTORY_DOESNT_EXIST);
std::vector<Poco::Path> dir_file_paths;
std::vector<fs::path> dir_file_paths;
for (const auto & file : files)
if (parentPath(file.first) == path)
dir_file_paths.emplace_back(file.first);

4
src/Disks/HDFS/DiskHDFS.cpp
View File

@ -172,8 +172,8 @@ void registerDiskHDFS(DiskFactory & factory)
const String & config_prefix,
ContextConstPtr context_) -> DiskPtr
{
Poco::File disk{context_->getPath() + "disks/" + name};
disk.createDirectories();
fs::path disk = fs::path(context_->getPath()) / "disks" / name;
fs::create_directories(disk);
String uri{config.getString(config_prefix + ".endpoint")};

10
src/Disks/IDisk.cpp
View File

@ -22,7 +22,8 @@ bool IDisk::isDirectoryEmpty(const String & path)
void copyFile(IDisk & from_disk, const String & from_path, IDisk & to_disk, const String & to_path)
{
LOG_DEBUG(&Poco::Logger::get("IDisk"), "Copying from {} {} to {} {}.", from_disk.getName(), from_path, to_disk.getName(), to_path);
LOG_DEBUG(&Poco::Logger::get("IDisk"), "Copying from {} (path: {}) {} to {} (path: {}) {}.",
from_disk.getName(), from_disk.getPath(), from_path, to_disk.getName(), to_disk.getPath(), to_path);
auto in = from_disk.readFile(from_path);
auto out = to_disk.writeFile(to_path);
@ -41,16 +42,15 @@ void asyncCopy(IDisk & from_disk, String from_path, IDisk & to_disk, String to_p
[&from_disk, from_path, &to_disk, to_path]()
{
setThreadName("DiskCopier");
DB::copyFile(from_disk, from_path, to_disk, to_path + fileName(from_path));
DB::copyFile(from_disk, from_path, to_disk, fs::path(to_path) / fileName(from_path));
});
results.push_back(std::move(result));
}
else
{
Poco::Path path(from_path);
const String & dir_name = path.directory(path.depth() - 1);
const String dest = to_path + dir_name + "/";
fs::path dir_name = fs::path(from_path).parent_path().filename();
fs::path dest(fs::path(to_path) / dir_name);
to_disk.createDirectories(dest);
for (auto it = from_disk.iterateDirectory(from_path); it->isValid(); it->next())

14
src/Disks/IDisk.h
View File

@ -7,16 +7,16 @@
#include <Common/Exception.h>
#include <Disks/Executor.h>
#include <Disks/DiskType.h>
#include "Disks/Executor.h"
#include <memory>
#include <mutex>
#include <utility>
#include <boost/noncopyable.hpp>
#include <Poco/Path.h>
#include <Poco/Timestamp.h>
#include <filesystem>
#include "Poco/Util/AbstractConfiguration.h"
namespace fs = std::filesystem;
namespace CurrentMetrics
{
@ -294,25 +294,27 @@ public:
/// Return full path to a file on disk.
inline String fullPath(const DiskPtr & disk, const String & path)
{
return disk->getPath() + path;
return fs::path(disk->getPath()) / path;
}
/// Return parent path for the specified path.
inline String parentPath(const String & path)
{
return Poco::Path(path).parent().toString();
if (path.ends_with('/'))
return fs::path(path).parent_path().parent_path() / "";
return fs::path(path).parent_path() / "";
}
/// Return file name for the specified path.
inline String fileName(const String & path)
{
return Poco::Path(path).getFileName();
return fs::path(path).filename();
}
/// Return directory path for the specified path.
inline String directoryPath(const String & path)
{
return Poco::Path(path).setFileName("").toString();
return fs::path(path).parent_path() / "";
}
}

38
src/Disks/IDiskRemote.cpp
View File

@ -6,12 +6,12 @@
#include <IO/WriteBufferFromFile.h>
#include <IO/WriteBufferFromS3.h>
#include <IO/WriteHelpers.h>
#include <Poco/File.h>
#include <Common/createHardLink.h>
#include <Common/quoteString.h>
#include <common/logger_useful.h>
#include <Common/checkStackSize.h>
#include <boost/algorithm/string.hpp>
#include <Common/filesystemHelpers.h>
namespace DB
@ -179,9 +179,9 @@ void IDiskRemote::removeMeta(const String & path, RemoteFSPathKeeperPtr fs_paths
{
LOG_DEBUG(log, "Remove file by path: {}", backQuote(metadata_path + path));
Poco::File file(metadata_path + path);
fs::path file(metadata_path + path);
if (!file.isFile())
if (!fs::is_regular_file(file))
throw Exception(ErrorCodes::CANNOT_DELETE_DIRECTORY, "Path '{}' is a directory", path);
try
@ -191,7 +191,7 @@ void IDiskRemote::removeMeta(const String & path, RemoteFSPathKeeperPtr fs_paths
/// If there is no references - delete content from remote FS.
if (metadata.ref_count == 0)
{
file.remove();
fs::remove(file);
for (const auto & [remote_fs_object_path, _] : metadata.remote_fs_objects)
fs_paths_keeper->addPath(remote_fs_root_path + remote_fs_object_path);
}
@ -199,7 +199,7 @@ void IDiskRemote::removeMeta(const String & path, RemoteFSPathKeeperPtr fs_paths
{
--metadata.ref_count;
metadata.save();
file.remove();
fs::remove(file);
}
}
catch (const Exception & e)
@ -210,7 +210,7 @@ void IDiskRemote::removeMeta(const String & path, RemoteFSPathKeeperPtr fs_paths
LOG_WARNING(log,
"Metadata file {} can't be read by reason: {}. Removing it forcibly.",
backQuote(path), e.nested() ? e.nested()->message() : e.message());
file.remove();
fs::remove(file);
}
else
throw;
@ -222,8 +222,8 @@ void IDiskRemote::removeMetaRecursive(const String & path, RemoteFSPathKeeperPtr
{
checkStackSize(); /// This is needed to prevent stack overflow in case of cyclic symlinks.
Poco::File file(metadata_path + path);
if (file.isFile())
fs::path file = fs::path(metadata_path) / path;
if (fs::is_regular_file(file))
{
removeMeta(path, fs_paths_keeper);
}
@ -231,7 +231,7 @@ void IDiskRemote::removeMetaRecursive(const String & path, RemoteFSPathKeeperPtr
{
for (auto it{iterateDirectory(path)}; it->isValid(); it->next())
removeMetaRecursive(it->path(), fs_paths_keeper);
file.remove();
fs::remove(file);
}
}
@ -296,13 +296,13 @@ IDiskRemote::IDiskRemote(
bool IDiskRemote::exists(const String & path) const
{
return Poco::File(metadata_path + path).exists();
return fs::exists(fs::path(metadata_path) / path);
}
bool IDiskRemote::isFile(const String & path) const
{
return Poco::File(metadata_path + path).isFile();
return fs::is_regular_file(fs::path(metadata_path) / path);
}
@ -326,7 +326,7 @@ void IDiskRemote::moveFile(const String & from_path, const String & to_path)
if (exists(to_path))
throw Exception("File already exists: " + to_path, ErrorCodes::FILE_ALREADY_EXISTS);
Poco::File(metadata_path + from_path).renameTo(metadata_path + to_path);
fs::rename(fs::path(metadata_path) / from_path, fs::path(metadata_path) / to_path);
}
@ -347,7 +347,7 @@ void IDiskRemote::replaceFile(const String & from_path, const String & to_path)
void IDiskRemote::removeFileIfExists(const String & path)
{
RemoteFSPathKeeperPtr fs_paths_keeper = createFSPathKeeper();
if (Poco::File(metadata_path + path).exists())
if (fs::exists(fs::path(metadata_path) / path))
{
removeMeta(path, fs_paths_keeper);
removeFromRemoteFS(fs_paths_keeper);
@ -385,19 +385,19 @@ void IDiskRemote::setReadOnly(const String & path)
bool IDiskRemote::isDirectory(const String & path) const
{
return Poco::File(metadata_path + path).isDirectory();
return fs::is_directory(fs::path(metadata_path) / path);
}
void IDiskRemote::createDirectory(const String & path)
{
Poco::File(metadata_path + path).createDirectory();
fs::create_directory(fs::path(metadata_path) / path);
}
void IDiskRemote::createDirectories(const String & path)
{
Poco::File(metadata_path + path).createDirectories();
fs::create_directories(fs::path(metadata_path) / path);
}
@ -411,7 +411,7 @@ void IDiskRemote::clearDirectory(const String & path)
void IDiskRemote::removeDirectory(const String & path)
{
Poco::File(metadata_path + path).remove();
fs::remove(fs::path(metadata_path) / path);
}
@ -430,13 +430,13 @@ void IDiskRemote::listFiles(const String & path, std::vector<String> & file_name
void IDiskRemote::setLastModified(const String & path, const Poco::Timestamp & timestamp)
{
Poco::File(metadata_path + path).setLastModified(timestamp);
FS::setModificationTime(fs::path(metadata_path) / path, timestamp.epochTime());
}
Poco::Timestamp IDiskRemote::getLastModified(const String & path)
{
return Poco::File(metadata_path + path).getLastModified();
return FS::getModificationTimestamp(fs::path(metadata_path) / path);
}

17
src/Disks/IDiskRemote.h
View File

@ -4,11 +4,12 @@
#include <atomic>
#include "Disks/DiskFactory.h"
#include "Disks/Executor.h"
#include <Poco/DirectoryIterator.h>
#include <utility>
#include <Common/MultiVersion.h>
#include <Common/ThreadPool.h>
#include <filesystem>
namespace fs = std::filesystem;
namespace DB
{
@ -193,21 +194,21 @@ public:
void next() override { ++iter; }
bool isValid() const override { return iter != Poco::DirectoryIterator(); }
bool isValid() const override { return iter != fs::directory_iterator(); }
String path() const override
{
if (iter->isDirectory())
return folder_path + iter.name() + '/';
if (fs::is_directory(iter->path()))
return folder_path / iter->path().filename().string() / "";
else
return folder_path + iter.name();
return folder_path / iter->path().filename().string();
}
String name() const override { return iter.name(); }
String name() const override { return iter->path().filename(); }
private:
Poco::DirectoryIterator iter;
String folder_path;
fs::directory_iterator iter;
fs::path folder_path;
};

1
src/Disks/ReadIndirectBufferFromRemoteFS.cpp
View File

@ -21,6 +21,7 @@ ReadIndirectBufferFromRemoteFS<T>::ReadIndirectBufferFromRemoteFS(
{
}
template<typename T>
off_t ReadIndirectBufferFromRemoteFS<T>::seek(off_t offset_, int whence)
{

25
src/Disks/S3/DiskS3.cpp
View File

@ -14,7 +14,6 @@
#include <IO/SeekAvoidingReadBuffer.h>
#include <IO/WriteBufferFromS3.h>
#include <IO/WriteHelpers.h>
#include <Poco/File.h>
#include <Common/createHardLink.h>
#include <Common/quoteString.h>
#include <Common/thread_local_rng.h>
@ -215,7 +214,7 @@ void DiskS3::moveFile(const String & from_path, const String & to_path, bool sen
createFileOperationObject("rename", revision, object_metadata);
}
Poco::File(metadata_path + from_path).renameTo(metadata_path + to_path);
fs::rename(fs::path(metadata_path) / from_path, fs::path(metadata_path) / to_path);
}
std::unique_ptr<ReadBufferFromFileBase> DiskS3::readFile(const String & path, size_t buf_size, size_t, size_t, size_t, MMappedFileCache *) const
@ -675,8 +674,8 @@ void DiskS3::restore()
restoreFiles(information);
restoreFileOperations(information);
Poco::File restore_file(metadata_path + RESTORE_FILE_NAME);
restore_file.remove();
fs::path restore_file = fs::path(metadata_path) / RESTORE_FILE_NAME;
fs::remove(restore_file);
saveSchemaVersion(RESTORABLE_SCHEMA_VERSION);
@ -863,8 +862,9 @@ void DiskS3::restoreFileOperations(const RestoreInformation & restore_informatio
continue;
/// Skip not finished parts. They shouldn't be in 'detached' directory, because CH wouldn't be able to finish processing them.
Poco::Path directory_path (path);
auto directory_name = directory_path.directory(directory_path.depth() - 1);
fs::path directory_path(path);
auto directory_name = directory_path.parent_path().filename().string();
auto predicate = [&directory_name](String & prefix) { return directory_name.starts_with(prefix); };
if (std::any_of(not_finished_prefixes.begin(), not_finished_prefixes.end(), predicate))
continue;
@ -873,7 +873,14 @@ void DiskS3::restoreFileOperations(const RestoreInformation & restore_informatio
LOG_DEBUG(log, "Move directory to 'detached' {} -> {}", path, detached_path);
Poco::File(metadata_path + path).moveTo(metadata_path + detached_path);
fs::path from_path = fs::path(metadata_path) / path;
fs::path to_path = fs::path(metadata_path) / detached_path;
if (path.ends_with('/'))
to_path /= from_path.parent_path().filename();
else
to_path /= from_path.filename();
fs::copy(from_path, to_path, fs::copy_options::recursive | fs::copy_options::overwrite_existing);
fs::remove_all(from_path);
}
}
@ -905,7 +912,9 @@ String DiskS3::revisionToString(UInt64 revision)
String DiskS3::pathToDetached(const String & source_path)
{
return Poco::Path(source_path).parent().append(Poco::Path("detached")).toString() + '/';
if (source_path.ends_with('/'))
return fs::path(source_path).parent_path().parent_path() / "detached/";
return fs::path(source_path).parent_path() / "detached/";
}
void DiskS3::onFreeze(const String & path)

2
src/Disks/S3/registerDiskS3.cpp
View File

@ -174,7 +174,7 @@ void registerDiskS3(DiskFactory & factory)
throw Exception("S3 path must ends with '/', but '" + uri.key + "' doesn't.", ErrorCodes::BAD_ARGUMENTS);
String metadata_path = config.getString(config_prefix + ".metadata_path", context->getPath() + "disks/" + name + "/");
Poco::File (metadata_path).createDirectories();
fs::create_directories(metadata_path);
std::shared_ptr<IDisk> s3disk = std::make_shared<DiskS3>(
name,

2
src/Disks/StoragePolicy.cpp
View File

@ -8,8 +8,6 @@
#include <set>
#include <Poco/File.h>
namespace
{

1
src/Disks/WriteIndirectBufferFromRemoteFS.cpp
View File

@ -8,7 +8,6 @@
namespace DB
{
/// Stores data in S3/HDFS and adds the object key (S3 path) and object size to metadata file on local FS.
template <typename T>
WriteIndirectBufferFromRemoteFS<T>::WriteIndirectBufferFromRemoteFS(
std::unique_ptr<T> impl_,

2
src/Disks/WriteIndirectBufferFromRemoteFS.h
View File

@ -10,7 +10,7 @@
namespace DB
{
/// Stores data in S3/HDFS and adds the object key (S3 path) and object size to metadata file on local FS.
/// Stores data in S3/HDFS and adds the object path and object size to metadata file on local FS.
template <typename T>
class WriteIndirectBufferFromRemoteFS final : public WriteBufferFromFileDecorator
{

7
src/Disks/tests/gtest_disk.cpp
View File

@ -2,6 +2,9 @@
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#include "gtest_disk.h"
#include <filesystem>
namespace fs = std::filesystem;
#if !defined(__clang__)
@ -22,7 +25,7 @@ DB::DiskPtr createDisk<DB::DiskMemory>()
template <>
DB::DiskPtr createDisk<DB::DiskLocal>()
{
Poco::File("tmp/").createDirectory();
fs::create_directory("tmp/");
return std::make_shared<DB::DiskLocal>("local_disk", "tmp/", 0);
}
@ -43,7 +46,7 @@ template <>
void destroyDisk<DB::DiskLocal>(DB::DiskPtr & disk)
{
disk.reset();
Poco::File("tmp/").remove(true);
fs::remove_all("tmp/");
}

60
src/Formats/FormatSchemaInfo.cpp
View File

@ -1,7 +1,7 @@
#include <Formats/FormatSchemaInfo.h>
#include <Poco/Path.h>
#include <Interpreters/Context.h>
#include <Common/Exception.h>
#include <filesystem>
namespace DB
@ -11,6 +11,7 @@ namespace ErrorCodes
extern const int BAD_ARGUMENTS;
}
namespace fs = std::filesystem;
namespace
{
@ -34,55 +35,66 @@ FormatSchemaInfo::FormatSchemaInfo(const String & format_schema, const String &
String default_file_extension = getFormatSchemaDefaultFileExtension(format);
Poco::Path path;
fs::path path;
if (require_message)
{
size_t colon_pos = format_schema.find(':');
if ((colon_pos == String::npos) || (colon_pos == 0) || (colon_pos == format_schema.length() - 1)
|| path.assign(format_schema.substr(0, colon_pos)).makeFile().getFileName().empty())
if ((colon_pos == String::npos) || (colon_pos == 0) || (colon_pos == format_schema.length() - 1))
{
throw Exception(
"Format schema requires the 'format_schema' setting to have the 'schema_file:message_name' format"
+ (default_file_extension.empty() ? "" : ", e.g. 'schema." + default_file_extension + ":Message'") +
". Got '" + format_schema
+ "'",
ErrorCodes::BAD_ARGUMENTS);
". Got '" + format_schema + "'", ErrorCodes::BAD_ARGUMENTS);
}
else
{
path = fs::path(format_schema.substr(0, colon_pos));
String filename = path.has_filename() ? path.filename() : path.parent_path().filename();
if (filename.empty())
throw Exception(
"Format schema requires the 'format_schema' setting to have the 'schema_file:message_name' format"
+ (default_file_extension.empty() ? "" : ", e.g. 'schema." + default_file_extension + ":Message'") +
". Got '" + format_schema + "'", ErrorCodes::BAD_ARGUMENTS);
}
message_name = format_schema.substr(colon_pos + 1);
}
else
path.assign(format_schema).makeFile().getFileName();
{
path = fs::path(format_schema);
if (!path.has_filename())
path = path.parent_path() / "";
}
auto default_schema_directory = [&format_schema_path]()
{
static const String str = Poco::Path(format_schema_path).makeAbsolute().makeDirectory().toString();
static const String str = fs::canonical(format_schema_path) / "";
return str;
};
if (path.getExtension().empty() && !default_file_extension.empty())
path.setExtension(default_file_extension);
if (!path.has_extension() && !default_file_extension.empty())
path = path.parent_path() / (path.stem().string() + '.' + default_file_extension);
if (path.isAbsolute())
fs::path default_schema_directory_path(default_schema_directory());
if (path.is_absolute())
{
if (is_server)
throw Exception("Absolute path in the 'format_schema' setting is prohibited: " + path.toString(), ErrorCodes::BAD_ARGUMENTS);
schema_path = path.getFileName();
schema_directory = path.makeParent().toString();
throw Exception("Absolute path in the 'format_schema' setting is prohibited: " + path.string(), ErrorCodes::BAD_ARGUMENTS);
schema_path = path.filename();
schema_directory = path.parent_path() / "";
}
else if (path.depth() >= 1 && path.directory(0) == "..")
else if (path.has_parent_path() && !fs::weakly_canonical(default_schema_directory_path / path).string().starts_with(fs::weakly_canonical(default_schema_directory_path).string()))
{
if (is_server)
throw Exception(
"Path in the 'format_schema' setting shouldn't go outside the 'format_schema_path' directory: " + path.toString(),
ErrorCodes::BAD_ARGUMENTS);
path = Poco::Path(default_schema_directory()).resolve(path).toString();
schema_path = path.getFileName();
schema_directory = path.makeParent().toString();
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Path in the 'format_schema' setting shouldn't go outside the 'format_schema_path' directory: {} ({} not in {})",
path.string());
path = default_schema_directory_path / path;
schema_path = path.filename();
schema_directory = path.parent_path() / "";
}
else
{
schema_path = path.toString();
schema_path = path;
schema_directory = default_schema_directory();
}
}

31
src/Functions/FunctionFile.cpp
View File

@ -3,10 +3,11 @@
#include <Functions/FunctionFactory.h>
#include <DataTypes/DataTypeString.h>
#include <IO/ReadBufferFromFile.h>
#include <Poco/File.h>
#include <Poco/Path.h>
#include <Interpreters/Context.h>
#include <unistd.h>
#include <filesystem>
namespace fs = std::filesystem;
namespace DB
{
@ -68,21 +69,19 @@ public:
{
const char * filename = reinterpret_cast<const char *>(&chars[source_offset]);
const String user_files_path = getContext()->getUserFilesPath();
String user_files_absolute_path = Poco::Path(user_files_path).makeAbsolute().makeDirectory().toString();
Poco::Path poco_filepath = Poco::Path(filename);
if (poco_filepath.isRelative())
poco_filepath = Poco::Path(user_files_absolute_path, poco_filepath);
const String file_absolute_path = poco_filepath.absolute().toString();
checkReadIsAllowedOrThrow(user_files_absolute_path, file_absolute_path);
fs::path user_files_absolute_path = fs::canonical(fs::path(getContext()->getUserFilesPath()));
fs::path file_path(filename);
if (file_path.is_relative())
file_path = user_files_absolute_path / file_path;
fs::path file_absolute_path = fs::canonical(file_path);
checkReadIsAllowedOrThrow(user_files_absolute_path.string(), file_absolute_path);
checked_filenames[row] = file_absolute_path;
auto file = Poco::File(file_absolute_path);
checked_filenames[row] = file_absolute_path.string();
if (!file.exists())
throw Exception(fmt::format("File {} doesn't exist.", file_absolute_path), ErrorCodes::FILE_DOESNT_EXIST);
if (!fs::exists(file_absolute_path))
throw Exception(fmt::format("File {} doesn't exist.", file_absolute_path.string()), ErrorCodes::FILE_DOESNT_EXIST);
const auto current_file_size = Poco::File(file_absolute_path).getSize();
const auto current_file_size = fs::file_size(file_absolute_path);
result_offset += current_file_size + 1;
res_offsets[row] = result_offset;
@ -117,8 +116,8 @@ private:
if (file_absolute_path.find(user_files_absolute_path) != 0)
throw Exception("File is not inside " + user_files_absolute_path, ErrorCodes::DATABASE_ACCESS_DENIED);
Poco::File path_poco_file = Poco::File(file_absolute_path);
if (path_poco_file.exists() && path_poco_file.isDirectory())
fs::path fs_path(file_absolute_path);
if (fs::exists(fs_path) && fs::is_directory(fs_path))
throw Exception("File can't be a directory", ErrorCodes::INCORRECT_FILE_NAME);
}
};

5
src/IO/tests/gtest_cascade_and_memory_write_buffer.cpp
View File

@ -1,7 +1,6 @@
#include <gtest/gtest.h>
#include <stdexcept>
#include <Poco/File.h>
#include <IO/CascadeWriteBuffer.h>
#include <IO/MemoryReadWriteBuffer.h>
#include <IO/WriteBufferFromTemporaryFile.h>
@ -9,7 +8,9 @@
#include <IO/ConcatReadBuffer.h>
#include <IO/copyData.h>
#include <Common/typeid_cast.h>
#include <filesystem>
namespace fs = std::filesystem;
using namespace DB;
@ -235,7 +236,7 @@ try
buf.reset();
reread_buf.reset();
ASSERT_TRUE(!Poco::File(tmp_filename).exists());
ASSERT_TRUE(!fs::exists(tmp_filename));
}
}
catch (...)

15
src/Interpreters/Context.cpp
View File

@ -74,8 +74,11 @@
#include <Interpreters/DatabaseCatalog.h>
#include <Storages/MergeTree/BackgroundJobsExecutor.h>
#include <Storages/MergeTree/MergeTreeDataPartUUID.h>
#include <filesystem>
namespace fs = std::filesystem;
namespace ProfileEvents
{
extern const Event ContextLock;
@ -2212,14 +2215,14 @@ void Context::checkCanBeDropped(const String & database, const String & table, c
if (!max_size_to_drop || size <= max_size_to_drop)
return;
Poco::File force_file(getFlagsPath() + "force_drop_table");
bool force_file_exists = force_file.exists();
fs::path force_file(getFlagsPath() + "force_drop_table");
bool force_file_exists = fs::exists(force_file);
if (force_file_exists)
{
try
{
force_file.remove();
fs::remove(force_file);
return;
}
catch (...)
@ -2241,9 +2244,9 @@ void Context::checkCanBeDropped(const String & database, const String & table, c
"Example:\nsudo touch '{}' && sudo chmod 666 '{}'",
backQuoteIfNeed(database), backQuoteIfNeed(table),
size_str, max_size_to_drop_str,
force_file.path(), force_file_exists ? "exists but not writeable (could not be removed)" : "doesn't exist",
force_file.path(),
force_file.path(), force_file.path());
force_file.string(), force_file_exists ? "exists but not writeable (could not be removed)" : "doesn't exist",
force_file.string(),
force_file.string(), force_file.string());
}

9
src/Interpreters/DDLTask.h
View File

@ -3,6 +3,7 @@
#include <Core/Types.h>
#include <Interpreters/Cluster.h>
#include <Common/ZooKeeper/Types.h>
#include <filesystem>
namespace Poco
{
@ -14,6 +15,8 @@ namespace zkutil
class ZooKeeper;
}
namespace fs = std::filesystem;
namespace DB
{
@ -100,9 +103,9 @@ struct DDLTaskBase
virtual ContextMutablePtr makeQueryContext(ContextPtr from_context, const ZooKeeperPtr & zookeeper);
inline String getActiveNodePath() const { return entry_path + "/active/" + host_id_str; }
inline String getFinishedNodePath() const { return entry_path + "/finished/" + host_id_str; }
inline String getShardNodePath() const { return entry_path + "/shards/" + getShardID(); }
inline String getActiveNodePath() const { return fs::path(entry_path) / "active" / host_id_str; }
inline String getFinishedNodePath() const { return fs::path(entry_path) / "finished" / host_id_str; }
inline String getShardNodePath() const { return fs::path(entry_path) / "shards" / getShardID(); }
static String getLogEntryName(UInt32 log_entry_number);
static UInt32 getLogEntryNumber(const String & log_entry_name);

26
src/Interpreters/DatabaseCatalog.cpp
View File

@ -5,7 +5,6 @@
#include <Databases/IDatabase.h>
#include <Databases/DatabaseMemory.h>
#include <Databases/DatabaseOnDisk.h>
#include <Poco/File.h>
#include <Common/quoteString.h>
#include <Storages/StorageMemory.h>
#include <Storages/LiveView/TemporaryLiveViewCleaner.h>
@ -17,6 +16,8 @@
#include <Common/CurrentMetrics.h>
#include <common/logger_useful.h>
#include <Poco/Util/AbstractConfiguration.h>
#include <filesystem>
#include <Common/filesystemHelpers.h>
#if !defined(ARCADIA_BUILD)
# include "config_core.h"
@ -27,8 +28,7 @@
# include <Storages/StorageMaterializeMySQL.h>
#endif
#include <filesystem>
namespace fs = std::filesystem;
namespace CurrentMetrics
{
@ -354,10 +354,9 @@ DatabasePtr DatabaseCatalog::detachDatabase(const String & database_name, bool d
db->drop(getContext());
/// Old ClickHouse versions did not store database.sql files
Poco::File database_metadata_file(
getContext()->getPath() + "metadata/" + escapeForFileName(database_name) + ".sql");
if (database_metadata_file.exists())
database_metadata_file.remove(false);
fs::path database_metadata_file = fs::path(getContext()->getPath()) / "metadata" / (escapeForFileName(database_name) + ".sql");
if (fs::exists(database_metadata_file))
fs::remove(database_metadata_file);
}
return db;
@ -783,7 +782,7 @@ void DatabaseCatalog::enqueueDroppedTableCleanup(StorageID table_id, StoragePtr
}
addUUIDMapping(table_id.uuid);
drop_time = Poco::File(dropped_metadata_path).getLastModified().epochTime();
drop_time = FS::getModificationTime(dropped_metadata_path);
}
std::lock_guard lock(tables_marked_dropped_mutex);
@ -892,16 +891,15 @@ void DatabaseCatalog::dropTableFinally(const TableMarkedAsDropped & table)
/// Even if table is not loaded, try remove its data from disk.
/// TODO remove data from all volumes
String data_path = getContext()->getPath() + "store/" + getPathForUUID(table.table_id.uuid);
Poco::File table_data_dir{data_path};
if (table_data_dir.exists())
fs::path data_path = fs::path(getContext()->getPath()) / "store" / getPathForUUID(table.table_id.uuid);
if (fs::exists(data_path))
{
LOG_INFO(log, "Removing data directory {} of dropped table {}", data_path, table.table_id.getNameForLogs());
table_data_dir.remove(true);
LOG_INFO(log, "Removing data directory {} of dropped table {}", data_path.string(), table.table_id.getNameForLogs());
fs::remove_all(data_path);
}
LOG_INFO(log, "Removing metadata {} of dropped table {}", table.metadata_path, table.table_id.getNameForLogs());
Poco::File(table.metadata_path).remove();
fs::remove(fs::path(table.metadata_path));
removeUUIDMappingFinally(table.table_id.uuid);
CurrentMetrics::sub(CurrentMetrics::TablesToDropQueueSize, 1);

15
src/Interpreters/ExternalLoaderXMLConfigRepository.cpp
View File

@ -3,12 +3,13 @@
#include <Common/StringUtils/StringUtils.h>
#include <Common/Config/ConfigProcessor.h>
#include <Common/getMultipleKeysFromConfig.h>
#include <Poco/Glob.h>
#include <Poco/File.h>
#include <Poco/Path.h>
#include <Common/filesystemHelpers.h>
#include <filesystem>
namespace fs = std::filesystem;
namespace DB
{
ExternalLoaderXMLConfigRepository::ExternalLoaderXMLConfigRepository(
@ -19,7 +20,7 @@ ExternalLoaderXMLConfigRepository::ExternalLoaderXMLConfigRepository(
Poco::Timestamp ExternalLoaderXMLConfigRepository::getUpdateTime(const std::string & definition_entity_name)
{
return Poco::File(definition_entity_name).getLastModified();
return FS::getModificationTimestamp(definition_entity_name);
}
std::set<std::string> ExternalLoaderXMLConfigRepository::getAllLoadablesDefinitionNames()
@ -36,8 +37,8 @@ std::set<std::string> ExternalLoaderXMLConfigRepository::getAllLoadablesDefiniti
if (pattern[0] != '/')
{
const auto app_config_path = main_config.getString("config-file", "config.xml");
const auto config_dir = Poco::Path{app_config_path}.parent().toString();
const auto absolute_path = config_dir + pattern;
const String config_dir = fs::path(app_config_path).parent_path();
const String absolute_path = fs::path(config_dir) / pattern;
Poco::Glob::glob(absolute_path, files, 0);
if (!files.empty())
continue;
@ -59,7 +60,7 @@ std::set<std::string> ExternalLoaderXMLConfigRepository::getAllLoadablesDefiniti
bool ExternalLoaderXMLConfigRepository::exists(const std::string & definition_entity_name)
{
return Poco::File(definition_entity_name).exists();
return fs::exists(fs::path(definition_entity_name));
}
Poco::AutoPtr<Poco::Util::AbstractConfiguration> ExternalLoaderXMLConfigRepository::load(

2
src/Interpreters/InterpreterDropQuery.cpp
View File

@ -1,5 +1,3 @@
#include <Poco/File.h>
#include <Databases/IDatabase.h>
#include <Interpreters/Context.h>
#include <Interpreters/executeDDLQueryOnCluster.h>

2
src/Interpreters/InterpreterSystemQuery.cpp
View File

@ -542,7 +542,7 @@ void InterpreterSystemQuery::dropReplica(ASTSystemQuery & query)
else if (!query.replica_zk_path.empty())
{
getContext()->checkAccess(AccessType::SYSTEM_DROP_REPLICA);
auto remote_replica_path = query.replica_zk_path + "/replicas/" + query.replica;
String remote_replica_path = fs::path(query.replica_zk_path) / "replicas" / query.replica;
/// This check is actually redundant, but it may prevent from some user mistakes
for (auto & elem : DatabaseCatalog::instance().getDatabases())

41
src/Interpreters/loadMetadata.cpp
View File

@ -18,7 +18,9 @@
#include <Common/typeid_cast.h>
#include <Common/StringUtils/StringUtils.h>
#include <filesystem>
namespace fs = std::filesystem;
namespace DB
{
@ -54,13 +56,13 @@ static void loadDatabase(
String database_attach_query;
String database_metadata_file = database_path + ".sql";
if (Poco::File(database_metadata_file).exists())
if (fs::exists(fs::path(database_metadata_file)))
{
/// There is .sql file with database creation statement.
ReadBufferFromFile in(database_metadata_file, 1024);
readStringUntilEOF(database_attach_query, in);
}
else if (Poco::File(database_path).exists())
else if (fs::exists(fs::path(database_path)))
{
/// Database exists, but .sql file is absent. It's old-style Ordinary database (e.g. system or default)
database_attach_query = "ATTACH DATABASE " + backQuoteIfNeed(database) + " ENGINE = Ordinary";
@ -95,34 +97,35 @@ void loadMetadata(ContextMutablePtr context, const String & default_database_nam
* This file is deleted after successful loading of tables.
* (flag is "one-shot")
*/
Poco::File force_restore_data_flag_file(context->getFlagsPath() + "force_restore_data");
bool has_force_restore_data_flag = force_restore_data_flag_file.exists();
auto force_restore_data_flag_file = fs::path(context->getFlagsPath()) / "force_restore_data";
bool has_force_restore_data_flag = fs::exists(force_restore_data_flag_file);
/// Loop over databases.
std::map<String, String> databases;
Poco::DirectoryIterator dir_end;
for (Poco::DirectoryIterator it(path); it != dir_end; ++it)
fs::directory_iterator dir_end;
for (fs::directory_iterator it(path); it != dir_end; ++it)
{
if (it->isLink())
if (it->is_symlink())
continue;
if (!it->isDirectory())
const auto current_file = it->path().filename().string();
if (!it->is_directory())
{
/// TODO: DETACH DATABASE PERMANENTLY ?
if (endsWith(it.name(), ".sql"))
if (fs::path(current_file).extension() == ".sql")
{
String db_name = it.name().substr(0, it.name().size() - 4);
String db_name = fs::path(current_file).stem();
if (db_name != DatabaseCatalog::SYSTEM_DATABASE)
databases.emplace(unescapeForFileName(db_name), path + "/" + db_name);
databases.emplace(unescapeForFileName(db_name), fs::path(path) / db_name);
}
/// Temporary fails may be left from previous server runs.
if (endsWith(it.name(), ".tmp"))
if (fs::path(current_file).extension() == ".tmp")
{
LOG_WARNING(log, "Removing temporary file {}", it->path());
LOG_WARNING(log, "Removing temporary file {}", it->path().string());
try
{
it->remove();
fs::remove(it->path());
}
catch (...)
{
@ -135,13 +138,13 @@ void loadMetadata(ContextMutablePtr context, const String & default_database_nam
}
/// For '.svn', '.gitignore' directory and similar.
if (it.name().at(0) == '.')
if (current_file.at(0) == '.')
continue;
if (it.name() == DatabaseCatalog::SYSTEM_DATABASE)
if (current_file == DatabaseCatalog::SYSTEM_DATABASE)
continue;
databases.emplace(unescapeForFileName(it.name()), it.path().toString());
databases.emplace(unescapeForFileName(current_file), it->path().string());
}
/// clickhouse-local creates DatabaseMemory as default database by itself
@ -158,7 +161,7 @@ void loadMetadata(ContextMutablePtr context, const String & default_database_nam
{
try
{
force_restore_data_flag_file.remove();
fs::remove(force_restore_data_flag_file);
}
catch (...)
{
@ -172,7 +175,7 @@ void loadMetadataSystem(ContextMutablePtr context)
{
String path = context->getPath() + "metadata/" + DatabaseCatalog::SYSTEM_DATABASE;
String metadata_file = path + ".sql";
if (Poco::File(path).exists() || Poco::File(metadata_file).exists())
if (fs::exists(fs::path(path)) || fs::exists(fs::path(metadata_file)))
{
/// 'has_force_restore_data_flag' is true, to not fail on loading query_log table, if it is corrupted.
loadDatabase(context, DatabaseCatalog::SYSTEM_DATABASE, path, true);

4
src/Server/HTTP/HTTPServerResponse.cpp
View File

@ -1,11 +1,8 @@
#include <Server/HTTP/HTTPServerResponse.h>
#include <Server/HTTP/HTTPServerRequest.h>
#include <Poco/CountingStream.h>
#include <Poco/DateTimeFormat.h>
#include <Poco/DateTimeFormatter.h>
#include <Poco/File.h>
#include <Poco/FileStream.h>
#include <Poco/Net/HTTPChunkedStream.h>
#include <Poco/Net/HTTPFixedLengthStream.h>
@ -13,6 +10,7 @@
#include <Poco/Net/HTTPStream.h>
#include <Poco/StreamCopier.h>
namespace DB
{

1
src/Server/HTTPHandler.cpp
View File

@ -39,7 +39,6 @@
#include <Poco/Base64Decoder.h>
#include <Poco/Base64Encoder.h>
#include <Poco/File.h>
#include <Poco/Net/HTTPBasicCredentials.h>
#include <Poco/Net/HTTPStream.h>
#include <Poco/Net/NetException.h>

16
src/Server/StaticRequestHandler.cpp
View File

@ -14,14 +14,15 @@
#include <Common/Exception.h>
#include <Poco/Path.h>
#include <Poco/File.h>
#include <Poco/Net/HTTPServerRequest.h>
#include <Poco/Net/HTTPServerResponse.h>
#include <Poco/Net/HTTPRequestHandlerFactory.h>
#include <Poco/Util/LayeredConfiguration.h>
#include <filesystem>
namespace fs = std::filesystem;
namespace DB
{
@ -137,11 +138,14 @@ void StaticRequestHandler::writeResponse(WriteBuffer & out)
if (startsWith(response_expression, file_prefix))
{
const auto & user_files_absolute_path = Poco::Path(server.context()->getUserFilesPath()).makeAbsolute().makeDirectory().toString();
const auto & file_name = response_expression.substr(file_prefix.size(), response_expression.size() - file_prefix.size());
auto file_name = response_expression.substr(file_prefix.size(), response_expression.size() - file_prefix.size());
if (file_name.starts_with('/'))
file_name = file_name.substr(1);
const auto & file_path = Poco::Path(user_files_absolute_path, file_name).makeAbsolute().toString();
if (!Poco::File(file_path).exists())
fs::path user_files_absolute_path = fs::canonical(fs::path(server.context()->getUserFilesPath()));
String file_path = fs::weakly_canonical(user_files_absolute_path / file_name);
if (!fs::exists(file_path))
throw Exception("Invalid file name " + file_path + " for static HTTPHandler. ", ErrorCodes::INCORRECT_FILE_NAME);
ReadBufferFromFile in(file_path);

59
src/Storages/Distributed/DirectoryMonitor.cpp
View File

@ -25,11 +25,9 @@
#include <IO/ConnectionTimeoutsContext.h>
#include <IO/Operators.h>
#include <Disks/IDisk.h>
#include <boost/algorithm/string/find_iterator.hpp>
#include <boost/algorithm/string/finder.hpp>
#include <Poco/DirectoryIterator.h>
#include <filesystem>
namespace CurrentMetrics
@ -39,6 +37,8 @@ namespace CurrentMetrics
extern const Metric BrokenDistributedFilesToInsert;
}
namespace fs = std::filesystem;
namespace DB
{
@ -293,7 +293,7 @@ StorageDistributedDirectoryMonitor::StorageDistributedDirectoryMonitor(
, pool(std::move(pool_))
, disk(disk_)
, relative_path(relative_path_)
, path(disk->getPath() + relative_path + '/')
, path(fs::path(disk->getPath()) / relative_path / "")
, should_batch_inserts(storage.getContext()->getSettingsRef().distributed_directory_monitor_batch_inserts)
, dir_fsync(storage.getDistributedSettingsRef().fsync_directories)
, min_batched_block_size_rows(storage.getContext()->getSettingsRef().min_insert_block_size_rows)
@ -347,7 +347,7 @@ void StorageDistributedDirectoryMonitor::shutdownAndDropAllData()
}
auto dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, relative_path);
Poco::File(path).remove(true);
fs::remove_all(path);
}
@ -490,16 +490,14 @@ std::map<UInt64, std::string> StorageDistributedDirectoryMonitor::getFiles()
std::map<UInt64, std::string> files;
size_t new_bytes_count = 0;
Poco::DirectoryIterator end;
for (Poco::DirectoryIterator it{path}; it != end; ++it)
fs::directory_iterator end;
for (fs::directory_iterator it{path}; it != end; ++it)
{
const auto & file_path_str = it->path();
Poco::Path file_path{file_path_str};
if (!it->isDirectory() && startsWith(file_path.getExtension(), "bin"))
if (!it->is_directory() && startsWith(fs::path(file_path_str).extension(), ".bin"))
{
files[parse<UInt64>(file_path.getBaseName())] = file_path_str;
new_bytes_count += Poco::File(file_path).getSize();
files[parse<UInt64>(fs::path(file_path_str).stem())] = file_path_str;
new_bytes_count += fs::file_size(fs::path(file_path_str));
}
}
@ -663,8 +661,7 @@ struct StorageDistributedDirectoryMonitor::Batch
String tmp_file{parent.current_batch_file_path + ".tmp"};
auto dir_sync_guard = getDirectorySyncGuard(dir_fsync, parent.disk, parent.relative_path);
if (Poco::File{tmp_file}.exists())
if (fs::exists(tmp_file))
LOG_ERROR(parent.log, "Temporary file {} exists. Unclean shutdown?", backQuote(tmp_file));
{
@ -676,7 +673,7 @@ struct StorageDistributedDirectoryMonitor::Batch
out.sync();
}
Poco::File{tmp_file}.renameTo(parent.current_batch_file_path);
fs::rename(tmp_file, parent.current_batch_file_path);
}
auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(parent.storage.getContext()->getSettingsRef());
auto connection = parent.pool->get(timeouts);
@ -757,7 +754,7 @@ struct StorageDistributedDirectoryMonitor::Batch
total_bytes = 0;
recovered = false;
Poco::File{parent.current_batch_file_path}.setSize(0);
fs::resize_file(parent.current_batch_file_path, 0);
}
void writeText(WriteBuffer & out)
@ -850,7 +847,7 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map
{
std::unordered_set<UInt64> file_indices_to_skip;
if (Poco::File{current_batch_file_path}.exists())
if (fs::exists(current_batch_file_path))
{
/// Possibly, we failed to send a batch on the previous iteration. Try to send exactly the same batch.
Batch batch(*this, files);
@ -951,8 +948,8 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map
/// current_batch.txt will not exist if there was no send
/// (this is the case when all batches that was pending has been marked as pending)
if (Poco::File{current_batch_file_path}.exists())
Poco::File{current_batch_file_path}.remove();
if (fs::exists(current_batch_file_path))
fs::remove(current_batch_file_path);
}
}
@ -961,20 +958,18 @@ void StorageDistributedDirectoryMonitor::markAsBroken(const std::string & file_p
const auto last_path_separator_pos = file_path.rfind('/');
const auto & base_path = file_path.substr(0, last_path_separator_pos + 1);
const auto & file_name = file_path.substr(last_path_separator_pos + 1);
const auto & broken_path = base_path + "broken/";
const auto & broken_file_path = broken_path + file_name;
const String & broken_path = fs::path(base_path) / "broken/";
const String & broken_file_path = fs::path(broken_path) / file_name;
Poco::File{broken_path}.createDirectory();
fs::create_directory(broken_path);
auto dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, relative_path);
auto broken_dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, relative_path + "/broken/");
Poco::File file(file_path);
auto broken_dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, fs::path(relative_path) / "broken/");
{
std::lock_guard status_lock(status_mutex);
size_t file_size = file.getSize();
size_t file_size = fs::file_size(file_path);
--status.files_count;
status.bytes_count -= file_size;
@ -985,15 +980,13 @@ void StorageDistributedDirectoryMonitor::markAsBroken(const std::string & file_p
metric_broken_files.add();
}
file.renameTo(broken_file_path);
fs::rename(file_path, broken_file_path);
LOG_ERROR(log, "Renamed `{}` to `{}`", file_path, broken_file_path);
}
void StorageDistributedDirectoryMonitor::markAsSend(const std::string & file_path)
{
Poco::File file(file_path);
size_t file_size = file.getSize();
size_t file_size = fs::file_size(file_path);
{
std::lock_guard status_lock(status_mutex);
@ -1002,7 +995,7 @@ void StorageDistributedDirectoryMonitor::markAsSend(const std::string & file_pat
status.bytes_count -= file_size;
}
file.remove();
fs::remove(file_path);
}
bool StorageDistributedDirectoryMonitor::maybeMarkAsBroken(const std::string & file_path, const Exception & e)
@ -1030,7 +1023,7 @@ void StorageDistributedDirectoryMonitor::updatePath(const std::string & new_rela
{
std::lock_guard status_lock(status_mutex);
relative_path = new_relative_path;
path = disk->getPath() + relative_path + '/';
path = fs::path(disk->getPath()) / relative_path / "";
}
current_batch_file_path = path + "current_batch.txt";

24
src/Storages/Distributed/DistributedBlockOutputStream.cpp
View File

@ -33,11 +33,10 @@
#include <ext/range.h>
#include <ext/scope_guard.h>
#include <Poco/DirectoryIterator.h>
#include <future>
#include <condition_variable>
#include <mutex>
#include <filesystem>
namespace CurrentMetrics
@ -50,10 +49,11 @@ namespace ProfileEvents
extern const Event DistributedSyncInsertionTimeoutExceeded;
}
namespace fs = std::filesystem;
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
@ -660,10 +660,10 @@ void DistributedBlockOutputStream::writeToShard(const Block & block, const std::
/// hardlinking to ensure the inode is not freed until we're done
{
const std::string path(disk_path + data_path + *it);
Poco::File(path).createDirectory();
const std::string tmp_path(path + "/tmp/");
Poco::File(tmp_path).createDirectory();
fs::create_directory(path);
fs::create_directory(tmp_path);
const std::string file_name(toString(storage.file_names_increment.get()) + ".bin");
@ -717,7 +717,7 @@ void DistributedBlockOutputStream::writeToShard(const Block & block, const std::
}
// Create hardlink here to reuse increment number
const std::string block_file_path(path + '/' + file_name);
const std::string block_file_path(fs::path(path) / file_name);
createHardLink(first_file_tmp_path, block_file_path);
auto dir_sync_guard = make_directory_sync_guard(*it);
}
@ -726,18 +726,18 @@ void DistributedBlockOutputStream::writeToShard(const Block & block, const std::
/// Make hardlinks
for (; it != dir_names.end(); ++it)
{
const std::string path(disk_path + data_path + *it);
Poco::File(path).createDirectory();
const std::string path(fs::path(disk_path) / (data_path + *it));
fs::create_directory(path);
const std::string block_file_path(path + '/' + toString(storage.file_names_increment.get()) + ".bin");
const std::string block_file_path(fs::path(path) / (toString(storage.file_names_increment.get()) + ".bin"));
createHardLink(first_file_tmp_path, block_file_path);
auto dir_sync_guard = make_directory_sync_guard(*it);
}
auto file_size = Poco::File(first_file_tmp_path).getSize();
auto file_size = fs::file_size(first_file_tmp_path);
/// remove the temporary file, enabling the OS to reclaim inode after all threads
/// have removed their corresponding files
Poco::File(first_file_tmp_path).remove();
fs::remove(first_file_tmp_path);
/// Notify
auto sleep_ms = context->getSettingsRef().distributed_directory_monitor_sleep_time_ms;

5
src/Storages/HDFS/StorageHDFS.cpp
View File

@ -18,7 +18,6 @@
#include <DataStreams/IBlockOutputStream.h>
#include <DataStreams/OwningBlockInputStream.h>
#include <DataStreams/IBlockInputStream.h>
#include <Common/parseGlobs.h>
#include <Poco/URI.h>
#include <re2/re2.h>
@ -26,7 +25,9 @@
#include <hdfs/hdfs.h>
#include <Processors/Sources/SourceWithProgress.h>
#include <Processors/Pipe.h>
#include <filesystem>
namespace fs = std::filesystem;
namespace DB
{
@ -257,7 +258,7 @@ Strings LSWithRegexpMatching(const String & path_for_ls, const HDFSFSPtr & fs, c
{
if (re2::RE2::FullMatch(file_name, matcher))
{
Strings result_part = LSWithRegexpMatching(full_path + "/", fs, suffix_with_globs.substr(next_slash));
Strings result_part = LSWithRegexpMatching(fs::path(full_path) / "", fs, suffix_with_globs.substr(next_slash));
/// Recursion depth is limited by pattern. '*' works only for depth = 1, for depth = 2 pattern path is '*/*'. So we do not need additional check.
std::move(result_part.begin(), result_part.end(), std::back_inserter(result));
}

30
src/Storages/MergeTree/DataPartsExchange.cpp
View File

@ -13,11 +13,11 @@
#include <Common/NetException.h>
#include <IO/createReadBufferFromFileBase.h>
#include <ext/scope_guard.h>
#include <Poco/File.h>
#include <Poco/Net/HTTPRequest.h>
namespace fs = std::filesystem;
namespace CurrentMetrics
{
extern const Metric ReplicatedSend;
@ -289,7 +289,7 @@ MergeTreeData::DataPart::Checksums Service::sendPartFromDisk(
{
String file_name = it.first;
String path = part->getFullRelativePath() + file_name;
String path = fs::path(part->getFullRelativePath()) / file_name;
UInt64 size = disk->getFileSize(path);
@ -339,15 +339,15 @@ void Service::sendPartS3Metadata(const MergeTreeData::DataPartPtr & part, WriteB
{
String file_name = it.first;
String metadata_file = disk->getPath() + part->getFullRelativePath() + file_name;
String metadata_file = fs::path(disk->getPath()) / part->getFullRelativePath() / file_name;
Poco::File metadata(metadata_file);
fs::path metadata(metadata_file);
if (!metadata.exists())
if (!fs::exists(metadata))
throw Exception("S3 metadata '" + file_name + "' is not exists", ErrorCodes::CORRUPTED_DATA);
if (!metadata.isFile())
if (!fs::is_regular_file(metadata))
throw Exception("S3 metadata '" + file_name + "' is not a file", ErrorCodes::CORRUPTED_DATA);
UInt64 file_size = metadata.getSize();
UInt64 file_size = fs::file_size(metadata);
writeStringBinary(it.first, out);
writeBinary(file_size, out);
@ -570,7 +570,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart(
readUUIDText(part_uuid, in);
auto storage_id = data.getStorageID();
String new_part_path = part_type == "InMemory" ? "memory" : data.getFullPathOnDisk(reservation->getDisk()) + part_name + "/";
String new_part_path = part_type == "InMemory" ? "memory" : fs::path(data.getFullPathOnDisk(reservation->getDisk())) / part_name / "";
auto entry = data.getContext()->getReplicatedFetchList().insert(
storage_id.getDatabaseName(), storage_id.getTableName(),
part_info.partition_id, part_name, new_part_path,
@ -681,13 +681,13 @@ void Fetcher::downloadBaseOrProjectionPartToDisk(
/// File must be inside "absolute_part_path" directory.
/// Otherwise malicious ClickHouse replica may force us to write to arbitrary path.
String absolute_file_path = Poco::Path(part_download_path + file_name).absolute().toString();
if (!startsWith(absolute_file_path, Poco::Path(part_download_path).absolute().toString()))
String absolute_file_path = fs::weakly_canonical(fs::path(part_download_path) / file_name);
if (!startsWith(absolute_file_path, fs::weakly_canonical(part_download_path).string()))
throw Exception("File path (" + absolute_file_path + ") doesn't appear to be inside part path (" + part_download_path + ")."
" This may happen if we are trying to download part from malicious replica or logical error.",
ErrorCodes::INSECURE_PATH);
auto file_out = disk->writeFile(part_download_path + file_name);
auto file_out = disk->writeFile(fs::path(part_download_path) / file_name);
HashingWriteBuffer hashing_out(*file_out);
copyData(in, hashing_out, file_size, blocker.getCounter());
@ -704,7 +704,7 @@ void Fetcher::downloadBaseOrProjectionPartToDisk(
readPODBinary(expected_hash, in);
if (expected_hash != hashing_out.getHash())
throw Exception("Checksum mismatch for file " + fullPath(disk, part_download_path + file_name) + " transferred from " + replica_path,
throw Exception("Checksum mismatch for file " + fullPath(disk, (fs::path(part_download_path) / file_name).string()) + " transferred from " + replica_path,
ErrorCodes::CHECKSUM_DOESNT_MATCH);
if (file_name != "checksums.txt" &&
@ -811,7 +811,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToS3(
String tmp_prefix = tmp_prefix_.empty() ? TMP_PREFIX : tmp_prefix_;
String part_relative_path = String(to_detached ? "detached/" : "") + tmp_prefix + part_name;
String part_download_path = data.getRelativeDataPath() + part_relative_path + "/";
String part_download_path = fs::path(data.getRelativeDataPath()) / part_relative_path / "";
if (disk->exists(part_download_path))
throw Exception("Directory " + fullPath(disk, part_download_path) + " already exists.", ErrorCodes::DIRECTORY_ALREADY_EXISTS);
@ -833,7 +833,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToS3(
readStringBinary(file_name, in);
readBinary(file_size, in);
String data_path = part_download_path + file_name;
String data_path = fs::path(part_download_path) / file_name;
String metadata_file = fullPath(disk, data_path);
{

85
src/Storages/MergeTree/IMergeTreeDataPart.cpp
View File

@ -69,7 +69,7 @@ void IMergeTreeDataPart::MinMaxIndex::load(const MergeTreeData & data, const Dis
hyperrectangle.reserve(minmax_idx_size);
for (size_t i = 0; i < minmax_idx_size; ++i)
{
String file_name = part_path + "minmax_" + escapeForFileName(minmax_column_names[i]) + ".idx";
String file_name = fs::path(part_path) / ("minmax_" + escapeForFileName(minmax_column_names[i]) + ".idx");
auto file = openForReading(disk_, file_name);
auto serialization = minmax_column_types[i]->getDefaultSerialization();
@ -111,7 +111,7 @@ void IMergeTreeDataPart::MinMaxIndex::store(
String file_name = "minmax_" + escapeForFileName(column_names[i]) + ".idx";
auto serialization = data_types.at(i)->getDefaultSerialization();
auto out = disk_->writeFile(part_path + file_name);
auto out = disk_->writeFile(fs::path(part_path) / file_name);
HashingWriteBuffer out_hashing(*out);
serialization->serializeBinary(hyperrectangle[i].left, out_hashing);
serialization->serializeBinary(hyperrectangle[i].right, out_hashing);
@ -560,7 +560,7 @@ String IMergeTreeDataPart::getFullPath() const
if (relative_path.empty())
throw Exception("Part relative_path cannot be empty. It's bug.", ErrorCodes::LOGICAL_ERROR);
return storage.getFullPathOnDisk(volume->getDisk()) + (parent_part ? parent_part->relative_path + "/" : "") + relative_path + "/";
return fs::path(storage.getFullPathOnDisk(volume->getDisk())) / (parent_part ? parent_part->relative_path : "") / relative_path / "";
}
String IMergeTreeDataPart::getFullRelativePath() const
@ -568,7 +568,7 @@ String IMergeTreeDataPart::getFullRelativePath() const
if (relative_path.empty())
throw Exception("Part relative_path cannot be empty. It's bug.", ErrorCodes::LOGICAL_ERROR);
return storage.relative_data_path + (parent_part ? parent_part->relative_path + "/" : "") + relative_path + "/";
return fs::path(storage.relative_data_path) / (parent_part ? parent_part->relative_path : "") / relative_path / "";
}
void IMergeTreeDataPart::loadColumnsChecksumsIndexes(bool require_columns_checksums, bool check_consistency)
@ -643,7 +643,7 @@ void IMergeTreeDataPart::loadIndex()
loaded_index[i]->reserve(index_granularity.getMarksCount());
}
String index_path = getFullRelativePath() + "primary.idx";
String index_path = fs::path(getFullRelativePath()) / "primary.idx";
auto index_file = openForReading(volume->getDisk(), index_path);
size_t marks_count = index_granularity.getMarksCount();
@ -678,7 +678,7 @@ NameSet IMergeTreeDataPart::getFileNamesWithoutChecksums() const
return {};
NameSet result = {"checksums.txt", "columns.txt"};
String default_codec_path = getFullRelativePath() + DEFAULT_COMPRESSION_CODEC_FILE_NAME;
String default_codec_path = fs::path(getFullRelativePath()) / DEFAULT_COMPRESSION_CODEC_FILE_NAME;
if (volume->getDisk()->exists(default_codec_path))
result.emplace(DEFAULT_COMPRESSION_CODEC_FILE_NAME);
@ -695,7 +695,7 @@ void IMergeTreeDataPart::loadDefaultCompressionCodec()
return;
}
String path = getFullRelativePath() + DEFAULT_COMPRESSION_CODEC_FILE_NAME;
String path = fs::path(getFullRelativePath()) / DEFAULT_COMPRESSION_CODEC_FILE_NAME;
if (!volume->getDisk()->exists(path))
{
default_codec = detectDefaultCompressionCodec();
@ -756,7 +756,7 @@ CompressionCodecPtr IMergeTreeDataPart::detectDefaultCompressionCodec() const
{
if (path_to_data_file.empty())
{
String candidate_path = getFullRelativePath() + ISerialization::getFileNameForStream(part_column, substream_path) + ".bin";
String candidate_path = fs::path(getFullRelativePath()) / (ISerialization::getFileNameForStream(part_column, substream_path) + ".bin");
/// We can have existing, but empty .bin files. Example: LowCardinality(Nullable(...)) columns and column_name.dict.null.bin file.
if (volume->getDisk()->exists(candidate_path) && volume->getDisk()->getFileSize(candidate_path) != 0)
@ -822,7 +822,7 @@ void IMergeTreeDataPart::loadPartitionAndMinMaxIndex()
void IMergeTreeDataPart::loadChecksums(bool require)
{
const String path = getFullRelativePath() + "checksums.txt";
const String path = fs::path(getFullRelativePath()) / "checksums.txt";
if (volume->getDisk()->exists(path))
{
@ -847,11 +847,11 @@ void IMergeTreeDataPart::loadChecksums(bool require)
checksums = checkDataPart(shared_from_this(), false);
{
auto out = volume->getDisk()->writeFile(getFullRelativePath() + "checksums.txt.tmp", 4096);
auto out = volume->getDisk()->writeFile(fs::path(getFullRelativePath()) / "checksums.txt.tmp", 4096);
checksums.write(*out);
}
volume->getDisk()->moveFile(getFullRelativePath() + "checksums.txt.tmp", getFullRelativePath() + "checksums.txt");
volume->getDisk()->moveFile(fs::path(getFullRelativePath()) / "checksums.txt.tmp", fs::path(getFullRelativePath()) / "checksums.txt");
bytes_on_disk = checksums.getTotalSizeOnDisk();
}
@ -859,7 +859,7 @@ void IMergeTreeDataPart::loadChecksums(bool require)
void IMergeTreeDataPart::loadRowsCount()
{
String path = getFullRelativePath() + "count.txt";
String path = fs::path(getFullRelativePath()) / "count.txt";
if (index_granularity.empty())
{
rows_count = 0;
@ -960,7 +960,7 @@ void IMergeTreeDataPart::loadRowsCount()
void IMergeTreeDataPart::loadTTLInfos()
{
String path = getFullRelativePath() + "ttl.txt";
String path = fs::path(getFullRelativePath()) / "ttl.txt";
if (volume->getDisk()->exists(path))
{
auto in = openForReading(volume->getDisk(), path);
@ -987,7 +987,7 @@ void IMergeTreeDataPart::loadTTLInfos()
void IMergeTreeDataPart::loadUUID()
{
String path = getFullRelativePath() + UUID_FILE_NAME;
String path = fs::path(getFullRelativePath()) / UUID_FILE_NAME;
if (volume->getDisk()->exists(path))
{
@ -1000,7 +1000,7 @@ void IMergeTreeDataPart::loadUUID()
void IMergeTreeDataPart::loadColumns(bool require)
{
String path = getFullRelativePath() + "columns.txt";
String path = fs::path(getFullRelativePath()) / "columns.txt";
auto metadata_snapshot = storage.getInMemoryMetadataPtr();
if (parent_part)
metadata_snapshot = metadata_snapshot->projections.get(name).metadata;
@ -1015,7 +1015,7 @@ void IMergeTreeDataPart::loadColumns(bool require)
/// If there is no file with a list of columns, write it down.
for (const NameAndTypePair & column : metadata_snapshot->getColumns().getAllPhysical())
if (volume->getDisk()->exists(getFullRelativePath() + getFileNameForColumn(column) + ".bin"))
if (volume->getDisk()->exists(fs::path(getFullRelativePath()) / (getFileNameForColumn(column) + ".bin")))
loaded_columns.push_back(column);
if (columns.empty())
@ -1053,7 +1053,7 @@ UInt64 IMergeTreeDataPart::calculateTotalSizeOnDisk(const DiskPtr & disk_, const
disk_->listFiles(from, files);
UInt64 res = 0;
for (const auto & file : files)
res += calculateTotalSizeOnDisk(disk_, from + "/" + file);
res += calculateTotalSizeOnDisk(disk_, fs::path(from) / file);
return res;
}
@ -1063,7 +1063,7 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_
assertOnDisk();
String from = getFullRelativePath();
String to = storage.relative_data_path + (parent_part ? parent_part->relative_path + "/" : "") + new_relative_path + "/";
String to = fs::path(storage.relative_data_path) / (parent_part ? parent_part->relative_path : "") / new_relative_path / "";
if (!volume->getDisk()->exists(from))
throw Exception("Part directory " + fullPath(volume->getDisk(), from) + " doesn't exist. Most likely it is a logical error.", ErrorCodes::FILE_DOESNT_EXIST);
@ -1124,8 +1124,8 @@ void IMergeTreeDataPart::remove(bool keep_s3) const
* And a race condition can happen that will lead to "File not found" error here.
*/
String from = storage.relative_data_path + relative_path;
String to = storage.relative_data_path + "delete_tmp_" + name;
fs::path from = fs::path(storage.relative_data_path) / relative_path;
fs::path to = fs::path(storage.relative_data_path) / ("delete_tmp_" + name);
// TODO directory delete_tmp_<name> is never removed if server crashes before returning from this function
auto disk = volume->getDisk();
@ -1134,7 +1134,7 @@ void IMergeTreeDataPart::remove(bool keep_s3) const
LOG_WARNING(storage.log, "Directory {} (to which part must be renamed before removing) already exists. Most likely this is due to unclean restart. Removing it.", fullPath(disk, to));
try
{
disk->removeSharedRecursive(to + "/", keep_s3);
disk->removeSharedRecursive(fs::path(to) / "", keep_s3);
}
catch (...)
{
@ -1147,12 +1147,15 @@ void IMergeTreeDataPart::remove(bool keep_s3) const
{
disk->moveDirectory(from, to);
}
catch (const Poco::FileNotFoundException &)
catch (const fs::filesystem_error & e)
{
if (e.code() == std::errc::no_such_file_or_directory)
{
LOG_ERROR(storage.log, "Directory {} (part to remove) doesn't exist or one of nested files has gone. Most likely this is due to manual removing. This should be discouraged. Ignoring.", fullPath(disk, to));
return;
}
throw;
}
// Record existing projection directories so we don't remove them twice
std::unordered_set<String> projection_directories;
@ -1166,7 +1169,7 @@ void IMergeTreeDataPart::remove(bool keep_s3) const
if (checksums.empty())
{
/// If the part is not completely written, we cannot use fast path by listing files.
disk->removeSharedRecursive(to + "/", keep_s3);
disk->removeSharedRecursive(fs::path(to) / "", keep_s3);
}
else
{
@ -1181,17 +1184,17 @@ void IMergeTreeDataPart::remove(bool keep_s3) const
for (const auto & [file, _] : checksums.files)
{
if (projection_directories.find(file) == projection_directories.end())
disk->removeSharedFile(to + "/" + file, keep_s3);
disk->removeSharedFile(fs::path(to) / file, keep_s3);
}
#if !defined(__clang__)
# pragma GCC diagnostic pop
#endif
for (const auto & file : {"checksums.txt", "columns.txt"})
disk->removeSharedFile(to + "/" + file, keep_s3);
disk->removeSharedFile(fs::path(to) / file, keep_s3);
disk->removeSharedFileIfExists(to + "/" + DEFAULT_COMPRESSION_CODEC_FILE_NAME, keep_s3);
disk->removeSharedFileIfExists(to + "/" + DELETE_ON_DESTROY_MARKER_FILE_NAME, keep_s3);
disk->removeSharedFileIfExists(fs::path(to) / DEFAULT_COMPRESSION_CODEC_FILE_NAME, keep_s3);
disk->removeSharedFileIfExists(fs::path(to) / DELETE_ON_DESTROY_MARKER_FILE_NAME, keep_s3);
disk->removeDirectory(to);
}
@ -1201,7 +1204,7 @@ void IMergeTreeDataPart::remove(bool keep_s3) const
LOG_ERROR(storage.log, "Cannot quickly remove directory {} by removing files; fallback to recursive removal. Reason: {}", fullPath(disk, to), getCurrentExceptionMessage(false));
disk->removeSharedRecursive(to + "/", keep_s3);
disk->removeSharedRecursive(fs::path(to) / "", keep_s3);
}
}
}
@ -1268,7 +1271,7 @@ String IMergeTreeDataPart::getRelativePathForPrefix(const String & prefix) const
{
res = (prefix.empty() ? "" : prefix + "_") + name + (try_no ? "_try" + DB::toString(try_no) : "");
if (!volume->getDisk()->exists(getFullRelativePath() + res))
if (!volume->getDisk()->exists(fs::path(getFullRelativePath()) / res))
return res;
LOG_WARNING(storage.log, "Directory {} (to detach to) already exists. Will detach to directory with '_tryN' suffix.", res);
@ -1291,11 +1294,11 @@ void IMergeTreeDataPart::renameToDetached(const String & prefix) const
void IMergeTreeDataPart::makeCloneInDetached(const String & prefix, const StorageMetadataPtr & /*metadata_snapshot*/) const
{
String destination_path = storage.relative_data_path + getRelativePathForDetachedPart(prefix);
String destination_path = fs::path(storage.relative_data_path) / getRelativePathForDetachedPart(prefix);
/// Backup is not recursive (max_level is 0), so do not copy inner directories
localBackup(volume->getDisk(), getFullRelativePath(), destination_path, 0);
volume->getDisk()->removeFileIfExists(destination_path + "/" + DELETE_ON_DESTROY_MARKER_FILE_NAME);
volume->getDisk()->removeFileIfExists(fs::path(destination_path) / DELETE_ON_DESTROY_MARKER_FILE_NAME);
}
void IMergeTreeDataPart::makeCloneOnDisk(const DiskPtr & disk, const String & directory_name) const
@ -1307,16 +1310,16 @@ void IMergeTreeDataPart::makeCloneOnDisk(const DiskPtr & disk, const String & di
if (directory_name.empty())
throw Exception("Can not clone data part " + name + " to empty directory.", ErrorCodes::LOGICAL_ERROR);
String path_to_clone = storage.relative_data_path + directory_name + '/';
String path_to_clone = fs::path(storage.relative_data_path) / directory_name / "";
if (disk->exists(path_to_clone + relative_path))
if (disk->exists(fs::path(path_to_clone) / relative_path))
{
LOG_WARNING(storage.log, "Path " + fullPath(disk, path_to_clone + relative_path) + " already exists. Will remove it and clone again.");
disk->removeRecursive(path_to_clone + relative_path + '/');
disk->removeRecursive(fs::path(path_to_clone) / relative_path / "");
}
disk->createDirectories(path_to_clone);
volume->getDisk()->copy(getFullRelativePath(), disk, path_to_clone);
volume->getDisk()->removeFileIfExists(path_to_clone + '/' + DELETE_ON_DESTROY_MARKER_FILE_NAME);
volume->getDisk()->removeFileIfExists(fs::path(path_to_clone) / DELETE_ON_DESTROY_MARKER_FILE_NAME);
}
void IMergeTreeDataPart::checkConsistencyBase() const
@ -1370,19 +1373,19 @@ void IMergeTreeDataPart::checkConsistencyBase() const
/// Check that the primary key index is not empty.
if (!pk.column_names.empty())
check_file_not_empty(volume->getDisk(), path + "primary.idx");
check_file_not_empty(volume->getDisk(), fs::path(path) / "primary.idx");
if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING)
{
check_file_not_empty(volume->getDisk(), path + "count.txt");
check_file_not_empty(volume->getDisk(), fs::path(path) / "count.txt");
if (metadata_snapshot->hasPartitionKey())
check_file_not_empty(volume->getDisk(), path + "partition.dat");
check_file_not_empty(volume->getDisk(), fs::path(path) / "partition.dat");
if (!parent_part)
{
for (const String & col_name : storage.getMinMaxColumnsNames(partition_key))
check_file_not_empty(volume->getDisk(), path + "minmax_" + escapeForFileName(col_name) + ".idx");
check_file_not_empty(volume->getDisk(), fs::path(path) / ("minmax_" + escapeForFileName(col_name) + ".idx"));
}
}
}
@ -1477,7 +1480,7 @@ String IMergeTreeDataPart::getUniqueId() const
auto disk = volume->getDisk();
if (disk->getType() == DB::DiskType::Type::S3)
id = disk->getUniqueId(getFullRelativePath() + "checksums.txt");
id = disk->getUniqueId(fs::path(getFullRelativePath()) / "checksums.txt");
if (id.empty())
throw Exception("Can't get unique S3 object", ErrorCodes::LOGICAL_ERROR);

2
src/Storages/MergeTree/IMergeTreeDataPart.h
View File

@ -17,8 +17,6 @@
#include <Storages/MergeTree/MergeTreeIOSettings.h>
#include <Storages/MergeTree/KeyCondition.h>
#include <Poco/Path.h>
#include <shared_mutex>
namespace zkutil

1
src/Storages/MergeTree/IMergeTreeReader.cpp
View File

@ -6,7 +6,6 @@
#include <Interpreters/inplaceBlockConversions.h>
#include <Storages/MergeTree/IMergeTreeReader.h>
#include <Common/typeid_cast.h>
#include <Poco/File.h>
namespace DB

2
src/Storages/MergeTree/LeaderElection.h
View File

@ -83,7 +83,7 @@ private:
void createNode()
{
shutdown_called = false;
node = EphemeralNodeHolder::createSequential(path + "/leader_election-", zookeeper, identifier);
node = EphemeralNodeHolder::createSequential(fs::path(path) / "leader_election-", zookeeper, identifier);
std::string node_path = node->getPath();
node_name = node_path.substr(node_path.find_last_of('/') + 1);

85
src/Storages/MergeTree/MergeTreeData.cpp
View File

@ -52,8 +52,6 @@
#include <Common/quoteString.h>
#include <Common/typeid_cast.h>
#include <Poco/DirectoryIterator.h>
#include <boost/range/adaptor/filtered.hpp>
#include <boost/algorithm/string/join.hpp>
@ -67,8 +65,11 @@
#include <typeinfo>
#include <typeindex>
#include <unordered_set>
#include <filesystem>
namespace fs = std::filesystem;
namespace ProfileEvents
{
extern const Event RejectedInserts;
@ -210,8 +211,8 @@ MergeTreeData::MergeTreeData(
for (const auto & [path, disk] : getRelativeDataPathsWithDisks())
{
disk->createDirectories(path);
disk->createDirectories(path + MergeTreeData::DETACHED_DIR_NAME);
auto current_version_file_path = path + MergeTreeData::FORMAT_VERSION_FILE_NAME;
disk->createDirectories(fs::path(path) / MergeTreeData::DETACHED_DIR_NAME);
String current_version_file_path = fs::path(path) / MergeTreeData::FORMAT_VERSION_FILE_NAME;
if (disk->exists(current_version_file_path))
{
if (!version_file.first.empty())
@ -225,7 +226,7 @@ MergeTreeData::MergeTreeData(
/// If not choose any
if (version_file.first.empty())
version_file = {relative_data_path + MergeTreeData::FORMAT_VERSION_FILE_NAME, getStoragePolicy()->getAnyDisk()};
version_file = {fs::path(relative_data_path) / MergeTreeData::FORMAT_VERSION_FILE_NAME, getStoragePolicy()->getAnyDisk()};
bool version_file_exists = version_file.second->exists(version_file.first);
@ -927,8 +928,8 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
auto part = createPart(part_name, part_info, single_disk_volume, part_name);
bool broken = false;
String part_path = relative_data_path + "/" + part_name;
String marker_path = part_path + "/" + IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME;
String part_path = fs::path(relative_data_path) / part_name;
String marker_path = fs::path(part_path) / IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME;
if (part_disk_ptr->exists(marker_path))
{
LOG_WARNING(log, "Detaching stale part {}{}, which should have been deleted after a move. That can only happen after unclean restart of ClickHouse after move of a part having an operation blocking that stale copy of part.", getFullPathOnDisk(part_disk_ptr), part_name);
@ -1016,7 +1017,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
else
has_adaptive_parts.store(true, std::memory_order_relaxed);
part->modification_time = part_disk_ptr->getLastModified(relative_data_path + part_name).epochTime();
part->modification_time = part_disk_ptr->getLastModified(fs::path(relative_data_path) / part_name).epochTime();
/// Assume that all parts are Committed, covered parts will be detected and marked as Outdated later
part->setState(DataPartState::Committed);
@ -1160,10 +1161,15 @@ void MergeTreeData::clearOldTemporaryDirectories(ssize_t custom_directories_life
disk->removeRecursive(it->path());
}
}
catch (const Poco::FileNotFoundException &)
catch (const fs::filesystem_error & e)
{
if (e.code() == std::errc::no_such_file_or_directory)
{
/// If the file is already deleted, do nothing.
}
else
throw;
}
}
}
}
@ -1448,11 +1454,16 @@ void MergeTreeData::dropAllData()
{
disk->removeRecursive(path);
}
catch (const Poco::FileNotFoundException &)
catch (const fs::filesystem_error & e)
{
if (e.code() == std::errc::no_such_file_or_directory)
{
/// If the file is already deleted, log the error message and do nothing.
tryLogCurrentException(__PRETTY_FUNCTION__);
}
else
throw;
}
}
setDataVolume(0, 0, 0);
@ -1474,8 +1485,8 @@ void MergeTreeData::dropIfEmpty()
for (const auto & [path, disk] : getRelativeDataPathsWithDisks())
{
/// Non recursive, exception is thrown if there are more files.
disk->removeFileIfExists(path + MergeTreeData::FORMAT_VERSION_FILE_NAME);
disk->removeDirectory(path + MergeTreeData::DETACHED_DIR_NAME);
disk->removeFileIfExists(fs::path(path) / MergeTreeData::FORMAT_VERSION_FILE_NAME);
disk->removeDirectory(fs::path(path) / MergeTreeData::DETACHED_DIR_NAME);
disk->removeDirectory(path);
}
}
@ -1925,7 +1936,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::createPart(
const VolumePtr & volume, const String & relative_path, const IMergeTreeDataPart * parent_part) const
{
MergeTreeDataPartType type;
auto full_path = relative_data_path + (parent_part ? parent_part->relative_path + "/" : "") + relative_path + "/";
auto full_path = fs::path(relative_data_path) / (parent_part ? parent_part->relative_path : "") / relative_path / "";
auto mrk_ext = MergeTreeIndexGranularityInfo::getMarksExtensionFromFilesystem(volume->getDisk(), full_path);
if (mrk_ext)
@ -1978,7 +1989,7 @@ void MergeTreeData::changeSettings(
{
auto disk = new_storage_policy->getDiskByName(disk_name);
disk->createDirectories(relative_data_path);
disk->createDirectories(relative_data_path + MergeTreeData::DETACHED_DIR_NAME);
disk->createDirectories(fs::path(relative_data_path) / MergeTreeData::DETACHED_DIR_NAME);
}
/// FIXME how would that be done while reloading configuration???
@ -2007,7 +2018,7 @@ void MergeTreeData::PartsTemporaryRename::addPart(const String & old_name, const
old_and_new_names.push_back({old_name, new_name});
for (const auto & [path, disk] : storage.getRelativeDataPathsWithDisks())
{
for (auto it = disk->iterateDirectory(path + source_dir); it->isValid(); it->next())
for (auto it = disk->iterateDirectory(fs::path(path) / source_dir); it->isValid(); it->next())
{
if (it->name() == old_name)
{
@ -2029,8 +2040,8 @@ void MergeTreeData::PartsTemporaryRename::tryRenameAll()
if (old_name.empty() || new_name.empty())
throw DB::Exception("Empty part name. Most likely it's a bug.", ErrorCodes::INCORRECT_FILE_NAME);
const auto & [path, disk] = old_part_name_to_path_and_disk[old_name];
const auto full_path = path + source_dir; /// for old_name
disk->moveFile(full_path + old_name, full_path + new_name);
const auto full_path = fs::path(path) / source_dir; /// for old_name
disk->moveFile(fs::path(full_path) / old_name, fs::path(full_path) / new_name);
}
catch (...)
{
@ -2054,8 +2065,8 @@ MergeTreeData::PartsTemporaryRename::~PartsTemporaryRename()
try
{
const auto & [path, disk] = old_part_name_to_path_and_disk[old_name];
const auto full_path = path + source_dir; /// for old_name
disk->moveFile(full_path + new_name, full_path + old_name);
const String full_path = fs::path(path) / source_dir; /// for old_name
disk->moveFile(fs::path(full_path) / new_name, fs::path(full_path) / old_name);
}
catch (...)
{
@ -2752,7 +2763,7 @@ void MergeTreeData::swapActivePart(MergeTreeData::DataPartPtr part_copy)
addPartContributionToDataVolume(part_copy);
auto disk = original_active_part->volume->getDisk();
String marker_path = original_active_part->getFullRelativePath() + IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME;
String marker_path = fs::path(original_active_part->getFullRelativePath()) / IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME;
try
{
disk->createFile(marker_path);
@ -3315,7 +3326,7 @@ MergeTreeData::getDetachedParts() const
for (const auto & [path, disk] : getRelativeDataPathsWithDisks())
{
for (auto it = disk->iterateDirectory(path + MergeTreeData::DETACHED_DIR_NAME); it->isValid(); it->next())
for (auto it = disk->iterateDirectory(fs::path(path) / MergeTreeData::DETACHED_DIR_NAME); it->isValid(); it->next())
{
res.emplace_back();
auto & part = res.back();
@ -3370,7 +3381,7 @@ void MergeTreeData::dropDetached(const ASTPtr & partition, bool part, ContextPtr
for (auto & [old_name, new_name] : renamed_parts.old_and_new_names)
{
const auto & [path, disk] = renamed_parts.old_part_name_to_path_and_disk[old_name];
disk->removeRecursive(path + "detached/" + new_name + "/");
disk->removeRecursive(fs::path(path) / "detached" / new_name / "");
LOG_DEBUG(log, "Dropped detached part {}", old_name);
old_name.clear();
}
@ -3431,8 +3442,8 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const
if (!containing_part.empty() && containing_part != name)
// TODO maybe use PartsTemporaryRename here?
disk->moveDirectory(relative_data_path + source_dir + name,
relative_data_path + source_dir + "inactive_" + name);
disk->moveDirectory(fs::path(relative_data_path) / source_dir / name,
fs::path(relative_data_path) / source_dir / ("inactive_" + name));
else
renamed_parts.addPart(name, "attaching_" + name);
}
@ -4294,12 +4305,12 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::cloneAndLoadDataPartOnSameDisk(
const auto & src_relative_data_path = src_part_in_memory->storage.relative_data_path;
auto flushed_part_path = src_part_in_memory->getRelativePathForPrefix(tmp_part_prefix);
src_part_in_memory->flushToDisk(src_relative_data_path, flushed_part_path, metadata_snapshot);
src_part_path = src_relative_data_path + flushed_part_path + "/";
src_part_path = fs::path(src_relative_data_path) / flushed_part_path / "";
}
LOG_DEBUG(log, "Cloning part {} to {}", fullPath(disk, src_part_path), fullPath(disk, dst_part_path));
localBackup(disk, src_part_path, dst_part_path);
disk->removeFileIfExists(dst_part_path + "/" + IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME);
disk->removeFileIfExists(fs::path(dst_part_path) / IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME);
auto single_disk_volume = std::make_shared<SingleDiskVolume>(disk->getName(), disk, 0);
auto dst_data_part = createPart(dst_part_name, dst_part_info, single_disk_volume, tmp_dst_part_name);
@ -4411,10 +4422,10 @@ PartitionCommandsResultInfo MergeTreeData::freezePartitionsByMatcher(
const String & with_name,
ContextPtr local_context)
{
String clickhouse_path = Poco::Path(local_context->getPath()).makeAbsolute().toString();
String default_shadow_path = clickhouse_path + "shadow/";
Poco::File(default_shadow_path).createDirectories();
auto increment = Increment(default_shadow_path + "increment.txt").get(true);
String clickhouse_path = fs::canonical(local_context->getPath());
String default_shadow_path = fs::path(clickhouse_path) / "shadow/";
fs::create_directories(default_shadow_path);
auto increment = Increment(fs::path(default_shadow_path) / "increment.txt").get(true);
const String shadow_path = "shadow/";
@ -4422,7 +4433,7 @@ PartitionCommandsResultInfo MergeTreeData::freezePartitionsByMatcher(
const auto data_parts = getDataParts();
String backup_name = (!with_name.empty() ? escapeForFileName(with_name) : toString(increment));
String backup_path = shadow_path + backup_name + "/";
String backup_path = fs::path(shadow_path) / backup_name / "";
for (const auto & disk : getStoragePolicy()->getDisks())
disk->onFreeze(backup_path);
@ -4439,20 +4450,20 @@ PartitionCommandsResultInfo MergeTreeData::freezePartitionsByMatcher(
part->volume->getDisk()->createDirectories(backup_path);
String backup_part_path = backup_path + relative_data_path + part->relative_path;
String backup_part_path = fs::path(backup_path) / relative_data_path / part->relative_path;
if (auto part_in_memory = asInMemoryPart(part))
part_in_memory->flushToDisk(backup_path + relative_data_path, part->relative_path, metadata_snapshot);
part_in_memory->flushToDisk(fs::path(backup_path) / relative_data_path, part->relative_path, metadata_snapshot);
else
localBackup(part->volume->getDisk(), part->getFullRelativePath(), backup_part_path);
part->volume->getDisk()->removeFileIfExists(backup_part_path + "/" + IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME);
part->volume->getDisk()->removeFileIfExists(fs::path(backup_part_path) / IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME);
part->is_frozen.store(true, std::memory_order_relaxed);
result.push_back(PartitionCommandResultInfo{
.partition_id = part->info.partition_id,
.part_name = part->name,
.backup_path = part->volume->getDisk()->getPath() + backup_path,
.part_backup_path = part->volume->getDisk()->getPath() + backup_part_path,
.backup_path = fs::path(part->volume->getDisk()->getPath()) / backup_path,
.part_backup_path = fs::path(part->volume->getDisk()->getPath()) / backup_part_path,
.backup_name = backup_name,
});
++parts_processed;
@ -4481,7 +4492,7 @@ PartitionCommandsResultInfo MergeTreeData::unfreezeAll(
PartitionCommandsResultInfo MergeTreeData::unfreezePartitionsByMatcher(MatcherFn matcher, const String & backup_name, ContextPtr)
{
auto backup_path = std::filesystem::path("shadow") / escapeForFileName(backup_name) / relative_data_path;
auto backup_path = fs::path("shadow") / escapeForFileName(backup_name) / relative_data_path;
LOG_DEBUG(log, "Unfreezing parts by path {}", backup_path.generic_string());

6
src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp
View File

@ -2268,7 +2268,7 @@ void MergeTreeDataMergerMutator::finalizeMutatedPart(
if (need_remove_expired_values)
{
/// Write a file with ttl infos in json format.
auto out_ttl = disk->writeFile(new_data_part->getFullRelativePath() + "ttl.txt", 4096);
auto out_ttl = disk->writeFile(fs::path(new_data_part->getFullRelativePath()) / "ttl.txt", 4096);
HashingWriteBuffer out_hashing(*out_ttl);
new_data_part->ttl_infos.write(out_hashing);
new_data_part->checksums.files["ttl.txt"].file_size = out_hashing.count();
@ -2277,7 +2277,7 @@ void MergeTreeDataMergerMutator::finalizeMutatedPart(
{
/// Write file with checksums.
auto out_checksums = disk->writeFile(new_data_part->getFullRelativePath() + "checksums.txt", 4096);
auto out_checksums = disk->writeFile(fs::path(new_data_part->getFullRelativePath()) / "checksums.txt", 4096);
new_data_part->checksums.write(*out_checksums);
} /// close fd
@ -2288,7 +2288,7 @@ void MergeTreeDataMergerMutator::finalizeMutatedPart(
{
/// Write a file with a description of columns.
auto out_columns = disk->writeFile(new_data_part->getFullRelativePath() + "columns.txt", 4096);
auto out_columns = disk->writeFile(fs::path(new_data_part->getFullRelativePath()) / "columns.txt", 4096);
new_data_part->getColumns().writeText(*out_columns);
} /// close fd

1
src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp
View File

@ -7,7 +7,6 @@
#include <IO/WriteBufferFromString.h>
#include <Compression/CompressedReadBuffer.h>
#include <Compression/CompressedWriteBuffer.h>
#include <Poco/File.h>
namespace DB

1
src/Storages/MergeTree/MergeTreeDataPartCompact.cpp
View File

@ -2,7 +2,6 @@
#include <DataTypes/NestedUtils.h>
#include <Storages/MergeTree/MergeTreeReaderCompact.h>
#include <Storages/MergeTree/MergeTreeDataPartWriterCompact.h>
#include <Poco/File.h>
namespace DB

1
src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp
View File

@ -5,7 +5,6 @@
#include <Storages/MergeTree/IMergeTreeReader.h>
#include <DataTypes/NestedUtils.h>
#include <Interpreters/Context.h>
#include <Poco/File.h>
#include <Poco/Logger.h>
#include <common/logger_useful.h>

1
src/Storages/MergeTree/MergeTreeDataPartWide.cpp
View File

@ -1,5 +1,4 @@
#include "MergeTreeDataPartWide.h"
#include <Poco/File.h>
#include <Storages/MergeTree/MergeTreeReaderWide.h>
#include <Storages/MergeTree/MergeTreeDataPartWriterWide.h>
#include <Storages/MergeTree/IMergeTreeDataPartWriter.h>

2
src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
View File

@ -3,8 +3,6 @@
#include <optional>
#include <unordered_set>
#include <Poco/File.h>
#include <Storages/MergeTree/MergeTreeDataSelectExecutor.h>
#include <Storages/MergeTree/MergeTreeSelectProcessor.h>
#include <Storages/MergeTree/MergeTreeReverseSelectProcessor.h>

1
src/Storages/MergeTree/MergeTreeDataWriter.cpp
View File

@ -11,7 +11,6 @@
#include <DataTypes/DataTypeDateTime.h>
#include <DataTypes/DataTypeDate.h>
#include <IO/WriteHelpers.h>
#include <Poco/File.h>
#include <Common/typeid_cast.h>
#include <DataStreams/ITTLAlgorithm.h>
#include <DataStreams/OneBlockInputStream.h>

7
src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp
View File

@ -1,6 +1,8 @@
#include <Storages/MergeTree/MergeTreeIndexGranularityInfo.h>
#include <Storages/MergeTree/MergeTreeData.h>
#include <Poco/Path.h>
namespace fs = std::filesystem;
namespace DB
{
@ -17,8 +19,7 @@ std::optional<std::string> MergeTreeIndexGranularityInfo::getMarksExtensionFromF
{
for (DiskDirectoryIteratorPtr it = disk->iterateDirectory(path_to_part); it->isValid(); it->next())
{
Poco::Path path(it->path());
const auto & ext = "." + path.getExtension();
const auto & ext = fs::path(it->path()).extension();
if (ext == getNonAdaptiveMrkExtension()
|| ext == getAdaptiveMrkExtension(MergeTreeDataPartType::WIDE)
|| ext == getAdaptiveMrkExtension(MergeTreeDataPartType::COMPACT))

1
src/Storages/MergeTree/MergeTreeMarksLoader.cpp
View File

@ -1,7 +1,6 @@
#include <Storages/MergeTree/MergeTreeMarksLoader.h>
#include <Storages/MergeTree/MergeTreeData.h>
#include <IO/ReadBufferFromFile.h>
#include <Poco/File.h>
#include <utility>

3
src/Storages/MergeTree/MergeTreeMutationEntry.cpp
View File

@ -4,9 +4,6 @@
#include <IO/ReadBufferFromFile.h>
#include <IO/ReadBufferFromString.h>
#include <Poco/File.h>
#include <Poco/Path.h>
#include <utility>

12
src/Storages/MergeTree/MergeTreePartsMover.cpp
View File

@ -206,18 +206,18 @@ MergeTreeData::DataPartPtr MergeTreePartsMover::clonePart(const MergeTreeMoveEnt
/// Try to fetch part from S3 without copy and fallback to default copy
/// if it's not possible
moving_part.part->assertOnDisk();
String path_to_clone = data->getRelativeDataPath() + directory_to_move + "/";
String path_to_clone = fs::path(data->getRelativeDataPath()) / directory_to_move / "";
String relative_path = part->relative_path;
if (disk->exists(path_to_clone + relative_path))
{
LOG_WARNING(log, "Path " + fullPath(disk, path_to_clone + relative_path) + " already exists. Will remove it and clone again.");
disk->removeRecursive(path_to_clone + relative_path + "/");
disk->removeRecursive(fs::path(path_to_clone) / relative_path / "");
}
disk->createDirectories(path_to_clone);
bool is_fetched = data->tryToFetchIfShared(*part, disk, path_to_clone + "/" + part->name);
bool is_fetched = data->tryToFetchIfShared(*part, disk, fs::path(path_to_clone) / part->name);
if (!is_fetched)
part->volume->getDisk()->copy(data->getRelativeDataPath() + relative_path + "/", disk, path_to_clone);
part->volume->getDisk()->removeFileIfExists(path_to_clone + "/" + IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME);
part->volume->getDisk()->copy(fs::path(data->getRelativeDataPath()) / relative_path / "", disk, path_to_clone);
part->volume->getDisk()->removeFileIfExists(fs::path(path_to_clone) / IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME);
}
else
{
@ -226,7 +226,7 @@ MergeTreeData::DataPartPtr MergeTreePartsMover::clonePart(const MergeTreeMoveEnt
auto single_disk_volume = std::make_shared<SingleDiskVolume>("volume_" + part->name, moving_part.reserved_space->getDisk(), 0);
MergeTreeData::MutableDataPartPtr cloned_part =
data->createPart(part->name, single_disk_volume, directory_to_move + '/' + part->name);
data->createPart(part->name, single_disk_volume, fs::path(directory_to_move) / part->name);
LOG_TRACE(log, "Part {} was cloned to {}", part->name, cloned_part->getFullPath());
cloned_part->loadColumnsChecksumsIndexes(true, true);

1
src/Storages/MergeTree/MergeTreeReaderCompact.cpp
View File

@ -2,7 +2,6 @@
#include <Storages/MergeTree/MergeTreeDataPartCompact.h>
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/NestedUtils.h>
#include <Poco/File.h>
namespace DB
{

1
src/Storages/MergeTree/MergeTreeReaderInMemory.cpp
View File

@ -3,7 +3,6 @@
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/NestedUtils.h>
#include <Columns/ColumnArray.h>
#include <Poco/File.h>
namespace DB
{

11
src/Storages/MergeTree/MergedBlockOutputStream.cpp
View File

@ -1,6 +1,5 @@
#include <Storages/MergeTree/MergedBlockOutputStream.h>
#include <Interpreters/Context.h>
#include <Poco/File.h>
#include <Parsers/queryToString.h>
@ -123,7 +122,7 @@ void MergedBlockOutputStream::finalizePartOnDisk(
{
if (new_part->uuid != UUIDHelpers::Nil)
{
auto out = volume->getDisk()->writeFile(part_path + IMergeTreeDataPart::UUID_FILE_NAME, 4096);
auto out = volume->getDisk()->writeFile(fs::path(part_path) / IMergeTreeDataPart::UUID_FILE_NAME, 4096);
HashingWriteBuffer out_hashing(*out);
writeUUIDText(new_part->uuid, out_hashing);
checksums.files[IMergeTreeDataPart::UUID_FILE_NAME].file_size = out_hashing.count();
@ -142,7 +141,7 @@ void MergedBlockOutputStream::finalizePartOnDisk(
throw Exception("MinMax index was not initialized for new non-empty part " + new_part->name
+ ". It is a bug.", ErrorCodes::LOGICAL_ERROR);
auto count_out = volume->getDisk()->writeFile(part_path + "count.txt", 4096);
auto count_out = volume->getDisk()->writeFile(fs::path(part_path) / "count.txt", 4096);
HashingWriteBuffer count_out_hashing(*count_out);
writeIntText(rows_count, count_out_hashing);
count_out_hashing.next();
@ -157,7 +156,7 @@ void MergedBlockOutputStream::finalizePartOnDisk(
if (!new_part->ttl_infos.empty())
{
/// Write a file with ttl infos in json format.
auto out = volume->getDisk()->writeFile(part_path + "ttl.txt", 4096);
auto out = volume->getDisk()->writeFile(fs::path(part_path) / "ttl.txt", 4096);
HashingWriteBuffer out_hashing(*out);
new_part->ttl_infos.write(out_hashing);
checksums.files["ttl.txt"].file_size = out_hashing.count();
@ -171,7 +170,7 @@ void MergedBlockOutputStream::finalizePartOnDisk(
{
/// Write a file with a description of columns.
auto out = volume->getDisk()->writeFile(part_path + "columns.txt", 4096);
auto out = volume->getDisk()->writeFile(fs::path(part_path) / "columns.txt", 4096);
part_columns.writeText(*out);
out->finalize();
if (sync)
@ -192,7 +191,7 @@ void MergedBlockOutputStream::finalizePartOnDisk(
{
/// Write file with checksums.
auto out = volume->getDisk()->writeFile(part_path + "checksums.txt", 4096);
auto out = volume->getDisk()->writeFile(fs::path(part_path) / "checksums.txt", 4096);
checksums.write(*out);
out->finalize();
if (sync)

52
src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp
View File

@ -62,7 +62,7 @@ bool ReplicatedMergeTreeQueue::isVirtualPart(const MergeTreeData::DataPartPtr &
bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper)
{
auto queue_path = replica_path + "/queue";
String queue_path = fs::path(replica_path) / "queue";
LOG_DEBUG(log, "Loading queue from {}", queue_path);
bool updated = false;
@ -74,7 +74,7 @@ bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper)
/// Reset batch size on initialization to recover from possible errors of too large batch size.
current_multi_batch_size = 1;
String log_pointer_str = zookeeper->get(replica_path + "/log_pointer");
String log_pointer_str = zookeeper->get(fs::path(replica_path) / "log_pointer");
log_pointer = log_pointer_str.empty() ? 0 : parse<UInt64>(log_pointer_str);
std::unordered_set<String> already_loaded_paths;
@ -101,7 +101,7 @@ bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper)
futures.reserve(children.size());
for (const String & child : children)
futures.emplace_back(child, zookeeper->asyncGet(queue_path + "/" + child));
futures.emplace_back(child, zookeeper->asyncGet(fs::path(queue_path) / child));
for (auto & future : futures)
{
@ -116,7 +116,7 @@ bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper)
updated = true;
}
zookeeper->tryGet(replica_path + "/mutation_pointer", mutation_pointer);
zookeeper->tryGet(fs::path(replica_path) / "mutation_pointer", mutation_pointer);
}
updateTimesInZooKeeper(zookeeper, min_unprocessed_insert_time_changed, {});
@ -399,7 +399,7 @@ void ReplicatedMergeTreeQueue::removeProcessedEntry(zkutil::ZooKeeperPtr zookeep
if (!need_remove_from_zk)
return;
auto code = zookeeper->tryRemove(replica_path + "/queue/" + entry->znode_name);
auto code = zookeeper->tryRemove(fs::path(replica_path) / "queue" / entry->znode_name);
if (code != Coordination::Error::ZOK)
LOG_ERROR(log, "Couldn't remove {}/queue/{}: {}. This shouldn't happen often.", replica_path, entry->znode_name, Coordination::errorMessage(code));
@ -455,7 +455,7 @@ bool ReplicatedMergeTreeQueue::remove(zkutil::ZooKeeperPtr zookeeper, const Stri
notifySubscribers(queue_size);
zookeeper->tryRemove(replica_path + "/queue/" + found->znode_name);
zookeeper->tryRemove(fs::path(replica_path) / "queue" / found->znode_name);
updateTimesInZooKeeper(zookeeper, min_unprocessed_insert_time_changed, max_processed_insert_time_changed);
return true;
@ -474,14 +474,14 @@ int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper
if (pull_log_blocker.isCancelled())
throw Exception("Log pulling is cancelled", ErrorCodes::ABORTED);
String index_str = zookeeper->get(replica_path + "/log_pointer");
String index_str = zookeeper->get(fs::path(replica_path) / "log_pointer");
UInt64 index;
/// The version of "/log" is modified when new entries to merge/mutate/drop appear.
Coordination::Stat stat;
zookeeper->get(zookeeper_path + "/log", &stat);
zookeeper->get(fs::path(zookeeper_path) / "log", &stat);
Strings log_entries = zookeeper->getChildrenWatch(zookeeper_path + "/log", nullptr, watch_callback);
Strings log_entries = zookeeper->getChildrenWatch(fs::path(zookeeper_path) / "log", nullptr, watch_callback);
/// We update mutations after we have loaded the list of log entries, but before we insert them
/// in the queue.
@ -494,7 +494,7 @@ int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper
/// If we do not already have a pointer to the log, put a pointer to the first entry in it.
index = log_entries.empty() ? 0 : parse<UInt64>(std::min_element(log_entries.begin(), log_entries.end())->substr(strlen("log-")));
zookeeper->set(replica_path + "/log_pointer", toString(index));
zookeeper->set(fs::path(replica_path) / "log_pointer", toString(index));
}
else
{
@ -541,7 +541,7 @@ int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper
futures.reserve(end - begin);
for (auto it = begin; it != end; ++it)
futures.emplace_back(*it, zookeeper->asyncGet(zookeeper_path + "/log/" + *it));
futures.emplace_back(*it, zookeeper->asyncGet(fs::path(zookeeper_path) / "log" / *it));
/// Simultaneously add all new entries to the queue and move the pointer to the log.
@ -558,7 +558,7 @@ int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper
copied_entries.emplace_back(LogEntry::parse(res.data, res.stat));
ops.emplace_back(zkutil::makeCreateRequest(
replica_path + "/queue/queue-", res.data, zkutil::CreateMode::PersistentSequential));
fs::path(replica_path) / "queue/queue-", res.data, zkutil::CreateMode::PersistentSequential));
const auto & entry = *copied_entries.back();
if (entry.type == LogEntry::GET_PART || entry.type == LogEntry::ATTACH_PART)
@ -573,11 +573,11 @@ int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper
}
ops.emplace_back(zkutil::makeSetRequest(
replica_path + "/log_pointer", toString(last_entry_index + 1), -1));
fs::path(replica_path) / "log_pointer", toString(last_entry_index + 1), -1));
if (min_unprocessed_insert_time_changed)
ops.emplace_back(zkutil::makeSetRequest(
replica_path + "/min_unprocessed_insert_time", toString(*min_unprocessed_insert_time_changed), -1));
fs::path(replica_path) / "min_unprocessed_insert_time", toString(*min_unprocessed_insert_time_changed), -1));
auto responses = zookeeper->multi(ops);
@ -655,7 +655,7 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C
{
std::lock_guard lock(update_mutations_mutex);
Strings entries_in_zk = zookeeper->getChildrenWatch(zookeeper_path + "/mutations", nullptr, watch_callback);
Strings entries_in_zk = zookeeper->getChildrenWatch(fs::path(zookeeper_path) / "mutations", nullptr, watch_callback);
StringSet entries_in_zk_set(entries_in_zk.begin(), entries_in_zk.end());
/// Compare with the local state, delete obsolete entries and determine which new entries to load.
@ -712,7 +712,7 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C
std::vector<std::future<Coordination::GetResponse>> futures;
for (const String & entry : entries_to_load)
futures.emplace_back(zookeeper->asyncGet(zookeeper_path + "/mutations/" + entry));
futures.emplace_back(zookeeper->asyncGet(fs::path(zookeeper_path) / "mutations" / entry));
std::vector<ReplicatedMergeTreeMutationEntryPtr> new_mutations;
for (size_t i = 0; i < entries_to_load.size(); ++i)
@ -796,7 +796,7 @@ ReplicatedMergeTreeMutationEntryPtr ReplicatedMergeTreeQueue::removeMutation(
{
std::lock_guard lock(update_mutations_mutex);
auto rc = zookeeper->tryRemove(zookeeper_path + "/mutations/" + mutation_id);
auto rc = zookeeper->tryRemove(fs::path(zookeeper_path) / "mutations" / mutation_id);
if (rc == Coordination::Error::ZOK)
LOG_DEBUG(log, "Removed mutation {} from ZooKeeper.", mutation_id);
@ -940,12 +940,12 @@ void ReplicatedMergeTreeQueue::removePartProducingOpsInRange(
{
if ((*it)->currently_executing)
to_wait.push_back(*it);
auto code = zookeeper->tryRemove(replica_path + "/queue/" + (*it)->znode_name);
auto code = zookeeper->tryRemove(fs::path(replica_path) / "queue" / (*it)->znode_name);
/// FIXME it's probably unsafe to remove entries non-atomically
/// when this method called directly from alter query (not from replication queue task),
/// because entries will be lost if ALTER fails.
if (code != Coordination::Error::ZOK)
LOG_INFO(log, "Couldn't remove {}: {}", replica_path + "/queue/" + (*it)->znode_name, Coordination::errorMessage(code));
LOG_INFO(log, "Couldn't remove {}: {}", (fs::path(replica_path) / "queue" / (*it)->znode_name).string(), Coordination::errorMessage(code));
updateStateOnQueueEntryRemoval(
*it, /* is_successful = */ false,
@ -1554,7 +1554,7 @@ bool ReplicatedMergeTreeQueue::tryFinalizeMutations(zkutil::ZooKeeperPtr zookeep
if (!finished.empty())
{
zookeeper->set(replica_path + "/mutation_pointer", finished.back()->znode_name);
zookeeper->set(fs::path(replica_path) / "mutation_pointer", finished.back()->znode_name);
std::lock_guard lock(state_mutex);
@ -1750,22 +1750,22 @@ ReplicatedMergeTreeMergePredicate::ReplicatedMergeTreeMergePredicate(
}
/// Load current quorum status.
auto quorum_status_future = zookeeper->asyncTryGet(queue.zookeeper_path + "/quorum/status");
auto quorum_status_future = zookeeper->asyncTryGet(fs::path(queue.zookeeper_path) / "quorum" / "status");
/// Load current inserts
std::unordered_set<String> lock_holder_paths;
for (const String & entry : zookeeper->getChildren(queue.zookeeper_path + "/temp"))
for (const String & entry : zookeeper->getChildren(fs::path(queue.zookeeper_path) / "temp"))
{
if (startsWith(entry, "abandonable_lock-"))
lock_holder_paths.insert(queue.zookeeper_path + "/temp/" + entry);
lock_holder_paths.insert(fs::path(queue.zookeeper_path) / "temp" / entry);
}
if (!lock_holder_paths.empty())
{
Strings partitions = zookeeper->getChildren(queue.zookeeper_path + "/block_numbers");
Strings partitions = zookeeper->getChildren(fs::path(queue.zookeeper_path) / "block_numbers");
std::vector<std::future<Coordination::ListResponse>> lock_futures;
for (const String & partition : partitions)
lock_futures.push_back(zookeeper->asyncGetChildren(queue.zookeeper_path + "/block_numbers/" + partition));
lock_futures.push_back(zookeeper->asyncGetChildren(fs::path(queue.zookeeper_path) / "block_numbers" / partition));
struct BlockInfoInZooKeeper
{
@ -1786,7 +1786,7 @@ ReplicatedMergeTreeMergePredicate::ReplicatedMergeTreeMergePredicate(
if (startsWith(entry, "block-"))
{
Int64 block_number = parse<Int64>(entry.substr(strlen("block-")));
String zk_path = queue.zookeeper_path + "/block_numbers/" + partitions[i] + "/" + entry;
String zk_path = fs::path(queue.zookeeper_path) / "block_numbers" / partitions[i] / entry;
block_infos.emplace_back(
BlockInfoInZooKeeper{partitions[i], block_number, zk_path, zookeeper->asyncTryGet(zk_path)});
}

12
src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp
View File

@ -238,7 +238,7 @@ void ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart()
auto zookeeper = storage.getZooKeeper();
String quorum_str;
if (zookeeper->tryGet(storage.zookeeper_path + "/quorum/status", quorum_str))
if (zookeeper->tryGet(fs::path(storage.zookeeper_path) / "quorum" / "status", quorum_str))
{
ReplicatedMergeTreeQuorumEntry quorum_entry(quorum_str);
@ -251,12 +251,12 @@ void ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart()
}
Strings part_names;
String parallel_quorum_parts_path = storage.zookeeper_path + "/quorum/parallel";
String parallel_quorum_parts_path = fs::path(storage.zookeeper_path) / "quorum" / "parallel";
if (zookeeper->tryGetChildren(parallel_quorum_parts_path, part_names) == Coordination::Error::ZOK)
{
for (auto & part_name : part_names)
{
if (zookeeper->tryGet(parallel_quorum_parts_path + "/" + part_name, quorum_str))
if (zookeeper->tryGet(fs::path(parallel_quorum_parts_path) / part_name, quorum_str))
{
ReplicatedMergeTreeQuorumEntry quorum_entry(quorum_str);
if (!quorum_entry.replicas.count(storage.replica_name)
@ -278,7 +278,7 @@ void ReplicatedMergeTreeRestartingThread::activateReplica()
/// How other replicas can access this one.
ReplicatedMergeTreeAddress address = storage.getReplicatedMergeTreeAddress();
String is_active_path = storage.replica_path + "/is_active";
String is_active_path = fs::path(storage.replica_path) / "is_active";
/** If the node is marked as active, but the mark is made in the same instance, delete it.
* This is possible only when session in ZooKeeper expires.
@ -302,7 +302,7 @@ void ReplicatedMergeTreeRestartingThread::activateReplica()
/// Simultaneously declare that this replica is active, and update the host.
Coordination::Requests ops;
ops.emplace_back(zkutil::makeCreateRequest(is_active_path, active_node_identifier, zkutil::CreateMode::Ephemeral));
ops.emplace_back(zkutil::makeSetRequest(storage.replica_path + "/host", address.toString(), -1));
ops.emplace_back(zkutil::makeSetRequest(fs::path(storage.replica_path) / "host", address.toString(), -1));
try
{
@ -311,7 +311,7 @@ void ReplicatedMergeTreeRestartingThread::activateReplica()
catch (const Coordination::Exception & e)
{
String existing_replica_host;
zookeeper->tryGet(storage.replica_path + "/host", existing_replica_host);
zookeeper->tryGet(fs::path(storage.replica_path) / "host", existing_replica_host);
if (existing_replica_host.empty())
existing_replica_host = "without host node";

7
src/Storages/MergeTree/checkDataPart.cpp
View File

@ -1,7 +1,6 @@
#include <algorithm>
#include <optional>
#include <Poco/File.h>
#include <Poco/DirectoryIterator.h>
#include <Storages/MergeTree/MergeTreeIndexGranularity.h>
@ -70,7 +69,7 @@ IMergeTreeDataPart::Checksums checkDataPart(
NamesAndTypesList columns_txt;
{
auto buf = disk->readFile(path + "columns.txt");
auto buf = disk->readFile(fs::path(path) / "columns.txt");
columns_txt.readText(*buf);
assertEOF(*buf);
}
@ -231,9 +230,9 @@ IMergeTreeDataPart::Checksums checkDataPart(
/// Checksums from the rest files listed in checksums.txt. May be absent. If present, they are subsequently compared with the actual data checksums.
IMergeTreeDataPart::Checksums checksums_txt;
if (require_checksums || disk->exists(path + "checksums.txt"))
if (require_checksums || disk->exists(fs::path(path) / "checksums.txt"))
{
auto buf = disk->readFile(path + "checksums.txt");
auto buf = disk->readFile(fs::path(path) / "checksums.txt");
checksums_txt.read(*buf);
assertEOF(*buf);
}

Some files were not shown because too many files have changed in this diff Show More