thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-24 08:32:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix clang-tidy

Browse Source
This commit is contained in:
Alexey Milovidov 2022-08-20 17:09:20 +02:00
parent c5a7e75c88
commit 74e1f4dc61
58 changed files with 163 additions and 1394 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
base/base/ReplxxLineReader.cpp
View File

@ -27,7 +27,7 @@ void trim(String & s)
std::string getEditor()
{
const char * editor = std::getenv("EDITOR");
const char * editor = std::getenv("EDITOR"); // NOLINT
if (!editor || !*editor)
editor = "vim";
@ -76,7 +76,7 @@ void convertHistoryFile(const std::string & path, replxx::Replxx & rx)
if (!in)
{
rx.print("Cannot open %s reading (for conversion): %s\n",
path.c_str(), errnoToString(errno).c_str());
path.c_str(), errnoToString().c_str());
return;
}
@ -84,7 +84,7 @@ void convertHistoryFile(const std::string & path, replxx::Replxx & rx)
if (getline(in, line).bad())
{
rx.print("Cannot read from %s (for conversion): %s\n",
path.c_str(), errnoToString(errno).c_str());
path.c_str(), errnoToString().c_str());
return;
}
@ -113,7 +113,7 @@ void convertHistoryFile(const std::string & path, replxx::Replxx & rx)
if (!out)
{
rx.print("Cannot open %s for writing (for conversion): %s\n",
path.c_str(), errnoToString(errno).c_str());
path.c_str(), errnoToString().c_str());
return;
}
@ -151,7 +151,7 @@ ReplxxLineReader::ReplxxLineReader(
history_file_fd = open(history_file_path.c_str(), O_RDWR);
if (history_file_fd < 0)
{
rx.print("Open of history file failed: %s\n", errnoToString(errno).c_str());
rx.print("Open of history file failed: %s\n", errnoToString().c_str());
}
else
{
@ -159,18 +159,18 @@ ReplxxLineReader::ReplxxLineReader(
if (flock(history_file_fd, LOCK_SH))
{
rx.print("Shared lock of history file failed: %s\n", errnoToString(errno).c_str());
rx.print("Shared lock of history file failed: %s\n", errnoToString().c_str());
}
else
{
if (!rx.history_load(history_file_path))
{
rx.print("Loading history failed: %s\n", errnoToString(errno).c_str());
rx.print("Loading history failed: %s\n", errnoToString().c_str());
}
if (flock(history_file_fd, LOCK_UN))
{
rx.print("Unlock of history file failed: %s\n", errnoToString(errno).c_str());
rx.print("Unlock of history file failed: %s\n", errnoToString().c_str());
}
}
}
@ -225,7 +225,7 @@ ReplxxLineReader::ReplxxLineReader(
ReplxxLineReader::~ReplxxLineReader()
{
if (close(history_file_fd))
rx.print("Close of history file failed: %s\n", errnoToString(errno).c_str());
rx.print("Close of history file failed: %s\n", errnoToString().c_str());
}
LineReader::InputStatus ReplxxLineReader::readOneLine(const String & prompt)
@ -250,7 +250,7 @@ void ReplxxLineReader::addToHistory(const String & line)
// and that is why flock() is added here.
bool locked = false;
if (flock(history_file_fd, LOCK_EX))
rx.print("Lock of history file failed: %s\n", errnoToString(errno).c_str());
rx.print("Lock of history file failed: %s\n", errnoToString().c_str());
else
locked = true;
@ -258,10 +258,10 @@ void ReplxxLineReader::addToHistory(const String & line)
// flush changes to the disk
if (!rx.history_save(history_file_path))
rx.print("Saving history failed: %s\n", errnoToString(errno).c_str());
rx.print("Saving history failed: %s\n", errnoToString().c_str());
if (locked && 0 != flock(history_file_fd, LOCK_UN))
rx.print("Unlock of history file failed: %s\n", errnoToString(errno).c_str());
rx.print("Unlock of history file failed: %s\n", errnoToString().c_str());
}
/// See comments in ShellCommand::executeImpl()
@ -275,7 +275,7 @@ int ReplxxLineReader::executeEditor(const std::string & path)
static void * real_vfork = dlsym(RTLD_DEFAULT, "vfork");
if (!real_vfork)
{
rx.print("Cannot find symbol vfork in myself: %s\n", errnoToString(errno).c_str());
rx.print("Cannot find symbol vfork in myself: %s\n", errnoToString().c_str());
return -1;
}
@ -283,7 +283,7 @@ int ReplxxLineReader::executeEditor(const std::string & path)
if (-1 == pid)
{
rx.print("Cannot vfork: %s\n", errnoToString(errno).c_str());
rx.print("Cannot vfork: %s\n", errnoToString().c_str());
return -1;
}
@ -292,11 +292,11 @@ int ReplxxLineReader::executeEditor(const std::string & path)
{
sigset_t mask;
sigemptyset(&mask);
sigprocmask(0, nullptr, &mask);
sigprocmask(SIG_UNBLOCK, &mask, nullptr);
sigprocmask(0, nullptr, &mask); // NOLINT (ok in newly created process)
sigprocmask(SIG_UNBLOCK, &mask, nullptr); // NOLINT (ok in newly created process)
execvp(editor.c_str(), argv);
rx.print("Cannot execute %s: %s\n", editor.c_str(), errnoToString(errno).c_str());
rx.print("Cannot execute %s: %s\n", editor.c_str(), errnoToString().c_str());
_exit(-1);
}
@ -309,7 +309,7 @@ int ReplxxLineReader::executeEditor(const std::string & path)
if (errno == EINTR)
continue;
rx.print("Cannot waitpid: %s\n", errnoToString(errno).c_str());
rx.print("Cannot waitpid: %s\n", errnoToString().c_str());
return -1;
}
else
@ -324,7 +324,7 @@ void ReplxxLineReader::openEditor()
int fd = ::mkstemps(filename, 4);
if (-1 == fd)
{
rx.print("Cannot create temporary file to edit query: %s\n", errnoToString(errno).c_str());
rx.print("Cannot create temporary file to edit query: %s\n", errnoToString().c_str());
return;
}
@ -338,7 +338,7 @@ void ReplxxLineReader::openEditor()
ssize_t res = ::write(fd, begin + bytes_written, offset - bytes_written);
if ((-1 == res || 0 == res) && errno != EINTR)
{
rx.print("Cannot write to temporary query file %s: %s\n", filename, errnoToString(errno).c_str());
rx.print("Cannot write to temporary query file %s: %s\n", filename, errnoToString().c_str());
break;
}
bytes_written += res;
@ -346,7 +346,7 @@ void ReplxxLineReader::openEditor()
if (0 != ::close(fd))
{
rx.print("Cannot close temporary query file %s: %s\n", filename, errnoToString(errno).c_str());
rx.print("Cannot close temporary query file %s: %s\n", filename, errnoToString().c_str());
return;
}
@ -364,7 +364,7 @@ void ReplxxLineReader::openEditor()
}
catch (...)
{
rx.print("Cannot read from temporary query file %s: %s\n", filename, errnoToString(errno).c_str());
rx.print("Cannot read from temporary query file %s: %s\n", filename, errnoToString().c_str());
return;
}
}
@ -373,7 +373,7 @@ void ReplxxLineReader::openEditor()
enableBracketedPaste();
if (0 != ::unlink(filename))
rx.print("Cannot remove temporary query file %s: %s\n", filename, errnoToString(errno).c_str());
rx.print("Cannot remove temporary query file %s: %s\n", filename, errnoToString().c_str());
}
void ReplxxLineReader::enableBracketedPaste()

6
base/base/errnoToString.cpp
View File

@ -3,10 +3,11 @@
#include <fmt/format.h>
std::string errnoToString(int code, int the_errno)
std::string errnoToString(int the_errno)
{
const size_t buf_size = 128;
char buf[buf_size];
#ifndef _GNU_SOURCE
int rc = strerror_r(the_errno, buf, buf_size);
#ifdef OS_DARWIN
@ -15,7 +16,7 @@ std::string errnoToString(int code, int the_errno)
if (rc != 0)
#endif
{
std::string tmp = std::to_string(code);
std::string tmp = std::to_string(the_errno);
const char * code_str = tmp.c_str();
const char * unknown_message = "Unknown error ";
strcpy(buf, unknown_message);
@ -23,7 +24,6 @@ std::string errnoToString(int code, int the_errno)
}
return fmt::format("errno: {}, strerror: {}", the_errno, buf);
#else
(void)code;
return fmt::format("errno: {}, strerror: {}", the_errno, strerror_r(the_errno, buf, sizeof(buf)));
#endif
}

2
base/base/errnoToString.h
View File

@ -3,4 +3,4 @@
#include <cerrno>
#include <string>
std::string errnoToString(int code, int the_errno = errno);
std::string errnoToString(int the_errno = errno);

4
base/base/setTerminalEcho.cpp
View File

@ -16,7 +16,7 @@ void setTerminalEcho(bool enable)
struct termios tty{};
if (0 != tcgetattr(STDIN_FILENO, &tty))
throw std::runtime_error(std::string("setTerminalEcho failed get: ") + errnoToString(errno));
throw std::runtime_error(std::string("setTerminalEcho failed get: ") + errnoToString());
if (enable)
tty.c_lflag |= ECHO;
@ -24,5 +24,5 @@ void setTerminalEcho(bool enable)
tty.c_lflag &= ~ECHO;
if (0 != tcsetattr(STDIN_FILENO, TCSANOW, &tty))
throw std::runtime_error(std::string("setTerminalEcho failed set: ") + errnoToString(errno));
throw std::runtime_error(std::string("setTerminalEcho failed set: ") + errnoToString());
}

8
programs/benchmark/Benchmark.cpp
View File

@ -120,7 +120,7 @@ public:
void initialize(Poco::Util::Application & self [[maybe_unused]]) override
{
std::string home_path;
const char * home_path_cstr = getenv("HOME");
const char * home_path_cstr = getenv("HOME"); // NOLINT
if (home_path_cstr)
home_path = home_path_cstr;
@ -613,15 +613,15 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv)
std::optional<std::string> env_password_str;
std::optional<std::string> env_quota_key_str;
const char * env_user = getenv("CLICKHOUSE_USER");
const char * env_user = getenv("CLICKHOUSE_USER"); // NOLINT
if (env_user != nullptr)
env_user_str.emplace(std::string(env_user));
const char * env_password = getenv("CLICKHOUSE_PASSWORD");
const char * env_password = getenv("CLICKHOUSE_PASSWORD"); // NOLINT
if (env_password != nullptr)
env_password_str.emplace(std::string(env_password));
const char * env_quota_key = getenv("CLICKHOUSE_QUOTA_KEY");
const char * env_quota_key = getenv("CLICKHOUSE_QUOTA_KEY"); // NOLINT
if (env_quota_key != nullptr)
env_quota_key_str.emplace(std::string(env_quota_key));

12
programs/client/Client.cpp
View File

@ -183,7 +183,7 @@ void Client::initialize(Poco::Util::Application & self)
{
Poco::Util::Application::initialize(self);
const char * home_path_cstr = getenv("HOME");
const char * home_path_cstr = getenv("HOME"); // NOLINT
if (home_path_cstr)
home_path = home_path_cstr;
@ -202,11 +202,11 @@ void Client::initialize(Poco::Util::Application & self)
* may be statically allocated, and can be modified by a subsequent call to getenv(), putenv(3), setenv(3), or unsetenv(3).
*/
const char * env_user = getenv("CLICKHOUSE_USER");
const char * env_user = getenv("CLICKHOUSE_USER"); // NOLINT
if (env_user)
config().setString("user", env_user);
const char * env_password = getenv("CLICKHOUSE_PASSWORD");
const char * env_password = getenv("CLICKHOUSE_PASSWORD"); // NOLINT
if (env_password)
config().setString("password", env_password);
@ -620,7 +620,7 @@ bool Client::processWithFuzzing(const String & full_query)
stderr,
"Found error: IAST::clone() is broken for some AST node. This is a bug. The original AST ('dump before fuzz') and its cloned copy ('dump of cloned AST') refer to the same nodes, which must never happen. This means that their parent node doesn't implement clone() correctly.");
exit(1);
_exit(1);
}
auto fuzzed_text = ast_to_process->formatForErrorMessage();
@ -770,7 +770,7 @@ bool Client::processWithFuzzing(const String & full_query)
fmt::print(stderr, "Text-3 (AST-3 formatted):\n'{}'\n", text_3);
fmt::print(stderr, "Text-3 must be equal to Text-2, but it is not.\n");
exit(1);
_exit(1);
}
}
}
@ -909,7 +909,7 @@ void Client::processOptions(const OptionsDescription & options_description,
auto exit_code = e.code() % 256;
if (exit_code == 0)
exit_code = 255;
exit(exit_code);
_exit(exit_code);
}
}

2
programs/disks/DisksApp.cpp
View File

@ -110,7 +110,7 @@ void DisksApp::init(std::vector<String> & common_arguments)
if (options.count("help"))
{
printHelpMessage(options_description);
exit(0);
exit(0); // NOLINT
}
if (!supported_commands.contains(command_name))

2
programs/install/Install.cpp
View File

@ -708,7 +708,7 @@ int mainEntryClickHouseInstall(int argc, char ** argv)
/// dpkg or apt installers can ask for non-interactive work explicitly.
const char * debian_frontend_var = getenv("DEBIAN_FRONTEND");
const char * debian_frontend_var = getenv("DEBIAN_FRONTEND"); // NOLINT
bool noninteractive = debian_frontend_var && debian_frontend_var == std::string_view("noninteractive");
bool is_interactive = !noninteractive && stdin_is_a_tty && stdout_is_a_tty;

4
programs/main.cpp
View File

@ -366,10 +366,10 @@ void checkHarmfulEnvironmentVariables(char ** argv)
bool require_reexec = false;
for (const auto * var : harmful_env_variables)
{
if (const char * value = getenv(var); value && value[0])
if (const char * value = getenv(var); value && value[0]) // NOLINT
{
/// NOTE: setenv() is used over unsetenv() since unsetenv() marked as harmful
if (setenv(var, "", true))
if (setenv(var, "", true)) // NOLINT (this is safe if not called concurrently)
{
fmt::print(stderr, "Cannot override {} environment variable", var);
_exit(1);

29
programs/server/Server.cpp
View File

@ -170,7 +170,7 @@ int mainEntryClickHouseServer(int argc, char ** argv)
/// Can be overridden by environment variable (cannot use server config at this moment).
if (argc > 0)
{
const char * env_watchdog = getenv("CLICKHOUSE_WATCHDOG_ENABLE");
const char * env_watchdog = getenv("CLICKHOUSE_WATCHDOG_ENABLE"); // NOLINT
if (env_watchdog)
{
if (0 == strcmp(env_watchdog, "1"))
@ -658,6 +658,24 @@ int Server::main(const std::vector<std::string> & /*args*/)
StackTrace::setShowAddresses(config().getBool("show_addresses_in_stack_traces", true));
#if USE_HDFS
/// This will point libhdfs3 to the right location for its config.
/// Note: this has to be done once at server initialization, because 'setenv' is not thread-safe.
String libhdfs3_conf = config().getString("hdfs.libhdfs3_conf", "");
if (!libhdfs3_conf.empty())
{
if (std::filesystem::path{libhdfs3_conf}.is_relative() && !std::filesystem::exists(libhdfs3_conf))
{
const String config_path = config().getString("config-file", "config.xml");
const auto config_dir = std::filesystem::path{config_path}.remove_filename();
if (std::filesystem::exists(config_dir / libhdfs3_conf))
libhdfs3_conf = std::filesystem::absolute(config_dir / libhdfs3_conf);
}
setenv("LIBHDFS3_CONF", libhdfs3_conf.c_str(), true /* overwrite */); // NOLINT
}
#endif
registerFunctions();
registerAggregateFunctions();
registerTableFunctions();
@ -698,8 +716,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
GlobalThreadPool::initialize(
config().getUInt("max_thread_pool_size", 10000),
config().getUInt("max_thread_pool_free_size", 1000),
config().getUInt("thread_pool_queue_size", 10000)
);
config().getUInt("thread_pool_queue_size", 10000));
IOThreadPool::initialize(
config().getUInt("max_io_thread_pool_size", 100),
@ -840,7 +857,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
LOG_TRACE(log, "Will do mlock to prevent executable memory from being paged out. It may take a few seconds.");
if (0 != mlock(addr, len))
LOG_WARNING(log, "Failed mlock: {}", errnoToString(ErrorCodes::SYSTEM_ERROR));
LOG_WARNING(log, "Failed mlock: {}", errnoToString());
else
LOG_TRACE(log, "The memory map of clickhouse executable has been mlock'ed, total {}", ReadableSize(len));
}
@ -908,7 +925,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
rlim.rlim_cur = config().getUInt("max_open_files", rlim.rlim_max);
int rc = setrlimit(RLIMIT_NOFILE, &rlim);
if (rc != 0)
LOG_WARNING(log, "Cannot set max number of file descriptors to {}. Try to specify max_open_files according to your system limits. error: {}", rlim.rlim_cur, strerror(errno));
LOG_WARNING(log, "Cannot set max number of file descriptors to {}. Try to specify max_open_files according to your system limits. error: {}", rlim.rlim_cur, errnoToString());
else
LOG_DEBUG(log, "Set max number of file descriptors to {} (was {}).", rlim.rlim_cur, old);
}
@ -931,7 +948,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
int rc = setrlimit(RLIMIT_NPROC, &rlim);
if (rc != 0)
{
LOG_WARNING(log, "Cannot set max number of threads to {}. error: {}", rlim.rlim_cur, strerror(errno));
LOG_WARNING(log, "Cannot set max number of threads to {}. error: {}", rlim.rlim_cur, errnoToString());
rlim.rlim_cur = old;
}
else

2
programs/static-files-disk-uploader/static-files-disk-uploader.cpp
View File

@ -160,7 +160,7 @@ try
if (options.empty() || options.count("help"))
{
std::cout << description << std::endl;
exit(0);
exit(0); // NOLINT
}
String metadata_path;

2
programs/su/su.cpp
View File

@ -108,7 +108,7 @@ try
if (argc < 3)
{
std::cout << "Usage: ./clickhouse su user:group ..." << std::endl;
exit(0);
exit(0); // NOLINT
}
std::string_view user_and_group = argv[1];

2
src/Bridge/IBridge.cpp
View File

@ -179,7 +179,7 @@ void IBridge::initialize(Application & self)
limit.rlim_max = limit.rlim_cur = gb;
if (setrlimit(RLIMIT_RSS, &limit))
LOG_WARNING(log, "Unable to set maximum RSS to 1GB: {} (current rlim_cur={}, rlim_max={})",
errnoToString(errno), limit.rlim_cur, limit.rlim_max);
errnoToString(), limit.rlim_cur, limit.rlim_max);
if (!getrlimit(RLIMIT_RSS, &limit))
LOG_INFO(log, "RSS limit: cur={}, max={}", limit.rlim_cur, limit.rlim_max);

10
src/Client/ClientBase.cpp
View File

@ -1952,7 +1952,7 @@ void ClientBase::runInteractive()
if (home_path.empty())
{
const char * home_path_cstr = getenv("HOME");
const char * home_path_cstr = getenv("HOME"); // NOLINT
if (home_path_cstr)
home_path = home_path_cstr;
}
@ -1962,7 +1962,7 @@ void ClientBase::runInteractive()
history_file = config().getString("history_file");
else
{
auto * history_file_from_env = getenv("CLICKHOUSE_HISTORY_FILE");
auto * history_file_from_env = getenv("CLICKHOUSE_HISTORY_FILE"); // NOLINT
if (history_file_from_env)
history_file = history_file_from_env;
else if (!home_path.empty())
@ -2260,13 +2260,13 @@ void ClientBase::init(int argc, char ** argv)
if (options.count("version") || options.count("V"))
{
showClientVersion();
exit(0);
exit(0); // NOLINT
}
if (options.count("version-clean"))
{
std::cout << VERSION_STRING;
exit(0);
exit(0); // NOLINT
}
/// Output of help message.
@ -2274,7 +2274,7 @@ void ClientBase::init(int argc, char ** argv)
|| (options.count("host") && options["host"].as<std::string>() == "elp")) /// If user writes -help instead of --help.
{
printHelpMessage(options_description);
exit(0);
exit(0); // NOLINT
}
/// Common options for clickhouse-client and clickhouse-local.

2
src/Common/Config/ConfigProcessor.cpp
View File

@ -419,7 +419,7 @@ void ConfigProcessor::doIncludesRecursive(
XMLDocumentPtr env_document;
auto get_env_node = [&](const std::string & name) -> const Node *
{
const char * env_val = std::getenv(name.c_str());
const char * env_val = std::getenv(name.c_str()); // NOLINT (this is safe on Linux glibc/Musl, but potentially not safe on other platforms)
if (env_val == nullptr)
return nullptr;

4
src/Common/DateLUT.cpp
View File

@ -30,12 +30,12 @@ std::string determineDefaultTimeZone()
{
namespace fs = std::filesystem;
const char * tzdir_env_var = std::getenv("TZDIR");
const char * tzdir_env_var = std::getenv("TZDIR"); // NOLINT (ok, because it does not run concurrently with other getenv calls)
fs::path tz_database_path = tzdir_env_var ? tzdir_env_var : "/usr/share/zoneinfo/";
fs::path tz_file_path;
std::string error_prefix;
const char * tz_env_var = std::getenv("TZ");
const char * tz_env_var = std::getenv("TZ"); // NOLINT (ok, because it does not run concurrently with other getenv calls)
/// In recent tzdata packages some files now are symlinks and canonical path resolution
/// may give wrong timezone names - store the name as it is, if possible.

4
src/Common/Exception.cpp
View File

@ -152,12 +152,12 @@ Exception::FramePointers Exception::getStackFramePointers() const
void throwFromErrno(const std::string & s, int code, int the_errno)
{
throw ErrnoException(s + ", " + errnoToString(code, the_errno), code, the_errno);
throw ErrnoException(s + ", " + errnoToString(the_errno), code, the_errno);
}
void throwFromErrnoWithPath(const std::string & s, const std::string & path, int code, int the_errno)
{
throw ErrnoException(s + ", " + errnoToString(code, the_errno), code, the_errno, path);
throw ErrnoException(s + ", " + errnoToString(the_errno), code, the_errno, path);
}
static void tryLogCurrentExceptionImpl(Poco::Logger * logger, const std::string & start_of_message)

2
src/Common/PipeFDs.cpp
View File

@ -106,7 +106,7 @@ void LazyPipeFDs::tryIncreaseSize(int desired_size)
{
if (errno == EINVAL)
{
LOG_INFO(log, "Cannot get pipe capacity, {}. Very old Linux kernels have no support for this fcntl.", errnoToString(ErrorCodes::CANNOT_FCNTL));
LOG_INFO(log, "Cannot get pipe capacity, {}. Very old Linux kernels have no support for this fcntl.", errnoToString());
/// It will work nevertheless.
}
else

2
src/Common/QueryProfiler.cpp
View File

@ -188,7 +188,7 @@ void QueryProfilerBase<ProfilerImpl>::tryCleanup()
if (timer_id.has_value())
{
if (timer_delete(*timer_id))
LOG_ERROR(log, "Failed to delete query profiler timer {}", errnoToString(ErrorCodes::CANNOT_DELETE_TIMER));
LOG_ERROR(log, "Failed to delete query profiler timer {}", errnoToString());
timer_id.reset();
}

6
src/Common/SharedLibrary.cpp
View File

@ -17,7 +17,7 @@ SharedLibrary::SharedLibrary(std::string_view path, int flags)
{
handle = dlopen(path.data(), flags);
if (!handle)
throw Exception(ErrorCodes::CANNOT_DLOPEN, "Cannot dlopen: ({})", dlerror());
throw Exception(ErrorCodes::CANNOT_DLOPEN, "Cannot dlopen: ({})", dlerror()); // NOLINT (MT-Safe on Linux, see man dlerror)
updatePHDRCache();
@ -33,11 +33,11 @@ SharedLibrary::~SharedLibrary()
void * SharedLibrary::getImpl(std::string_view name, bool no_throw)
{
dlerror();
dlerror(); // NOLINT (MT-Safe on Linux, see man dlerror)
auto * res = dlsym(handle, name.data());
if (char * error = dlerror())
if (char * error = dlerror()) // NOLINT (MT-Safe on Linux, see man dlerror)
{
if (no_throw)
return nullptr;

6
src/Common/ShellCommand.cpp
View File

@ -76,7 +76,7 @@ ShellCommand::~ShellCommand()
int retcode = kill(pid, SIGTERM);
if (retcode != 0)
LOG_WARNING(getLogger(), "Cannot kill shell command pid {} errno '{}'", pid, errnoToString(retcode));
LOG_WARNING(getLogger(), "Cannot kill shell command pid {} errno '{}'", pid, errnoToString());
}
else
{
@ -201,8 +201,8 @@ std::unique_ptr<ShellCommand> ShellCommand::executeImpl(
// by the child process, which might not expect this.
sigset_t mask;
sigemptyset(&mask);
sigprocmask(0, nullptr, &mask);
sigprocmask(SIG_UNBLOCK, &mask, nullptr);
sigprocmask(0, nullptr, &mask); // NOLINT
sigprocmask(SIG_UNBLOCK, &mask, nullptr); // NOLINT
execv(filename, argv);
/// If the process is running, then `execv` does not return here.

4
src/Common/StatusFile.cpp
View File

@ -98,10 +98,10 @@ StatusFile::StatusFile(std::string path_, FillFunction fill_)
StatusFile::~StatusFile()
{
if (0 != close(fd))
LOG_ERROR(&Poco::Logger::get("StatusFile"), "Cannot close file {}, {}", path, errnoToString(ErrorCodes::CANNOT_CLOSE_FILE));
LOG_ERROR(&Poco::Logger::get("StatusFile"), "Cannot close file {}, {}", path, errnoToString());
if (0 != unlink(path.c_str()))
LOG_ERROR(&Poco::Logger::get("StatusFile"), "Cannot unlink file {}, {}", path, errnoToString(ErrorCodes::CANNOT_CLOSE_FILE));
LOG_ERROR(&Poco::Logger::get("StatusFile"), "Cannot unlink file {}, {}", path, errnoToString());
}
}

4
src/Common/ThreadFuzzer.cpp
View File

@ -82,7 +82,7 @@ ThreadFuzzer::ThreadFuzzer()
template <typename T>
static void initFromEnv(T & what, const char * name)
{
const char * env = getenv(name);
const char * env = getenv(name); // NOLINT
if (!env)
return;
what = parse<T>(env);
@ -91,7 +91,7 @@ static void initFromEnv(T & what, const char * name)
template <typename T>
static void initFromEnv(std::atomic<T> & what, const char * name)
{
const char * env = getenv(name);
const char * env = getenv(name); // NOLINT
if (!env)
return;
what.store(parse<T>(env), std::memory_order_relaxed);

12
src/Common/ThreadProfileEvents.cpp
View File

@ -301,7 +301,7 @@ static void enablePerfEvent(int event_fd)
{
LOG_WARNING(&Poco::Logger::get("PerfEvents"),
"Can't enable perf event with file descriptor {}: '{}' ({})",
event_fd, errnoToString(errno), errno);
event_fd, errnoToString(), errno);
}
}
@ -311,7 +311,7 @@ static void disablePerfEvent(int event_fd)
{
LOG_WARNING(&Poco::Logger::get("PerfEvents"),
"Can't disable perf event with file descriptor {}: '{}' ({})",
event_fd, errnoToString(errno), errno);
event_fd, errnoToString(), errno);
}
}
@ -321,7 +321,7 @@ static void releasePerfEvent(int event_fd)
{
LOG_WARNING(&Poco::Logger::get("PerfEvents"),
"Can't close perf event file descriptor {}: {} ({})",
event_fd, errnoToString(errno), errno);
event_fd, errnoToString(), errno);
}
}
@ -339,7 +339,7 @@ static bool validatePerfEventDescriptor(int & fd)
{
LOG_WARNING(&Poco::Logger::get("PerfEvents"),
"Error while checking availability of event descriptor {}: {} ({})",
fd, errnoToString(errno), errno);
fd, errnoToString(), errno);
disablePerfEvent(fd);
releasePerfEvent(fd);
@ -446,7 +446,7 @@ bool PerfEventsCounters::processThreadLocalChanges(const std::string & needed_ev
LOG_WARNING(&Poco::Logger::get("PerfEvents"),
"Failed to open perf event {} (event_type={}, event_config={}): "
"'{}' ({})", event_info.settings_name, event_info.event_type,
event_info.event_config, errnoToString(errno), errno);
event_info.event_config, errnoToString(), errno);
}
}
@ -532,7 +532,7 @@ void PerfEventsCounters::finalizeProfileEvents(ProfileEvents::Counters & profile
{
LOG_WARNING(&Poco::Logger::get("PerfEvents"),
"Can't read event value from file descriptor {}: '{}' ({})",
fd, errnoToString(errno), errno);
fd, errnoToString(), errno);
current_values[i] = {};
}
}

6
src/Common/ThreadStatus.cpp
View File

@ -120,7 +120,7 @@ ThreadStatus::ThreadStatus()
if (0 != sigaltstack(&altstack_description, nullptr))
{
LOG_WARNING(log, "Cannot set alternative signal stack for thread, {}", errnoToString(errno));
LOG_WARNING(log, "Cannot set alternative signal stack for thread, {}", errnoToString());
}
else
{
@ -128,7 +128,7 @@ ThreadStatus::ThreadStatus()
struct sigaction action{};
if (0 != sigaction(SIGSEGV, nullptr, &action))
{
LOG_WARNING(log, "Cannot obtain previous signal action to set alternative signal stack for thread, {}", errnoToString(errno));
LOG_WARNING(log, "Cannot obtain previous signal action to set alternative signal stack for thread, {}", errnoToString());
}
else if (!(action.sa_flags & SA_ONSTACK))
{
@ -136,7 +136,7 @@ ThreadStatus::ThreadStatus()
if (0 != sigaction(SIGSEGV, &action, nullptr))
{
LOG_WARNING(log, "Cannot set action with alternative signal stack for thread, {}", errnoToString(errno));
LOG_WARNING(log, "Cannot set action with alternative signal stack for thread, {}", errnoToString());
}
}
}

3
src/Common/ZooKeeper/examples/CMakeLists.txt
View File

@ -6,6 +6,3 @@ target_link_libraries(zkutil_test_commands_new_lib PRIVATE clickhouse_common_zoo
clickhouse_add_executable(zkutil_test_async zkutil_test_async.cpp)
target_link_libraries(zkutil_test_async PRIVATE clickhouse_common_zookeeper_no_log)
clickhouse_add_executable (zookeeper_impl zookeeper_impl.cpp)
target_link_libraries (zookeeper_impl PRIVATE clickhouse_common_zookeeper_no_log)

26
src/Common/ZooKeeper/examples/zookeeper_impl.cpp
View File

@ -1,26 +0,0 @@
#include <Common/ZooKeeper/ZooKeeperImpl.h>
#include <iostream>
int main()
try
{
Coordination::ZooKeeper zookeeper({Coordination::ZooKeeper::Node{Poco::Net::SocketAddress{"localhost:2181"}, false}}, "", "", "", {30, 0}, {0, 50000}, {0, 50000}, nullptr);
zookeeper.create("/test", "hello", false, false, {}, [](const Coordination::CreateResponse & response)
{
if (response.error != Coordination::Error::ZOK)
std::cerr << "Error: " << Coordination::errorMessage(response.error) << "\n";
else
std::cerr << "Path created: " << response.path_created << "\n";
});
sleep(100);
return 0;
}
catch (...)
{
DB::tryLogCurrentException(__PRETTY_FUNCTION__);
return 1;
}

11
src/Common/examples/arena_with_free_lists.cpp
View File

@ -13,6 +13,8 @@
#include <array>
#include <sys/resource.h>
#include <base/bit_cast.h>
#include <Common/randomSeed.h>
#include <pcg_random.hpp>
#include <base/StringRef.h>
#include <base/arraySize.h>
@ -218,6 +220,7 @@ int main(int argc, char ** argv)
}
std::cerr << std::fixed << std::setprecision(2);
pcg64 rng(randomSeed());
size_t n = parse<size_t>(argv[1]);
std::vector<std::string> data;
@ -281,8 +284,8 @@ int main(int argc, char ** argv)
size_t bytes = 0;
for (size_t i = 0, size = data.size(); i < size; ++i)
{
size_t index_from = lrand48() % size;
size_t index_to = lrand48() % size;
size_t index_from = rng() % size;
size_t index_to = rng() % size;
arena.free(const_cast<char *>(refs[index_to].data), refs[index_to].size);
const auto & s = data[index_from];
@ -318,8 +321,8 @@ int main(int argc, char ** argv)
size_t bytes = 0;
for (size_t i = 0, size = data.size(); i < size; ++i)
{
size_t index_from = lrand48() % size;
size_t index_to = lrand48() % cache_size;
size_t index_from = rng() % size;
size_t index_to = rng() % cache_size;
dictionary.setAttributeValue(attr, index_to, data[index_from]);

6
src/Common/examples/int_hashes_perf.cpp
View File

@ -8,9 +8,11 @@
#include <iostream>
#include <iomanip>
#include <pcg_random.hpp>
#include <Poco/Exception.h>
#include <Common/HashTable/Hash.h>
#include <Common/Stopwatch.h>
#include <Common/randomSeed.h>
#include <Core/Defines.h>
@ -266,9 +268,9 @@ int main(int argc, char ** argv)
{
Stopwatch watch;
srand48(rdtsc());
pcg64 rng(randomSeed());
for (size_t i = 0; i < BUF_SIZE; ++i)
data[i] = lrand48();
data[i] = rng();
watch.stop();
double elapsed = watch.elapsedSeconds();

6
src/Coordination/KeeperStateMachine.cpp
View File

@ -1,4 +1,5 @@
#include <cerrno>
#include <base/errnoToString.h>
#include <future>
#include <Coordination/KeeperSnapshotManager.h>
#include <Coordination/KeeperStateMachine.h>
@ -11,6 +12,7 @@
#include <Common/ProfileEvents.h>
#include "Coordination/KeeperStorage.h"
namespace ProfileEvents
{
extern const Event KeeperCommits;
@ -446,7 +448,7 @@ static int bufferFromFile(Poco::Logger * log, const std::string & path, nuraft::
LOG_INFO(log, "Opening file {} for read_logical_snp_obj", path);
if (fd < 0)
{
LOG_WARNING(log, "Error opening {}, error: {}, errno: {}", path, std::strerror(errno), errno);
LOG_WARNING(log, "Error opening {}, error: {}, errno: {}", path, errnoToString(), errno);
return errno;
}
auto file_size = ::lseek(fd, 0, SEEK_END);
@ -454,7 +456,7 @@ static int bufferFromFile(Poco::Logger * log, const std::string & path, nuraft::
auto * chunk = reinterpret_cast<nuraft::byte *>(::mmap(nullptr, file_size, PROT_READ, MAP_FILE | MAP_SHARED, fd, 0));
if (chunk == MAP_FAILED)
{
LOG_WARNING(log, "Error mmapping {}, error: {}, errno: {}", path, std::strerror(errno), errno);
LOG_WARNING(log, "Error mmapping {}, error: {}, errno: {}", path, errnoToString(), errno);
::close(fd);
return errno;
}

2
src/Coordination/tests/gtest_coordination.cpp
View File

@ -137,7 +137,7 @@ struct SimpliestRaftServer
if (!raft_instance)
{
std::cerr << "Failed to initialize launcher" << std::endl;
exit(-1);
_exit(1);
}
std::cout << "init Raft instance " << server_id;

14
src/Daemon/BaseDaemon.cpp
View File

@ -4,12 +4,14 @@
#include <Daemon/BaseDaemon.h>
#include <Daemon/SentryWriter.h>
#include <base/errnoToString.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/time.h>
#include <sys/wait.h>
#include <sys/resource.h>
#if defined(OS_LINUX)
#include <sys/prctl.h>
#endif
@ -315,13 +317,13 @@ private:
{
LOG_FATAL(log, "(version {}{}, {}) (from thread {}) (no query) Received signal {} ({})",
VERSION_STRING, VERSION_OFFICIAL, daemon.build_id_info,
thread_num, strsignal(sig), sig);
thread_num, strsignal(sig), sig); // NOLINT (it is not thread-safe but ok in this context)
}
else
{
LOG_FATAL(log, "(version {}{}, {}) (from thread {}) (query_id: {}) (query: {}) Received signal {} ({})",
VERSION_STRING, VERSION_OFFICIAL, daemon.build_id_info,
thread_num, query_id, query, strsignal(sig), sig);
thread_num, query_id, query, strsignal(sig), sig); // NOLINT (it is not thread-safe but ok in this context)
}
String error_message;
@ -665,7 +667,7 @@ void BaseDaemon::initialize(Application & self)
if (config().has("timezone"))
{
const std::string config_timezone = config().getString("timezone");
if (0 != setenv("TZ", config_timezone.data(), 1))
if (0 != setenv("TZ", config_timezone.data(), 1)) // NOLINT (ok if not called concurrently with other setenv/getenv)
throw Poco::Exception("Cannot setenv TZ variable");
tzset();
@ -940,13 +942,13 @@ void BaseDaemon::handleSignal(int signal_id)
onInterruptSignals(signal_id);
}
else
throw DB::Exception(std::string("Unsupported signal: ") + strsignal(signal_id), 0);
throw DB::Exception(std::string("Unsupported signal: ") + strsignal(signal_id), 0); // NOLINT (it is not thread-safe but ok in this context)
}
void BaseDaemon::onInterruptSignals(int signal_id)
{
is_cancelled = true;
LOG_INFO(&logger(), "Received termination signal ({})", strsignal(signal_id));
LOG_INFO(&logger(), "Received termination signal ({})", strsignal(signal_id)); // NOLINT (it is not thread-safe but ok in this context)
if (sigint_signals_counter >= 2)
{
@ -1064,7 +1066,7 @@ void BaseDaemon::setupWatchdog()
break;
}
else if (errno != EINTR)
throw Poco::Exception("Cannot waitpid, errno: " + std::string(strerror(errno)));
throw Poco::Exception("Cannot waitpid, errno: " + errnoToString());
} while (true);
if (errno == ECHILD)

2
src/Daemon/SentryWriter.cpp
View File

@ -146,7 +146,7 @@ void SentryWriter::onFault(int sig, const std::string & error_message, const Sta
if (initialized)
{
sentry_value_t event = sentry_value_new_message_event(SENTRY_LEVEL_FATAL, "fault", error_message.c_str());
sentry_set_tag("signal", strsignal(sig));
sentry_set_tag("signal", strsignal(sig)); // NOLINT (not thread-safe but ok in this context)
sentry_set_extra("signal_number", sentry_value_new_int32(sig));
#if defined(__ELF__) && !defined(OS_FREEBSD)

3
src/Disks/IO/ThreadPoolReader.cpp
View File

@ -165,8 +165,7 @@ std::future<IAsynchronousReader::Result> ThreadPoolReader::submit(Request reques
{
ProfileEvents::increment(ProfileEvents::ReadBufferFromFileDescriptorReadFailed);
promise.set_exception(std::make_exception_ptr(ErrnoException(
fmt::format("Cannot read from file {}, {}", fd,
errnoToString(ErrorCodes::CANNOT_READ_FROM_FILE_DESCRIPTOR, errno)),
fmt::format("Cannot read from file {}, {}", fd, errnoToString()),
ErrorCodes::CANNOT_READ_FROM_FILE_DESCRIPTOR, errno)));
return future;
}

3
src/IO/Archives/ZipArchiveReader.cpp
View File

@ -4,6 +4,7 @@
#include <IO/Archives/ZipArchiveWriter.h>
#include <IO/ReadBufferFromFileBase.h>
#include <Common/quoteString.h>
#include <base/errnoToString.h>
#include <unzip.h>
@ -557,7 +558,7 @@ void ZipArchiveReader::checkResult(int code) const
switch (code)
{
case UNZ_OK: return;
case UNZ_ERRNO: message += "ERRNO, errno= " + String{strerror(errno)}; break;
case UNZ_ERRNO: message += "ERRNO, errno = " + errnoToString(); break;
case UNZ_PARAMERROR: message += "PARAMERROR"; break;
case UNZ_BADZIPFILE: message += "BADZIPFILE"; break;
case UNZ_INTERNALERROR: message += "INTERNALERROR"; break;

3
src/IO/Archives/ZipArchiveWriter.cpp
View File

@ -3,6 +3,7 @@
#if USE_MINIZIP
#include <IO/WriteBufferFromFileBase.h>
#include <Common/quoteString.h>
#include <base/errnoToString.h>
#include <zip.h>
#include <boost/algorithm/string/predicate.hpp>
@ -383,7 +384,7 @@ void ZipArchiveWriter::checkResult(int code) const
String message = "Code = ";
switch (code)
{
case ZIP_ERRNO: message += "ERRNO, errno= " + String{strerror(errno)}; break;
case ZIP_ERRNO: message += "ERRNO, errno = " + errnoToString(); break;
case ZIP_PARAMERROR: message += "PARAMERROR"; break;
case ZIP_BADZIPFILE: message += "BADZIPFILE"; break;
case ZIP_INTERNALERROR: message += "INTERNALERROR"; break;

6
src/IO/examples/CMakeLists.txt
View File

@ -37,12 +37,6 @@ target_link_libraries (read_write_int PRIVATE clickhouse_common_io)
clickhouse_add_executable (o_direct_and_dirty_pages o_direct_and_dirty_pages.cpp)
target_link_libraries (o_direct_and_dirty_pages PRIVATE clickhouse_common_io)
clickhouse_add_executable (hashing_write_buffer hashing_write_buffer.cpp)
target_link_libraries (hashing_write_buffer PRIVATE clickhouse_common_io)
clickhouse_add_executable (hashing_read_buffer hashing_read_buffer.cpp)
target_link_libraries (hashing_read_buffer PRIVATE clickhouse_common_io)
clickhouse_add_executable (io_operators io_operators.cpp)
target_link_libraries (io_operators PRIVATE clickhouse_common_io)

21
src/IO/examples/hashing_buffer.h
View File

@ -1,21 +0,0 @@
#pragma once
#include <IO/HashingWriteBuffer.h>
#include <IO/WriteBufferFromFile.h>
#define FAIL(msg) do { std::cout << msg; exit(1); } while (false)
static CityHash_v1_0_2::uint128 referenceHash(const char * data, size_t len)
{
const size_t block_size = DBMS_DEFAULT_HASHING_BLOCK_SIZE;
CityHash_v1_0_2::uint128 state(0, 0);
size_t pos;
for (pos = 0; pos + block_size <= len; pos += block_size)
state = CityHash_v1_0_2::CityHash128WithSeed(data + pos, block_size, state);
if (pos < len)
state = CityHash_v1_0_2::CityHash128WithSeed(data + pos, len - pos, state);
return state;
}

69
src/IO/examples/hashing_read_buffer.cpp
View File

@ -1,69 +0,0 @@
#include <IO/ReadBufferFromIStream.h>
#include <IO/HashingReadBuffer.h>
#include <IO/WriteBufferFromOStream.h>
#include "hashing_buffer.h"
#include <iostream>
#include <pcg_random.hpp>
static void test(size_t data_size)
{
pcg64 rng;
std::vector<char> vec(data_size);
char * data = vec.data();
for (size_t i = 0; i < data_size; ++i)
data[i] = rng() & 255;
CityHash_v1_0_2::uint128 reference = referenceHash(data, data_size);
std::vector<size_t> block_sizes = {56, 128, 513, 2048, 3055, 4097, 4096};
for (size_t read_buffer_block_size : block_sizes)
{
std::cout << "block size " << read_buffer_block_size << std::endl;
std::stringstream io; // STYLE_CHECK_ALLOW_STD_STRING_STREAM
io.exceptions(std::ios::failbit);
DB::WriteBufferFromOStream out_impl(io);
DB::HashingWriteBuffer out(out_impl);
out.write(data, data_size);
out.next();
DB::ReadBufferFromIStream source(io, read_buffer_block_size);
DB::HashingReadBuffer buf(source);
std::vector<char> read_buf(data_size);
buf.read(read_buf.data(), data_size);
bool failed_to_read = false;
for (size_t i = 0; i < data_size; ++i)
if (read_buf[i] != vec[i])
failed_to_read = true;
if (failed_to_read)
{
std::cout.write(data, data_size);
std::cout << std::endl;
std::cout.write(read_buf.data(), data_size);
std::cout << std::endl;
FAIL("Fail to read data");
}
if (buf.getHash() != reference)
FAIL("failed on data size " << data_size << " reading by blocks of size " << read_buffer_block_size);
if (buf.getHash() != out.getHash())
FAIL("Hash of HashingReadBuffer doesn't match with hash of HashingWriteBuffer on data size " << data_size << " reading by blocks of size " << read_buffer_block_size);
}
}
int main()
{
test(5);
test(100);
test(2048);
test(2049);
test(100000);
test(1 << 17);
return 0;
}

86
src/IO/examples/hashing_write_buffer.cpp
View File

@ -1,86 +0,0 @@
#include <IO/HashingWriteBuffer.h>
#include <IO/WriteBufferFromFile.h>
#include <pcg_random.hpp>
#include "hashing_buffer.h"
static void test(size_t data_size)
{
pcg64 rng;
std::vector<char> vec(data_size);
char * data = vec.data();
for (size_t i = 0; i < data_size; ++i)
data[i] = rng() & 255;
CityHash_v1_0_2::uint128 reference = referenceHash(data, data_size);
DB::WriteBufferFromFile sink("/dev/null", 1 << 16);
{
DB::HashingWriteBuffer buf(sink);
for (size_t pos = 0; pos < data_size;)
{
size_t len = std::min(static_cast<size_t>(rng() % 10000 + 1), data_size - pos);
buf.write(data + pos, len);
buf.next();
pos += len;
}
if (buf.getHash() != reference)
FAIL("failed on data size " << data_size << " writing rngom chunks of up to 10000 bytes");
}
{
DB::HashingWriteBuffer buf(sink);
for (size_t pos = 0; pos < data_size;)
{
size_t len = std::min(static_cast<size_t>(rng() % 5 + 1), data_size - pos);
buf.write(data + pos, len);
buf.next();
pos += len;
}
if (buf.getHash() != reference)
FAIL("failed on data size " << data_size << " writing rngom chunks of up to 5 bytes");
}
{
DB::HashingWriteBuffer buf(sink);
for (size_t pos = 0; pos < data_size;)
{
size_t len = std::min(static_cast<size_t>(2048 + rng() % 3 - 1), data_size - pos);
buf.write(data + pos, len);
buf.next();
pos += len;
}
if (buf.getHash() != reference)
FAIL("failed on data size " << data_size << " writing rngom chunks of 2048 +-1 bytes");
}
{
DB::HashingWriteBuffer buf(sink);
buf.write(data, data_size);
if (buf.getHash() != reference)
FAIL("failed on data size " << data_size << " writing all at once");
}
}
int main()
{
test(5);
test(100);
test(2048);
test(2049);
test(100000);
test(1 << 17);
return 0;
}

6
src/IO/examples/parse_int_perf.cpp
View File

@ -1,5 +1,6 @@
#include <iostream>
#include <iomanip>
#include <pcg_random.hpp>
#include <base/types.h>
@ -7,7 +8,6 @@
#include <IO/WriteHelpers.h>
#include <IO/WriteIntText.h>
#include <IO/WriteBufferFromVector.h>
#include <Compression/CompressedReadBuffer.h>
#include <Common/Stopwatch.h>
@ -27,6 +27,8 @@ static UInt64 rdtsc()
int main(int argc, char ** argv)
{
pcg64 rng;
try
{
if (argc < 2)
@ -47,7 +49,7 @@ int main(int argc, char ** argv)
Stopwatch watch;
for (size_t i = 0; i < n; ++i)
data[i] = lrand48();// / lrand48();// ^ (lrand48() << 24) ^ (lrand48() << 48);
data[i] = rng();
watch.stop();
std::cerr << std::fixed << std::setprecision(2)

24
src/IO/examples/var_uint.cpp
View File

@ -8,32 +8,8 @@
#include <Poco/HexBinaryEncoder.h>
static void parse_trash_string_as_uint_must_fail(const std::string & str)
{
using namespace DB;
unsigned x = 0xFF;
try
{
x = parse<unsigned>(str);
}
catch (...)
{
/// Ok
return;
}
std::cerr << "Parsing must fail, but finished successfully x=" << x;
exit(-1);
}
int main(int argc, char ** argv)
{
parse_trash_string_as_uint_must_fail("trash");
parse_trash_string_as_uint_must_fail("-1");
if (argc != 2)
{
std::cerr << "Usage: " << std::endl

2
src/Interpreters/ThreadStatusExt.cpp
View File

@ -416,7 +416,7 @@ void ThreadStatus::detachQuery(bool exit_if_already_detached, bool thread_exits)
LOG_TRACE(log, "Resetting nice");
if (0 != setpriority(PRIO_PROCESS, thread_id, 0))
LOG_ERROR(log, "Cannot 'setpriority' back to zero: {}", errnoToString(ErrorCodes::CANNOT_SET_THREAD_PRIORITY, errno));
LOG_ERROR(log, "Cannot 'setpriority' back to zero: {}", errnoToString());
os_thread_priority = 0;
}

6
src/Processors/Merges/Algorithms/tests/gtest_graphite.cpp
View File

@ -12,6 +12,8 @@
#include <AggregateFunctions/registerAggregateFunctions.h>
#include <Processors/Merges/Algorithms/Graphite.h>
#include <Common/Config/ConfigProcessor.h>
#include <base/errnoToString.h>
using namespace DB;
@ -43,7 +45,7 @@ static ConfigProcessor::LoadedConfig loadConfigurationFromString(std::string & s
int fd = mkstemp(tmp_file);
if (fd == -1)
{
throw std::runtime_error(strerror(errno));
throw std::runtime_error(errnoToString());
}
try
{
@ -61,7 +63,7 @@ static ConfigProcessor::LoadedConfig loadConfigurationFromString(std::string & s
{
int err = errno;
(void)remove(tmp_file);
throw std::runtime_error(strerror(err));
throw std::runtime_error(errnoToString(err));
}
ConfigProcessor::LoadedConfig config = loadConfiguration(config_path);
(void)remove(tmp_file);

2
src/Server/CertificateReloader.cpp
View File

@ -118,7 +118,7 @@ bool CertificateReloader::File::changeIfModified(std::string new_path, Poco::Log
if (ec)
{
LOG_ERROR(logger, "Cannot obtain modification time for {} file {}, skipping update. {}",
description, new_path, errnoToString(ErrorCodes::CANNOT_STAT, ec.value()));
description, new_path, errnoToString(ec.value()));
return false;
}

42
src/Storages/HDFS/HDFSCommon.cpp
View File

@ -7,15 +7,17 @@
#if USE_HDFS
#include <Common/ShellCommand.h>
#include <Common/Exception.h>
#include <IO/WriteBufferFromString.h>
#include <IO/Operators.h>
#include <Common/logger_useful.h>
#if USE_KRB5
#include <Access/KerberosInit.h>
#endif // USE_KRB5
#endif
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
@ -26,10 +28,15 @@ namespace ErrorCodes
#endif // USE_KRB5
}
const String HDFSBuilderWrapper::CONFIG_PREFIX = "hdfs";
const String HDFS_URL_REGEXP = "^hdfs://[^/]*/.*";
static constexpr std::string_view CONFIG_PREFIX = "hdfs";
static constexpr std::string_view HDFS_URL_REGEXP = "^hdfs://[^/]*/.*";
HDFSFileInfo::~HDFSFileInfo()
{
hdfsFreeFileInfo(file_info, length);
}
std::once_flag init_libhdfs3_conf_flag;
void HDFSBuilderWrapper::loadFromConfig(const Poco::Util::AbstractConfiguration & config,
const String & prefix, bool isUser)
@ -111,23 +118,6 @@ HDFSBuilderWrapper createHDFSBuilder(const String & uri_str, const Poco::Util::A
if (host.empty())
throw Exception("Illegal HDFS URI: " + uri.toString(), ErrorCodes::BAD_ARGUMENTS);
// Shall set env LIBHDFS3_CONF *before* HDFSBuilderWrapper construction.
std::call_once(init_libhdfs3_conf_flag, [&config]()
{
String libhdfs3_conf = config.getString(HDFSBuilderWrapper::CONFIG_PREFIX + ".libhdfs3_conf", "");
if (!libhdfs3_conf.empty())
{
if (std::filesystem::path{libhdfs3_conf}.is_relative() && !std::filesystem::exists(libhdfs3_conf))
{
const String config_path = config.getString("config-file", "config.xml");
const auto config_dir = std::filesystem::path{config_path}.remove_filename();
if (std::filesystem::exists(config_dir / libhdfs3_conf))
libhdfs3_conf = std::filesystem::absolute(config_dir / libhdfs3_conf);
}
setenv("LIBHDFS3_CONF", libhdfs3_conf.c_str(), 1);
}
});
HDFSBuilderWrapper builder;
if (builder.get() == nullptr)
throw Exception("Unable to create builder to connect to HDFS: " +
@ -157,14 +147,14 @@ HDFSBuilderWrapper createHDFSBuilder(const String & uri_str, const Poco::Util::A
hdfsBuilderSetNameNodePort(builder.get(), port);
}
if (config.has(HDFSBuilderWrapper::CONFIG_PREFIX))
if (config.has(std::string(CONFIG_PREFIX)))
{
builder.loadFromConfig(config, HDFSBuilderWrapper::CONFIG_PREFIX);
builder.loadFromConfig(config, std::string(CONFIG_PREFIX));
}
if (!user.empty())
{
String user_config_prefix = HDFSBuilderWrapper::CONFIG_PREFIX + "_" + user;
String user_config_prefix = std::string(CONFIG_PREFIX) + "_" + user;
if (config.has(user_config_prefix))
{
builder.loadFromConfig(config, user_config_prefix, true);
@ -208,7 +198,7 @@ String getNameNodeCluster(const String &hdfs_url)
void checkHDFSURL(const String & url)
{
if (!re2::RE2::FullMatch(url, HDFS_URL_REGEXP))
if (!re2::RE2::FullMatch(url, std::string(HDFS_URL_REGEXP)))
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad hdfs url: {}. It should have structure 'hdfs://<host_name>:<port>/<path>'", url);
}

6
src/Storages/HDFS/HDFSCommon.h
View File

@ -40,11 +40,7 @@ struct HDFSFileInfo
HDFSFileInfo(HDFSFileInfo && other) = default;
HDFSFileInfo & operator=(const HDFSFileInfo & other) = delete;
HDFSFileInfo & operator=(HDFSFileInfo && other) = default;
~HDFSFileInfo()
{
hdfsFreeFileInfo(file_info, length);
}
~HDFSFileInfo();
};

2
src/Storages/NATS/NATSConnection.cpp
View File

@ -18,7 +18,7 @@ NATSConnectionManager::NATSConnectionManager(const NATSConfiguration & configura
, log(log_)
, event_handler(loop.getLoop(), log)
{
const char * val = std::getenv("CLICKHOUSE_NATS_TLS_SECURE");
const char * val = std::getenv("CLICKHOUSE_NATS_TLS_SECURE"); // NOLINT (this is safe on Linux glibc/Musl, but potentially not safe on other platforms)
std::string tls_secure = val == nullptr ? std::string("1") : std::string(val);
if (tls_secure == "0")
skip_verification = true;

2
utils/CMakeLists.txt
View File

@ -23,7 +23,6 @@ if (NOT DEFINED ENABLE_UTILS OR ENABLE_UTILS)
add_subdirectory (iotest)
add_subdirectory (corrector_utf8)
add_subdirectory (zookeeper-cli)
add_subdirectory (zookeeper-test)
add_subdirectory (zookeeper-dump-tree)
add_subdirectory (zookeeper-remove-by-list)
add_subdirectory (zookeeper-create-entry-to-download-part)
@ -35,7 +34,6 @@ if (NOT DEFINED ENABLE_UTILS OR ENABLE_UTILS)
add_subdirectory (wal-dump)
add_subdirectory (check-mysql-binlog)
add_subdirectory (keeper-bench)
add_subdirectory (graphite-rollup)
if (TARGET ch_contrib::nuraft)
add_subdirectory (keeper-data-dumper)

23
utils/graphite-rollup/CMakeLists.txt
View File

@ -1,23 +0,0 @@
clickhouse_add_executable(graphite-rollup-bench graphite-rollup-bench.cpp)
target_link_libraries(
graphite-rollup-bench
PRIVATE
clickhouse_storages_system
clickhouse_aggregate_functions
clickhouse_common_config
dbms
)
target_include_directories(
graphite-rollup-bench
SYSTEM PRIVATE
${ClickHouse_SOURCE_DIR}/src ${CMAKE_BINARY_DIR}/src
${ClickHouse_SOURCE_DIR}/base ${ClickHouse_SOURCE_DIR}/base/pcg-random
${CMAKE_BINARY_DIR}/src/Core/include
${POCO_INCLUDE_DIR}
${ClickHouse_SOURCE_DIR}/contrib/double-conversion ${ClickHouse_SOURCE_DIR}/contrib/dragonbox/include
${ClickHouse_SOURCE_DIR}/contrib/fmtlib/include
${ClickHouse_SOURCE_DIR}/contrib/cityhash102/include
${RE2_INCLUDE_DIR} ${CMAKE_BINARY_DIR}/contrib/re2-cmake
)
target_compile_definitions(graphite-rollup-bench PRIVATE RULES_DIR="${CMAKE_CURRENT_SOURCE_DIR}")

130
utils/graphite-rollup/graphite-rollup-bench.cpp
View File

@ -1,130 +0,0 @@
#include <chrono>
#include <cstdio>
#include <cstdlib>
#include <iomanip>
#include <iostream>
#include <stdexcept>
#include <boost/program_options.hpp>
#include <AggregateFunctions/registerAggregateFunctions.h>
#include <Processors/Merges/Algorithms/Graphite.h>
#include <Processors/Merges/Algorithms/GraphiteRollupSortedAlgorithm.h>
#include <Storages/System/StorageSystemGraphite.h>
#include <Common/Config/ConfigProcessor.h>
#include <Common/Exception.h>
#include <Interpreters/Context.h>
#include <IO/ReadBufferFromFile.h>
#include <IO/ReadHelpers.h>
using namespace DB;
static SharedContextHolder shared_context = Context::createShared();
auto loadMetrics(const std::string & metrics_file)
{
std::vector<std::string> metrics;
ReadBufferFromFile in(metrics_file);
String line;
while (!in.eof())
{
readEscapedStringUntilEOL(line, in);
if (!in.eof())
{
++in.position();
}
if (!line.empty() && line.back() == '\n')
{
line.pop_back();
}
if (!line.empty())
{
metrics.emplace_back(line);
}
}
return metrics;
}
ConfigProcessor::LoadedConfig loadConfiguration(const std::string & config_path)
{
ConfigProcessor config_processor(config_path, true, true);
ConfigProcessor::LoadedConfig config = config_processor.loadConfig(false);
return config;
}
void bench(const std::string & config_path, const std::string & metrics_file, size_t n, bool verbose)
{
auto config = loadConfiguration(config_path);
auto context = Context::createGlobal(shared_context.get());
context->setConfig(config.configuration.get());
Graphite::Params params;
setGraphitePatternsFromConfig(context, "graphite_rollup", params);
auto metrics = loadMetrics(metrics_file);
std::vector<double> durations(metrics.size());
size_t j, i;
for (j = 0; j < n; j++)
{
for (i = 0; i < metrics.size(); i++)
{
auto start = std::chrono::high_resolution_clock::now();
auto rule = DB::Graphite::selectPatternForPath(params, metrics[i]);
(void)rule;
auto end = std::chrono::high_resolution_clock::now();
double duration = (duration_cast<std::chrono::duration<double>>(end - start)).count() * 1E9;
durations[i] += duration;
if (j == 0 && verbose)
{
std::cout << metrics[i] << ": rule with regexp '" << rule.second->regexp_str << "' found\n";
}
}
}
for (i = 0; i < metrics.size(); i++)
{
std::cout << metrics[i] << " " << durations[i] / n << " ns\n";
}
}
int main(int argc, char ** argv)
{
registerAggregateFunctions();
std::string config_file, metrics_file;
using namespace std::literals;
std::string config_default = RULES_DIR + "/rollup.xml"s;
std::string metrics_default = RULES_DIR + "/metrics.txt"s;
namespace po = boost::program_options;
po::variables_map vm;
po::options_description desc;
desc.add_options()("help,h", "produce help")(
"config,c", po::value<std::string>()->default_value(config_default), "XML config with rollup rules")(
"metrics,m", po::value<std::string>()->default_value(metrics_default), "metrcis files (one metric per line) for run benchmark")(
"verbose,V", po::bool_switch()->default_value(false), "verbose output (print found rule)");
po::parsed_options parsed = po::command_line_parser(argc, argv).options(desc).run();
po::store(parsed, vm);
po::notify(vm);
if (vm.count("help"))
{
std::cout << desc << '\n';
exit(1);
}
bench(vm["config"].as<std::string>(), vm["metrics"].as<std::string>(), 10000, vm["verbose"].as<bool>());
return 0;
}

11
utils/graphite-rollup/metrics.txt
View File

@ -1,11 +0,0 @@
test.sum
sum?env=test&tag=Fake3
test.max
max?env=test&tag=Fake4
test.min
min?env=test&tag=Fake5
fake5?env=test&tag=Fake5
test.p95
p95?env=test&tag=FakeNo
default
default?env=test&tag=FakeNo

167
utils/graphite-rollup/rollup-tag-list.xml
View File

@ -1,167 +0,0 @@
<clickhouse>
<graphite_rollup>
<pattern>
<rule_type>plain</rule_type>
<regexp>\.sum$</regexp>
<function>sum</function>
<retention>
<age>0</age>
<precision>60</precision>
</retention>
<retention>
<age>86400</age>
<precision>3600</precision>
</retention>
</pattern>
<pattern>
<rule_type>tagged</rule_type>
<regexp>^((.*)|.)sum\?</regexp>
<function>sum</function>
<retention>
<age>0</age>
<precision>60</precision>
</retention>
<retention>
<age>86400</age>
<precision>3600</precision>
</retention>
</pattern>
<pattern>
<rule_type>plain</rule_type>
<regexp>\.max$</regexp>
<function>max</function>
<retention>
<age>0</age>
<precision>60</precision>
</retention>
<retention>
<age>86400</age>
<precision>3600</precision>
</retention>
</pattern>
<pattern>
<rule_type>tagged</rule_type>
<regexp>^((.*)|.)max\?</regexp>
<function>max</function>
<retention>
<age>0</age>
<precision>60</precision>
</retention>
<retention>
<age>86400</age>
<precision>3600</precision>
</retention>
</pattern>
<pattern>
<rule_type>plain</rule_type>
<regexp>\.min$</regexp>
<function>min</function>
<retention>
<age>0</age>
<precision>60</precision>
</retention>
<retention>
<age>86400</age>
<precision>3600</precision>
</retention>
</pattern>
<pattern>
<rule_type>tagged</rule_type>
<regexp>^((.*)|.)min\?</regexp>
<function>min</function>
<retention>
<age>0</age>
<precision>60</precision>
</retention>
<retention>
<age>86400</age>
<precision>3600</precision>
</retention>
</pattern>
<pattern>
<rule_type>plain</rule_type>
<regexp>\.fake1\..*\.Fake1\.</regexp>
<function>sum</function>
</pattern>
<pattern>
<rule_type>tag_list</rule_type>
<regexp>fake1;tag=Fake1</regexp>
<function>sum</function>
</pattern>
<pattern>
<rule_type>plain</rule_type>
<regexp>\.fake2\..*\.Fake2\.</regexp>
<function>sum</function>
</pattern>
<pattern>
<rule_type>tag_list</rule_type>
<regexp>fake2;tag=Fake2</regexp>
<function>sum</function>
</pattern>
<pattern>
<rule_type>plain</rule_type>
<regexp>\.fake3\..*\.Fake3\.</regexp>
<function>sum</function>
</pattern>
<pattern>
<rule_type>tag_list</rule_type>
<regexp>fake3;tag=Fake3</regexp>
<function>sum</function>
</pattern>
<pattern>
<rule_type>plain</rule_type>
<regexp>\.fake4\..*\.Fake4\.</regexp>
<function>sum</function>
</pattern>
<pattern>
<rule_type>tag_list</rule_type>
<regexp>fake4;tag=Fake4</regexp>
<function>sum</function>
</pattern>
<pattern>
<rule_type>plain</rule_type>
<regexp>\.fake5\..*\.Fake5\.</regexp>
<function>sum</function>
</pattern>
<pattern>
<rule_type>tag_list</rule_type>
<regexp>fake5;tag=Fake5</regexp>
<function>sum</function>
</pattern>
<pattern>
<rule_type>plain</rule_type>
<regexp>\.fake6\..*\.Fake6\.</regexp>
<function>sum</function>
</pattern>
<pattern>
<rule_type>tag_list</rule_type>
<regexp>fake6;tag=Fake6</regexp>
<function>sum</function>
</pattern>
<pattern>
<rule_type>plain</rule_type>
<regexp>\.fake7\..*\.Fake7\.</regexp>
<function>sum</function>
</pattern>
<pattern>
<rule_type>tag_list</rule_type>
<regexp>fake7;tag=Fake7</regexp>
<function>sum</function>
</pattern>
<default>
<function>avg</function>
<retention>
<age>0</age>
<precision>60</precision>
</retention>
<retention>
<age>3600</age>
<precision>300</precision>
</retention>
<retention>
<age>86400</age>
<precision>3600</precision>
</retention>
</default>
</graphite_rollup>
</clickhouse>

167
utils/graphite-rollup/rollup-typed.xml
View File

@ -1,167 +0,0 @@
<clickhouse>
<graphite_rollup>
<pattern>
<rule_type>plain</rule_type>
<regexp>\.sum$</regexp>
<function>sum</function>
<retention>
<age>0</age>
<precision>60</precision>
</retention>
<retention>
<age>86400</age>
<precision>3600</precision>
</retention>
</pattern>
<pattern>
<rule_type>tagged</rule_type>
<regexp>^((.*)|.)sum\?</regexp>
<function>sum</function>
<retention>
<age>0</age>
<precision>60</precision>
</retention>
<retention>
<age>86400</age>
<precision>3600</precision>
</retention>
</pattern>
<pattern>
<rule_type>plain</rule_type>
<regexp>\.max$</regexp>
<function>max</function>
<retention>
<age>0</age>
<precision>60</precision>
</retention>
<retention>
<age>86400</age>
<precision>3600</precision>
</retention>
</pattern>
<pattern>
<rule_type>tagged</rule_type>
<regexp>^((.*)|.)max\?</regexp>
<function>max</function>
<retention>
<age>0</age>
<precision>60</precision>
</retention>
<retention>
<age>86400</age>
<precision>3600</precision>
</retention>
</pattern>
<pattern>
<rule_type>plain</rule_type>
<regexp>\.min$</regexp>
<function>min</function>
<retention>
<age>0</age>
<precision>60</precision>
</retention>
<retention>
<age>86400</age>
<precision>3600</precision>
</retention>
</pattern>
<pattern>
<rule_type>tagged</rule_type>
<regexp>^((.*)|.)min\?</regexp>
<function>min</function>
<retention>
<age>0</age>
<precision>60</precision>
</retention>
<retention>
<age>86400</age>
<precision>3600</precision>
</retention>
</pattern>
<pattern>
<rule_type>plain</rule_type>
<regexp>\.fake1\..*\.Fake1\.</regexp>
<function>sum</function>
</pattern>
<pattern>
<rule_type>tagged</rule_type>
<regexp><![CDATA[^fake1\?(.*&)*tag=Fake1(&|$)]]></regexp>
<function>sum</function>
</pattern>
<pattern>
<rule_type>plain</rule_type>
<regexp>\.fake2\..*\.Fake2\.</regexp>
<function>sum</function>
</pattern>
<pattern>
<rule_type>tagged</rule_type>
<regexp><![CDATA[^fake2\?(.*&)*tag=Fake2(&|$)]]></regexp>
<function>sum</function>
</pattern>
<pattern>
<rule_type>plain</rule_type>
<regexp>\.fake3\..*\.Fake3\.</regexp>
<function>sum</function>
</pattern>
<pattern>
<rule_type>tagged</rule_type>
<regexp><![CDATA[^fake3\?(.*&)*tag=Fake3(&|$)]]></regexp>
<function>sum</function>
</pattern>
<pattern>
<rule_type>plain</rule_type>
<regexp>\.fake4\..*\.Fake4\.</regexp>
<function>sum</function>
</pattern>
<pattern>
<rule_type>tagged</rule_type>
<regexp><![CDATA[^fake4\?(.*&)*tag=Fake4(&|$)]]></regexp>
<function>sum</function>
</pattern>
<pattern>
<rule_type>plain</rule_type>
<regexp>\.fake5\..*\.Fake5\.</regexp>
<function>sum</function>
</pattern>
<pattern>
<rule_type>tagged</rule_type>
<regexp><![CDATA[^fake5\?(.*&)*tag=Fake5(&|$)]]></regexp>
<function>sum</function>
</pattern>
<pattern>
<rule_type>plain</rule_type>
<regexp>\.fake6\..*\.Fake6\.</regexp>
<function>sum</function>
</pattern>
<pattern>
<rule_type>tagged</rule_type>
<regexp><![CDATA[^fake6\?(.*&)*tag=Fake6(&|$)]]></regexp>
<function>sum</function>
</pattern>
<pattern>
<rule_type>plain</rule_type>
<regexp>\.fake7\..*\.Fake7\.</regexp>
<function>sum</function>
</pattern>
<pattern>
<rule_type>tagged</rule_type>
<regexp><![CDATA[^fake7\?(.*&)*tag=Fake7(&|$)]]></regexp>
<function>sum</function>
</pattern>
<default>
<function>avg</function>
<retention>
<age>0</age>
<precision>60</precision>
</retention>
<retention>
<age>3600</age>
<precision>300</precision>
</retention>
<retention>
<age>86400</age>
<precision>3600</precision>
</retention>
</default>
</graphite_rollup>
</clickhouse>

147
utils/graphite-rollup/rollup.xml
View File

@ -1,147 +0,0 @@
<clickhouse>
<graphite_rollup>
<pattern>
<regexp>\.sum$</regexp>
<function>sum</function>
<retention>
<age>0</age>
<precision>60</precision>
</retention>
<retention>
<age>86400</age>
<precision>3600</precision>
</retention>
</pattern>
<pattern>
<regexp>^((.*)|.)sum\?</regexp>
<function>sum</function>
<retention>
<age>0</age>
<precision>60</precision>
</retention>
<retention>
<age>86400</age>
<precision>3600</precision>
</retention>
</pattern>
<pattern>
<regexp>\.max$</regexp>
<function>max</function>
<retention>
<age>0</age>
<precision>60</precision>
</retention>
<retention>
<age>86400</age>
<precision>3600</precision>
</retention>
</pattern>
<pattern>
<regexp>^((.*)|.)max\?</regexp>
<function>max</function>
<retention>
<age>0</age>
<precision>60</precision>
</retention>
<retention>
<age>86400</age>
<precision>3600</precision>
</retention>
</pattern>
<pattern>
<regexp>\.min$</regexp>
<function>min</function>
<retention>
<age>0</age>
<precision>60</precision>
</retention>
<retention>
<age>86400</age>
<precision>3600</precision>
</retention>
</pattern>
<pattern>
<regexp>^((.*)|.)min\?</regexp>
<function>min</function>
<retention>
<age>0</age>
<precision>60</precision>
</retention>
<retention>
<age>86400</age>
<precision>3600</precision>
</retention>
</pattern>
<pattern>
<regexp>\.fake1\..*\.Fake1\.</regexp>
<function>sum</function>
</pattern>
<pattern>
<regexp><![CDATA[^fake1\?(.*&)*tag=Fake1(&|$)]]></regexp>
<function>sum</function>
</pattern>
<pattern>
<regexp>\.fake2\..*\.Fake2\.</regexp>
<function>sum</function>
</pattern>
<pattern>
<regexp><![CDATA[^fake2\?(.*&)*tag=Fake2(&|$)]]></regexp>
<function>sum</function>
</pattern>
<pattern>
<regexp>\.fake3\..*\.Fake3\.</regexp>
<function>sum</function>
</pattern>
<pattern>
<regexp><![CDATA[^fake3\?(.*&)*tag=Fake3(&|$)]]></regexp>
<function>sum</function>
</pattern>
<pattern>
<regexp>\.fake4\..*\.Fake4\.</regexp>
<function>sum</function>
</pattern>
<pattern>
<regexp><![CDATA[^fake4\?(.*&)*tag=Fake4(&|$)]]></regexp>
<function>sum</function>
</pattern>
<pattern>
<regexp>\.fake5\..*\.Fake5\.</regexp>
<function>sum</function>
</pattern>
<pattern>
<regexp><![CDATA[^fake5\?(.*&)*tag=Fake5(&|$)]]></regexp>
<function>sum</function>
</pattern>
<pattern>
<regexp>\.fake6\..*\.Fake6\.</regexp>
<function>sum</function>
</pattern>
<pattern>
<regexp><![CDATA[^fake6\?(.*&)*tag=Fake6(&|$)]]></regexp>
<function>sum</function>
</pattern>
<pattern>
<regexp>\.fake7\..*\.Fake7\.</regexp>
<function>sum</function>
</pattern>
<pattern>
<regexp><![CDATA[^fake7\?(.*&)*tag=Fake7(&|$)]]></regexp>
<function>sum</function>
</pattern>
<default>
<function>avg</function>
<retention>
<age>0</age>
<precision>60</precision>
</retention>
<retention>
<age>3600</age>
<precision>300</precision>
</retention>
<retention>
<age>86400</age>
<precision>3600</precision>
</retention>
</default>
</graphite_rollup>
</clickhouse>

4
utils/self-extracting-executable/decompressor.cpp
View File

@ -101,8 +101,8 @@ int decompress(char * input, char * output, off_t start, off_t end, size_t max_n
{
/// Decompress data in child process.
if (0 != doDecompress(input, output, in_pointer, out_pointer, size, decompressed_size, dctx))
exit(1);
exit(0);
_exit(1);
_exit(0);
}
else
{

2
utils/zookeeper-test/CMakeLists.txt
View File

@ -1,2 +0,0 @@
clickhouse_add_executable(zk-test main.cpp)
target_link_libraries(zk-test PRIVATE clickhouse_common_zookeeper_no_log)

364
utils/zookeeper-test/main.cpp
View File

@ -1,364 +0,0 @@
#include <IO/ReadBufferFromString.h>
#include <IO/ReadHelpers.h>
#include <Poco/ConsoleChannel.h>
#include <Common/ZooKeeper/KeeperException.h>
#include <Common/ZooKeeper/ZooKeeper.h>
#include <base/LineReader.h>
#include <Common/logger_useful.h>
#include <fmt/format.h>
#include <random>
#include <iterator>
#include <algorithm>
#include <chrono>
#include <Common/Stopwatch.h>
#include <iostream>
#include <sstream>
#include <exception>
#include <future>
using namespace std;
/// TODO: Remove ME
void checkEq(zkutil::ZooKeeper & zk, const std::string & path, const std::string & expected)
{
auto result = zk.get(path);
if (result != expected)
throw std::runtime_error(fmt::format("Data on path '{}' = '{}' doesn't match expected '{}'",
path, result, expected));
}
void checkExists(zkutil::ZooKeeper & zk, const std::string & path)
{
if (!zk.exists(path))
throw std::runtime_error(fmt::format("Path '{}' doesn't exists", path));
}
void testCreateGetExistsNode(zkutil::ZooKeeper & zk)
{
zk.create("/data", "test_string", zkutil::CreateMode::Persistent);
zk.create("/data/seq-", "another_string", zkutil::CreateMode::PersistentSequential);
checkEq(zk, "/data", "test_string");
checkExists(zk, "/data/seq-0000000000");
checkEq(zk, "/data/seq-0000000000", "another_string");
}
void testCreateSetNode(zkutil::ZooKeeper & zk)
{
zk.create("/data/set", "sssss", zkutil::CreateMode::Persistent);
checkEq(zk, "/data/set", "sssss");
zk.set("/data/set", "qqqqq");
checkEq(zk, "/data/set", "qqqqq");
}
void testCreateList(zkutil::ZooKeeper & zk)
{
zk.create("/data/lst", "", zkutil::CreateMode::Persistent);
zk.create("/data/lst/d1", "", zkutil::CreateMode::Persistent);
zk.create("/data/lst/d2", "", zkutil::CreateMode::Persistent);
zk.create("/data/lst/d3", "", zkutil::CreateMode::Persistent);
auto children = zk.getChildren("/data/lst");
if (children.size() != 3)
throw std::runtime_error("Children of /data/lst doesn't equal to three");
for (size_t i = 0; i < children.size(); ++i)
{
std::cerr << "children:" << children[i] << std::endl;
std::cerr << "children size:" << children[i].size() << std::endl;
if (children[i] != "d" + std::to_string(i + 1))
throw std::runtime_error(fmt::format("Incorrect children #{} got {}, expected {}", i, children[i], "d" + std::to_string(i + 1)));
}
}
void testCreateSetVersionRequest(zkutil::ZooKeeper & zk)
{
zk.create("/data/check_data", "d", zkutil::CreateMode::Persistent);
Coordination::Stat stat{};
try
{
zk.set("/data/check_data", "e", stat.version + 2);
std::terminate();
}
catch (...)
{
std::cerr << "Got exception on incorrect version (it's ok)\n";
}
checkEq(zk, "/data/check_data", "d");
zk.set("/data/check_data", "e", stat.version);
checkEq(zk, "/data/check_data", "e");
}
void testCreateSetWatchEvent(zkutil::ZooKeeper & zk)
{
std::shared_ptr<Poco::Event> event = std::make_shared<Poco::Event>();
zk.create("/data/nodeforwatch", "", zkutil::CreateMode::Persistent);
Coordination::Stat stat;
zk.get("/data/nodeforwatch", &stat, event);
if (event->tryWait(300))
throw std::runtime_error(fmt::format("Event for path {} was set without any actions", "/data/nodeforwatch"));
zk.set("/data/nodeforwatch", "x");
if (!event->tryWait(300))
throw std::runtime_error(fmt::format("Event for path {} was not set after set", "/data/nodeforwatch"));
else
std::cerr << "Event was set well\n";
}
void testCreateListWatchEvent(zkutil::ZooKeeper & zk)
{
std::shared_ptr<Poco::Event> event = std::make_shared<Poco::Event>();
std::string path = "/data/pathforwatch";
zk.create(path, "", zkutil::CreateMode::Persistent);
zk.create(path + "/n1", "", zkutil::CreateMode::Persistent);
zk.create(path + "/n2", "", zkutil::CreateMode::Persistent);
zk.getChildren(path, nullptr, event);
if (event->tryWait(300))
throw std::runtime_error(fmt::format("ListEvent for path {} was set without any actions", path));
zk.create(path + "/n3", "", zkutil::CreateMode::Persistent);
if (!event->tryWait(300))
throw std::runtime_error(fmt::format("ListEvent for path {} was not set after create", path));
else
std::cerr << "ListEvent was set well\n";
}
void testMultiRequest(zkutil::ZooKeeper & zk)
{
std::cerr << "Testing multi request\n";
Coordination::Requests requests;
requests.push_back(zkutil::makeCreateRequest("/data/multirequest", "aaa", zkutil::CreateMode::Persistent));
requests.push_back(zkutil::makeSetRequest("/data/multirequest", "bbb", -1));
zk.multi(requests);
std::cerr << "Multi executed\n";
try
{
requests.clear();
std::cerr << "Testing bad multi\n";
requests.push_back(zkutil::makeCreateRequest("/data/multirequest", "qweqwe", zkutil::CreateMode::Persistent));
requests.push_back(zkutil::makeSetRequest("/data/multirequest", "bbb", -1));
requests.push_back(zkutil::makeSetRequest("/data/multirequest", "ccc", -1));
zk.multi(requests);
std::cerr << "Bad multi executed\n";
std::terminate();
}
catch (...)
{
std::cerr << "Got exception on multy request (it's ok)\n";
}
checkEq(zk, "/data/multirequest", "bbb");
std::cerr << "Multi request finished\n";
}
std::mutex elements_mutex;
std::vector<int> current_elements;
std::atomic<int> watches_triggered = 0;
void triggerWatch(const Coordination::WatchResponse &)
{
watches_triggered++;
}
template<typename Iter, typename RandomGenerator>
Iter select_randomly(Iter start, Iter end, RandomGenerator& g)
{
std::uniform_int_distribution<> dis(0, std::distance(start, end) - 1);
std::advance(start, dis(g));
return start;
}
template<typename Iter>
Iter select_randomly(Iter start, Iter end)
{
static std::random_device rd;
static std::mt19937 gen(rd());
return select_randomly(start, end, gen);
}
std::atomic<int> element_counter = 0;
std::atomic<int> failed_setup_counter = 0;
void createPathAndSetWatch(zkutil::ZooKeeper & zk, const String & path_prefix, size_t total)
{
for (size_t i = 0; i < total; ++i)
{
int element = element_counter++;
zk.createIfNotExists(path_prefix + "/" + std::to_string(element), "");
std::string result;
if (!zk.tryGetWatch(path_prefix + "/" + std::to_string(element), result, nullptr, triggerWatch))
failed_setup_counter++;
{
std::lock_guard lock(elements_mutex);
current_elements.push_back(element);
}
std::this_thread::sleep_for(std::chrono::milliseconds(200));
{
std::lock_guard lock(elements_mutex);
if (current_elements.empty())
continue;
element = *select_randomly(current_elements.begin(), current_elements.end());
current_elements.erase(std::remove(current_elements.begin(), current_elements.end(), element), current_elements.end());
}
zk.tryRemove(path_prefix + "/" + std::to_string(element));
}
}
std::string random_string(size_t length)
{
auto randchar = []() -> char
{
const char charset[] =
"0123456789"
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"abcdefghijklmnopqrstuvwxyz";
const size_t max_index = (sizeof(charset) - 1);
return charset[rand() % max_index]; /// NOLINT
};
std::string str(length, 0);
std::generate_n(str.begin(), length, randchar);
return str;
}
std::string currentDateTime()
{
time_t now = time(nullptr);
tm tstruct;
char buf[80];
tstruct = *localtime(&now);
// Visit http://en.cppreference.com/w/cpp/chrono/c/strftime
// for more information about date/time format
size_t size = strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", &tstruct);
return std::string(buf, size);
}
void createOnPrefix(const std::string & zkhost, const String & path_prefix, size_t datasize, size_t total)
{
zkutil::ZooKeeper zk(zkhost);
std::vector<std::future<Coordination::CreateResponse>> holder_futures;
using namespace std::chrono;
try
{
for (size_t i = 0; i < total; ++i)
{
std::cerr << currentDateTime() << "] Request:" << i << std::endl;
std::string path = path_prefix + "/element" + std::to_string(i);
holder_futures.push_back(zk.asyncCreate(path, random_string(datasize), zkutil::CreateMode::Persistent));
}
for (auto & future : holder_futures)
future.get();
}
catch (...)
{
::exit(-1);
}
}
void createConcurrent(zkutil::ZooKeeper & testzk, const std::string & zkhost, size_t threads, size_t requests, size_t blobsize)
{
std::vector<std::future<void>> asyncs;
for (size_t i = 0; i < threads; ++i)
{
std::string path_prefix = "/data/create_test" + std::to_string(i);
testzk.createIfNotExists(path_prefix, "");
auto callback = [&zkhost, path_prefix, requests, blobsize] ()
{
createOnPrefix(zkhost, path_prefix, blobsize, requests);
};
asyncs.push_back(std::async(std::launch::async, callback));
}
for (auto & async : asyncs)
{
async.wait();
}
}
void tryConcurrentWatches(zkutil::ZooKeeper & zk)
{
std::string path_prefix = "/concurrent_watches";
std::vector<std::future<void>> asyncs;
zk.createIfNotExists(path_prefix, "");
for (size_t i = 0; i < 100; ++i)
{
auto callback = [&zk, path_prefix] ()
{
createPathAndSetWatch(zk, path_prefix, 100);
};
asyncs.push_back(std::async(std::launch::async, callback));
}
for (auto & async : asyncs)
{
async.wait();
}
size_t counter = 0;
while (watches_triggered != 100 * 100)
{
std::this_thread::sleep_for(std::chrono::milliseconds(100));
if (counter++ > 20)
break;
}
std::cerr << "Failed setup counter:" << failed_setup_counter << std::endl;
std::cerr << "Current elements size:" << current_elements.size() << std::endl;
std::cerr << "WatchesTriggered:" << watches_triggered << std::endl;
}
int main(int argc, char *argv[])
{
if (argc != 2)
{
std::cerr << "usage: " << argv[0] << " hosts" << std::endl;
return 2;
}
Poco::AutoPtr<Poco::ConsoleChannel> channel = new Poco::ConsoleChannel(std::cerr);
Poco::Logger::root().setChannel(channel);
Poco::Logger::root().setLevel("trace");
zkutil::ZooKeeper zk(argv[1]);
try
{
std::cerr << "Removing\n";
zk.tryRemoveRecursive("/data");
std::cerr << "Creating\n";
zk.createIfNotExists("/data", "");
std::cerr << "Created\n";
Stopwatch watch;
createConcurrent(zk, argv[1], 1, 1005000, 10);
std::cerr << "Finished in: " << watch.elapsedMilliseconds() << "ms" << std::endl;
//testCreateGetExistsNode(zk);
//testCreateSetNode(zk);
//testCreateList(zk);
//testCreateSetVersionRequest(zk);
//testMultiRequest(zk);
//testCreateSetWatchEvent(zk);
//testCreateListWatchEvent(zk);
//tryConcurrentWatches(zk);
}
catch (...)
{
zk.tryRemoveRecursive("/data");
throw;
}
return 0;
}