Fix clang-tidy

base/base/ReplxxLineReader.cpp

@@ -27,7 +27,7 @@ void trim(String & s)
 
 std::string getEditor()
 {
-    const char * editor = std::getenv("EDITOR");
+    const char * editor = std::getenv("EDITOR"); // NOLINT
 
     if (!editor || !*editor)
         editor = "vim";
@@ -76,7 +76,7 @@ void convertHistoryFile(const std::string & path, replxx::Replxx & rx)
     if (!in)
     {
         rx.print("Cannot open %s reading (for conversion): %s\n",
-            path.c_str(), errnoToString(errno).c_str());
+            path.c_str(), errnoToString().c_str());
         return;
     }
 
@@ -84,7 +84,7 @@ void convertHistoryFile(const std::string & path, replxx::Replxx & rx)
     if (getline(in, line).bad())
     {
         rx.print("Cannot read from %s (for conversion): %s\n",
-            path.c_str(), errnoToString(errno).c_str());
+            path.c_str(), errnoToString().c_str());
         return;
     }
 
@@ -113,7 +113,7 @@ void convertHistoryFile(const std::string & path, replxx::Replxx & rx)
     if (!out)
     {
         rx.print("Cannot open %s for writing (for conversion): %s\n",
-            path.c_str(), errnoToString(errno).c_str());
+            path.c_str(), errnoToString().c_str());
         return;
     }
 
@@ -151,7 +151,7 @@ ReplxxLineReader::ReplxxLineReader(
         history_file_fd = open(history_file_path.c_str(), O_RDWR);
         if (history_file_fd < 0)
         {
-            rx.print("Open of history file failed: %s\n", errnoToString(errno).c_str());
+            rx.print("Open of history file failed: %s\n", errnoToString().c_str());
         }
         else
         {
@@ -159,18 +159,18 @@ ReplxxLineReader::ReplxxLineReader(
 
             if (flock(history_file_fd, LOCK_SH))
             {
-                rx.print("Shared lock of history file failed: %s\n", errnoToString(errno).c_str());
+                rx.print("Shared lock of history file failed: %s\n", errnoToString().c_str());
             }
             else
             {
                 if (!rx.history_load(history_file_path))
                 {
-                    rx.print("Loading history failed: %s\n", errnoToString(errno).c_str());
+                    rx.print("Loading history failed: %s\n", errnoToString().c_str());
                 }
 
                 if (flock(history_file_fd, LOCK_UN))
                 {
-                    rx.print("Unlock of history file failed: %s\n", errnoToString(errno).c_str());
+                    rx.print("Unlock of history file failed: %s\n", errnoToString().c_str());
                 }
             }
         }
@@ -225,7 +225,7 @@ ReplxxLineReader::ReplxxLineReader(
 ReplxxLineReader::~ReplxxLineReader()
 {
     if (close(history_file_fd))
-        rx.print("Close of history file failed: %s\n", errnoToString(errno).c_str());
+        rx.print("Close of history file failed: %s\n", errnoToString().c_str());
 }
 
 LineReader::InputStatus ReplxxLineReader::readOneLine(const String & prompt)
@@ -250,7 +250,7 @@ void ReplxxLineReader::addToHistory(const String & line)
     // and that is why flock() is added here.
     bool locked = false;
     if (flock(history_file_fd, LOCK_EX))
-        rx.print("Lock of history file failed: %s\n", errnoToString(errno).c_str());
+        rx.print("Lock of history file failed: %s\n", errnoToString().c_str());
     else
         locked = true;
 
@@ -258,10 +258,10 @@ void ReplxxLineReader::addToHistory(const String & line)
 
     // flush changes to the disk
     if (!rx.history_save(history_file_path))
-        rx.print("Saving history failed: %s\n", errnoToString(errno).c_str());
+        rx.print("Saving history failed: %s\n", errnoToString().c_str());
 
     if (locked && 0 != flock(history_file_fd, LOCK_UN))
-        rx.print("Unlock of history file failed: %s\n", errnoToString(errno).c_str());
+        rx.print("Unlock of history file failed: %s\n", errnoToString().c_str());
 }
 
 /// See comments in ShellCommand::executeImpl()
@@ -275,7 +275,7 @@ int ReplxxLineReader::executeEditor(const std::string & path)
     static void * real_vfork = dlsym(RTLD_DEFAULT, "vfork");
     if (!real_vfork)
     {
-        rx.print("Cannot find symbol vfork in myself: %s\n", errnoToString(errno).c_str());
+        rx.print("Cannot find symbol vfork in myself: %s\n", errnoToString().c_str());
         return -1;
     }
 
@@ -283,7 +283,7 @@ int ReplxxLineReader::executeEditor(const std::string & path)
 
     if (-1 == pid)
     {
-        rx.print("Cannot vfork: %s\n", errnoToString(errno).c_str());
+        rx.print("Cannot vfork: %s\n", errnoToString().c_str());
         return -1;
     }
 
@@ -292,11 +292,11 @@ int ReplxxLineReader::executeEditor(const std::string & path)
     {
         sigset_t mask;
         sigemptyset(&mask);
-        sigprocmask(0, nullptr, &mask);
+        sigprocmask(0, nullptr, &mask); // NOLINT (ok in newly created process)
-        sigprocmask(SIG_UNBLOCK, &mask, nullptr);
+        sigprocmask(SIG_UNBLOCK, &mask, nullptr); // NOLINT (ok in newly created process)
 
         execvp(editor.c_str(), argv);
-        rx.print("Cannot execute %s: %s\n", editor.c_str(), errnoToString(errno).c_str());
+        rx.print("Cannot execute %s: %s\n", editor.c_str(), errnoToString().c_str());
         _exit(-1);
     }
 
@@ -309,7 +309,7 @@ int ReplxxLineReader::executeEditor(const std::string & path)
             if (errno == EINTR)
                 continue;
 
-            rx.print("Cannot waitpid: %s\n", errnoToString(errno).c_str());
+            rx.print("Cannot waitpid: %s\n", errnoToString().c_str());
             return -1;
         }
         else
@@ -324,7 +324,7 @@ void ReplxxLineReader::openEditor()
     int fd = ::mkstemps(filename, 4);
     if (-1 == fd)
     {
-        rx.print("Cannot create temporary file to edit query: %s\n", errnoToString(errno).c_str());
+        rx.print("Cannot create temporary file to edit query: %s\n", errnoToString().c_str());
         return;
     }
 
@@ -338,7 +338,7 @@ void ReplxxLineReader::openEditor()
         ssize_t res = ::write(fd, begin + bytes_written, offset - bytes_written);
         if ((-1 == res || 0 == res) && errno != EINTR)
         {
-            rx.print("Cannot write to temporary query file %s: %s\n", filename, errnoToString(errno).c_str());
+            rx.print("Cannot write to temporary query file %s: %s\n", filename, errnoToString().c_str());
             break;
         }
         bytes_written += res;
@@ -346,7 +346,7 @@ void ReplxxLineReader::openEditor()
 
     if (0 != ::close(fd))
     {
-        rx.print("Cannot close temporary query file %s: %s\n", filename, errnoToString(errno).c_str());
+        rx.print("Cannot close temporary query file %s: %s\n", filename, errnoToString().c_str());
         return;
     }
 
@@ -364,7 +364,7 @@ void ReplxxLineReader::openEditor()
         }
         catch (...)
         {
-            rx.print("Cannot read from temporary query file %s: %s\n", filename, errnoToString(errno).c_str());
+            rx.print("Cannot read from temporary query file %s: %s\n", filename, errnoToString().c_str());
             return;
         }
     }
@@ -373,7 +373,7 @@ void ReplxxLineReader::openEditor()
         enableBracketedPaste();
 
     if (0 != ::unlink(filename))
-        rx.print("Cannot remove temporary query file %s: %s\n", filename, errnoToString(errno).c_str());
+        rx.print("Cannot remove temporary query file %s: %s\n", filename, errnoToString().c_str());
 }
 
 void ReplxxLineReader::enableBracketedPaste()

base/base/errnoToString.cpp

@@ -3,10 +3,11 @@
 #include <fmt/format.h>
 
 
-std::string errnoToString(int code, int the_errno)
+std::string errnoToString(int the_errno)
 {
     const size_t buf_size = 128;
     char buf[buf_size];
+
 #ifndef _GNU_SOURCE
     int rc = strerror_r(the_errno, buf, buf_size);
 #ifdef OS_DARWIN
@@ -15,7 +16,7 @@ std::string errnoToString(int code, int the_errno)
     if (rc != 0)
 #endif
     {
-        std::string tmp = std::to_string(code);
+        std::string tmp = std::to_string(the_errno);
         const char * code_str = tmp.c_str();
         const char * unknown_message = "Unknown error ";
         strcpy(buf, unknown_message);
@@ -23,7 +24,6 @@ std::string errnoToString(int code, int the_errno)
     }
     return fmt::format("errno: {}, strerror: {}", the_errno, buf);
 #else
-    (void)code;
     return fmt::format("errno: {}, strerror: {}", the_errno, strerror_r(the_errno, buf, sizeof(buf)));
 #endif
 }

base/base/errnoToString.h

@@ -3,4 +3,4 @@
 #include <cerrno>
 #include <string>
 
-std::string errnoToString(int code, int the_errno = errno);
+std::string errnoToString(int the_errno = errno);

base/base/setTerminalEcho.cpp

@@ -16,7 +16,7 @@ void setTerminalEcho(bool enable)
     struct termios tty{};
 
     if (0 != tcgetattr(STDIN_FILENO, &tty))
-        throw std::runtime_error(std::string("setTerminalEcho failed get: ") + errnoToString(errno));
+        throw std::runtime_error(std::string("setTerminalEcho failed get: ") + errnoToString());
 
     if (enable)
         tty.c_lflag |= ECHO;
@@ -24,5 +24,5 @@ void setTerminalEcho(bool enable)
         tty.c_lflag &= ~ECHO;
 
     if (0 != tcsetattr(STDIN_FILENO, TCSANOW, &tty))
-        throw std::runtime_error(std::string("setTerminalEcho failed set: ") + errnoToString(errno));
+        throw std::runtime_error(std::string("setTerminalEcho failed set: ") + errnoToString());
 }

programs/benchmark/Benchmark.cpp

@@ -120,7 +120,7 @@ public:
     void initialize(Poco::Util::Application & self [[maybe_unused]]) override
     {
         std::string home_path;
-        const char * home_path_cstr = getenv("HOME");
+        const char * home_path_cstr = getenv("HOME"); // NOLINT
         if (home_path_cstr)
             home_path = home_path_cstr;
 
@@ -613,15 +613,15 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv)
         std::optional<std::string> env_password_str;
         std::optional<std::string> env_quota_key_str;
 
-        const char * env_user = getenv("CLICKHOUSE_USER");
+        const char * env_user = getenv("CLICKHOUSE_USER"); // NOLINT
         if (env_user != nullptr)
             env_user_str.emplace(std::string(env_user));
 
-        const char * env_password = getenv("CLICKHOUSE_PASSWORD");
+        const char * env_password = getenv("CLICKHOUSE_PASSWORD"); // NOLINT
         if (env_password != nullptr)
             env_password_str.emplace(std::string(env_password));
 
-        const char * env_quota_key = getenv("CLICKHOUSE_QUOTA_KEY");
+        const char * env_quota_key = getenv("CLICKHOUSE_QUOTA_KEY"); // NOLINT
         if (env_quota_key != nullptr)
             env_quota_key_str.emplace(std::string(env_quota_key));
 

programs/client/Client.cpp

@@ -183,7 +183,7 @@ void Client::initialize(Poco::Util::Application & self)
 {
     Poco::Util::Application::initialize(self);
 
-    const char * home_path_cstr = getenv("HOME");
+    const char * home_path_cstr = getenv("HOME"); // NOLINT
     if (home_path_cstr)
         home_path = home_path_cstr;
 
@@ -202,11 +202,11 @@ void Client::initialize(Poco::Util::Application & self)
       * may be statically allocated, and can be modified by a subsequent call to getenv(), putenv(3), setenv(3), or unsetenv(3).
       */
 
-    const char * env_user = getenv("CLICKHOUSE_USER");
+    const char * env_user = getenv("CLICKHOUSE_USER"); // NOLINT
     if (env_user)
         config().setString("user", env_user);
 
-    const char * env_password = getenv("CLICKHOUSE_PASSWORD");
+    const char * env_password = getenv("CLICKHOUSE_PASSWORD"); // NOLINT
     if (env_password)
         config().setString("password", env_password);
 
@@ -620,7 +620,7 @@ bool Client::processWithFuzzing(const String & full_query)
                     stderr,
                     "Found error: IAST::clone() is broken for some AST node. This is a bug. The original AST ('dump before fuzz') and its cloned copy ('dump of cloned AST') refer to the same nodes, which must never happen. This means that their parent node doesn't implement clone() correctly.");
 
-                exit(1);
+                _exit(1);
             }
 
             auto fuzzed_text = ast_to_process->formatForErrorMessage();
@@ -770,7 +770,7 @@ bool Client::processWithFuzzing(const String & full_query)
                     fmt::print(stderr, "Text-3 (AST-3 formatted):\n'{}'\n", text_3);
                     fmt::print(stderr, "Text-3 must be equal to Text-2, but it is not.\n");
 
-                    exit(1);
+                    _exit(1);
                 }
             }
         }
@@ -909,7 +909,7 @@ void Client::processOptions(const OptionsDescription & options_description,
             auto exit_code = e.code() % 256;
             if (exit_code == 0)
                 exit_code = 255;
-            exit(exit_code);
+            _exit(exit_code);
         }
     }
 

programs/disks/DisksApp.cpp

@@ -110,7 +110,7 @@ void DisksApp::init(std::vector<String> & common_arguments)
     if (options.count("help"))
     {
         printHelpMessage(options_description);
-        exit(0);
+        exit(0); // NOLINT
     }
 
     if (!supported_commands.contains(command_name))

programs/install/Install.cpp

@@ -708,7 +708,7 @@ int mainEntryClickHouseInstall(int argc, char ** argv)
 
         /// dpkg or apt installers can ask for non-interactive work explicitly.
 
-        const char * debian_frontend_var = getenv("DEBIAN_FRONTEND");
+        const char * debian_frontend_var = getenv("DEBIAN_FRONTEND"); // NOLINT
         bool noninteractive = debian_frontend_var && debian_frontend_var == std::string_view("noninteractive");
 
         bool is_interactive = !noninteractive && stdin_is_a_tty && stdout_is_a_tty;

programs/main.cpp

@@ -366,10 +366,10 @@ void checkHarmfulEnvironmentVariables(char ** argv)
     bool require_reexec = false;
     for (const auto * var : harmful_env_variables)
     {
-        if (const char * value = getenv(var); value && value[0])
+        if (const char * value = getenv(var); value && value[0]) // NOLINT
         {
             /// NOTE: setenv() is used over unsetenv() since unsetenv() marked as harmful
-            if (setenv(var, "", true))
+            if (setenv(var, "", true)) // NOLINT (this is safe if not called concurrently)
             {
                 fmt::print(stderr, "Cannot override {} environment variable", var);
                 _exit(1);

programs/server/Server.cpp

@@ -170,7 +170,7 @@ int mainEntryClickHouseServer(int argc, char ** argv)
     /// Can be overridden by environment variable (cannot use server config at this moment).
     if (argc > 0)
     {
-        const char * env_watchdog = getenv("CLICKHOUSE_WATCHDOG_ENABLE");
+        const char * env_watchdog = getenv("CLICKHOUSE_WATCHDOG_ENABLE"); // NOLINT
         if (env_watchdog)
         {
             if (0 == strcmp(env_watchdog, "1"))
@@ -658,6 +658,24 @@ int Server::main(const std::vector<std::string> & /*args*/)
 
     StackTrace::setShowAddresses(config().getBool("show_addresses_in_stack_traces", true));
 
+#if USE_HDFS
+    /// This will point libhdfs3 to the right location for its config.
+    /// Note: this has to be done once at server initialization, because 'setenv' is not thread-safe.
+
+    String libhdfs3_conf = config().getString("hdfs.libhdfs3_conf", "");
+    if (!libhdfs3_conf.empty())
+    {
+        if (std::filesystem::path{libhdfs3_conf}.is_relative() && !std::filesystem::exists(libhdfs3_conf))
+        {
+            const String config_path = config().getString("config-file", "config.xml");
+            const auto config_dir = std::filesystem::path{config_path}.remove_filename();
+            if (std::filesystem::exists(config_dir / libhdfs3_conf))
+                libhdfs3_conf = std::filesystem::absolute(config_dir / libhdfs3_conf);
+        }
+        setenv("LIBHDFS3_CONF", libhdfs3_conf.c_str(), true /* overwrite */); // NOLINT
+    }
+#endif
+
     registerFunctions();
     registerAggregateFunctions();
     registerTableFunctions();
@@ -698,8 +716,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
     GlobalThreadPool::initialize(
         config().getUInt("max_thread_pool_size", 10000),
         config().getUInt("max_thread_pool_free_size", 1000),
-        config().getUInt("thread_pool_queue_size", 10000)
+        config().getUInt("thread_pool_queue_size", 10000));
-    );
 
     IOThreadPool::initialize(
         config().getUInt("max_io_thread_pool_size", 100),
@@ -840,7 +857,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
 
                     LOG_TRACE(log, "Will do mlock to prevent executable memory from being paged out. It may take a few seconds.");
                     if (0 != mlock(addr, len))
-                        LOG_WARNING(log, "Failed mlock: {}", errnoToString(ErrorCodes::SYSTEM_ERROR));
+                        LOG_WARNING(log, "Failed mlock: {}", errnoToString());
                     else
                         LOG_TRACE(log, "The memory map of clickhouse executable has been mlock'ed, total {}", ReadableSize(len));
                 }
@@ -908,7 +925,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
             rlim.rlim_cur = config().getUInt("max_open_files", rlim.rlim_max);
             int rc = setrlimit(RLIMIT_NOFILE, &rlim);
             if (rc != 0)
-                LOG_WARNING(log, "Cannot set max number of file descriptors to {}. Try to specify max_open_files according to your system limits. error: {}", rlim.rlim_cur, strerror(errno));
+                LOG_WARNING(log, "Cannot set max number of file descriptors to {}. Try to specify max_open_files according to your system limits. error: {}", rlim.rlim_cur, errnoToString());
             else
                 LOG_DEBUG(log, "Set max number of file descriptors to {} (was {}).", rlim.rlim_cur, old);
         }
@@ -931,7 +948,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
             int rc = setrlimit(RLIMIT_NPROC, &rlim);
             if (rc != 0)
             {
-                LOG_WARNING(log, "Cannot set max number of threads to {}. error: {}", rlim.rlim_cur, strerror(errno));
+                LOG_WARNING(log, "Cannot set max number of threads to {}. error: {}", rlim.rlim_cur, errnoToString());
                 rlim.rlim_cur = old;
             }
             else

programs/static-files-disk-uploader/static-files-disk-uploader.cpp

@@ -160,7 +160,7 @@ try
     if (options.empty() || options.count("help"))
     {
         std::cout << description << std::endl;
-        exit(0);
+        exit(0); // NOLINT
     }
 
     String metadata_path;

programs/su/su.cpp

@@ -108,7 +108,7 @@ try
     if (argc < 3)
     {
         std::cout << "Usage: ./clickhouse su user:group ..." << std::endl;
-        exit(0);
+        exit(0); // NOLINT
     }
 
     std::string_view user_and_group = argv[1];

src/Bridge/IBridge.cpp

@@ -179,7 +179,7 @@ void IBridge::initialize(Application & self)
     limit.rlim_max = limit.rlim_cur = gb;
     if (setrlimit(RLIMIT_RSS, &limit))
         LOG_WARNING(log, "Unable to set maximum RSS to 1GB: {} (current rlim_cur={}, rlim_max={})",
-                    errnoToString(errno), limit.rlim_cur, limit.rlim_max);
+                    errnoToString(), limit.rlim_cur, limit.rlim_max);
 
     if (!getrlimit(RLIMIT_RSS, &limit))
         LOG_INFO(log, "RSS limit: cur={}, max={}", limit.rlim_cur, limit.rlim_max);

src/Client/ClientBase.cpp

@@ -1952,7 +1952,7 @@ void ClientBase::runInteractive()
 
     if (home_path.empty())
     {
-        const char * home_path_cstr = getenv("HOME");
+        const char * home_path_cstr = getenv("HOME"); // NOLINT
         if (home_path_cstr)
             home_path = home_path_cstr;
     }
@@ -1962,7 +1962,7 @@ void ClientBase::runInteractive()
         history_file = config().getString("history_file");
     else
     {
-        auto * history_file_from_env = getenv("CLICKHOUSE_HISTORY_FILE");
+        auto * history_file_from_env = getenv("CLICKHOUSE_HISTORY_FILE"); // NOLINT
         if (history_file_from_env)
             history_file = history_file_from_env;
         else if (!home_path.empty())
@@ -2260,13 +2260,13 @@ void ClientBase::init(int argc, char ** argv)
     if (options.count("version") || options.count("V"))
     {
         showClientVersion();
-        exit(0);
+        exit(0); // NOLINT
     }
 
     if (options.count("version-clean"))
     {
         std::cout << VERSION_STRING;
-        exit(0);
+        exit(0); // NOLINT
     }
 
     /// Output of help message.
@@ -2274,7 +2274,7 @@ void ClientBase::init(int argc, char ** argv)
         || (options.count("host") && options["host"].as<std::string>() == "elp")) /// If user writes -help instead of --help.
     {
         printHelpMessage(options_description);
-        exit(0);
+        exit(0); // NOLINT
     }
 
     /// Common options for clickhouse-client and clickhouse-local.

src/Common/Config/ConfigProcessor.cpp

@@ -419,7 +419,7 @@ void ConfigProcessor::doIncludesRecursive(
         XMLDocumentPtr env_document;
         auto get_env_node = [&](const std::string & name) -> const Node *
         {
-            const char * env_val = std::getenv(name.c_str());
+            const char * env_val = std::getenv(name.c_str()); // NOLINT (this is safe on Linux glibc/Musl, but potentially not safe on other platforms)
             if (env_val == nullptr)
                 return nullptr;
 

src/Common/DateLUT.cpp

@@ -30,12 +30,12 @@ std::string determineDefaultTimeZone()
 {
     namespace fs = std::filesystem;
 
-    const char * tzdir_env_var = std::getenv("TZDIR");
+    const char * tzdir_env_var = std::getenv("TZDIR"); // NOLINT (ok, because it does not run concurrently with other getenv calls)
     fs::path tz_database_path = tzdir_env_var ? tzdir_env_var : "/usr/share/zoneinfo/";
 
     fs::path tz_file_path;
     std::string error_prefix;
-    const char * tz_env_var = std::getenv("TZ");
+    const char * tz_env_var = std::getenv("TZ"); // NOLINT (ok, because it does not run concurrently with other getenv calls)
 
     /// In recent tzdata packages some files now are symlinks and canonical path resolution
     /// may give wrong timezone names - store the name as it is, if possible.

src/Common/Exception.cpp

@@ -152,12 +152,12 @@ Exception::FramePointers Exception::getStackFramePointers() const
 
 void throwFromErrno(const std::string & s, int code, int the_errno)
 {
-    throw ErrnoException(s + ", " + errnoToString(code, the_errno), code, the_errno);
+    throw ErrnoException(s + ", " + errnoToString(the_errno), code, the_errno);
 }
 
 void throwFromErrnoWithPath(const std::string & s, const std::string & path, int code, int the_errno)
 {
-    throw ErrnoException(s + ", " + errnoToString(code, the_errno), code, the_errno, path);
+    throw ErrnoException(s + ", " + errnoToString(the_errno), code, the_errno, path);
 }
 
 static void tryLogCurrentExceptionImpl(Poco::Logger * logger, const std::string & start_of_message)

src/Common/PipeFDs.cpp

@@ -106,7 +106,7 @@ void LazyPipeFDs::tryIncreaseSize(int desired_size)
     {
         if (errno == EINVAL)
         {
-            LOG_INFO(log, "Cannot get pipe capacity, {}. Very old Linux kernels have no support for this fcntl.", errnoToString(ErrorCodes::CANNOT_FCNTL));
+            LOG_INFO(log, "Cannot get pipe capacity, {}. Very old Linux kernels have no support for this fcntl.", errnoToString());
             /// It will work nevertheless.
         }
         else

src/Common/QueryProfiler.cpp

@@ -188,7 +188,7 @@ void QueryProfilerBase<ProfilerImpl>::tryCleanup()
     if (timer_id.has_value())
     {
         if (timer_delete(*timer_id))
-            LOG_ERROR(log, "Failed to delete query profiler timer {}", errnoToString(ErrorCodes::CANNOT_DELETE_TIMER));
+            LOG_ERROR(log, "Failed to delete query profiler timer {}", errnoToString());
         timer_id.reset();
     }
 

src/Common/SharedLibrary.cpp

@@ -17,7 +17,7 @@ SharedLibrary::SharedLibrary(std::string_view path, int flags)
 {
     handle = dlopen(path.data(), flags);
     if (!handle)
-        throw Exception(ErrorCodes::CANNOT_DLOPEN, "Cannot dlopen: ({})", dlerror());
+        throw Exception(ErrorCodes::CANNOT_DLOPEN, "Cannot dlopen: ({})", dlerror()); // NOLINT (MT-Safe on Linux, see man dlerror)
 
     updatePHDRCache();
 
@@ -33,11 +33,11 @@ SharedLibrary::~SharedLibrary()
 
 void * SharedLibrary::getImpl(std::string_view name, bool no_throw)
 {
-    dlerror();
+    dlerror(); // NOLINT (MT-Safe on Linux, see man dlerror)
 
     auto * res = dlsym(handle, name.data());
 
-    if (char * error = dlerror())
+    if (char * error = dlerror()) // NOLINT (MT-Safe on Linux, see man dlerror)
     {
         if (no_throw)
             return nullptr;

src/Common/ShellCommand.cpp

@@ -76,7 +76,7 @@ ShellCommand::~ShellCommand()
 
         int retcode = kill(pid, SIGTERM);
         if (retcode != 0)
-            LOG_WARNING(getLogger(), "Cannot kill shell command pid {} errno '{}'", pid, errnoToString(retcode));
+            LOG_WARNING(getLogger(), "Cannot kill shell command pid {} errno '{}'", pid, errnoToString());
     }
     else
     {
@@ -201,8 +201,8 @@ std::unique_ptr<ShellCommand> ShellCommand::executeImpl(
         // by the child process, which might not expect this.
         sigset_t mask;
         sigemptyset(&mask);
-        sigprocmask(0, nullptr, &mask);
+        sigprocmask(0, nullptr, &mask); // NOLINT
-        sigprocmask(SIG_UNBLOCK, &mask, nullptr);
+        sigprocmask(SIG_UNBLOCK, &mask, nullptr); // NOLINT
 
         execv(filename, argv);
         /// If the process is running, then `execv` does not return here.

src/Common/StatusFile.cpp

@@ -98,10 +98,10 @@ StatusFile::StatusFile(std::string path_, FillFunction fill_)
 StatusFile::~StatusFile()
 {
     if (0 != close(fd))
-        LOG_ERROR(&Poco::Logger::get("StatusFile"), "Cannot close file {}, {}", path, errnoToString(ErrorCodes::CANNOT_CLOSE_FILE));
+        LOG_ERROR(&Poco::Logger::get("StatusFile"), "Cannot close file {}, {}", path, errnoToString());
 
     if (0 != unlink(path.c_str()))
-        LOG_ERROR(&Poco::Logger::get("StatusFile"), "Cannot unlink file {}, {}", path, errnoToString(ErrorCodes::CANNOT_CLOSE_FILE));
+        LOG_ERROR(&Poco::Logger::get("StatusFile"), "Cannot unlink file {}, {}", path, errnoToString());
 }
 
 }

src/Common/ThreadFuzzer.cpp

@@ -82,7 +82,7 @@ ThreadFuzzer::ThreadFuzzer()
 template <typename T>
 static void initFromEnv(T & what, const char * name)
 {
-    const char * env = getenv(name);
+    const char * env = getenv(name); // NOLINT
     if (!env)
         return;
     what = parse<T>(env);
@@ -91,7 +91,7 @@ static void initFromEnv(T & what, const char * name)
 template <typename T>
 static void initFromEnv(std::atomic<T> & what, const char * name)
 {
-    const char * env = getenv(name);
+    const char * env = getenv(name); // NOLINT
     if (!env)
         return;
     what.store(parse<T>(env), std::memory_order_relaxed);

src/Common/ThreadProfileEvents.cpp

@@ -301,7 +301,7 @@ static void enablePerfEvent(int event_fd)
     {
         LOG_WARNING(&Poco::Logger::get("PerfEvents"),
             "Can't enable perf event with file descriptor {}: '{}' ({})",
-            event_fd, errnoToString(errno), errno);
+            event_fd, errnoToString(), errno);
     }
 }
 
@@ -311,7 +311,7 @@ static void disablePerfEvent(int event_fd)
     {
         LOG_WARNING(&Poco::Logger::get("PerfEvents"),
             "Can't disable perf event with file descriptor {}: '{}' ({})",
-            event_fd, errnoToString(errno), errno);
+            event_fd, errnoToString(), errno);
     }
 }
 
@@ -321,7 +321,7 @@ static void releasePerfEvent(int event_fd)
     {
         LOG_WARNING(&Poco::Logger::get("PerfEvents"),
             "Can't close perf event file descriptor {}: {} ({})",
-            event_fd, errnoToString(errno), errno);
+            event_fd, errnoToString(), errno);
     }
 }
 
@@ -339,7 +339,7 @@ static bool validatePerfEventDescriptor(int & fd)
     {
         LOG_WARNING(&Poco::Logger::get("PerfEvents"),
             "Error while checking availability of event descriptor {}: {} ({})",
-            fd, errnoToString(errno), errno);
+            fd, errnoToString(), errno);
 
         disablePerfEvent(fd);
         releasePerfEvent(fd);
@@ -446,7 +446,7 @@ bool PerfEventsCounters::processThreadLocalChanges(const std::string & needed_ev
             LOG_WARNING(&Poco::Logger::get("PerfEvents"),
                 "Failed to open perf event {} (event_type={}, event_config={}): "
                 "'{}' ({})", event_info.settings_name, event_info.event_type,
-                event_info.event_config, errnoToString(errno), errno);
+                event_info.event_config, errnoToString(), errno);
         }
     }
 
@@ -532,7 +532,7 @@ void PerfEventsCounters::finalizeProfileEvents(ProfileEvents::Counters & profile
         {
             LOG_WARNING(&Poco::Logger::get("PerfEvents"),
                 "Can't read event value from file descriptor {}: '{}' ({})",
-                fd, errnoToString(errno), errno);
+                fd, errnoToString(), errno);
             current_values[i] = {};
         }
     }

src/Common/ThreadStatus.cpp

@@ -120,7 +120,7 @@ ThreadStatus::ThreadStatus()
 
         if (0 != sigaltstack(&altstack_description, nullptr))
         {
-            LOG_WARNING(log, "Cannot set alternative signal stack for thread, {}", errnoToString(errno));
+            LOG_WARNING(log, "Cannot set alternative signal stack for thread, {}", errnoToString());
         }
         else
         {
@@ -128,7 +128,7 @@ ThreadStatus::ThreadStatus()
             struct sigaction action{};
             if (0 != sigaction(SIGSEGV, nullptr, &action))
             {
-                LOG_WARNING(log, "Cannot obtain previous signal action to set alternative signal stack for thread, {}", errnoToString(errno));
+                LOG_WARNING(log, "Cannot obtain previous signal action to set alternative signal stack for thread, {}", errnoToString());
             }
             else if (!(action.sa_flags & SA_ONSTACK))
             {
@@ -136,7 +136,7 @@ ThreadStatus::ThreadStatus()
 
                 if (0 != sigaction(SIGSEGV, &action, nullptr))
                 {
-                    LOG_WARNING(log, "Cannot set action with alternative signal stack for thread, {}", errnoToString(errno));
+                    LOG_WARNING(log, "Cannot set action with alternative signal stack for thread, {}", errnoToString());
                 }
             }
         }

src/Common/ZooKeeper/examples/CMakeLists.txt

@@ -6,6 +6,3 @@ target_link_libraries(zkutil_test_commands_new_lib PRIVATE clickhouse_common_zoo
 
 clickhouse_add_executable(zkutil_test_async zkutil_test_async.cpp)
 target_link_libraries(zkutil_test_async PRIVATE clickhouse_common_zookeeper_no_log)
-
-clickhouse_add_executable (zookeeper_impl zookeeper_impl.cpp)
-target_link_libraries (zookeeper_impl PRIVATE clickhouse_common_zookeeper_no_log)

src/Common/ZooKeeper/examples/zookeeper_impl.cpp

@@ -1,26 +0,0 @@
-#include <Common/ZooKeeper/ZooKeeperImpl.h>
-#include <iostream>
-
-
-int main()
-try
-{
-    Coordination::ZooKeeper zookeeper({Coordination::ZooKeeper::Node{Poco::Net::SocketAddress{"localhost:2181"}, false}}, "", "", "", {30, 0}, {0, 50000}, {0, 50000}, nullptr);
-
-    zookeeper.create("/test", "hello", false, false, {}, [](const Coordination::CreateResponse & response)
-    {
-        if (response.error != Coordination::Error::ZOK)
-            std::cerr << "Error: " << Coordination::errorMessage(response.error) << "\n";
-        else
-            std::cerr << "Path created: " << response.path_created << "\n";
-    });
-
-    sleep(100);
-
-    return 0;
-}
-catch (...)
-{
-    DB::tryLogCurrentException(__PRETTY_FUNCTION__);
-    return 1;
-}

src/Common/examples/arena_with_free_lists.cpp

@@ -13,6 +13,8 @@
 #include <array>
 #include <sys/resource.h>
 #include <base/bit_cast.h>
+#include <Common/randomSeed.h>
+#include <pcg_random.hpp>
 
 #include <base/StringRef.h>
 #include <base/arraySize.h>
@@ -218,6 +220,7 @@ int main(int argc, char ** argv)
     }
 
     std::cerr << std::fixed << std::setprecision(2);
+    pcg64 rng(randomSeed());
 
     size_t n = parse<size_t>(argv[1]);
     std::vector<std::string> data;
@@ -281,8 +284,8 @@ int main(int argc, char ** argv)
         size_t bytes = 0;
         for (size_t i = 0, size = data.size(); i < size; ++i)
         {
-            size_t index_from = lrand48() % size;
+            size_t index_from = rng() % size;
-            size_t index_to = lrand48() % size;
+            size_t index_to = rng() % size;
 
             arena.free(const_cast<char *>(refs[index_to].data), refs[index_to].size);
             const auto & s = data[index_from];
@@ -318,8 +321,8 @@ int main(int argc, char ** argv)
         size_t bytes = 0;
         for (size_t i = 0, size = data.size(); i < size; ++i)
         {
-            size_t index_from = lrand48() % size;
+            size_t index_from = rng() % size;
-            size_t index_to = lrand48() % cache_size;
+            size_t index_to = rng() % cache_size;
 
             dictionary.setAttributeValue(attr, index_to, data[index_from]);
 

src/Common/examples/int_hashes_perf.cpp

@@ -8,9 +8,11 @@
 
 #include <iostream>
 #include <iomanip>
+#include <pcg_random.hpp>
 #include <Poco/Exception.h>
 #include <Common/HashTable/Hash.h>
 #include <Common/Stopwatch.h>
+#include <Common/randomSeed.h>
 #include <Core/Defines.h>
 
 
@@ -266,9 +268,9 @@ int main(int argc, char ** argv)
     {
         Stopwatch watch;
 
-        srand48(rdtsc());
+        pcg64 rng(randomSeed());
         for (size_t i = 0; i < BUF_SIZE; ++i)
-            data[i] = lrand48();
+            data[i] = rng();
 
         watch.stop();
         double elapsed = watch.elapsedSeconds();

src/Coordination/KeeperStateMachine.cpp

@@ -1,4 +1,5 @@
 #include <cerrno>
+#include <base/errnoToString.h>
 #include <future>
 #include <Coordination/KeeperSnapshotManager.h>
 #include <Coordination/KeeperStateMachine.h>
@@ -11,6 +12,7 @@
 #include <Common/ProfileEvents.h>
 #include "Coordination/KeeperStorage.h"
 
+
 namespace ProfileEvents
 {
     extern const Event KeeperCommits;
@@ -446,7 +448,7 @@ static int bufferFromFile(Poco::Logger * log, const std::string & path, nuraft::
     LOG_INFO(log, "Opening file {} for read_logical_snp_obj", path);
     if (fd < 0)
     {
-        LOG_WARNING(log, "Error opening {}, error: {}, errno: {}", path, std::strerror(errno), errno);
+        LOG_WARNING(log, "Error opening {}, error: {}, errno: {}", path, errnoToString(), errno);
         return errno;
     }
     auto file_size = ::lseek(fd, 0, SEEK_END);
@@ -454,7 +456,7 @@ static int bufferFromFile(Poco::Logger * log, const std::string & path, nuraft::
     auto * chunk = reinterpret_cast<nuraft::byte *>(::mmap(nullptr, file_size, PROT_READ, MAP_FILE | MAP_SHARED, fd, 0));
     if (chunk == MAP_FAILED)
     {
-        LOG_WARNING(log, "Error mmapping {}, error: {}, errno: {}", path, std::strerror(errno), errno);
+        LOG_WARNING(log, "Error mmapping {}, error: {}, errno: {}", path, errnoToString(), errno);
         ::close(fd);
         return errno;
     }

src/Coordination/tests/gtest_coordination.cpp

@@ -137,7 +137,7 @@ struct SimpliestRaftServer
         if (!raft_instance)
         {
             std::cerr << "Failed to initialize launcher" << std::endl;
-            exit(-1);
+            _exit(1);
         }
 
         std::cout << "init Raft instance " << server_id;

src/Daemon/BaseDaemon.cpp

@@ -4,12 +4,14 @@
 
 #include <Daemon/BaseDaemon.h>
 #include <Daemon/SentryWriter.h>
+#include <base/errnoToString.h>
 
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <sys/time.h>
 #include <sys/wait.h>
 #include <sys/resource.h>
+
 #if defined(OS_LINUX)
     #include <sys/prctl.h>
 #endif
@@ -315,13 +317,13 @@ private:
         {
             LOG_FATAL(log, "(version {}{}, {}) (from thread {}) (no query) Received signal {} ({})",
                 VERSION_STRING, VERSION_OFFICIAL, daemon.build_id_info,
-                thread_num, strsignal(sig), sig);
+                thread_num, strsignal(sig), sig); // NOLINT (it is not thread-safe but ok in this context)
         }
         else
         {
             LOG_FATAL(log, "(version {}{}, {}) (from thread {}) (query_id: {}) (query: {}) Received signal {} ({})",
                 VERSION_STRING, VERSION_OFFICIAL, daemon.build_id_info,
-                thread_num, query_id, query, strsignal(sig), sig);
+                thread_num, query_id, query, strsignal(sig), sig); // NOLINT (it is not thread-safe but ok in this context)
         }
 
         String error_message;
@@ -665,7 +667,7 @@ void BaseDaemon::initialize(Application & self)
     if (config().has("timezone"))
     {
         const std::string config_timezone = config().getString("timezone");
-        if (0 != setenv("TZ", config_timezone.data(), 1))
+        if (0 != setenv("TZ", config_timezone.data(), 1)) // NOLINT (ok if not called concurrently with other setenv/getenv)
             throw Poco::Exception("Cannot setenv TZ variable");
 
         tzset();
@@ -940,13 +942,13 @@ void BaseDaemon::handleSignal(int signal_id)
         onInterruptSignals(signal_id);
     }
     else
-        throw DB::Exception(std::string("Unsupported signal: ") + strsignal(signal_id), 0);
+        throw DB::Exception(std::string("Unsupported signal: ") + strsignal(signal_id), 0); // NOLINT (it is not thread-safe but ok in this context)
 }
 
 void BaseDaemon::onInterruptSignals(int signal_id)
 {
     is_cancelled = true;
-    LOG_INFO(&logger(), "Received termination signal ({})", strsignal(signal_id));
+    LOG_INFO(&logger(), "Received termination signal ({})", strsignal(signal_id)); // NOLINT (it is not thread-safe but ok in this context)
 
     if (sigint_signals_counter >= 2)
     {
@@ -1064,7 +1066,7 @@ void BaseDaemon::setupWatchdog()
                     break;
             }
             else if (errno != EINTR)
-                throw Poco::Exception("Cannot waitpid, errno: " + std::string(strerror(errno)));
+                throw Poco::Exception("Cannot waitpid, errno: " + errnoToString());
         } while (true);
 
         if (errno == ECHILD)

src/Daemon/SentryWriter.cpp

@@ -146,7 +146,7 @@ void SentryWriter::onFault(int sig, const std::string & error_message, const Sta
     if (initialized)
     {
         sentry_value_t event = sentry_value_new_message_event(SENTRY_LEVEL_FATAL, "fault", error_message.c_str());
-        sentry_set_tag("signal", strsignal(sig));
+        sentry_set_tag("signal", strsignal(sig)); // NOLINT (not thread-safe but ok in this context)
         sentry_set_extra("signal_number", sentry_value_new_int32(sig));
 
         #if defined(__ELF__) && !defined(OS_FREEBSD)

src/Disks/IO/ThreadPoolReader.cpp

@@ -165,8 +165,7 @@ std::future<IAsynchronousReader::Result> ThreadPoolReader::submit(Request reques
                 {
                     ProfileEvents::increment(ProfileEvents::ReadBufferFromFileDescriptorReadFailed);
                     promise.set_exception(std::make_exception_ptr(ErrnoException(
-                        fmt::format("Cannot read from file {}, {}", fd,
+                        fmt::format("Cannot read from file {}, {}", fd, errnoToString()),
-                            errnoToString(ErrorCodes::CANNOT_READ_FROM_FILE_DESCRIPTOR, errno)),
                         ErrorCodes::CANNOT_READ_FROM_FILE_DESCRIPTOR, errno)));
                     return future;
                 }

src/IO/Archives/ZipArchiveReader.cpp

@@ -4,6 +4,7 @@
 #include <IO/Archives/ZipArchiveWriter.h>
 #include <IO/ReadBufferFromFileBase.h>
 #include <Common/quoteString.h>
+#include <base/errnoToString.h>
 #include <unzip.h>
 
 
@@ -553,11 +554,11 @@ void ZipArchiveReader::checkResult(int code) const
     if (code >= UNZ_OK)
         return;
 
-    String message = "Code= ";
+    String message = "Code = ";
     switch (code)
     {
         case UNZ_OK: return;
-        case UNZ_ERRNO: message += "ERRNO, errno= " + String{strerror(errno)}; break;
+        case UNZ_ERRNO: message += "ERRNO, errno = " + errnoToString(); break;
         case UNZ_PARAMERROR: message += "PARAMERROR"; break;
         case UNZ_BADZIPFILE: message += "BADZIPFILE"; break;
         case UNZ_INTERNALERROR: message += "INTERNALERROR"; break;

src/IO/Archives/ZipArchiveWriter.cpp

@@ -3,6 +3,7 @@
 #if USE_MINIZIP
 #include <IO/WriteBufferFromFileBase.h>
 #include <Common/quoteString.h>
+#include <base/errnoToString.h>
 #include <zip.h>
 #include <boost/algorithm/string/predicate.hpp>
 
@@ -380,10 +381,10 @@ void ZipArchiveWriter::checkResult(int code) const
     if (code >= ZIP_OK)
         return;
 
-    String message = "Code= ";
+    String message = "Code = ";
     switch (code)
     {
-        case ZIP_ERRNO: message += "ERRNO, errno= " + String{strerror(errno)}; break;
+        case ZIP_ERRNO: message += "ERRNO, errno = " + errnoToString(); break;
         case ZIP_PARAMERROR: message += "PARAMERROR"; break;
         case ZIP_BADZIPFILE: message += "BADZIPFILE"; break;
         case ZIP_INTERNALERROR: message += "INTERNALERROR"; break;

src/IO/examples/CMakeLists.txt

@@ -37,12 +37,6 @@ target_link_libraries (read_write_int PRIVATE clickhouse_common_io)
 clickhouse_add_executable (o_direct_and_dirty_pages o_direct_and_dirty_pages.cpp)
 target_link_libraries (o_direct_and_dirty_pages PRIVATE clickhouse_common_io)
 
-clickhouse_add_executable (hashing_write_buffer hashing_write_buffer.cpp)
-target_link_libraries (hashing_write_buffer PRIVATE clickhouse_common_io)
-
-clickhouse_add_executable (hashing_read_buffer hashing_read_buffer.cpp)
-target_link_libraries (hashing_read_buffer PRIVATE clickhouse_common_io)
-
 clickhouse_add_executable (io_operators io_operators.cpp)
 target_link_libraries (io_operators PRIVATE clickhouse_common_io)
 

src/IO/examples/hashing_buffer.h

@@ -1,21 +0,0 @@
-#pragma once
-#include <IO/HashingWriteBuffer.h>
-#include <IO/WriteBufferFromFile.h>
-
-#define FAIL(msg) do { std::cout << msg; exit(1); } while (false)
-
-
-static CityHash_v1_0_2::uint128 referenceHash(const char * data, size_t len)
-{
-    const size_t block_size = DBMS_DEFAULT_HASHING_BLOCK_SIZE;
-    CityHash_v1_0_2::uint128 state(0, 0);
-    size_t pos;
-
-    for (pos = 0; pos + block_size <= len; pos += block_size)
-        state = CityHash_v1_0_2::CityHash128WithSeed(data + pos, block_size, state);
-
-    if (pos < len)
-        state = CityHash_v1_0_2::CityHash128WithSeed(data + pos, len - pos, state);
-
-    return state;
-}

src/IO/examples/hashing_read_buffer.cpp

@@ -1,69 +0,0 @@
-#include <IO/ReadBufferFromIStream.h>
-#include <IO/HashingReadBuffer.h>
-#include <IO/WriteBufferFromOStream.h>
-#include "hashing_buffer.h"
-#include <iostream>
-#include <pcg_random.hpp>
-
-
-static void test(size_t data_size)
-{
-    pcg64 rng;
-
-    std::vector<char> vec(data_size);
-    char * data = vec.data();
-
-    for (size_t i = 0; i < data_size; ++i)
-        data[i] = rng() & 255;
-
-    CityHash_v1_0_2::uint128 reference = referenceHash(data, data_size);
-
-    std::vector<size_t> block_sizes = {56, 128, 513, 2048, 3055, 4097, 4096};
-    for (size_t read_buffer_block_size : block_sizes)
-    {
-        std::cout << "block size " << read_buffer_block_size << std::endl;
-        std::stringstream io;       // STYLE_CHECK_ALLOW_STD_STRING_STREAM
-        io.exceptions(std::ios::failbit);
-        DB::WriteBufferFromOStream out_impl(io);
-        DB::HashingWriteBuffer out(out_impl);
-        out.write(data, data_size);
-        out.next();
-
-        DB::ReadBufferFromIStream source(io, read_buffer_block_size);
-        DB::HashingReadBuffer buf(source);
-
-        std::vector<char> read_buf(data_size);
-        buf.read(read_buf.data(), data_size);
-
-        bool failed_to_read = false;
-        for (size_t i = 0; i < data_size; ++i)
-            if (read_buf[i] != vec[i])
-                failed_to_read = true;
-
-        if (failed_to_read)
-        {
-            std::cout.write(data, data_size);
-            std::cout << std::endl;
-            std::cout.write(read_buf.data(), data_size);
-            std::cout << std::endl;
-            FAIL("Fail to read data");
-        }
-
-        if (buf.getHash() != reference)
-            FAIL("failed on data size " << data_size << " reading by blocks of size " << read_buffer_block_size);
-        if (buf.getHash() != out.getHash())
-            FAIL("Hash of HashingReadBuffer doesn't match with hash of HashingWriteBuffer on data size " << data_size << " reading by blocks of size " << read_buffer_block_size);
-    }
-}
-
-int main()
-{
-    test(5);
-    test(100);
-    test(2048);
-    test(2049);
-    test(100000);
-    test(1 << 17);
-
-    return 0;
-}

src/IO/examples/hashing_write_buffer.cpp

@@ -1,86 +0,0 @@
-#include <IO/HashingWriteBuffer.h>
-#include <IO/WriteBufferFromFile.h>
-#include <pcg_random.hpp>
-
-#include "hashing_buffer.h"
-
-static void test(size_t data_size)
-{
-    pcg64 rng;
-
-    std::vector<char> vec(data_size);
-    char * data = vec.data();
-
-    for (size_t i = 0; i < data_size; ++i)
-        data[i] = rng() & 255;
-
-    CityHash_v1_0_2::uint128 reference = referenceHash(data, data_size);
-
-    DB::WriteBufferFromFile sink("/dev/null", 1 << 16);
-
-    {
-        DB::HashingWriteBuffer buf(sink);
-
-        for (size_t pos = 0; pos < data_size;)
-        {
-            size_t len = std::min(static_cast<size_t>(rng() % 10000 + 1), data_size - pos);
-            buf.write(data + pos, len);
-            buf.next();
-            pos += len;
-        }
-
-        if (buf.getHash() != reference)
-            FAIL("failed on data size " << data_size << " writing rngom chunks of up to 10000 bytes");
-    }
-
-    {
-        DB::HashingWriteBuffer buf(sink);
-
-        for (size_t pos = 0; pos < data_size;)
-        {
-            size_t len = std::min(static_cast<size_t>(rng() % 5 + 1), data_size - pos);
-            buf.write(data + pos, len);
-            buf.next();
-            pos += len;
-        }
-
-        if (buf.getHash() != reference)
-            FAIL("failed on data size " << data_size << " writing rngom chunks of up to 5 bytes");
-    }
-
-    {
-        DB::HashingWriteBuffer buf(sink);
-
-        for (size_t pos = 0; pos < data_size;)
-        {
-            size_t len = std::min(static_cast<size_t>(2048 + rng() % 3 - 1), data_size - pos);
-            buf.write(data + pos, len);
-            buf.next();
-            pos += len;
-        }
-
-        if (buf.getHash() != reference)
-            FAIL("failed on data size " << data_size << " writing rngom chunks of 2048 +-1 bytes");
-    }
-
-    {
-        DB::HashingWriteBuffer buf(sink);
-
-        buf.write(data, data_size);
-
-        if (buf.getHash() != reference)
-            FAIL("failed on data size " << data_size << " writing all at once");
-    }
-}
-
-int main()
-{
-    test(5);
-    test(100);
-    test(2048);
-    test(2049);
-    test(100000);
-    test(1 << 17);
-
-    return 0;
-}

src/IO/examples/parse_int_perf.cpp

@@ -1,5 +1,6 @@
 #include <iostream>
 #include <iomanip>
+#include <pcg_random.hpp>
 
 #include <base/types.h>
 
@@ -7,7 +8,6 @@
 #include <IO/WriteHelpers.h>
 #include <IO/WriteIntText.h>
 #include <IO/WriteBufferFromVector.h>
-#include <Compression/CompressedReadBuffer.h>
 
 #include <Common/Stopwatch.h>
 
@@ -27,6 +27,8 @@ static UInt64 rdtsc()
 
 int main(int argc, char ** argv)
 {
+    pcg64 rng;
+
     try
     {
         if (argc < 2)
@@ -47,7 +49,7 @@ int main(int argc, char ** argv)
             Stopwatch watch;
 
             for (size_t i = 0; i < n; ++i)
-                data[i] = lrand48();// / lrand48();// ^ (lrand48() << 24) ^ (lrand48() << 48);
+                data[i] = rng();
 
             watch.stop();
             std::cerr << std::fixed << std::setprecision(2)

src/IO/examples/var_uint.cpp

@@ -8,32 +8,8 @@
 #include <Poco/HexBinaryEncoder.h>
 
 
-static void parse_trash_string_as_uint_must_fail(const std::string & str)
-{
-    using namespace DB;
-
-    unsigned x = 0xFF;
-
-    try
-    {
-        x = parse<unsigned>(str);
-    }
-    catch (...)
-    {
-        /// Ok
-        return;
-    }
-
-    std::cerr << "Parsing must fail, but finished successfully x=" << x;
-    exit(-1);
-}
-
-
 int main(int argc, char ** argv)
 {
-    parse_trash_string_as_uint_must_fail("trash");
-    parse_trash_string_as_uint_must_fail("-1");
-
     if (argc != 2)
     {
         std::cerr << "Usage: " << std::endl

src/Interpreters/ThreadStatusExt.cpp

@@ -416,7 +416,7 @@ void ThreadStatus::detachQuery(bool exit_if_already_detached, bool thread_exits)
         LOG_TRACE(log, "Resetting nice");
 
         if (0 != setpriority(PRIO_PROCESS, thread_id, 0))
-            LOG_ERROR(log, "Cannot 'setpriority' back to zero: {}", errnoToString(ErrorCodes::CANNOT_SET_THREAD_PRIORITY, errno));
+            LOG_ERROR(log, "Cannot 'setpriority' back to zero: {}", errnoToString());
 
         os_thread_priority = 0;
     }

src/Processors/Merges/Algorithms/tests/gtest_graphite.cpp

@@ -12,6 +12,8 @@
 #include <AggregateFunctions/registerAggregateFunctions.h>
 #include <Processors/Merges/Algorithms/Graphite.h>
 #include <Common/Config/ConfigProcessor.h>
+#include <base/errnoToString.h>
+
 
 using namespace DB;
 
@@ -43,7 +45,7 @@ static ConfigProcessor::LoadedConfig loadConfigurationFromString(std::string & s
     int fd = mkstemp(tmp_file);
     if (fd == -1)
     {
-        throw std::runtime_error(strerror(errno));
+        throw std::runtime_error(errnoToString());
     }
     try
     {
@@ -61,7 +63,7 @@ static ConfigProcessor::LoadedConfig loadConfigurationFromString(std::string & s
         {
             int err = errno;
             (void)remove(tmp_file);
-            throw std::runtime_error(strerror(err));
+            throw std::runtime_error(errnoToString(err));
         }
         ConfigProcessor::LoadedConfig config = loadConfiguration(config_path);
         (void)remove(tmp_file);

src/Server/CertificateReloader.cpp

@@ -118,7 +118,7 @@ bool CertificateReloader::File::changeIfModified(std::string new_path, Poco::Log
     if (ec)
     {
         LOG_ERROR(logger, "Cannot obtain modification time for {} file {}, skipping update. {}",
-            description, new_path, errnoToString(ErrorCodes::CANNOT_STAT, ec.value()));
+            description, new_path, errnoToString(ec.value()));
         return false;
     }
 

src/Storages/HDFS/HDFSCommon.cpp

@@ -7,15 +7,17 @@
 #if USE_HDFS
 #include <Common/ShellCommand.h>
 #include <Common/Exception.h>
-#include <IO/WriteBufferFromString.h>
 #include <IO/Operators.h>
 #include <Common/logger_useful.h>
+
 #if USE_KRB5
-#include <Access/KerberosInit.h>
+    #include <Access/KerberosInit.h>
-#endif // USE_KRB5
+#endif
+
 
 namespace DB
 {
+
 namespace ErrorCodes
 {
     extern const int BAD_ARGUMENTS;
@@ -26,10 +28,15 @@ namespace ErrorCodes
     #endif // USE_KRB5
 }
 
-const String HDFSBuilderWrapper::CONFIG_PREFIX = "hdfs";
+static constexpr std::string_view CONFIG_PREFIX = "hdfs";
-const String HDFS_URL_REGEXP = "^hdfs://[^/]*/.*";
+static constexpr std::string_view HDFS_URL_REGEXP = "^hdfs://[^/]*/.*";
+
+
+HDFSFileInfo::~HDFSFileInfo()
+{
+    hdfsFreeFileInfo(file_info, length);
+}
 
-std::once_flag init_libhdfs3_conf_flag;
 
 void HDFSBuilderWrapper::loadFromConfig(const Poco::Util::AbstractConfiguration & config,
     const String & prefix, bool isUser)
@@ -111,23 +118,6 @@ HDFSBuilderWrapper createHDFSBuilder(const String & uri_str, const Poco::Util::A
     if (host.empty())
         throw Exception("Illegal HDFS URI: " + uri.toString(), ErrorCodes::BAD_ARGUMENTS);
 
-    // Shall set env LIBHDFS3_CONF *before* HDFSBuilderWrapper construction.
-    std::call_once(init_libhdfs3_conf_flag, [&config]()
-    {
-        String libhdfs3_conf = config.getString(HDFSBuilderWrapper::CONFIG_PREFIX + ".libhdfs3_conf", "");
-        if (!libhdfs3_conf.empty())
-        {
-            if (std::filesystem::path{libhdfs3_conf}.is_relative() && !std::filesystem::exists(libhdfs3_conf))
-            {
-                const String config_path = config.getString("config-file", "config.xml");
-                const auto config_dir = std::filesystem::path{config_path}.remove_filename();
-                if (std::filesystem::exists(config_dir / libhdfs3_conf))
-                    libhdfs3_conf = std::filesystem::absolute(config_dir / libhdfs3_conf);
-            }
-            setenv("LIBHDFS3_CONF", libhdfs3_conf.c_str(), 1);
-        }
-    });
-
     HDFSBuilderWrapper builder;
     if (builder.get() == nullptr)
         throw Exception("Unable to create builder to connect to HDFS: " +
@@ -157,14 +147,14 @@ HDFSBuilderWrapper createHDFSBuilder(const String & uri_str, const Poco::Util::A
         hdfsBuilderSetNameNodePort(builder.get(), port);
     }
 
-    if (config.has(HDFSBuilderWrapper::CONFIG_PREFIX))
+    if (config.has(std::string(CONFIG_PREFIX)))
     {
-        builder.loadFromConfig(config, HDFSBuilderWrapper::CONFIG_PREFIX);
+        builder.loadFromConfig(config, std::string(CONFIG_PREFIX));
     }
 
     if (!user.empty())
     {
-        String user_config_prefix = HDFSBuilderWrapper::CONFIG_PREFIX + "_" + user;
+        String user_config_prefix = std::string(CONFIG_PREFIX) + "_" + user;
         if (config.has(user_config_prefix))
         {
             builder.loadFromConfig(config, user_config_prefix, true);
@@ -208,7 +198,7 @@ String getNameNodeCluster(const String &hdfs_url)
 
 void checkHDFSURL(const String & url)
 {
-    if (!re2::RE2::FullMatch(url, HDFS_URL_REGEXP))
+    if (!re2::RE2::FullMatch(url, std::string(HDFS_URL_REGEXP)))
         throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad hdfs url: {}. It should have structure 'hdfs://<host_name>:<port>/<path>'", url);
 }
 

src/Storages/HDFS/HDFSCommon.h

@@ -40,11 +40,7 @@ struct HDFSFileInfo
     HDFSFileInfo(HDFSFileInfo && other) = default;
     HDFSFileInfo & operator=(const HDFSFileInfo & other) = delete;
     HDFSFileInfo & operator=(HDFSFileInfo && other) = default;
-
+    ~HDFSFileInfo();
-    ~HDFSFileInfo()
-    {
-        hdfsFreeFileInfo(file_info, length);
-    }
 };
 
 

src/Storages/NATS/NATSConnection.cpp

@@ -18,7 +18,7 @@ NATSConnectionManager::NATSConnectionManager(const NATSConfiguration & configura
     , log(log_)
     , event_handler(loop.getLoop(), log)
 {
-    const char * val = std::getenv("CLICKHOUSE_NATS_TLS_SECURE");
+    const char * val = std::getenv("CLICKHOUSE_NATS_TLS_SECURE"); // NOLINT (this is safe on Linux glibc/Musl, but potentially not safe on other platforms)
     std::string tls_secure = val == nullptr ? std::string("1") : std::string(val);
     if (tls_secure == "0")
         skip_verification = true;

utils/CMakeLists.txt

@@ -23,7 +23,6 @@ if (NOT DEFINED ENABLE_UTILS OR ENABLE_UTILS)
     add_subdirectory (iotest)
     add_subdirectory (corrector_utf8)
     add_subdirectory (zookeeper-cli)
-    add_subdirectory (zookeeper-test)
     add_subdirectory (zookeeper-dump-tree)
     add_subdirectory (zookeeper-remove-by-list)
     add_subdirectory (zookeeper-create-entry-to-download-part)
@@ -35,7 +34,6 @@ if (NOT DEFINED ENABLE_UTILS OR ENABLE_UTILS)
     add_subdirectory (wal-dump)
     add_subdirectory (check-mysql-binlog)
     add_subdirectory (keeper-bench)
-    add_subdirectory (graphite-rollup)
 
     if (TARGET ch_contrib::nuraft)
         add_subdirectory (keeper-data-dumper)

utils/graphite-rollup/CMakeLists.txt

@@ -1,23 +0,0 @@
-clickhouse_add_executable(graphite-rollup-bench graphite-rollup-bench.cpp)
-target_link_libraries(
-    graphite-rollup-bench
-    PRIVATE
-    clickhouse_storages_system
-    clickhouse_aggregate_functions
-    clickhouse_common_config
-    dbms
-)
-target_include_directories(
-    graphite-rollup-bench
-    SYSTEM PRIVATE
-    ${ClickHouse_SOURCE_DIR}/src ${CMAKE_BINARY_DIR}/src
-    ${ClickHouse_SOURCE_DIR}/base ${ClickHouse_SOURCE_DIR}/base/pcg-random
-    ${CMAKE_BINARY_DIR}/src/Core/include
-    ${POCO_INCLUDE_DIR}
-    ${ClickHouse_SOURCE_DIR}/contrib/double-conversion ${ClickHouse_SOURCE_DIR}/contrib/dragonbox/include
-    ${ClickHouse_SOURCE_DIR}/contrib/fmtlib/include
-    ${ClickHouse_SOURCE_DIR}/contrib/cityhash102/include
-    ${RE2_INCLUDE_DIR} ${CMAKE_BINARY_DIR}/contrib/re2-cmake
-)
-
-target_compile_definitions(graphite-rollup-bench PRIVATE RULES_DIR="${CMAKE_CURRENT_SOURCE_DIR}")

utils/graphite-rollup/graphite-rollup-bench.cpp

@@ -1,130 +0,0 @@
-#include <chrono>
-#include <cstdio>
-#include <cstdlib>
-#include <iomanip>
-#include <iostream>
-#include <stdexcept>
-
-#include <boost/program_options.hpp>
-
-#include <AggregateFunctions/registerAggregateFunctions.h>
-#include <Processors/Merges/Algorithms/Graphite.h>
-#include <Processors/Merges/Algorithms/GraphiteRollupSortedAlgorithm.h>
-#include <Storages/System/StorageSystemGraphite.h>
-#include <Common/Config/ConfigProcessor.h>
-#include <Common/Exception.h>
-#include <Interpreters/Context.h>
-#include <IO/ReadBufferFromFile.h>
-#include <IO/ReadHelpers.h>
-
-using namespace DB;
-
-static SharedContextHolder shared_context = Context::createShared();
-
-auto loadMetrics(const std::string & metrics_file)
-{
-    std::vector<std::string> metrics;
-    ReadBufferFromFile in(metrics_file);
-    String line;
-
-    while (!in.eof())
-    {
-        readEscapedStringUntilEOL(line, in);
-        if (!in.eof())
-        {
-            ++in.position();
-        }
-        if (!line.empty() && line.back() == '\n')
-        {
-            line.pop_back();
-        }
-        if (!line.empty())
-        {
-            metrics.emplace_back(line);
-        }
-    }
-
-    return metrics;
-}
-
-ConfigProcessor::LoadedConfig loadConfiguration(const std::string & config_path)
-{
-    ConfigProcessor config_processor(config_path, true, true);
-    ConfigProcessor::LoadedConfig config = config_processor.loadConfig(false);
-    return config;
-}
-
-void bench(const std::string & config_path, const std::string & metrics_file, size_t n, bool verbose)
-{
-    auto config = loadConfiguration(config_path);
-
-    auto context = Context::createGlobal(shared_context.get());
-    context->setConfig(config.configuration.get());
-
-    Graphite::Params params;
-    setGraphitePatternsFromConfig(context, "graphite_rollup", params);
-
-    auto metrics = loadMetrics(metrics_file);
-
-    std::vector<double> durations(metrics.size());
-    size_t j, i;
-    for (j = 0; j < n; j++)
-    {
-        for (i = 0; i < metrics.size(); i++)
-        {
-            auto start = std::chrono::high_resolution_clock::now();
-
-            auto rule = DB::Graphite::selectPatternForPath(params, metrics[i]);
-            (void)rule;
-
-            auto end = std::chrono::high_resolution_clock::now();
-            double duration = (duration_cast<std::chrono::duration<double>>(end - start)).count() * 1E9;
-            durations[i] += duration;
-
-            if (j == 0 && verbose)
-            {
-                std::cout << metrics[i] << ": rule with regexp '" << rule.second->regexp_str << "' found\n";
-            }
-        }
-    }
-
-    for (i = 0; i < metrics.size(); i++)
-    {
-        std::cout << metrics[i] << " " << durations[i] / n << " ns\n";
-    }
-}
-
-int main(int argc, char ** argv)
-{
-    registerAggregateFunctions();
-
-    std::string config_file, metrics_file;
-
-    using namespace std::literals;
-
-    std::string config_default = RULES_DIR + "/rollup.xml"s;
-    std::string metrics_default = RULES_DIR + "/metrics.txt"s;
-
-    namespace po = boost::program_options;
-    po::variables_map vm;
-
-    po::options_description desc;
-    desc.add_options()("help,h", "produce help")(
-        "config,c", po::value<std::string>()->default_value(config_default), "XML config with rollup rules")(
-        "metrics,m", po::value<std::string>()->default_value(metrics_default), "metrcis files (one metric per line) for run benchmark")(
-        "verbose,V", po::bool_switch()->default_value(false), "verbose output (print found rule)");
-
-    po::parsed_options parsed = po::command_line_parser(argc, argv).options(desc).run();
-    po::store(parsed, vm);
-    po::notify(vm);
-
-    if (vm.count("help"))
-    {
-        std::cout << desc << '\n';
-        exit(1);
-    }
-
-    bench(vm["config"].as<std::string>(), vm["metrics"].as<std::string>(), 10000, vm["verbose"].as<bool>());
-
-    return 0;
-}

utils/graphite-rollup/metrics.txt

@@ -1,11 +0,0 @@
-test.sum
-sum?env=test&tag=Fake3
-test.max
-max?env=test&tag=Fake4
-test.min
-min?env=test&tag=Fake5
-fake5?env=test&tag=Fake5
-test.p95
-p95?env=test&tag=FakeNo
-default
-default?env=test&tag=FakeNo

utils/graphite-rollup/rollup-tag-list.xml

@@ -1,167 +0,0 @@
-<clickhouse>
-<graphite_rollup>
-	<pattern>
-        <rule_type>plain</rule_type>
- 		<regexp>\.sum$</regexp>
- 		<function>sum</function>
- 		<retention>
- 			<age>0</age>
- 			<precision>60</precision>
- 		</retention>
- 		<retention>
- 			<age>86400</age>
- 			<precision>3600</precision>
- 		</retention>
-	</pattern>
-	<pattern>
-        <rule_type>tagged</rule_type>
-		<regexp>^((.*)|.)sum\?</regexp>
-		<function>sum</function>
-		<retention>
- 			<age>0</age>
- 			<precision>60</precision>
- 		</retention>
- 		<retention>
- 			<age>86400</age>
- 			<precision>3600</precision>
- 		</retention>
-	</pattern>
-	<pattern>
-        <rule_type>plain</rule_type>
- 		<regexp>\.max$</regexp>
- 		<function>max</function>
- 		<retention>
- 			<age>0</age>
- 			<precision>60</precision>
- 		</retention>
- 		<retention>
- 			<age>86400</age>
- 			<precision>3600</precision>
- 		</retention>
-	</pattern>
-	<pattern>
-        <rule_type>tagged</rule_type>
-		<regexp>^((.*)|.)max\?</regexp>
-		<function>max</function>
-		<retention>
- 			<age>0</age>
- 			<precision>60</precision>
- 		</retention>
- 		<retention>
- 			<age>86400</age>
- 			<precision>3600</precision>
- 		</retention>
-	</pattern>
-	<pattern>
-        <rule_type>plain</rule_type>
- 		<regexp>\.min$</regexp>
- 		<function>min</function>
- 		<retention>
- 			<age>0</age>
- 			<precision>60</precision>
- 		</retention>
- 		<retention>
- 			<age>86400</age>
- 			<precision>3600</precision>
- 		</retention>
-	</pattern>
-	<pattern>
-    <rule_type>tagged</rule_type>
- 		<regexp>^((.*)|.)min\?</regexp>
- 		<function>min</function>
- 		<retention>
- 			<age>0</age>
- 			<precision>60</precision>
- 		</retention>
- 		<retention>
- 			<age>86400</age>
- 			<precision>3600</precision>
- 		</retention>
-	</pattern>
-	<pattern>
-        <rule_type>plain</rule_type>
-		<regexp>\.fake1\..*\.Fake1\.</regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-        <rule_type>tag_list</rule_type>
-		<regexp>fake1;tag=Fake1</regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-        <rule_type>plain</rule_type>
-		<regexp>\.fake2\..*\.Fake2\.</regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-        <rule_type>tag_list</rule_type>
-		<regexp>fake2;tag=Fake2</regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-        <rule_type>plain</rule_type>
-		<regexp>\.fake3\..*\.Fake3\.</regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-        <rule_type>tag_list</rule_type>
-		<regexp>fake3;tag=Fake3</regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-        <rule_type>plain</rule_type>
-		<regexp>\.fake4\..*\.Fake4\.</regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-        <rule_type>tag_list</rule_type>
-		<regexp>fake4;tag=Fake4</regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-        <rule_type>plain</rule_type>
-		<regexp>\.fake5\..*\.Fake5\.</regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-        <rule_type>tag_list</rule_type>
-		<regexp>fake5;tag=Fake5</regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-        <rule_type>plain</rule_type>
-		<regexp>\.fake6\..*\.Fake6\.</regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-        <rule_type>tag_list</rule_type>
-		<regexp>fake6;tag=Fake6</regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-        <rule_type>plain</rule_type>
-		<regexp>\.fake7\..*\.Fake7\.</regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-        <rule_type>tag_list</rule_type>
-		<regexp>fake7;tag=Fake7</regexp>
-		<function>sum</function>
-	</pattern>
- 	<default>
- 		<function>avg</function>
- 		<retention>
- 			<age>0</age>
- 			<precision>60</precision>
- 		</retention>
- 		<retention>
- 			<age>3600</age>
- 			<precision>300</precision>
- 		</retention>
- 		<retention>
- 			<age>86400</age>
- 			<precision>3600</precision>
- 		</retention>
- 	</default>
-</graphite_rollup>
-</clickhouse>

utils/graphite-rollup/rollup-typed.xml

@@ -1,167 +0,0 @@
-<clickhouse>
-<graphite_rollup>
-	<pattern>
-        <rule_type>plain</rule_type>
- 		<regexp>\.sum$</regexp>
- 		<function>sum</function>
- 		<retention>
- 			<age>0</age>
- 			<precision>60</precision>
- 		</retention>
- 		<retention>
- 			<age>86400</age>
- 			<precision>3600</precision>
- 		</retention>
-	</pattern>
-	<pattern>
-        <rule_type>tagged</rule_type>
-		<regexp>^((.*)|.)sum\?</regexp>
-		<function>sum</function>
-		<retention>
- 			<age>0</age>
- 			<precision>60</precision>
- 		</retention>
- 		<retention>
- 			<age>86400</age>
- 			<precision>3600</precision>
- 		</retention>
-	</pattern>
-	<pattern>
-        <rule_type>plain</rule_type>
- 		<regexp>\.max$</regexp>
- 		<function>max</function>
- 		<retention>
- 			<age>0</age>
- 			<precision>60</precision>
- 		</retention>
- 		<retention>
- 			<age>86400</age>
- 			<precision>3600</precision>
- 		</retention>
-	</pattern>
-	<pattern>
-        <rule_type>tagged</rule_type>
-		<regexp>^((.*)|.)max\?</regexp>
-		<function>max</function>
-		<retention>
- 			<age>0</age>
- 			<precision>60</precision>
- 		</retention>
- 		<retention>
- 			<age>86400</age>
- 			<precision>3600</precision>
- 		</retention>
-	</pattern>
-	<pattern>
-        <rule_type>plain</rule_type>
- 		<regexp>\.min$</regexp>
- 		<function>min</function>
- 		<retention>
- 			<age>0</age>
- 			<precision>60</precision>
- 		</retention>
- 		<retention>
- 			<age>86400</age>
- 			<precision>3600</precision>
- 		</retention>
-	</pattern>
-	<pattern>
-    <rule_type>tagged</rule_type>
- 		<regexp>^((.*)|.)min\?</regexp>
- 		<function>min</function>
- 		<retention>
- 			<age>0</age>
- 			<precision>60</precision>
- 		</retention>
- 		<retention>
- 			<age>86400</age>
- 			<precision>3600</precision>
- 		</retention>
-	</pattern>
-	<pattern>
-        <rule_type>plain</rule_type>
-		<regexp>\.fake1\..*\.Fake1\.</regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-        <rule_type>tagged</rule_type>
-		<regexp><![CDATA[^fake1\?(.*&)*tag=Fake1(&|$)]]></regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-        <rule_type>plain</rule_type>
-		<regexp>\.fake2\..*\.Fake2\.</regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-        <rule_type>tagged</rule_type>
-		<regexp><![CDATA[^fake2\?(.*&)*tag=Fake2(&|$)]]></regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-        <rule_type>plain</rule_type>
-		<regexp>\.fake3\..*\.Fake3\.</regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-        <rule_type>tagged</rule_type>
-		<regexp><![CDATA[^fake3\?(.*&)*tag=Fake3(&|$)]]></regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-        <rule_type>plain</rule_type>
-		<regexp>\.fake4\..*\.Fake4\.</regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-        <rule_type>tagged</rule_type>
-		<regexp><![CDATA[^fake4\?(.*&)*tag=Fake4(&|$)]]></regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-        <rule_type>plain</rule_type>
-		<regexp>\.fake5\..*\.Fake5\.</regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-        <rule_type>tagged</rule_type>
-		<regexp><![CDATA[^fake5\?(.*&)*tag=Fake5(&|$)]]></regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-        <rule_type>plain</rule_type>
-		<regexp>\.fake6\..*\.Fake6\.</regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-        <rule_type>tagged</rule_type>
-		<regexp><![CDATA[^fake6\?(.*&)*tag=Fake6(&|$)]]></regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-        <rule_type>plain</rule_type>
-		<regexp>\.fake7\..*\.Fake7\.</regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-        <rule_type>tagged</rule_type>
-		<regexp><![CDATA[^fake7\?(.*&)*tag=Fake7(&|$)]]></regexp>
-		<function>sum</function>
-	</pattern>
- 	<default>
- 		<function>avg</function>
- 		<retention>
- 			<age>0</age>
- 			<precision>60</precision>
- 		</retention>
- 		<retention>
- 			<age>3600</age>
- 			<precision>300</precision>
- 		</retention>
- 		<retention>
- 			<age>86400</age>
- 			<precision>3600</precision>
- 		</retention>
- 	</default>
-</graphite_rollup>
-</clickhouse>

utils/graphite-rollup/rollup.xml

@@ -1,147 +0,0 @@
-<clickhouse>
-<graphite_rollup>
-	<pattern>
- 		<regexp>\.sum$</regexp>
- 		<function>sum</function>
- 		<retention>
- 			<age>0</age>
- 			<precision>60</precision>
- 		</retention>
- 		<retention>
- 			<age>86400</age>
- 			<precision>3600</precision>
- 		</retention>
-	</pattern>
-	<pattern>
-		<regexp>^((.*)|.)sum\?</regexp>
-		<function>sum</function>
-		<retention>
- 			<age>0</age>
- 			<precision>60</precision>
- 		</retention>
- 		<retention>
- 			<age>86400</age>
- 			<precision>3600</precision>
- 		</retention>
-	</pattern>
-	<pattern>
- 		<regexp>\.max$</regexp>
- 		<function>max</function>
- 		<retention>
- 			<age>0</age>
- 			<precision>60</precision>
- 		</retention>
- 		<retention>
- 			<age>86400</age>
- 			<precision>3600</precision>
- 		</retention>
-	</pattern>
-	<pattern>
-		<regexp>^((.*)|.)max\?</regexp>
-		<function>max</function>
-		<retention>
- 			<age>0</age>
- 			<precision>60</precision>
- 		</retention>
- 		<retention>
- 			<age>86400</age>
- 			<precision>3600</precision>
- 		</retention>
-	</pattern>
-	<pattern>
- 		<regexp>\.min$</regexp>
- 		<function>min</function>
- 		<retention>
- 			<age>0</age>
- 			<precision>60</precision>
- 		</retention>
- 		<retention>
- 			<age>86400</age>
- 			<precision>3600</precision>
- 		</retention>
-	</pattern>
-	<pattern>
- 		<regexp>^((.*)|.)min\?</regexp>
- 		<function>min</function>
- 		<retention>
- 			<age>0</age>
- 			<precision>60</precision>
- 		</retention>
- 		<retention>
- 			<age>86400</age>
- 			<precision>3600</precision>
- 		</retention>
-	</pattern>
-	<pattern>
-		<regexp>\.fake1\..*\.Fake1\.</regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-		<regexp><![CDATA[^fake1\?(.*&)*tag=Fake1(&|$)]]></regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-		<regexp>\.fake2\..*\.Fake2\.</regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-		<regexp><![CDATA[^fake2\?(.*&)*tag=Fake2(&|$)]]></regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-		<regexp>\.fake3\..*\.Fake3\.</regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-		<regexp><![CDATA[^fake3\?(.*&)*tag=Fake3(&|$)]]></regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-		<regexp>\.fake4\..*\.Fake4\.</regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-		<regexp><![CDATA[^fake4\?(.*&)*tag=Fake4(&|$)]]></regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-		<regexp>\.fake5\..*\.Fake5\.</regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-		<regexp><![CDATA[^fake5\?(.*&)*tag=Fake5(&|$)]]></regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-		<regexp>\.fake6\..*\.Fake6\.</regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-		<regexp><![CDATA[^fake6\?(.*&)*tag=Fake6(&|$)]]></regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-		<regexp>\.fake7\..*\.Fake7\.</regexp>
-		<function>sum</function>
-	</pattern>
-	<pattern>
-		<regexp><![CDATA[^fake7\?(.*&)*tag=Fake7(&|$)]]></regexp>
-		<function>sum</function>
-	</pattern>
- 	<default>
- 		<function>avg</function>
- 		<retention>
- 			<age>0</age>
- 			<precision>60</precision>
- 		</retention>
- 		<retention>
- 			<age>3600</age>
- 			<precision>300</precision>
- 		</retention>
- 		<retention>
- 			<age>86400</age>
- 			<precision>3600</precision>
- 		</retention>
- 	</default>
-</graphite_rollup>
-</clickhouse>

utils/self-extracting-executable/decompressor.cpp

@@ -101,8 +101,8 @@ int decompress(char * input, char * output, off_t start, off_t end, size_t max_n
         {
             /// Decompress data in child process.
             if (0 != doDecompress(input, output, in_pointer, out_pointer, size, decompressed_size, dctx))
-                exit(1);
+                _exit(1);
-            exit(0);
+            _exit(0);
         }
         else
         {

utils/zookeeper-test/CMakeLists.txt

@@ -1,2 +0,0 @@
-clickhouse_add_executable(zk-test main.cpp)
-target_link_libraries(zk-test PRIVATE clickhouse_common_zookeeper_no_log)

utils/zookeeper-test/main.cpp

@@ -1,364 +0,0 @@
-#include <IO/ReadBufferFromString.h>
-#include <IO/ReadHelpers.h>
-#include <Poco/ConsoleChannel.h>
-#include <Common/ZooKeeper/KeeperException.h>
-#include <Common/ZooKeeper/ZooKeeper.h>
-#include <base/LineReader.h>
-#include <Common/logger_useful.h>
-#include <fmt/format.h>
-#include <random>
-#include <iterator>
-#include <algorithm>
-#include <chrono>
-#include <Common/Stopwatch.h>
-
-#include <iostream>
-#include <sstream>
-#include <exception>
-#include <future>
-
-using namespace std;
-
-/// TODO: Remove ME
-
-void checkEq(zkutil::ZooKeeper & zk, const std::string & path, const std::string & expected)
-{
-    auto result = zk.get(path);
-    if (result != expected)
-        throw std::runtime_error(fmt::format("Data on path '{}' = '{}' doesn't match expected '{}'",
-                                             path, result, expected));
-}
-
-void checkExists(zkutil::ZooKeeper & zk, const std::string & path)
-{
-    if (!zk.exists(path))
-        throw std::runtime_error(fmt::format("Path '{}' doesn't exists", path));
-}
-
-void testCreateGetExistsNode(zkutil::ZooKeeper & zk)
-{
-    zk.create("/data", "test_string", zkutil::CreateMode::Persistent);
-    zk.create("/data/seq-", "another_string", zkutil::CreateMode::PersistentSequential);
-    checkEq(zk, "/data", "test_string");
-    checkExists(zk, "/data/seq-0000000000");
-    checkEq(zk, "/data/seq-0000000000", "another_string");
-}
-
-void testCreateSetNode(zkutil::ZooKeeper & zk)
-{
-    zk.create("/data/set", "sssss", zkutil::CreateMode::Persistent);
-    checkEq(zk, "/data/set", "sssss");
-    zk.set("/data/set", "qqqqq");
-    checkEq(zk, "/data/set", "qqqqq");
-}
-
-void testCreateList(zkutil::ZooKeeper & zk)
-{
-    zk.create("/data/lst", "", zkutil::CreateMode::Persistent);
-    zk.create("/data/lst/d1", "", zkutil::CreateMode::Persistent);
-    zk.create("/data/lst/d2", "", zkutil::CreateMode::Persistent);
-    zk.create("/data/lst/d3", "", zkutil::CreateMode::Persistent);
-    auto children = zk.getChildren("/data/lst");
-    if (children.size() != 3)
-        throw std::runtime_error("Children of /data/lst doesn't equal to three");
-    for (size_t i = 0; i < children.size(); ++i)
-    {
-        std::cerr << "children:" << children[i] << std::endl;
-        std::cerr << "children size:" << children[i].size() << std::endl;
-        if (children[i] != "d" + std::to_string(i + 1))
-            throw std::runtime_error(fmt::format("Incorrect children #{} got {}, expected {}", i, children[i], "d" + std::to_string(i + 1)));
-    }
-}
-
-void testCreateSetVersionRequest(zkutil::ZooKeeper & zk)
-{
-    zk.create("/data/check_data", "d", zkutil::CreateMode::Persistent);
-    Coordination::Stat stat{};
-    try
-    {
-        zk.set("/data/check_data", "e", stat.version + 2);
-        std::terminate();
-    }
-    catch (...)
-    {
-        std::cerr << "Got exception on incorrect version (it's ok)\n";
-    }
-
-    checkEq(zk, "/data/check_data", "d");
-    zk.set("/data/check_data", "e", stat.version);
-
-    checkEq(zk, "/data/check_data", "e");
-}
-
-void testCreateSetWatchEvent(zkutil::ZooKeeper & zk)
-{
-
-    std::shared_ptr<Poco::Event> event = std::make_shared<Poco::Event>();
-    zk.create("/data/nodeforwatch", "", zkutil::CreateMode::Persistent);
-    Coordination::Stat stat;
-    zk.get("/data/nodeforwatch", &stat, event);
-
-    if (event->tryWait(300))
-        throw std::runtime_error(fmt::format("Event for path {} was set without any actions", "/data/nodeforwatch"));
-
-    zk.set("/data/nodeforwatch", "x");
-    if (!event->tryWait(300))
-        throw std::runtime_error(fmt::format("Event for path {} was not set after set", "/data/nodeforwatch"));
-    else
-        std::cerr << "Event was set well\n";
-}
-
-void testCreateListWatchEvent(zkutil::ZooKeeper & zk)
-{
-    std::shared_ptr<Poco::Event> event = std::make_shared<Poco::Event>();
-    std::string path = "/data/pathforwatch";
-    zk.create(path, "", zkutil::CreateMode::Persistent);
-    zk.create(path + "/n1", "", zkutil::CreateMode::Persistent);
-    zk.create(path + "/n2", "", zkutil::CreateMode::Persistent);
-    zk.getChildren(path, nullptr, event);
-
-    if (event->tryWait(300))
-        throw std::runtime_error(fmt::format("ListEvent for path {} was set without any actions", path));
-
-    zk.create(path + "/n3", "", zkutil::CreateMode::Persistent);
-    if (!event->tryWait(300))
-        throw std::runtime_error(fmt::format("ListEvent for path {} was not set after create", path));
-    else
-        std::cerr << "ListEvent was set well\n";
-}
-
-void testMultiRequest(zkutil::ZooKeeper & zk)
-{
-    std::cerr << "Testing multi request\n";
-    Coordination::Requests requests;
-    requests.push_back(zkutil::makeCreateRequest("/data/multirequest", "aaa", zkutil::CreateMode::Persistent));
-    requests.push_back(zkutil::makeSetRequest("/data/multirequest", "bbb", -1));
-    zk.multi(requests);
-    std::cerr << "Multi executed\n";
-
-    try
-    {
-        requests.clear();
-        std::cerr << "Testing bad multi\n";
-        requests.push_back(zkutil::makeCreateRequest("/data/multirequest", "qweqwe", zkutil::CreateMode::Persistent));
-        requests.push_back(zkutil::makeSetRequest("/data/multirequest", "bbb", -1));
-        requests.push_back(zkutil::makeSetRequest("/data/multirequest", "ccc", -1));
-        zk.multi(requests);
-        std::cerr << "Bad multi executed\n";
-        std::terminate();
-    }
-    catch (...)
-    {
-        std::cerr << "Got exception on multy request (it's ok)\n";
-    }
-
-    checkEq(zk, "/data/multirequest", "bbb");
-    std::cerr << "Multi request finished\n";
-}
-
-std::mutex elements_mutex;
-std::vector<int> current_elements;
-std::atomic<int> watches_triggered = 0;
-
-void triggerWatch(const Coordination::WatchResponse &)
-{
-    watches_triggered++;
-}
-
-template<typename Iter, typename RandomGenerator>
-Iter select_randomly(Iter start, Iter end, RandomGenerator& g)
-{
-    std::uniform_int_distribution<> dis(0, std::distance(start, end) - 1);
-    std::advance(start, dis(g));
-    return start;
-}
-
-template<typename Iter>
-Iter select_randomly(Iter start, Iter end)
-{
-    static std::random_device rd;
-    static std::mt19937 gen(rd());
-    return select_randomly(start, end, gen);
-}
-
-std::atomic<int> element_counter = 0;
-std::atomic<int> failed_setup_counter = 0;
-
-void createPathAndSetWatch(zkutil::ZooKeeper & zk, const String & path_prefix, size_t total)
-{
-    for (size_t i = 0; i < total; ++i)
-    {
-        int element = element_counter++;
-        zk.createIfNotExists(path_prefix + "/" + std::to_string(element), "");
-
-        std::string result;
-        if (!zk.tryGetWatch(path_prefix + "/" + std::to_string(element), result, nullptr, triggerWatch))
-            failed_setup_counter++;
-
-        {
-            std::lock_guard lock(elements_mutex);
-            current_elements.push_back(element);
-        }
-
-        std::this_thread::sleep_for(std::chrono::milliseconds(200));
-
-        {
-            std::lock_guard lock(elements_mutex);
-            if (current_elements.empty())
-                continue;
-            element = *select_randomly(current_elements.begin(), current_elements.end());
-            current_elements.erase(std::remove(current_elements.begin(), current_elements.end(), element), current_elements.end());
-        }
-        zk.tryRemove(path_prefix + "/" + std::to_string(element));
-    }
-
-}
-
-std::string random_string(size_t length)
-{
-    auto randchar = []() -> char
-    {
-        const char charset[] =
-        "0123456789"
-        "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
-        "abcdefghijklmnopqrstuvwxyz";
-        const size_t max_index = (sizeof(charset) - 1);
-        return charset[rand() % max_index]; /// NOLINT
-    };
-    std::string str(length, 0);
-    std::generate_n(str.begin(), length, randchar);
-    return str;
-}
-
-std::string currentDateTime()
-{
-    time_t now = time(nullptr);
-    tm tstruct;
-    char buf[80];
-    tstruct = *localtime(&now);
-    // Visit http://en.cppreference.com/w/cpp/chrono/c/strftime
-    // for more information about date/time format
-    size_t size = strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", &tstruct);
-
-    return std::string(buf, size);
-}
-
-
-void createOnPrefix(const std::string & zkhost, const String & path_prefix, size_t datasize, size_t total)
-{
-    zkutil::ZooKeeper zk(zkhost);
-    std::vector<std::future<Coordination::CreateResponse>> holder_futures;
-    using namespace std::chrono;
-    try
-    {
-        for (size_t i = 0; i < total; ++i)
-        {
-            std::cerr << currentDateTime() << "] Request:" << i << std::endl;
-            std::string path = path_prefix + "/element" + std::to_string(i);
-            holder_futures.push_back(zk.asyncCreate(path, random_string(datasize), zkutil::CreateMode::Persistent));
-        }
-
-        for (auto & future : holder_futures)
-            future.get();
-    }
-    catch (...)
-    {
-        ::exit(-1);
-    }
-}
-
-
-void createConcurrent(zkutil::ZooKeeper & testzk, const std::string & zkhost, size_t threads, size_t requests, size_t blobsize)
-{
-    std::vector<std::future<void>> asyncs;
-    for (size_t i = 0; i < threads; ++i)
-    {
-        std::string path_prefix = "/data/create_test" + std::to_string(i);
-        testzk.createIfNotExists(path_prefix, "");
-        auto callback = [&zkhost, path_prefix, requests, blobsize] ()
-        {
-            createOnPrefix(zkhost, path_prefix, blobsize, requests);
-        };
-        asyncs.push_back(std::async(std::launch::async, callback));
-    }
-
-    for (auto & async : asyncs)
-    {
-        async.wait();
-    }
-}
-
-void tryConcurrentWatches(zkutil::ZooKeeper & zk)
-{
-    std::string path_prefix = "/concurrent_watches";
-    std::vector<std::future<void>> asyncs;
-    zk.createIfNotExists(path_prefix, "");
-    for (size_t i = 0; i < 100; ++i)
-    {
-        auto callback = [&zk, path_prefix] ()
-        {
-            createPathAndSetWatch(zk, path_prefix, 100);
-        };
-        asyncs.push_back(std::async(std::launch::async, callback));
-    }
-
-    for (auto & async : asyncs)
-    {
-        async.wait();
-    }
-
-    size_t counter = 0;
-    while (watches_triggered != 100 * 100)
-    {
-        std::this_thread::sleep_for(std::chrono::milliseconds(100));
-        if (counter++ > 20)
-            break;
-    }
-
-    std::cerr << "Failed setup counter:" << failed_setup_counter << std::endl;
-    std::cerr << "Current elements size:" << current_elements.size() << std::endl;
-    std::cerr << "WatchesTriggered:" << watches_triggered << std::endl;
-}
-
-
-int main(int argc, char *argv[])
-{
-    if (argc != 2)
-    {
-        std::cerr << "usage: " << argv[0] << " hosts" << std::endl;
-        return 2;
-    }
-    Poco::AutoPtr<Poco::ConsoleChannel> channel = new Poco::ConsoleChannel(std::cerr);
-    Poco::Logger::root().setChannel(channel);
-    Poco::Logger::root().setLevel("trace");
-
-    zkutil::ZooKeeper zk(argv[1]);
-
-    try
-    {
-        std::cerr << "Removing\n";
-        zk.tryRemoveRecursive("/data");
-        std::cerr << "Creating\n";
-        zk.createIfNotExists("/data", "");
-        std::cerr << "Created\n";
-
-        Stopwatch watch;
-        createConcurrent(zk, argv[1], 1, 1005000, 10);
-        std::cerr << "Finished in: " << watch.elapsedMilliseconds() << "ms" << std::endl;
-
-        //testCreateGetExistsNode(zk);
-        //testCreateSetNode(zk);
-        //testCreateList(zk);
-        //testCreateSetVersionRequest(zk);
-        //testMultiRequest(zk);
-        //testCreateSetWatchEvent(zk);
-        //testCreateListWatchEvent(zk);
-        //tryConcurrentWatches(zk);
-    }
-    catch (...)
-    {
-        zk.tryRemoveRecursive("/data");
-        throw;
-    }
-    return 0;
-}