diff --git a/.gitmodules b/.gitmodules index ab7c8a7c94d..be44e3268e3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -210,9 +210,6 @@ [submodule "contrib/fast_float"] path = contrib/fast_float url = https://github.com/fastfloat/fast_float -[submodule "contrib/libpqxx"] - path = contrib/libpqxx - url = https://github.com/jtv/libpqxx [submodule "contrib/libpq"] path = contrib/libpq url = https://github.com/ClickHouse-Extras/libpq @@ -231,3 +228,6 @@ [submodule "contrib/yaml-cpp"] path = contrib/yaml-cpp url = https://github.com/ClickHouse-Extras/yaml-cpp.git +[submodule "contrib/libpqxx"] + path = contrib/libpqxx + url = https://github.com/ClickHouse-Extras/libpqxx.git diff --git a/base/bridge/IBridge.cpp b/base/bridge/IBridge.cpp index b2ec53158b1..35a9b95c97f 100644 --- a/base/bridge/IBridge.cpp +++ b/base/bridge/IBridge.cpp @@ -1,14 +1,22 @@ #include "IBridge.h" -#include #include #include #include -#include -#include + #include +#include + +#include #include +#include +#include +#include #include +#include +#include +#include +#include #if USE_ODBC # include @@ -163,6 +171,31 @@ void IBridge::initialize(Application & self) max_server_connections = config().getUInt("max-server-connections", 1024); keep_alive_timeout = config().getUInt64("keep-alive-timeout", 10); + struct rlimit limit; + const UInt64 gb = 1024 * 1024 * 1024; + + /// Set maximum RSS to 1 GiB. + limit.rlim_max = limit.rlim_cur = gb; + if (setrlimit(RLIMIT_RSS, &limit)) + LOG_WARNING(log, "Unable to set maximum RSS to 1GB: {} (current rlim_cur={}, rlim_max={})", + errnoToString(errno), limit.rlim_cur, limit.rlim_max); + + if (!getrlimit(RLIMIT_RSS, &limit)) + LOG_INFO(log, "RSS limit: cur={}, max={}", limit.rlim_cur, limit.rlim_max); + + try + { + const auto oom_score = toString(config().getUInt64("bridge_oom_score", 500)); + WriteBufferFromFile buf("/proc/self/oom_score_adj"); + buf.write(oom_score.data(), oom_score.size()); + buf.close(); + LOG_INFO(log, "OOM score is set to {}", oom_score); + } + catch (const Exception & e) + { + LOG_WARNING(log, "Failed to set OOM score, error: {}", e.what()); + } + initializeTerminationAndSignalProcessing(); ServerApplication::initialize(self); // NOLINT @@ -214,7 +247,7 @@ int IBridge::main(const std::vector & /*args*/) server.stop(); - for (size_t count : ext::range(1, 6)) + for (size_t count : collections::range(1, 6)) { if (server.currentConnections() == 0) break; diff --git a/base/common/DecomposedFloat.h b/base/common/DecomposedFloat.h index 078ba823c15..21034908fe7 100644 --- a/base/common/DecomposedFloat.h +++ b/base/common/DecomposedFloat.h @@ -91,10 +91,12 @@ struct DecomposedFloat /// Compare float with integer of arbitrary width (both signed and unsigned are supported). Assuming two's complement arithmetic. + /// This function is generic, big integers (128, 256 bit) are supported as well. /// Infinities are compared correctly. NaNs are treat similarly to infinities, so they can be less than all numbers. /// (note that we need total order) + /// Returns -1, 0 or 1. template - int compare(Int rhs) + int compare(Int rhs) const { if (rhs == 0) return sign(); @@ -137,10 +139,11 @@ struct DecomposedFloat if (normalized_exponent() >= static_cast(8 * sizeof(Int) - is_signed_v)) return is_negative() ? -1 : 1; - using UInt = make_unsigned_t; + using UInt = std::conditional_t<(sizeof(Int) > sizeof(typename Traits::UInt)), make_unsigned_t, typename Traits::UInt>; UInt uint_rhs = rhs < 0 ? -rhs : rhs; /// Smaller octave: abs(rhs) < abs(float) + /// FYI, TIL: octave is also called "binade", https://en.wikipedia.org/wiki/Binade if (uint_rhs < (static_cast(1) << normalized_exponent())) return is_negative() ? -1 : 1; @@ -154,11 +157,11 @@ struct DecomposedFloat bool large_and_always_integer = normalized_exponent() >= static_cast(Traits::mantissa_bits); - typename Traits::UInt a = large_and_always_integer - ? mantissa() << (normalized_exponent() - Traits::mantissa_bits) - : mantissa() >> (Traits::mantissa_bits - normalized_exponent()); + UInt a = large_and_always_integer + ? static_cast(mantissa()) << (normalized_exponent() - Traits::mantissa_bits) + : static_cast(mantissa()) >> (Traits::mantissa_bits - normalized_exponent()); - typename Traits::UInt b = uint_rhs - (static_cast(1) << normalized_exponent()); + UInt b = uint_rhs - (static_cast(1) << normalized_exponent()); if (a < b) return is_negative() ? 1 : -1; @@ -175,37 +178,37 @@ struct DecomposedFloat template - bool equals(Int rhs) + bool equals(Int rhs) const { return compare(rhs) == 0; } template - bool notEquals(Int rhs) + bool notEquals(Int rhs) const { return compare(rhs) != 0; } template - bool less(Int rhs) + bool less(Int rhs) const { return compare(rhs) < 0; } template - bool greater(Int rhs) + bool greater(Int rhs) const { return compare(rhs) > 0; } template - bool lessOrEquals(Int rhs) + bool lessOrEquals(Int rhs) const { return compare(rhs) <= 0; } template - bool greaterOrEquals(Int rhs) + bool greaterOrEquals(Int rhs) const { return compare(rhs) >= 0; } diff --git a/base/common/ReadlineLineReader.cpp b/base/common/ReadlineLineReader.cpp index 397a7dd7543..f2c2b60f327 100644 --- a/base/common/ReadlineLineReader.cpp +++ b/base/common/ReadlineLineReader.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include diff --git a/base/common/SimpleCache.h b/base/common/SimpleCache.h index 57247de696a..c3bf019c226 100644 --- a/base/common/SimpleCache.h +++ b/base/common/SimpleCache.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include /** The simplest cache for a free function. @@ -32,10 +32,11 @@ public: template Result operator() (Args &&... args) { + Key key{std::forward(args)...}; + { std::lock_guard lock(mutex); - Key key{std::forward(args)...}; auto it = cache.find(key); if (cache.end() != it) @@ -43,7 +44,7 @@ public: } /// The calculations themselves are not done under mutex. - Result res = f(std::forward(args)...); + Result res = std::apply(f, key); { std::lock_guard lock(mutex); @@ -57,11 +58,12 @@ public: template void update(Args &&... args) { - Result res = f(std::forward(args)...); + Key key{std::forward(args)...}; + + Result res = std::apply(f, key); + { std::lock_guard lock(mutex); - - Key key{std::forward(args)...}; cache[key] = std::move(res); } } diff --git a/base/common/arraySize.h b/base/common/arraySize.h new file mode 100644 index 00000000000..d6245257ad0 --- /dev/null +++ b/base/common/arraySize.h @@ -0,0 +1,7 @@ +#pragma once + +#include + +/** \brief Returns number of elements in an automatic array. */ +template +constexpr size_t arraySize(const T (&)[N]) noexcept { return N; } diff --git a/base/common/bit_cast.h b/base/common/bit_cast.h new file mode 100644 index 00000000000..5b4b0931b62 --- /dev/null +++ b/base/common/bit_cast.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include +#include + + +/** \brief Returns value `from` converted to type `To` while retaining bit representation. + * `To` and `From` must satisfy `CopyConstructible`. + */ +template +std::decay_t bit_cast(const From & from) +{ + To res {}; + memcpy(static_cast(&res), &from, std::min(sizeof(res), sizeof(from))); + return res; +} + +/** \brief Returns value `from` converted to type `To` while retaining bit representation. + * `To` and `From` must satisfy `CopyConstructible`. + */ +template +std::decay_t safe_bit_cast(const From & from) +{ + static_assert(sizeof(To) == sizeof(From), "bit cast on types of different width"); + return bit_cast(from); +} diff --git a/base/common/chrono_io.h b/base/common/chrono_io.h new file mode 100644 index 00000000000..4ee8dec6634 --- /dev/null +++ b/base/common/chrono_io.h @@ -0,0 +1,46 @@ +#pragma once + +#include +#include +#include +#include + + +inline std::string to_string(const std::time_t & time) +{ + return cctz::format("%Y-%m-%d %H:%M:%S", std::chrono::system_clock::from_time_t(time), cctz::local_time_zone()); +} + +template +std::string to_string(const std::chrono::time_point & tp) +{ + // Don't use DateLUT because it shows weird characters for + // TimePoint::max(). I wish we could use C++20 format, but it's not + // there yet. + // return DateLUT::instance().timeToString(std::chrono::system_clock::to_time_t(tp)); + + auto in_time_t = std::chrono::system_clock::to_time_t(tp); + return to_string(in_time_t); +} + +template > +std::string to_string(const std::chrono::duration & duration) +{ + auto seconds_as_int = std::chrono::duration_cast(duration); + if (seconds_as_int == duration) + return std::to_string(seconds_as_int.count()) + "s"; + auto seconds_as_double = std::chrono::duration_cast>(duration); + return std::to_string(seconds_as_double.count()) + "s"; +} + +template +std::ostream & operator<<(std::ostream & o, const std::chrono::time_point & tp) +{ + return o << to_string(tp); +} + +template > +std::ostream & operator<<(std::ostream & o, const std::chrono::duration & duration) +{ + return o << to_string(duration); +} diff --git a/base/ext/function_traits.h b/base/common/function_traits.h similarity index 100% rename from base/ext/function_traits.h rename to base/common/function_traits.h diff --git a/base/common/map.h b/base/common/map.h new file mode 100644 index 00000000000..043d8363619 --- /dev/null +++ b/base/common/map.h @@ -0,0 +1,52 @@ +#pragma once + +#include +#include + +namespace collections +{ + +/// \brief Strip type off top level reference and cv-qualifiers thus allowing storage in containers +template +using unqualified_t = std::remove_cv_t>; + +/** \brief Returns collection of the same container-type as the input collection, + * with each element transformed by the application of `mapper`. + */ +template