diff --git a/.gitignore b/.gitignore index 1db6e0a78c9..a469ff7bca1 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,11 @@ /build-* /tests/venv +# logs +*.log +*.stderr +*.stdout + /docs/build /docs/publish /docs/edit diff --git a/.gitmodules b/.gitmodules index 2ccce88e5e4..1d9d4d25baf 100644 --- a/.gitmodules +++ b/.gitmodules @@ -103,7 +103,7 @@ url = https://github.com/ClickHouse-Extras/fastops [submodule "contrib/orc"] path = contrib/orc - url = https://github.com/apache/orc + url = https://github.com/ClickHouse-Extras/orc [submodule "contrib/sparsehash-c11"] path = contrib/sparsehash-c11 url = https://github.com/sparsehash/sparsehash-c11.git @@ -168,9 +168,6 @@ [submodule "contrib/fmtlib"] path = contrib/fmtlib url = https://github.com/fmtlib/fmt.git -[submodule "contrib/antlr4-runtime"] - path = contrib/antlr4-runtime - url = https://github.com/ClickHouse-Extras/antlr4-runtime.git [submodule "contrib/sentry-native"] path = contrib/sentry-native url = https://github.com/ClickHouse-Extras/sentry-native.git @@ -210,9 +207,6 @@ [submodule "contrib/fast_float"] path = contrib/fast_float url = https://github.com/fastfloat/fast_float -[submodule "contrib/libpqxx"] - path = contrib/libpqxx - url = https://github.com/jtv/libpqxx [submodule "contrib/libpq"] path = contrib/libpq url = https://github.com/ClickHouse-Extras/libpq @@ -228,7 +222,9 @@ [submodule "contrib/datasketches-cpp"] path = contrib/datasketches-cpp url = https://github.com/ClickHouse-Extras/datasketches-cpp.git - [submodule "contrib/yaml-cpp"] path = contrib/yaml-cpp url = https://github.com/ClickHouse-Extras/yaml-cpp.git +[submodule "contrib/libpqxx"] + path = contrib/libpqxx + url = https://github.com/ClickHouse-Extras/libpqxx.git diff --git a/CHANGELOG.md b/CHANGELOG.md index cad82fd1e28..8987082db30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -763,6 +763,7 @@ * Allow using extended integer types (`Int128`, `Int256`, `UInt256`) in `avg` and `avgWeighted` functions. Also allow using different types (integer, decimal, floating point) for value and for weight in `avgWeighted` function. This is a backward-incompatible change: now the `avg` and `avgWeighted` functions always return `Float64` (as documented). Before this change the return type for `Decimal` arguments was also `Decimal`. [#15419](https://github.com/ClickHouse/ClickHouse/pull/15419) ([Mike](https://github.com/myrrc)). * Expression `toUUID(N)` no longer works. Replace with `toUUID('00000000-0000-0000-0000-000000000000')`. This change is motivated by non-obvious results of `toUUID(N)` where N is non zero. * SSL Certificates with incorrect "key usage" are rejected. In previous versions they are used to work. See [#19262](https://github.com/ClickHouse/ClickHouse/issues/19262). +* `incl` references to substitutions file (`/etc/metrika.xml`) were removed from the default config (``, ``, ``, ``, ``). If you were using substitutions file and were relying on those implicit references, you should put them back manually and explicitly by adding corresponding sections with `incl="..."` attributes before the update. See [#18740](https://github.com/ClickHouse/ClickHouse/pull/18740) ([alexey-milovidov](https://github.com/alexey-milovidov)). #### New Feature diff --git a/CMakeLists.txt b/CMakeLists.txt index 9c62748ff95..d23e5f540d3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -183,24 +183,37 @@ endif () # Make sure the final executable has symbols exported set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -rdynamic") -if (OS_LINUX) - find_program (OBJCOPY_PATH NAMES "llvm-objcopy" "llvm-objcopy-12" "llvm-objcopy-11" "llvm-objcopy-10" "llvm-objcopy-9" "llvm-objcopy-8" "objcopy") - if (OBJCOPY_PATH) - message(STATUS "Using objcopy: ${OBJCOPY_PATH}.") +find_program (OBJCOPY_PATH NAMES "llvm-objcopy" "llvm-objcopy-12" "llvm-objcopy-11" "llvm-objcopy-10" "llvm-objcopy-9" "llvm-objcopy-8" "objcopy") - if (ARCH_AMD64) - set(OBJCOPY_ARCH_OPTIONS -O elf64-x86-64 -B i386) - elseif (ARCH_AARCH64) - set(OBJCOPY_ARCH_OPTIONS -O elf64-aarch64 -B aarch64) +if (NOT OBJCOPY_PATH AND OS_DARWIN) + find_program (BREW_PATH NAMES "brew") + if (BREW_PATH) + execute_process (COMMAND ${BREW_PATH} --prefix llvm ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE LLVM_PREFIX) + if (LLVM_PREFIX) + find_program (OBJCOPY_PATH NAMES "llvm-objcopy" PATHS "${LLVM_PREFIX}/bin" NO_DEFAULT_PATH) + endif () + if (NOT OBJCOPY_PATH) + execute_process (COMMAND ${BREW_PATH} --prefix binutils ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE BINUTILS_PREFIX) + if (BINUTILS_PREFIX) + find_program (OBJCOPY_PATH NAMES "objcopy" PATHS "${BINUTILS_PREFIX}/bin" NO_DEFAULT_PATH) + endif () endif () - else () - message(FATAL_ERROR "Cannot find objcopy.") endif () endif () +if (OBJCOPY_PATH) + message (STATUS "Using objcopy: ${OBJCOPY_PATH}") +else () + message (FATAL_ERROR "Cannot find objcopy.") +endif () + if (OS_DARWIN) - set(WHOLE_ARCHIVE -all_load) - set(NO_WHOLE_ARCHIVE -noall_load) + # The `-all_load` flag forces loading of all symbols from all libraries, + # and leads to multiply-defined symbols. This flag allows force loading + # from a _specific_ library, which is what we need. + set(WHOLE_ARCHIVE -force_load) + # The `-noall_load` flag is the default and now obsolete. + set(NO_WHOLE_ARCHIVE "") else () set(WHOLE_ARCHIVE --whole-archive) set(NO_WHOLE_ARCHIVE --no-whole-archive) @@ -528,7 +541,6 @@ include (cmake/find/libpqxx.cmake) include (cmake/find/nuraft.cmake) include (cmake/find/yaml-cpp.cmake) - if(NOT USE_INTERNAL_PARQUET_LIBRARY) set (ENABLE_ORC OFF CACHE INTERNAL "") endif() diff --git a/README.md b/README.md index 5677837815c..a915570122d 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,11 @@ ClickHouse® is an open-source column-oriented database management system that a * [Tutorial](https://clickhouse.tech/docs/en/getting_started/tutorial/) shows how to set up and query small ClickHouse cluster. * [Documentation](https://clickhouse.tech/docs/en/) provides more in-depth information. * [YouTube channel](https://www.youtube.com/c/ClickHouseDB) has a lot of content about ClickHouse in video format. -* [Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-qfort0u8-TWqK4wIP0YSdoDE0btKa1w) and [Telegram](https://telegram.me/clickhouse_en) allow to chat with ClickHouse users in real-time. +* [Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-rxm3rdrk-lIUmhLC3V8WTaL0TGxsOmg) and [Telegram](https://telegram.me/clickhouse_en) allow to chat with ClickHouse users in real-time. * [Blog](https://clickhouse.yandex/blog/en/) contains various ClickHouse-related articles, as well as announcements and reports about events. * [Code Browser](https://clickhouse.tech/codebrowser/html_report/ClickHouse/index.html) with syntax highlight and navigation. * [Contacts](https://clickhouse.tech/#contacts) can help to get your questions answered if there are any. * You can also [fill this form](https://clickhouse.tech/#meet) to meet Yandex ClickHouse team in person. ## Upcoming Events -* [SF Bay Area ClickHouse Community Meetup (online)](https://www.meetup.com/San-Francisco-Bay-Area-ClickHouse-Meetup/events/278144089/) on 16 June 2021. +* [China ClickHouse Community Meetup (online)](http://hdxu.cn/rhbfZ) on 26 June 2021. diff --git a/base/bridge/IBridge.cpp b/base/bridge/IBridge.cpp index b2ec53158b1..35a9b95c97f 100644 --- a/base/bridge/IBridge.cpp +++ b/base/bridge/IBridge.cpp @@ -1,14 +1,22 @@ #include "IBridge.h" -#include #include #include #include -#include -#include + #include +#include + +#include #include +#include +#include +#include #include +#include +#include +#include +#include #if USE_ODBC # include @@ -163,6 +171,31 @@ void IBridge::initialize(Application & self) max_server_connections = config().getUInt("max-server-connections", 1024); keep_alive_timeout = config().getUInt64("keep-alive-timeout", 10); + struct rlimit limit; + const UInt64 gb = 1024 * 1024 * 1024; + + /// Set maximum RSS to 1 GiB. + limit.rlim_max = limit.rlim_cur = gb; + if (setrlimit(RLIMIT_RSS, &limit)) + LOG_WARNING(log, "Unable to set maximum RSS to 1GB: {} (current rlim_cur={}, rlim_max={})", + errnoToString(errno), limit.rlim_cur, limit.rlim_max); + + if (!getrlimit(RLIMIT_RSS, &limit)) + LOG_INFO(log, "RSS limit: cur={}, max={}", limit.rlim_cur, limit.rlim_max); + + try + { + const auto oom_score = toString(config().getUInt64("bridge_oom_score", 500)); + WriteBufferFromFile buf("/proc/self/oom_score_adj"); + buf.write(oom_score.data(), oom_score.size()); + buf.close(); + LOG_INFO(log, "OOM score is set to {}", oom_score); + } + catch (const Exception & e) + { + LOG_WARNING(log, "Failed to set OOM score, error: {}", e.what()); + } + initializeTerminationAndSignalProcessing(); ServerApplication::initialize(self); // NOLINT @@ -214,7 +247,7 @@ int IBridge::main(const std::vector & /*args*/) server.stop(); - for (size_t count : ext::range(1, 6)) + for (size_t count : collections::range(1, 6)) { if (server.currentConnections() == 0) break; diff --git a/base/common/DateLUT.h b/base/common/DateLUT.h index 378b4360f3b..31fc6b1e24b 100644 --- a/base/common/DateLUT.h +++ b/base/common/DateLUT.h @@ -17,7 +17,7 @@ class DateLUT : private boost::noncopyable { public: /// Return singleton DateLUTImpl instance for the default time zone. - static ALWAYS_INLINE const DateLUTImpl & instance() + static ALWAYS_INLINE const DateLUTImpl & instance() // -V1071 { const auto & date_lut = getInstance(); return *date_lut.default_impl.load(std::memory_order_acquire); diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 9e60181e802..2ccad4be348 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -119,11 +119,16 @@ private: } public: + /// We use Int64 instead of time_t because time_t is mapped to the different types (long or long long) + /// on Linux and Darwin (on both of them, long and long long are 64 bit and behaves identically, + /// but they are different types in C++ and this affects function overload resolution). + using Time = Int64; + /// The order of fields matters for alignment and sizeof. struct Values { - /// time_t at beginning of the day. - Int64 date; + /// Time at beginning of the day. + Time date; /// Properties of the day. UInt16 year; @@ -182,20 +187,20 @@ private: LUTIndex years_months_lut[DATE_LUT_YEARS * 12]; /// UTC offset at beginning of the Unix epoch. The same as unix timestamp of 1970-01-01 00:00:00 local time. - time_t offset_at_start_of_epoch; + Time offset_at_start_of_epoch; /// UTC offset at the beginning of the first supported year. - time_t offset_at_start_of_lut; + Time offset_at_start_of_lut; bool offset_is_whole_number_of_hours_during_epoch; /// Time zone name. std::string time_zone; - inline LUTIndex findIndex(time_t t) const + inline LUTIndex findIndex(Time t) const { /// First guess. - Int64 guess = (t / 86400) + daynum_offset_epoch; + Time guess = (t / 86400) + daynum_offset_epoch; - /// For negative time_t the integer division was rounded up, so the guess is offset by one. + /// For negative Time the integer division was rounded up, so the guess is offset by one. if (unlikely(t < 0)) --guess; @@ -227,7 +232,7 @@ private: return LUTIndex{static_cast(d + daynum_offset_epoch) & date_lut_mask}; } - inline LUTIndex toLUTIndex(time_t t) const + inline LUTIndex toLUTIndex(Time t) const { return findIndex(t); } @@ -280,7 +285,7 @@ public: /// Round down to start of monday. template - inline time_t toFirstDayOfWeek(DateOrTime v) const + inline Time toFirstDayOfWeek(DateOrTime v) const { const LUTIndex i = toLUTIndex(v); return lut[i - (lut[i].day_of_week - 1)].date; @@ -295,7 +300,7 @@ public: /// Round down to start of month. template - inline time_t toFirstDayOfMonth(DateOrTime v) const + inline Time toFirstDayOfMonth(DateOrTime v) const { const LUTIndex i = toLUTIndex(v); return lut[i - (lut[i].day_of_month - 1)].date; @@ -332,13 +337,13 @@ public: } template - inline time_t toFirstDayOfQuarter(DateOrTime v) const + inline Time toFirstDayOfQuarter(DateOrTime v) const { return toDate(toFirstDayOfQuarterIndex(v)); } /// Round down to start of year. - inline time_t toFirstDayOfYear(time_t t) const + inline Time toFirstDayOfYear(Time t) const { return lut[years_lut[lut[findIndex(t)].year - DATE_LUT_MIN_YEAR]].date; } @@ -355,14 +360,14 @@ public: return toDayNum(toFirstDayNumOfYearIndex(v)); } - inline time_t toFirstDayOfNextMonth(time_t t) const + inline Time toFirstDayOfNextMonth(Time t) const { LUTIndex index = findIndex(t); index += 32 - lut[index].day_of_month; return lut[index - (lut[index].day_of_month - 1)].date; } - inline time_t toFirstDayOfPrevMonth(time_t t) const + inline Time toFirstDayOfPrevMonth(Time t) const { LUTIndex index = findIndex(t); index -= lut[index].day_of_month; @@ -389,16 +394,16 @@ public: /** Round to start of day, then shift for specified amount of days. */ - inline time_t toDateAndShift(time_t t, Int32 days) const + inline Time toDateAndShift(Time t, Int32 days) const { return lut[findIndex(t) + days].date; } - inline time_t toTime(time_t t) const + inline Time toTime(Time t) const { const LUTIndex index = findIndex(t); - time_t res = t - lut[index].date; + Time res = t - lut[index].date; if (res >= lut[index].time_at_offset_change()) res += lut[index].amount_of_offset_change(); @@ -406,11 +411,11 @@ public: return res - offset_at_start_of_epoch; /// Starting at 1970-01-01 00:00:00 local time. } - inline unsigned toHour(time_t t) const + inline unsigned toHour(Time t) const { const LUTIndex index = findIndex(t); - time_t time = t - lut[index].date; + Time time = t - lut[index].date; if (time >= lut[index].time_at_offset_change()) time += lut[index].amount_of_offset_change(); @@ -426,7 +431,7 @@ public: * then subtract the former from the latter to get the offset result. * The boundaries when meets DST(daylight saving time) change should be handled very carefully. */ - inline time_t timezoneOffset(time_t t) const + inline Time timezoneOffset(Time t) const { const LUTIndex index = findIndex(t); @@ -434,7 +439,7 @@ public: /// Because the "amount_of_offset_change" in LUT entry only exists in the change day, it's costly to scan it from the very begin. /// but we can figure out all the accumulated offsets from 1970-01-01 to that day just by get the whole difference between lut[].date, /// and then, we can directly subtract multiple 86400s to get the real DST offsets for the leap seconds is not considered now. - time_t res = (lut[index].date - lut[daynum_offset_epoch].date) % 86400; + Time res = (lut[index].date - lut[daynum_offset_epoch].date) % 86400; /// As so far to know, the maximal DST offset couldn't be more than 2 hours, so after the modulo operation the remainder /// will sits between [-offset --> 0 --> offset] which respectively corresponds to moving clock forward or backward. @@ -448,7 +453,7 @@ public: } - inline unsigned toSecond(time_t t) const + inline unsigned toSecond(Time t) const { auto res = t % 60; if (likely(res >= 0)) @@ -456,7 +461,7 @@ public: return res + 60; } - inline unsigned toMinute(time_t t) const + inline unsigned toMinute(Time t) const { if (t >= 0 && offset_is_whole_number_of_hours_during_epoch) return (t / 60) % 60; @@ -474,27 +479,27 @@ public: } /// NOTE: Assuming timezone offset is a multiple of 15 minutes. - inline time_t toStartOfMinute(time_t t) const { return roundDown(t, 60); } - inline time_t toStartOfFiveMinute(time_t t) const { return roundDown(t, 300); } - inline time_t toStartOfFifteenMinutes(time_t t) const { return roundDown(t, 900); } + inline Time toStartOfMinute(Time t) const { return roundDown(t, 60); } + inline Time toStartOfFiveMinute(Time t) const { return roundDown(t, 300); } + inline Time toStartOfFifteenMinutes(Time t) const { return roundDown(t, 900); } - inline time_t toStartOfTenMinutes(time_t t) const + inline Time toStartOfTenMinutes(Time t) const { if (t >= 0 && offset_is_whole_number_of_hours_during_epoch) return t / 600 * 600; /// More complex logic is for Nepal - it has offset 05:45. Australia/Eucla is also unfortunate. - Int64 date = find(t).date; + Time date = find(t).date; return date + (t - date) / 600 * 600; } /// NOTE: Assuming timezone transitions are multiple of hours. Lord Howe Island in Australia is a notable exception. - inline time_t toStartOfHour(time_t t) const + inline Time toStartOfHour(Time t) const { if (t >= 0 && offset_is_whole_number_of_hours_during_epoch) return t / 3600 * 3600; - Int64 date = find(t).date; + Time date = find(t).date; return date + (t - date) / 3600 * 3600; } @@ -506,11 +511,11 @@ public: * because the same calendar day starts/ends at different timestamps in different time zones) */ - inline time_t fromDayNum(DayNum d) const { return lut[toLUTIndex(d)].date; } - inline time_t fromDayNum(ExtendedDayNum d) const { return lut[toLUTIndex(d)].date; } + inline Time fromDayNum(DayNum d) const { return lut[toLUTIndex(d)].date; } + inline Time fromDayNum(ExtendedDayNum d) const { return lut[toLUTIndex(d)].date; } template - inline time_t toDate(DateOrTime v) const { return lut[toLUTIndex(v)].date; } + inline Time toDate(DateOrTime v) const { return lut[toLUTIndex(v)].date; } template inline unsigned toMonth(DateOrTime v) const { return lut[toLUTIndex(v)].month; } @@ -578,7 +583,7 @@ public: return toDayNum(toFirstDayNumOfISOYearIndex(v)); } - inline time_t toFirstDayOfISOYear(time_t t) const + inline Time toFirstDayOfISOYear(Time t) const { return lut[toFirstDayNumOfISOYearIndex(t)].date; } @@ -773,7 +778,7 @@ public: } /// We count all hour-length intervals, unrelated to offset changes. - inline time_t toRelativeHourNum(time_t t) const + inline Time toRelativeHourNum(Time t) const { if (t >= 0 && offset_is_whole_number_of_hours_during_epoch) return t / 3600; @@ -784,18 +789,18 @@ public: } template - inline time_t toRelativeHourNum(DateOrTime v) const + inline Time toRelativeHourNum(DateOrTime v) const { return toRelativeHourNum(lut[toLUTIndex(v)].date); } - inline time_t toRelativeMinuteNum(time_t t) const + inline Time toRelativeMinuteNum(Time t) const { return (t + DATE_LUT_ADD) / 60 - (DATE_LUT_ADD / 60); } template - inline time_t toRelativeMinuteNum(DateOrTime v) const + inline Time toRelativeMinuteNum(DateOrTime v) const { return toRelativeMinuteNum(lut[toLUTIndex(v)].date); } @@ -842,14 +847,14 @@ public: return ExtendedDayNum(4 + (d - 4) / days * days); } - inline time_t toStartOfDayInterval(ExtendedDayNum d, UInt64 days) const + inline Time toStartOfDayInterval(ExtendedDayNum d, UInt64 days) const { if (days == 1) return toDate(d); return lut[toLUTIndex(ExtendedDayNum(d / days * days))].date; } - inline time_t toStartOfHourInterval(time_t t, UInt64 hours) const + inline Time toStartOfHourInterval(Time t, UInt64 hours) const { if (hours == 1) return toStartOfHour(t); @@ -867,7 +872,7 @@ public: const LUTIndex index = findIndex(t); const Values & values = lut[index]; - time_t time = t - values.date; + Time time = t - values.date; if (time >= values.time_at_offset_change()) { /// Align to new hour numbers before rounding. @@ -892,7 +897,7 @@ public: return values.date + time; } - inline time_t toStartOfMinuteInterval(time_t t, UInt64 minutes) const + inline Time toStartOfMinuteInterval(Time t, UInt64 minutes) const { if (minutes == 1) return toStartOfMinute(t); @@ -909,7 +914,7 @@ public: return roundDown(t, seconds); } - inline time_t toStartOfSecondInterval(time_t t, UInt64 seconds) const + inline Time toStartOfSecondInterval(Time t, UInt64 seconds) const { if (seconds == 1) return t; @@ -934,14 +939,14 @@ public: return toDayNum(makeLUTIndex(year, month, day_of_month)); } - inline time_t makeDate(Int16 year, UInt8 month, UInt8 day_of_month) const + inline Time makeDate(Int16 year, UInt8 month, UInt8 day_of_month) const { return lut[makeLUTIndex(year, month, day_of_month)].date; } /** Does not accept daylight saving time as argument: in case of ambiguity, it choose greater timestamp. */ - inline time_t makeDateTime(Int16 year, UInt8 month, UInt8 day_of_month, UInt8 hour, UInt8 minute, UInt8 second) const + inline Time makeDateTime(Int16 year, UInt8 month, UInt8 day_of_month, UInt8 hour, UInt8 minute, UInt8 second) const { size_t index = makeLUTIndex(year, month, day_of_month); UInt32 time_offset = hour * 3600 + minute * 60 + second; @@ -969,7 +974,7 @@ public: return values.year * 10000 + values.month * 100 + values.day_of_month; } - inline time_t YYYYMMDDToDate(UInt32 num) const + inline Time YYYYMMDDToDate(UInt32 num) const { return makeDate(num / 10000, num / 100 % 100, num % 100); } @@ -1000,13 +1005,13 @@ public: TimeComponents time; }; - inline DateComponents toDateComponents(time_t t) const + inline DateComponents toDateComponents(Time t) const { const Values & values = getValues(t); return { values.year, values.month, values.day_of_month }; } - inline DateTimeComponents toDateTimeComponents(time_t t) const + inline DateTimeComponents toDateTimeComponents(Time t) const { const LUTIndex index = findIndex(t); const Values & values = lut[index]; @@ -1017,7 +1022,7 @@ public: res.date.month = values.month; res.date.day = values.day_of_month; - time_t time = t - values.date; + Time time = t - values.date; if (time >= values.time_at_offset_change()) time += values.amount_of_offset_change(); @@ -1042,7 +1047,7 @@ public: } - inline UInt64 toNumYYYYMMDDhhmmss(time_t t) const + inline UInt64 toNumYYYYMMDDhhmmss(Time t) const { DateTimeComponents components = toDateTimeComponents(t); @@ -1055,7 +1060,7 @@ public: + UInt64(components.date.year) * 10000000000; } - inline time_t YYYYMMDDhhmmssToTime(UInt64 num) const + inline Time YYYYMMDDhhmmssToTime(UInt64 num) const { return makeDateTime( num / 10000000000, @@ -1069,12 +1074,12 @@ public: /// Adding calendar intervals. /// Implementation specific behaviour when delta is too big. - inline NO_SANITIZE_UNDEFINED time_t addDays(time_t t, Int64 delta) const + inline NO_SANITIZE_UNDEFINED Time addDays(Time t, Int64 delta) const { const LUTIndex index = findIndex(t); const Values & values = lut[index]; - time_t time = t - values.date; + Time time = t - values.date; if (time >= values.time_at_offset_change()) time += values.amount_of_offset_change(); @@ -1086,7 +1091,7 @@ public: return lut[new_index].date + time; } - inline NO_SANITIZE_UNDEFINED time_t addWeeks(time_t t, Int64 delta) const + inline NO_SANITIZE_UNDEFINED Time addWeeks(Time t, Int64 delta) const { return addDays(t, delta * 7); } @@ -1131,14 +1136,14 @@ public: /// If resulting month has less deys than source month, then saturation can happen. /// Example: 31 Aug + 1 month = 30 Sep. - inline time_t NO_SANITIZE_UNDEFINED addMonths(time_t t, Int64 delta) const + inline Time NO_SANITIZE_UNDEFINED addMonths(Time t, Int64 delta) const { const auto result_day = addMonthsIndex(t, delta); const LUTIndex index = findIndex(t); const Values & values = lut[index]; - time_t time = t - values.date; + Time time = t - values.date; if (time >= values.time_at_offset_change()) time += values.amount_of_offset_change(); @@ -1153,7 +1158,7 @@ public: return toDayNum(addMonthsIndex(d, delta)); } - inline time_t NO_SANITIZE_UNDEFINED addQuarters(time_t t, Int64 delta) const + inline Time NO_SANITIZE_UNDEFINED addQuarters(Time t, Int64 delta) const { return addMonths(t, delta * 3); } @@ -1180,14 +1185,14 @@ public: } /// Saturation can occur if 29 Feb is mapped to non-leap year. - inline time_t addYears(time_t t, Int64 delta) const + inline Time addYears(Time t, Int64 delta) const { auto result_day = addYearsIndex(t, delta); const LUTIndex index = findIndex(t); const Values & values = lut[index]; - time_t time = t - values.date; + Time time = t - values.date; if (time >= values.time_at_offset_change()) time += values.amount_of_offset_change(); @@ -1203,7 +1208,7 @@ public: } - inline std::string timeToString(time_t t) const + inline std::string timeToString(Time t) const { DateTimeComponents components = toDateTimeComponents(t); @@ -1228,7 +1233,7 @@ public: return s; } - inline std::string dateToString(time_t t) const + inline std::string dateToString(Time t) const { const Values & values = getValues(t); diff --git a/base/common/DecomposedFloat.h b/base/common/DecomposedFloat.h index 078ba823c15..21034908fe7 100644 --- a/base/common/DecomposedFloat.h +++ b/base/common/DecomposedFloat.h @@ -91,10 +91,12 @@ struct DecomposedFloat /// Compare float with integer of arbitrary width (both signed and unsigned are supported). Assuming two's complement arithmetic. + /// This function is generic, big integers (128, 256 bit) are supported as well. /// Infinities are compared correctly. NaNs are treat similarly to infinities, so they can be less than all numbers. /// (note that we need total order) + /// Returns -1, 0 or 1. template - int compare(Int rhs) + int compare(Int rhs) const { if (rhs == 0) return sign(); @@ -137,10 +139,11 @@ struct DecomposedFloat if (normalized_exponent() >= static_cast(8 * sizeof(Int) - is_signed_v)) return is_negative() ? -1 : 1; - using UInt = make_unsigned_t; + using UInt = std::conditional_t<(sizeof(Int) > sizeof(typename Traits::UInt)), make_unsigned_t, typename Traits::UInt>; UInt uint_rhs = rhs < 0 ? -rhs : rhs; /// Smaller octave: abs(rhs) < abs(float) + /// FYI, TIL: octave is also called "binade", https://en.wikipedia.org/wiki/Binade if (uint_rhs < (static_cast(1) << normalized_exponent())) return is_negative() ? -1 : 1; @@ -154,11 +157,11 @@ struct DecomposedFloat bool large_and_always_integer = normalized_exponent() >= static_cast(Traits::mantissa_bits); - typename Traits::UInt a = large_and_always_integer - ? mantissa() << (normalized_exponent() - Traits::mantissa_bits) - : mantissa() >> (Traits::mantissa_bits - normalized_exponent()); + UInt a = large_and_always_integer + ? static_cast(mantissa()) << (normalized_exponent() - Traits::mantissa_bits) + : static_cast(mantissa()) >> (Traits::mantissa_bits - normalized_exponent()); - typename Traits::UInt b = uint_rhs - (static_cast(1) << normalized_exponent()); + UInt b = uint_rhs - (static_cast(1) << normalized_exponent()); if (a < b) return is_negative() ? 1 : -1; @@ -175,37 +178,37 @@ struct DecomposedFloat template - bool equals(Int rhs) + bool equals(Int rhs) const { return compare(rhs) == 0; } template - bool notEquals(Int rhs) + bool notEquals(Int rhs) const { return compare(rhs) != 0; } template - bool less(Int rhs) + bool less(Int rhs) const { return compare(rhs) < 0; } template - bool greater(Int rhs) + bool greater(Int rhs) const { return compare(rhs) > 0; } template - bool lessOrEquals(Int rhs) + bool lessOrEquals(Int rhs) const { return compare(rhs) <= 0; } template - bool greaterOrEquals(Int rhs) + bool greaterOrEquals(Int rhs) const { return compare(rhs) >= 0; } diff --git a/base/common/ReadlineLineReader.cpp b/base/common/ReadlineLineReader.cpp index 397a7dd7543..f2c2b60f327 100644 --- a/base/common/ReadlineLineReader.cpp +++ b/base/common/ReadlineLineReader.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include diff --git a/base/common/ReplxxLineReader.cpp b/base/common/ReplxxLineReader.cpp index 7893e56d751..9c65b1dfe4c 100644 --- a/base/common/ReplxxLineReader.cpp +++ b/base/common/ReplxxLineReader.cpp @@ -1,8 +1,9 @@ #include #include -#include -#include +#include +#include +#include #include #include #include @@ -24,6 +25,94 @@ void trim(String & s) s.erase(std::find_if(s.rbegin(), s.rend(), [](int ch) { return !std::isspace(ch); }).base(), s.end()); } +/// Copied from replxx::src/util.cxx::now_ms_str() under the terms of 3-clause BSD license of Replxx. +/// Copyright (c) 2017-2018, Marcin Konarski (amok at codestation.org) +/// Copyright (c) 2010, Salvatore Sanfilippo (antirez at gmail dot com) +/// Copyright (c) 2010, Pieter Noordhuis (pcnoordhuis at gmail dot com) +std::string replxx_now_ms_str() +{ + std::chrono::milliseconds ms(std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch())); + time_t t = ms.count() / 1000; + tm broken; + if (!localtime_r(&t, &broken)) + { + return std::string(); + } + + static int const BUFF_SIZE(32); + char str[BUFF_SIZE]; + strftime(str, BUFF_SIZE, "%Y-%m-%d %H:%M:%S.", &broken); + snprintf(str + sizeof("YYYY-mm-dd HH:MM:SS"), 5, "%03d", static_cast(ms.count() % 1000)); + return str; +} + +/// Convert from readline to replxx format. +/// +/// replxx requires each history line to prepended with time line: +/// +/// ### YYYY-MM-DD HH:MM:SS.SSS +/// select 1 +/// +/// And w/o those service lines it will load all lines from history file as +/// one history line for suggestion. And if there are lots of lines in file it +/// will take lots of time (getline() + tons of reallocations). +/// +/// NOTE: this code uses std::ifstream/std::ofstream like original replxx code. +void convertHistoryFile(const std::string & path, replxx::Replxx & rx) +{ + std::ifstream in(path); + if (!in) + { + rx.print("Cannot open %s reading (for conversion): %s\n", + path.c_str(), errnoToString(errno).c_str()); + return; + } + + std::string line; + if (!getline(in, line).good()) + { + rx.print("Cannot read from %s (for conversion): %s\n", + path.c_str(), errnoToString(errno).c_str()); + return; + } + + /// This is the marker of the date, no need to convert. + static char const REPLXX_TIMESTAMP_PATTERN[] = "### dddd-dd-dd dd:dd:dd.ddd"; + if (line.starts_with("### ") && line.size() == strlen(REPLXX_TIMESTAMP_PATTERN)) + { + return; + } + + std::vector lines; + in.seekg(0); + while (getline(in, line).good()) + { + lines.push_back(line); + } + in.close(); + + size_t lines_size = lines.size(); + std::sort(lines.begin(), lines.end()); + lines.erase(std::unique(lines.begin(), lines.end()), lines.end()); + rx.print("The history file (%s) is in old format. %zu lines, %zu unique lines.\n", + path.c_str(), lines_size, lines.size()); + + std::ofstream out(path); + if (!out) + { + rx.print("Cannot open %s for writing (for conversion): %s\n", + path.c_str(), errnoToString(errno).c_str()); + return; + } + + const std::string & timestamp = replxx_now_ms_str(); + for (const auto & out_line : lines) + { + out << "### " << timestamp << "\n" << out_line << std::endl; + } + out.close(); +} + } ReplxxLineReader::ReplxxLineReader( @@ -47,6 +136,8 @@ ReplxxLineReader::ReplxxLineReader( } else { + convertHistoryFile(history_file_path, rx); + if (flock(history_file_fd, LOCK_SH)) { rx.print("Shared lock of history file failed: %s\n", errnoToString(errno).c_str()); diff --git a/base/common/SimpleCache.h b/base/common/SimpleCache.h index 57247de696a..c3bf019c226 100644 --- a/base/common/SimpleCache.h +++ b/base/common/SimpleCache.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include /** The simplest cache for a free function. @@ -32,10 +32,11 @@ public: template Result operator() (Args &&... args) { + Key key{std::forward(args)...}; + { std::lock_guard lock(mutex); - Key key{std::forward(args)...}; auto it = cache.find(key); if (cache.end() != it) @@ -43,7 +44,7 @@ public: } /// The calculations themselves are not done under mutex. - Result res = f(std::forward(args)...); + Result res = std::apply(f, key); { std::lock_guard lock(mutex); @@ -57,11 +58,12 @@ public: template void update(Args &&... args) { - Result res = f(std::forward(args)...); + Key key{std::forward(args)...}; + + Result res = std::apply(f, key); + { std::lock_guard lock(mutex); - - Key key{std::forward(args)...}; cache[key] = std::move(res); } } diff --git a/base/common/arraySize.h b/base/common/arraySize.h new file mode 100644 index 00000000000..d6245257ad0 --- /dev/null +++ b/base/common/arraySize.h @@ -0,0 +1,7 @@ +#pragma once + +#include + +/** \brief Returns number of elements in an automatic array. */ +template +constexpr size_t arraySize(const T (&)[N]) noexcept { return N; } diff --git a/base/common/bit_cast.h b/base/common/bit_cast.h new file mode 100644 index 00000000000..5b4b0931b62 --- /dev/null +++ b/base/common/bit_cast.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include +#include + + +/** \brief Returns value `from` converted to type `To` while retaining bit representation. + * `To` and `From` must satisfy `CopyConstructible`. + */ +template +std::decay_t bit_cast(const From & from) +{ + To res {}; + memcpy(static_cast(&res), &from, std::min(sizeof(res), sizeof(from))); + return res; +} + +/** \brief Returns value `from` converted to type `To` while retaining bit representation. + * `To` and `From` must satisfy `CopyConstructible`. + */ +template +std::decay_t safe_bit_cast(const From & from) +{ + static_assert(sizeof(To) == sizeof(From), "bit cast on types of different width"); + return bit_cast(from); +} diff --git a/base/common/chrono_io.h b/base/common/chrono_io.h new file mode 100644 index 00000000000..4ee8dec6634 --- /dev/null +++ b/base/common/chrono_io.h @@ -0,0 +1,46 @@ +#pragma once + +#include +#include +#include +#include + + +inline std::string to_string(const std::time_t & time) +{ + return cctz::format("%Y-%m-%d %H:%M:%S", std::chrono::system_clock::from_time_t(time), cctz::local_time_zone()); +} + +template +std::string to_string(const std::chrono::time_point & tp) +{ + // Don't use DateLUT because it shows weird characters for + // TimePoint::max(). I wish we could use C++20 format, but it's not + // there yet. + // return DateLUT::instance().timeToString(std::chrono::system_clock::to_time_t(tp)); + + auto in_time_t = std::chrono::system_clock::to_time_t(tp); + return to_string(in_time_t); +} + +template > +std::string to_string(const std::chrono::duration & duration) +{ + auto seconds_as_int = std::chrono::duration_cast(duration); + if (seconds_as_int == duration) + return std::to_string(seconds_as_int.count()) + "s"; + auto seconds_as_double = std::chrono::duration_cast>(duration); + return std::to_string(seconds_as_double.count()) + "s"; +} + +template +std::ostream & operator<<(std::ostream & o, const std::chrono::time_point & tp) +{ + return o << to_string(tp); +} + +template > +std::ostream & operator<<(std::ostream & o, const std::chrono::duration & duration) +{ + return o << to_string(duration); +} diff --git a/base/ext/function_traits.h b/base/common/function_traits.h similarity index 100% rename from base/ext/function_traits.h rename to base/common/function_traits.h diff --git a/base/common/getResource.cpp b/base/common/getResource.cpp index 5d5f18047b3..6682ae0a01f 100644 --- a/base/common/getResource.cpp +++ b/base/common/getResource.cpp @@ -4,23 +4,42 @@ #include #include - std::string_view getResource(std::string_view name) { + // Convert the resource file name into the form generated by `ld -r -b binary`. std::string name_replaced(name); std::replace(name_replaced.begin(), name_replaced.end(), '/', '_'); std::replace(name_replaced.begin(), name_replaced.end(), '-', '_'); std::replace(name_replaced.begin(), name_replaced.end(), '.', '_'); boost::replace_all(name_replaced, "+", "_PLUS_"); - /// These are the names that are generated by "ld -r -b binary" - std::string symbol_name_data = "_binary_" + name_replaced + "_start"; - std::string symbol_name_size = "_binary_" + name_replaced + "_size"; + // In most `dlsym(3)` APIs, one passes the symbol name as it appears via + // something like `nm` or `objdump -t`. For example, a symbol `_foo` would be + // looked up with the string `"_foo"`. + // + // Apple's linker is confusingly different. The NOTES on the man page for + // `dlsym(3)` claim that one looks up the symbol with "the name used in C + // source code". In this example, that would mean using the string `"foo"`. + // This apparently applies even in the case where the symbol did not originate + // from C source, such as the embedded binary resource files used here. So + // the symbol name must not have a leading `_` on Apple platforms. It's not + // clear how this applies to other symbols, such as those which _have_ a leading + // underscore in them by design, many leading underscores, etc. +#if defined OS_DARWIN + std::string prefix = "binary_"; +#else + std::string prefix = "_binary_"; +#endif + std::string symbol_name_start = prefix + name_replaced + "_start"; + std::string symbol_name_end = prefix + name_replaced + "_end"; - const void * sym_data = dlsym(RTLD_DEFAULT, symbol_name_data.c_str()); - const void * sym_size = dlsym(RTLD_DEFAULT, symbol_name_size.c_str()); + const char* sym_start = reinterpret_cast(dlsym(RTLD_DEFAULT, symbol_name_start.c_str())); + const char* sym_end = reinterpret_cast(dlsym(RTLD_DEFAULT, symbol_name_end.c_str())); - if (sym_data && sym_size) - return { static_cast(sym_data), unalignedLoad(&sym_size) }; + if (sym_start && sym_end) + { + auto resource_size = static_cast(std::distance(sym_start, sym_end)); + return { sym_start, resource_size }; + } return {}; } diff --git a/base/common/map.h b/base/common/map.h new file mode 100644 index 00000000000..043d8363619 --- /dev/null +++ b/base/common/map.h @@ -0,0 +1,52 @@ +#pragma once + +#include +#include + +namespace collections +{ + +/// \brief Strip type off top level reference and cv-qualifiers thus allowing storage in containers +template +using unqualified_t = std::remove_cv_t>; + +/** \brief Returns collection of the same container-type as the input collection, + * with each element transformed by the application of `mapper`. + */ +template