Merge branch 'master' into fix-backward-compatibility-after-21196

This commit is contained in:
Nikolai Kochetov 2021-10-04 17:35:32 +03:00 committed by GitHub
commit e2d07fbdf5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2228 changed files with 13909 additions and 8648 deletions

View File

@ -1355,7 +1355,7 @@
* Export current max ddl entry executed by DDLWorker via server metric. It's useful to check if DDLWorker hangs somewhere. [#17464](https://github.com/ClickHouse/ClickHouse/pull/17464) ([Amos Bird](https://github.com/amosbird)).
* Export asynchronous metrics of all servers current threads. It's useful to track down issues like [this](https://github.com/ClickHouse-Extras/poco/pull/28). [#17463](https://github.com/ClickHouse/ClickHouse/pull/17463) ([Amos Bird](https://github.com/amosbird)).
* Include dynamic columns like MATERIALIZED / ALIAS for wildcard query when settings `asterisk_include_materialized_columns` and `asterisk_include_alias_columns` are turned on. [#17462](https://github.com/ClickHouse/ClickHouse/pull/17462) ([Ken Chen](https://github.com/chenziliang)).
* Allow specifying [TTL](https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/mergetree/#mergetree-table-ttl) to remove old entries from [system log tables](https://clickhouse.tech/docs/en/operations/system-tables/), using the `<ttl>` attribute in `config.xml`. [#17438](https://github.com/ClickHouse/ClickHouse/pull/17438) ([Du Chuan](https://github.com/spongedu)).
* Allow specifying [TTL](https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree/#mergetree-table-ttl) to remove old entries from [system log tables](https://clickhouse.com/docs/en/operations/system-tables/), using the `<ttl>` attribute in `config.xml`. [#17438](https://github.com/ClickHouse/ClickHouse/pull/17438) ([Du Chuan](https://github.com/spongedu)).
* Now queries coming to the server via MySQL and PostgreSQL protocols have distinctive interface types (which can be seen in the `interface` column of the table`system.query_log`): `4` for MySQL, and `5` for PostgreSQL, instead of formerly used `1` which is now used for the native protocol only. [#17437](https://github.com/ClickHouse/ClickHouse/pull/17437) ([Vitaly Baranov](https://github.com/vitlibar)).
* Fix parsing of SETTINGS clause of the `INSERT ... SELECT ... SETTINGS` query. [#17414](https://github.com/ClickHouse/ClickHouse/pull/17414) ([Azat Khuzhin](https://github.com/azat)).
* Correctly account memory in RadixSort. [#17412](https://github.com/ClickHouse/ClickHouse/pull/17412) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).

View File

@ -176,6 +176,13 @@ if (COMPILER_CLANG)
endif()
endif ()
# If compiler has support for -Wreserved-identifier. It is difficult to detect by clang version,
# because there are two different branches of clang: clang and AppleClang.
# (AppleClang is not supported by ClickHouse, but some developers have misfortune to use it).
if (HAS_RESERVED_IDENTIFIER)
add_compile_definitions (HAS_RESERVED_IDENTIFIER)
endif ()
# If turned `ON`, assumes the user has either the system GTest library or the bundled one.
option(ENABLE_TESTS "Provide unit_test_dbms target with Google.Test unit tests" ON)
option(ENABLE_EXAMPLES "Build all example programs in 'examples' subdirectories" OFF)
@ -187,6 +194,14 @@ elseif(GLIBC_COMPATIBILITY)
message (${RECONFIGURE_MESSAGE_LEVEL} "Glibc compatibility cannot be enabled in current configuration")
endif ()
if (GLIBC_COMPATIBILITY)
# NOTE: we may also want to check glibc version and add -include only for 2.32+
# however this is extra complexity, especially for cross compiling.
# And anyway it should not break anything for <2.32.
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -include ${CMAKE_CURRENT_SOURCE_DIR}/base/glibc-compatibility/glibc-compat-2.32.h")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -include ${CMAKE_CURRENT_SOURCE_DIR}/base/glibc-compatibility/glibc-compat-2.32.h")
endif()
if (NOT CMAKE_VERSION VERSION_GREATER "3.9.0")
message (WARNING "CMake version must be greater than 3.9.0 for production builds.")
endif ()

View File

@ -188,7 +188,7 @@ Copyright 2016-2021 ClickHouse, Inc.
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2016-2021 Yandex LLC
Copyright 2016-2021 ClickHouse, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.

View File

@ -1,14 +1,14 @@
[![ClickHouse — open source distributed column-oriented DBMS](https://github.com/ClickHouse/ClickHouse/raw/master/website/images/logo-400x240.png)](https://clickhouse.tech)
[![ClickHouse — open source distributed column-oriented DBMS](https://github.com/ClickHouse/ClickHouse/raw/master/website/images/logo-400x240.png)](https://clickhouse.com)
ClickHouse® is an open-source column-oriented database management system that allows generating analytical data reports in real time.
## Useful Links
* [Official website](https://clickhouse.tech/) has quick high-level overview of ClickHouse on main page.
* [Tutorial](https://clickhouse.tech/docs/en/getting_started/tutorial/) shows how to set up and query small ClickHouse cluster.
* [Documentation](https://clickhouse.tech/docs/en/) provides more in-depth information.
* [Official website](https://clickhouse.com/) has quick high-level overview of ClickHouse on main page.
* [Tutorial](https://clickhouse.com/docs/en/getting_started/tutorial/) shows how to set up and query small ClickHouse cluster.
* [Documentation](https://clickhouse.com/docs/en/) provides more in-depth information.
* [YouTube channel](https://www.youtube.com/c/ClickHouseDB) has a lot of content about ClickHouse in video format.
* [Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-rxm3rdrk-lIUmhLC3V8WTaL0TGxsOmg) and [Telegram](https://telegram.me/clickhouse_en) allow to chat with ClickHouse users in real-time.
* [Blog](https://clickhouse.yandex/blog/en/) contains various ClickHouse-related articles, as well as announcements and reports about events.
* [Code Browser](https://clickhouse.tech/codebrowser/html_report/ClickHouse/index.html) with syntax highlight and navigation.
* [Contacts](https://clickhouse.tech/#contacts) can help to get your questions answered if there are any.
* [Blog](https://clickhouse.com/blog/en/) contains various ClickHouse-related articles, as well as announcements and reports about events.
* [Code Browser](https://clickhouse.com/codebrowser/html_report/ClickHouse/index.html) with syntax highlight and navigation.
* [Contacts](https://clickhouse.com/company/#contact) can help to get your questions answered if there are any.

View File

@ -1,7 +1,7 @@
# Security Policy
## Security Announcements
Security fixes will be announced by posting them in the [security changelog](https://clickhouse.tech/docs/en/whats-new/security-changelog/)
Security fixes will be announced by posting them in the [security changelog](https://clickhouse.com/docs/en/whats-new/security-changelog/)
## Scope and Supported Versions

View File

@ -2,7 +2,7 @@ if (USE_CLANG_TIDY)
set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}")
endif ()
add_subdirectory (common)
add_subdirectory (base)
add_subdirectory (daemon)
add_subdirectory (loggers)
add_subdirectory (pcg-random)

View File

@ -6,8 +6,8 @@
#include <mutex>
#include <condition_variable>
#include <common/defines.h>
#include <common/MoveOrCopyIfThrow.h>
#include <base/defines.h>
#include <base/MoveOrCopyIfThrow.h>
/** Pool for limited size objects that cannot be used from different threads simultaneously.
* The main use case is to have fixed size of objects that can be reused in difference threads during their lifetime

View File

@ -29,7 +29,7 @@ elseif (ENABLE_READLINE)
endif ()
if (USE_DEBUG_HELPERS)
set (INCLUDE_DEBUG_HELPERS "-include \"${ClickHouse_SOURCE_DIR}/base/common/iostream_debug_helpers.h\"")
set (INCLUDE_DEBUG_HELPERS "-include \"${ClickHouse_SOURCE_DIR}/base/base/iostream_debug_helpers.h\"")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${INCLUDE_DEBUG_HELPERS}")
endif ()

73
base/base/CachedFn.h Normal file
View File

@ -0,0 +1,73 @@
#pragma once
#include <map>
#include <mutex>
#include "FnTraits.h"
/**
* Caching proxy for a functor that decays to a pointer-to-function.
* Saves pairs (func args, func result on args).
* Cache size is unlimited. Cache items are evicted only on manual drop.
* Invocation/update is O(log(saved cache values)).
*
* See Common/tests/cached_fn.cpp for examples.
*/
template <auto * Func>
struct CachedFn
{
private:
using Traits = FnTraits<decltype(Func)>;
using Key = typename Traits::DecayedArgs;
using Result = typename Traits::Ret;
std::map<Key, Result> cache; // Can't use hashmap as tuples are unhashable by default
mutable std::mutex mutex;
public:
template <class ...Args>
Result operator()(Args && ...args)
{
Key key{std::forward<Args>(args)...};
{
std::lock_guard lock(mutex);
if (auto it = cache.find(key); it != cache.end())
return it->second;
}
Result res = std::apply(Func, key);
{
std::lock_guard lock(mutex);
cache.emplace(std::move(key), res);
}
return res;
}
template <class ...Args>
void update(Args && ...args)
{
Key key{std::forward<Args>(args)...};
Result res = std::apply(Func, key);
{
std::lock_guard lock(mutex);
// TODO Can't use emplace(std::move(key), ..), causes test_host_ip_change errors.
cache[key] = std::move(res);
}
}
size_t size() const
{
std::lock_guard lock(mutex);
return cache.size();
}
void drop()
{
std::lock_guard lock(mutex);
cache.clear();
}
};

View File

@ -3,7 +3,7 @@
#include <cctz/civil_time.h>
#include <cctz/time_zone.h>
#include <cctz/zone_info_source.h>
#include <common/getResource.h>
#include <base/getResource.h>
#include <Poco/Exception.h>
#include <algorithm>

View File

@ -1,7 +1,7 @@
#pragma once
#include <common/types.h>
#include <common/strong_typedef.h>
#include <base/types.h>
#include <base/strong_typedef.h>
/** Represents number of days since 1970-01-01.
* See DateLUTImpl for usage examples.

View File

@ -1,5 +1,5 @@
#pragma once
#include "common/extended_types.h"
#include <base/extended_types.h>
#if !defined(NO_SANITIZE_UNDEFINED)
#if defined(__clang__)

View File

@ -3,7 +3,7 @@
#include <cstdint>
#include <cstddef>
#include <cstring>
#include <common/extended_types.h>
#include <base/extended_types.h>
/// Allows to check the internals of IEEE-754 floating point number.

View File

@ -1,7 +1,7 @@
#pragma once
#include <Poco/ErrorHandler.h>
#include <common/logger_useful.h>
#include <base/logger_useful.h>
#include <Common/Exception.h>

37
base/base/FnTraits.h Normal file
View File

@ -0,0 +1,37 @@
#pragma once
#include <tuple>
#include <concepts>
#include <type_traits>
namespace detail
{
template <class T>
struct FnTraits { template <class> static constexpr bool value = false; };
template <class R, class ...A>
struct FnTraits<R(A...)>
{
template <class F>
static constexpr bool value = std::is_invocable_r_v<R, F, A...>;
using Ret = R;
using Args = std::tuple<A...>;
using DecayedArgs = std::tuple<typename std::decay<A>::type...>;
};
template <class R, class ...A>
struct FnTraits<R(*)(A...)> : FnTraits<R(A...)> {};
}
template <class T> using FnTraits = detail::FnTraits<T>;
/**
* A less-typing alias for std::is_invokable_r_v.
* @example void foo(Fn<bool(int, char)> auto && functor)
*/
template <class F, class FS>
concept Fn = FnTraits<FS>::template value<F>;
template <auto Value>
using Constant = std::integral_constant<decltype(Value), Value>;

View File

@ -3,9 +3,9 @@
#include <Poco/UTF8Encoding.h>
#include <Poco/NumberParser.h>
#include <common/JSON.h>
#include <common/find_symbols.h>
#include <common/preciseExp10.h>
#include <base/JSON.h>
#include <base/find_symbols.h>
#include <base/preciseExp10.h>
#include <iostream>

View File

@ -2,8 +2,8 @@
#include <typeinfo>
#include <Poco/Exception.h>
#include <common/StringRef.h>
#include <common/types.h>
#include <base/StringRef.h>
#include <base/types.h>
/** Очень простой класс для чтения JSON (или его кусочков).

View File

@ -1,4 +1,4 @@
#include <common/LineReader.h>
#include <base/LineReader.h>
#include <iostream>
#include <string_view>
@ -16,7 +16,7 @@ extern "C"
}
#endif
#if defined(__clang__) && __clang_major__ >= 13
#ifdef HAS_RESERVED_IDENTIFIER
#pragma clang diagnostic ignored "-Wreserved-identifier"
#endif

View File

@ -1,6 +1,6 @@
#pragma once
#include <common/types.h>
#include <base/types.h>
#include <atomic>
#include <vector>

View File

@ -4,7 +4,7 @@
#include <string>
#include <sstream>
#include <exception>
#include <common/DateLUT.h>
#include <base/DateLUT.h>
/** Stores a calendar date in broken-down form (year, month, day-in-month).

View File

@ -3,8 +3,8 @@
#include <string>
#include <iomanip>
#include <exception>
#include <common/DateLUT.h>
#include <common/LocalDate.h>
#include <base/DateLUT.h>
#include <base/LocalDate.h>
/** Stores calendar date and time in broken-down form.

View File

@ -1,6 +1,6 @@
#pragma once
#include <common/types.h>
#include <base/types.h>
namespace detail
{

View File

@ -1,6 +1,6 @@
#include <common/ReadlineLineReader.h>
#include <common/errnoToString.h>
#include <common/scope_guard.h>
#include <base/ReadlineLineReader.h>
#include <base/errnoToString.h>
#include <base/scope_guard.h>
#include <errno.h>
#include <signal.h>

View File

@ -1,5 +1,5 @@
#include <common/ReplxxLineReader.h>
#include <common/errnoToString.h>
#include <base/ReplxxLineReader.h>
#include <base/errnoToString.h>
#include <chrono>
#include <cerrno>

View File

@ -7,8 +7,8 @@
#include <functional>
#include <iosfwd>
#include <common/types.h>
#include <common/unaligned.h>
#include <base/types.h>
#include <base/unaligned.h>
#include <city.h>

View File

@ -1,7 +1,7 @@
#pragma once
#include <common/extended_types.h>
#include <common/defines.h>
#include <base/extended_types.h>
#include <base/defines.h>
namespace common

View File

@ -8,7 +8,7 @@
/// NOTE:
/// - __has_feature cannot be simply undefined,
/// since this will be broken if some C++ header will be included after
/// including <common/defines.h>
/// including <base/defines.h>
/// - it should not have fallback to 0,
/// since this may create false-positive detection (common problem)
#if defined(__clang__) && defined(__has_feature)

View File

@ -1,4 +1,4 @@
#include <common/demangle.h>
#include <base/demangle.h>
#if defined(_MSC_VER)

View File

@ -2,8 +2,8 @@
#include <type_traits>
#include <common/types.h>
#include <common/wide_integer.h>
#include <base/types.h>
#include <base/wide_integer.h>
using Int128 = wide::integer<128, signed>;

View File

@ -1,5 +1,5 @@
#include <Poco/Net/DNS.h>
#include <common/getFQDNOrHostName.h>
#include <base/getFQDNOrHostName.h>
namespace

View File

@ -1,6 +1,6 @@
#include <stdexcept>
#include "common/getMemoryAmount.h"
#include "common/getPageSize.h"
#include <base/getMemoryAmount.h>
#include <base/getPageSize.h>
#include <unistd.h>
#include <sys/types.h>

View File

@ -1,7 +1,7 @@
#include "common/getPageSize.h"
#include <base/getPageSize.h>
#include <unistd.h>
Int64 getPageSize()
{
return sysconf(_SC_PAGESIZE);

View File

@ -1,6 +1,7 @@
#pragma once
#include "common/types.h"
#include <base/types.h>
/// Get memory page size
Int64 getPageSize();

View File

@ -1,4 +1,4 @@
#include <common/getThreadId.h>
#include <base/getThreadId.h>
#if defined(OS_ANDROID)
#include <sys/types.h>

View File

@ -30,7 +30,7 @@
#include <cstddef>
#include <cstring>
#include <type_traits>
#include <common/extended_types.h>
#include <base/extended_types.h>
namespace impl

View File

@ -1,4 +1,4 @@
#include <common/mremap.h>
#include <base/mremap.h>
#include <cstddef>
#include <cstdlib>

View File

@ -1,10 +1,10 @@
#if defined(__clang__) && __clang_major__ >= 13
#ifdef HAS_RESERVED_IDENTIFIER
#pragma clang diagnostic ignored "-Wreserved-identifier"
#endif
/// This code was based on the code by Fedor Korotkiy (prime@yandex-team.ru) for YT product in Yandex.
#include <common/defines.h>
#include <base/defines.h>
#if defined(__linux__) && !defined(THREAD_SANITIZER)
#define USE_PHDR_CACHE 1

View File

@ -1,7 +1,7 @@
#pragma once
#include <common/scope_guard.h>
#include <common/logger_useful.h>
#include <base/scope_guard.h>
#include <base/logger_useful.h>
#include <Common/MemoryTracker.h>
/// Same as SCOPE_EXIT() but block the MEMORY_LIMIT_EXCEEDED errors.

View File

@ -1,5 +1,5 @@
#include <common/setTerminalEcho.h>
#include <common/errnoToString.h>
#include <base/setTerminalEcho.h>
#include <base/errnoToString.h>
#include <stdexcept>
#include <cstring>
#include <string>

View File

@ -1,4 +1,4 @@
#include <common/shift10.h>
#include <base/shift10.h>
#include "defines.h"

View File

@ -1,6 +1,6 @@
#pragma once
#include <common/types.h>
#include <base/types.h>
/** Almost the same as x = x * exp10(exponent), but gives more accurate result.
* Example:

View File

@ -1,4 +1,4 @@
#include "common/sleep.h"
#include <base/sleep.h>
#include <time.h>
#include <errno.h>

View File

@ -1,5 +1,5 @@
#include <string>
#include <common/terminalColors.h>
#include <base/terminalColors.h>
std::string setColor(UInt64 hash)

View File

@ -1,7 +1,7 @@
#pragma once
#include <string>
#include <common/types.h>
#include <base/types.h>
/** Set color in terminal based on 64-bit hash value.

View File

@ -1,4 +1,4 @@
#include <common/iostream_debug_helpers.h>
#include <base/iostream_debug_helpers.h>
#include <iostream>
#include <memory>

View File

@ -1,7 +1,7 @@
#pragma once
#include <cstddef>
#if defined(__clang__) && __clang_major__ >= 13
#ifdef HAS_RESERVED_IDENTIFIER
#pragma clang diagnostic ignored "-Wreserved-identifier"
#endif

View File

@ -4,12 +4,12 @@
#include <Poco/Net/NetException.h>
#include <Poco/Util/HelpFormatter.h>
#include <common/logger_useful.h>
#include <common/range.h>
#include <base/logger_useful.h>
#include <base/range.h>
#include <Common/StringUtils/StringUtils.h>
#include <Common/SensitiveDataMasker.h>
#include <common/errnoToString.h>
#include <base/errnoToString.h>
#include <IO/ReadHelpers.h>
#include <Formats/registerFormats.h>
#include <Server/HTTP/HTTPServer.h>

View File

@ -1,82 +0,0 @@
#pragma once
#include <map>
#include <tuple>
#include <mutex>
#include <common/function_traits.h>
/** The simplest cache for a free function.
* You can also pass a static class method or lambda without captures.
* The size is unlimited. Values are stored permanently and never evicted.
* But single record or all cache can be manually dropped.
* Mutex is used for synchronization.
* Suitable only for the simplest cases.
*
* Usage
*
* SimpleCache<decltype(func), &func> func_cached;
* std::cerr << func_cached(args...);
*/
template <typename F, F* f>
class SimpleCache
{
private:
using Key = typename function_traits<F>::arguments_decay;
using Result = typename function_traits<F>::result;
std::map<Key, Result> cache;
mutable std::mutex mutex;
public:
template <typename... Args>
Result operator() (Args &&... args)
{
Key key{std::forward<Args>(args)...};
{
std::lock_guard lock(mutex);
auto it = cache.find(key);
if (cache.end() != it)
return it->second;
}
/// The calculations themselves are not done under mutex.
Result res = std::apply(f, key);
{
std::lock_guard lock(mutex);
cache.emplace(std::forward_as_tuple(args...), res);
}
return res;
}
template <typename... Args>
void update(Args &&... args)
{
Key key{std::forward<Args>(args)...};
Result res = std::apply(f, key);
{
std::lock_guard lock(mutex);
cache[key] = std::move(res);
}
}
size_t size() const
{
std::lock_guard lock(mutex);
return cache.size();
}
void drop()
{
std::lock_guard lock(mutex);
cache.clear();
}
};

View File

@ -1,16 +0,0 @@
#pragma once
#include <tuple>
#include <type_traits>
template <typename T>
struct function_traits;
template <typename ReturnType, typename... Args>
struct function_traits<ReturnType(Args...)>
{
using result = ReturnType;
using arguments = std::tuple<Args...>;
using arguments_decay = std::tuple<typename std::decay<Args>::type...>;
};

View File

@ -1,4 +1,4 @@
#if defined(__clang__) && __clang_major__ >= 13
#ifdef HAS_RESERVED_IDENTIFIER
#pragma clang diagnostic ignored "-Wreserved-identifier"
#endif
@ -25,7 +25,7 @@
#include <fstream>
#include <sstream>
#include <memory>
#include <common/scope_guard.h>
#include <base/scope_guard.h>
#include <Poco/Observer.h>
#include <Poco/AutoPtr.h>
@ -38,12 +38,12 @@
#include <Poco/SyslogChannel.h>
#include <Poco/DirectoryIterator.h>
#include <common/logger_useful.h>
#include <common/ErrorHandlers.h>
#include <common/argsToConfig.h>
#include <common/getThreadId.h>
#include <common/coverage.h>
#include <common/sleep.h>
#include <base/logger_useful.h>
#include <base/ErrorHandlers.h>
#include <base/argsToConfig.h>
#include <base/getThreadId.h>
#include <base/coverage.h>
#include <base/sleep.h>
#include <IO/WriteBufferFromFile.h>
#include <IO/WriteBufferFromFileDescriptorDiscardOnFailure.h>

View File

@ -16,9 +16,9 @@
#include <Poco/Util/ServerApplication.h>
#include <Poco/Net/SocketAddress.h>
#include <Poco/Version.h>
#include <common/types.h>
#include <common/logger_useful.h>
#include <common/getThreadId.h>
#include <base/types.h>
#include <base/logger_useful.h>
#include <base/getThreadId.h>
#include <daemon/GraphiteWriter.h>
#include <Common/Config/ConfigProcessor.h>
#include <Common/StatusFile.h>

Some files were not shown because too many files have changed in this diff Show More