Merge branch 'ClickHouse:master' into new_func

This commit is contained in:
Memo 2021-10-05 09:02:37 +08:00 committed by GitHub
commit 0e9bbc4d43
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1216 changed files with 4826 additions and 2764 deletions

View File

@ -176,6 +176,13 @@ if (COMPILER_CLANG)
endif()
endif ()
# If compiler has support for -Wreserved-identifier. It is difficult to detect by clang version,
# because there are two different branches of clang: clang and AppleClang.
# (AppleClang is not supported by ClickHouse, but some developers have misfortune to use it).
if (HAS_RESERVED_IDENTIFIER)
add_compile_definitions (HAS_RESERVED_IDENTIFIER)
endif ()
# If turned `ON`, assumes the user has either the system GTest library or the bundled one.
option(ENABLE_TESTS "Provide unit_test_dbms target with Google.Test unit tests" ON)
option(ENABLE_EXAMPLES "Build all example programs in 'examples' subdirectories" OFF)
@ -187,6 +194,14 @@ elseif(GLIBC_COMPATIBILITY)
message (${RECONFIGURE_MESSAGE_LEVEL} "Glibc compatibility cannot be enabled in current configuration")
endif ()
if (GLIBC_COMPATIBILITY)
# NOTE: we may also want to check glibc version and add -include only for 2.32+
# however this is extra complexity, especially for cross compiling.
# And anyway it should not break anything for <2.32.
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -include ${CMAKE_CURRENT_SOURCE_DIR}/base/glibc-compatibility/glibc-compat-2.32.h")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -include ${CMAKE_CURRENT_SOURCE_DIR}/base/glibc-compatibility/glibc-compat-2.32.h")
endif()
if (NOT CMAKE_VERSION VERSION_GREATER "3.9.0")
message (WARNING "CMake version must be greater than 3.9.0 for production builds.")
endif ()

View File

@ -188,7 +188,7 @@ Copyright 2016-2021 ClickHouse, Inc.
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2016-2021 Yandex LLC
Copyright 2016-2021 ClickHouse, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.

View File

@ -11,4 +11,4 @@ ClickHouse® is an open-source column-oriented database management system that a
* [Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-rxm3rdrk-lIUmhLC3V8WTaL0TGxsOmg) and [Telegram](https://telegram.me/clickhouse_en) allow to chat with ClickHouse users in real-time.
* [Blog](https://clickhouse.com/blog/en/) contains various ClickHouse-related articles, as well as announcements and reports about events.
* [Code Browser](https://clickhouse.com/codebrowser/html_report/ClickHouse/index.html) with syntax highlight and navigation.
* [Contacts](https://clickhouse.com/#contacts) can help to get your questions answered if there are any.
* [Contacts](https://clickhouse.com/company/#contact) can help to get your questions answered if there are any.

View File

@ -2,7 +2,7 @@ if (USE_CLANG_TIDY)
set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}")
endif ()
add_subdirectory (common)
add_subdirectory (base)
add_subdirectory (daemon)
add_subdirectory (loggers)
add_subdirectory (pcg-random)

View File

@ -6,8 +6,8 @@
#include <mutex>
#include <condition_variable>
#include <common/defines.h>
#include <common/MoveOrCopyIfThrow.h>
#include <base/defines.h>
#include <base/MoveOrCopyIfThrow.h>
/** Pool for limited size objects that cannot be used from different threads simultaneously.
* The main use case is to have fixed size of objects that can be reused in difference threads during their lifetime
@ -88,10 +88,12 @@ public:
/// Return object into pool. Client must return same object that was borrowed.
inline void returnObject(T && object_to_return)
{
std::unique_lock<std::mutex> lock(objects_mutex);
{
std::lock_guard<std::mutex> lock(objects_mutex);
objects.emplace_back(std::move(object_to_return));
--borrowed_objects_size;
objects.emplace_back(std::move(object_to_return));
--borrowed_objects_size;
}
condition_variable.notify_one();
}

View File

@ -29,7 +29,7 @@ elseif (ENABLE_READLINE)
endif ()
if (USE_DEBUG_HELPERS)
set (INCLUDE_DEBUG_HELPERS "-include \"${ClickHouse_SOURCE_DIR}/base/common/iostream_debug_helpers.h\"")
set (INCLUDE_DEBUG_HELPERS "-include \"${ClickHouse_SOURCE_DIR}/base/base/iostream_debug_helpers.h\"")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${INCLUDE_DEBUG_HELPERS}")
endif ()

73
base/base/CachedFn.h Normal file
View File

@ -0,0 +1,73 @@
#pragma once
#include <map>
#include <mutex>
#include "FnTraits.h"
/**
* Caching proxy for a functor that decays to a pointer-to-function.
* Saves pairs (func args, func result on args).
* Cache size is unlimited. Cache items are evicted only on manual drop.
* Invocation/update is O(log(saved cache values)).
*
* See Common/tests/cached_fn.cpp for examples.
*/
template <auto * Func>
struct CachedFn
{
private:
using Traits = FnTraits<decltype(Func)>;
using Key = typename Traits::DecayedArgs;
using Result = typename Traits::Ret;
std::map<Key, Result> cache; // Can't use hashmap as tuples are unhashable by default
mutable std::mutex mutex;
public:
template <class ...Args>
Result operator()(Args && ...args)
{
Key key{std::forward<Args>(args)...};
{
std::lock_guard lock(mutex);
if (auto it = cache.find(key); it != cache.end())
return it->second;
}
Result res = std::apply(Func, key);
{
std::lock_guard lock(mutex);
cache.emplace(std::move(key), res);
}
return res;
}
template <class ...Args>
void update(Args && ...args)
{
Key key{std::forward<Args>(args)...};
Result res = std::apply(Func, key);
{
std::lock_guard lock(mutex);
// TODO Can't use emplace(std::move(key), ..), causes test_host_ip_change errors.
cache[key] = std::move(res);
}
}
size_t size() const
{
std::lock_guard lock(mutex);
return cache.size();
}
void drop()
{
std::lock_guard lock(mutex);
cache.clear();
}
};

View File

@ -3,7 +3,7 @@
#include <cctz/civil_time.h>
#include <cctz/time_zone.h>
#include <cctz/zone_info_source.h>
#include <common/getResource.h>
#include <base/getResource.h>
#include <Poco/Exception.h>
#include <algorithm>

View File

@ -1,7 +1,7 @@
#pragma once
#include <common/types.h>
#include <common/strong_typedef.h>
#include <base/types.h>
#include <base/strong_typedef.h>
/** Represents number of days since 1970-01-01.
* See DateLUTImpl for usage examples.

View File

@ -1,5 +1,5 @@
#pragma once
#include "common/extended_types.h"
#include <base/extended_types.h>
#if !defined(NO_SANITIZE_UNDEFINED)
#if defined(__clang__)

View File

@ -3,7 +3,7 @@
#include <cstdint>
#include <cstddef>
#include <cstring>
#include <common/extended_types.h>
#include <base/extended_types.h>
/// Allows to check the internals of IEEE-754 floating point number.

View File

@ -1,7 +1,7 @@
#pragma once
#include <Poco/ErrorHandler.h>
#include <common/logger_useful.h>
#include <base/logger_useful.h>
#include <Common/Exception.h>

37
base/base/FnTraits.h Normal file
View File

@ -0,0 +1,37 @@
#pragma once
#include <tuple>
#include <concepts>
#include <type_traits>
namespace detail
{
template <class T>
struct FnTraits { template <class> static constexpr bool value = false; };
template <class R, class ...A>
struct FnTraits<R(A...)>
{
template <class F>
static constexpr bool value = std::is_invocable_r_v<R, F, A...>;
using Ret = R;
using Args = std::tuple<A...>;
using DecayedArgs = std::tuple<typename std::decay<A>::type...>;
};
template <class R, class ...A>
struct FnTraits<R(*)(A...)> : FnTraits<R(A...)> {};
}
template <class T> using FnTraits = detail::FnTraits<T>;
/**
* A less-typing alias for std::is_invokable_r_v.
* @example void foo(Fn<bool(int, char)> auto && functor)
*/
template <class F, class FS>
concept Fn = FnTraits<FS>::template value<F>;
template <auto Value>
using Constant = std::integral_constant<decltype(Value), Value>;

View File

@ -3,9 +3,9 @@
#include <Poco/UTF8Encoding.h>
#include <Poco/NumberParser.h>
#include <common/JSON.h>
#include <common/find_symbols.h>
#include <common/preciseExp10.h>
#include <base/JSON.h>
#include <base/find_symbols.h>
#include <base/preciseExp10.h>
#include <iostream>

View File

@ -2,8 +2,8 @@
#include <typeinfo>
#include <Poco/Exception.h>
#include <common/StringRef.h>
#include <common/types.h>
#include <base/StringRef.h>
#include <base/types.h>
/** Очень простой класс для чтения JSON (или его кусочков).

View File

@ -1,4 +1,4 @@
#include <common/LineReader.h>
#include <base/LineReader.h>
#include <iostream>
#include <string_view>
@ -16,7 +16,7 @@ extern "C"
}
#endif
#if defined(__clang__) && __clang_major__ >= 13
#ifdef HAS_RESERVED_IDENTIFIER
#pragma clang diagnostic ignored "-Wreserved-identifier"
#endif

View File

@ -1,6 +1,6 @@
#pragma once
#include <common/types.h>
#include <base/types.h>
#include <atomic>
#include <vector>

View File

@ -4,7 +4,7 @@
#include <string>
#include <sstream>
#include <exception>
#include <common/DateLUT.h>
#include <base/DateLUT.h>
/** Stores a calendar date in broken-down form (year, month, day-in-month).

View File

@ -3,8 +3,8 @@
#include <string>
#include <iomanip>
#include <exception>
#include <common/DateLUT.h>
#include <common/LocalDate.h>
#include <base/DateLUT.h>
#include <base/LocalDate.h>
/** Stores calendar date and time in broken-down form.

View File

@ -1,6 +1,6 @@
#pragma once
#include <common/types.h>
#include <base/types.h>
namespace detail
{

View File

@ -1,6 +1,6 @@
#include <common/ReadlineLineReader.h>
#include <common/errnoToString.h>
#include <common/scope_guard.h>
#include <base/ReadlineLineReader.h>
#include <base/errnoToString.h>
#include <base/scope_guard.h>
#include <errno.h>
#include <signal.h>

View File

@ -1,5 +1,5 @@
#include <common/ReplxxLineReader.h>
#include <common/errnoToString.h>
#include <base/ReplxxLineReader.h>
#include <base/errnoToString.h>
#include <chrono>
#include <cerrno>

View File

@ -7,8 +7,8 @@
#include <functional>
#include <iosfwd>
#include <common/types.h>
#include <common/unaligned.h>
#include <base/types.h>
#include <base/unaligned.h>
#include <city.h>

View File

@ -1,7 +1,7 @@
#pragma once
#include <common/extended_types.h>
#include <common/defines.h>
#include <base/extended_types.h>
#include <base/defines.h>
namespace common

View File

@ -8,38 +8,25 @@
/// NOTE:
/// - __has_feature cannot be simply undefined,
/// since this will be broken if some C++ header will be included after
/// including <common/defines.h>
/// including <base/defines.h>
/// - it should not have fallback to 0,
/// since this may create false-positive detection (common problem)
#if defined(__clang__) && defined(__has_feature)
# define ch_has_feature __has_feature
#endif
#if defined(_MSC_VER)
# if !defined(likely)
# define likely(x) (x)
# endif
# if !defined(unlikely)
# define unlikely(x) (x)
# endif
#else
# if !defined(likely)
# define likely(x) (__builtin_expect(!!(x), 1))
# endif
# if !defined(unlikely)
# define unlikely(x) (__builtin_expect(!!(x), 0))
# endif
#if !defined(likely)
# define likely(x) (__builtin_expect(!!(x), 1))
#endif
#if !defined(unlikely)
# define unlikely(x) (__builtin_expect(!!(x), 0))
#endif
#if defined(_MSC_VER)
# define ALWAYS_INLINE __forceinline
# define NO_INLINE static __declspec(noinline)
# define MAY_ALIAS
#else
# define ALWAYS_INLINE __attribute__((__always_inline__))
# define NO_INLINE __attribute__((__noinline__))
# define MAY_ALIAS __attribute__((__may_alias__))
#endif
// more aliases: https://mailman.videolan.org/pipermail/x264-devel/2014-May/010660.html
#define ALWAYS_INLINE __attribute__((__always_inline__))
#define NO_INLINE __attribute__((__noinline__))
#define MAY_ALIAS __attribute__((__may_alias__))
#if !defined(__x86_64__) && !defined(__aarch64__) && !defined(__PPC__)
# error "The only supported platforms are x86_64 and AArch64, PowerPC (work in progress)"
@ -117,6 +104,11 @@
# define ALWAYS_INLINE_NO_SANITIZE_UNDEFINED ALWAYS_INLINE
#endif
#if !__has_include(<sanitizer/asan_interface.h>) || !defined(ADDRESS_SANITIZER)
# define ASAN_UNPOISON_MEMORY_REGION(a, b)
# define ASAN_POISON_MEMORY_REGION(a, b)
#endif
/// A template function for suppressing warnings about unused variables or function results.
template <typename... Args>
constexpr void UNUSED(Args &&... args [[maybe_unused]])

View File

@ -1,19 +1,4 @@
#include <common/demangle.h>
#if defined(_MSC_VER)
DemangleResult tryDemangle(const char *)
{
return DemangleResult{};
}
std::string demangle(const char * name, int & status)
{
status = 0;
return name;
}
#else
#include <base/demangle.h>
#include <stdlib.h>
#include <cxxabi.h>
@ -39,6 +24,3 @@ std::string demangle(const char * name, int & status)
return name;
}
#endif

View File

@ -2,8 +2,8 @@
#include <type_traits>
#include <common/types.h>
#include <common/wide_integer.h>
#include <base/types.h>
#include <base/wide_integer.h>
using Int128 = wide::integer<128, signed>;

View File

@ -1,5 +1,5 @@
#include <Poco/Net/DNS.h>
#include <common/getFQDNOrHostName.h>
#include <base/getFQDNOrHostName.h>
namespace

View File

@ -1,6 +1,6 @@
#include <stdexcept>
#include "common/getMemoryAmount.h"
#include "common/getPageSize.h"
#include <base/getMemoryAmount.h>
#include <base/getPageSize.h>
#include <unistd.h>
#include <sys/types.h>

View File

@ -1,7 +1,7 @@
#include "common/getPageSize.h"
#include <base/getPageSize.h>
#include <unistd.h>
Int64 getPageSize()
{
return sysconf(_SC_PAGESIZE);

View File

@ -1,6 +1,7 @@
#pragma once
#include "common/types.h"
#include <base/types.h>
/// Get memory page size
Int64 getPageSize();

View File

@ -1,4 +1,4 @@
#include <common/getThreadId.h>
#include <base/getThreadId.h>
#if defined(OS_ANDROID)
#include <sys/types.h>

View File

@ -30,7 +30,7 @@
#include <cstddef>
#include <cstring>
#include <type_traits>
#include <common/extended_types.h>
#include <base/extended_types.h>
namespace impl

View File

@ -1,4 +1,4 @@
#include <common/mremap.h>
#include <base/mremap.h>
#include <cstddef>
#include <cstdlib>
@ -19,22 +19,14 @@ void * mremap_fallback(
return MAP_FAILED;
}
#if defined(_MSC_VER)
void * new_address = ::operator new(new_size);
#else
void * new_address = mmap(nullptr, new_size, mmap_prot, mmap_flags, mmap_fd, mmap_offset);
if (MAP_FAILED == new_address)
return MAP_FAILED;
#endif
memcpy(new_address, old_address, old_size);
#if defined(_MSC_VER)
delete old_address;
#else
if (munmap(old_address, old_size))
abort();
#endif
return new_address;
}

View File

@ -2,9 +2,7 @@
#include <cstddef>
#include <sys/types.h>
#if !defined(_MSC_VER)
#include <sys/mman.h>
#endif
#ifdef MREMAP_MAYMOVE

View File

@ -1,10 +1,10 @@
#if defined(__clang__) && __clang_major__ >= 13
#ifdef HAS_RESERVED_IDENTIFIER
#pragma clang diagnostic ignored "-Wreserved-identifier"
#endif
/// This code was based on the code by Fedor Korotkiy (prime@yandex-team.ru) for YT product in Yandex.
#include <common/defines.h>
#include <base/defines.h>
#if defined(__linux__) && !defined(THREAD_SANITIZER)
#define USE_PHDR_CACHE 1

View File

@ -1,7 +1,7 @@
#pragma once
#include <common/scope_guard.h>
#include <common/logger_useful.h>
#include <base/scope_guard.h>
#include <base/logger_useful.h>
#include <Common/MemoryTracker.h>
/// Same as SCOPE_EXIT() but block the MEMORY_LIMIT_EXCEEDED errors.

View File

@ -1,5 +1,5 @@
#include <common/setTerminalEcho.h>
#include <common/errnoToString.h>
#include <base/setTerminalEcho.h>
#include <base/errnoToString.h>
#include <stdexcept>
#include <cstring>
#include <string>

View File

@ -1,4 +1,4 @@
#include <common/shift10.h>
#include <base/shift10.h>
#include "defines.h"

View File

@ -1,6 +1,6 @@
#pragma once
#include <common/types.h>
#include <base/types.h>
/** Almost the same as x = x * exp10(exponent), but gives more accurate result.
* Example:

View File

@ -1,4 +1,4 @@
#include "common/sleep.h"
#include <base/sleep.h>
#include <time.h>
#include <errno.h>

View File

@ -1,5 +1,8 @@
#pragma once
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wold-style-cast"
#if !defined(ARCADIA_BUILD)
# include <miniselect/floyd_rivest_select.h> // Y_IGNORE
#else
@ -34,4 +37,7 @@ void partial_sort(RandomIt first, RandomIt middle, RandomIt last, Compare compar
#else
::std::partial_sort(first, middle, last, compare);
#endif
#pragma GCC diagnostic pop
}

View File

@ -1,5 +1,5 @@
#include <string>
#include <common/terminalColors.h>
#include <base/terminalColors.h>
std::string setColor(UInt64 hash)

View File

@ -1,7 +1,7 @@
#pragma once
#include <string>
#include <common/types.h>
#include <base/types.h>
/** Set color in terminal based on 64-bit hash value.

View File

@ -1,4 +1,4 @@
#include <common/iostream_debug_helpers.h>
#include <base/iostream_debug_helpers.h>
#include <iostream>
#include <memory>

View File

@ -1,7 +1,7 @@
#pragma once
#include <cstddef>
#if defined(__clang__) && __clang_major__ >= 13
#ifdef HAS_RESERVED_IDENTIFIER
#pragma clang diagnostic ignored "-Wreserved-identifier"
#endif

View File

@ -4,12 +4,12 @@
#include <Poco/Net/NetException.h>
#include <Poco/Util/HelpFormatter.h>
#include <common/logger_useful.h>
#include <common/range.h>
#include <base/logger_useful.h>
#include <base/range.h>
#include <Common/StringUtils/StringUtils.h>
#include <Common/SensitiveDataMasker.h>
#include <common/errnoToString.h>
#include <base/errnoToString.h>
#include <IO/ReadHelpers.h>
#include <Formats/registerFormats.h>
#include <Server/HTTP/HTTPServer.h>

View File

@ -1,82 +0,0 @@
#pragma once
#include <map>
#include <tuple>
#include <mutex>
#include <common/function_traits.h>
/** The simplest cache for a free function.
* You can also pass a static class method or lambda without captures.
* The size is unlimited. Values are stored permanently and never evicted.
* But single record or all cache can be manually dropped.
* Mutex is used for synchronization.
* Suitable only for the simplest cases.
*
* Usage
*
* SimpleCache<decltype(func), &func> func_cached;
* std::cerr << func_cached(args...);
*/
template <typename F, F* f>
class SimpleCache
{
private:
using Key = typename function_traits<F>::arguments_decay;
using Result = typename function_traits<F>::result;
std::map<Key, Result> cache;
mutable std::mutex mutex;
public:
template <typename... Args>
Result operator() (Args &&... args)
{
Key key{std::forward<Args>(args)...};
{
std::lock_guard lock(mutex);
auto it = cache.find(key);
if (cache.end() != it)
return it->second;
}
/// The calculations themselves are not done under mutex.
Result res = std::apply(f, key);
{
std::lock_guard lock(mutex);
cache.emplace(std::forward_as_tuple(args...), res);
}
return res;
}
template <typename... Args>
void update(Args &&... args)
{
Key key{std::forward<Args>(args)...};
Result res = std::apply(f, key);
{
std::lock_guard lock(mutex);
cache[key] = std::move(res);
}
}
size_t size() const
{
std::lock_guard lock(mutex);
return cache.size();
}
void drop()
{
std::lock_guard lock(mutex);
cache.clear();
}
};

View File

@ -1,16 +0,0 @@
#pragma once
#include <tuple>
#include <type_traits>
template <typename T>
struct function_traits;
template <typename ReturnType, typename... Args>
struct function_traits<ReturnType(Args...)>
{
using result = ReturnType;
using arguments = std::tuple<Args...>;
using arguments_decay = std::tuple<typename std::decay<Args>::type...>;
};

View File

@ -1,4 +1,4 @@
#if defined(__clang__) && __clang_major__ >= 13
#ifdef HAS_RESERVED_IDENTIFIER
#pragma clang diagnostic ignored "-Wreserved-identifier"
#endif
@ -25,7 +25,7 @@
#include <fstream>
#include <sstream>
#include <memory>
#include <common/scope_guard.h>
#include <base/scope_guard.h>
#include <Poco/Observer.h>
#include <Poco/AutoPtr.h>
@ -38,12 +38,12 @@
#include <Poco/SyslogChannel.h>
#include <Poco/DirectoryIterator.h>
#include <common/logger_useful.h>
#include <common/ErrorHandlers.h>
#include <common/argsToConfig.h>
#include <common/getThreadId.h>
#include <common/coverage.h>
#include <common/sleep.h>
#include <base/logger_useful.h>
#include <base/ErrorHandlers.h>
#include <base/argsToConfig.h>
#include <base/getThreadId.h>
#include <base/coverage.h>
#include <base/sleep.h>
#include <IO/WriteBufferFromFile.h>
#include <IO/WriteBufferFromFileDescriptorDiscardOnFailure.h>

View File

@ -16,9 +16,9 @@
#include <Poco/Util/ServerApplication.h>
#include <Poco/Net/SocketAddress.h>
#include <Poco/Version.h>
#include <common/types.h>
#include <common/logger_useful.h>
#include <common/getThreadId.h>
#include <base/types.h>
#include <base/logger_useful.h>
#include <base/getThreadId.h>
#include <daemon/GraphiteWriter.h>
#include <Common/Config/ConfigProcessor.h>
#include <Common/StatusFile.h>

View File

@ -2,7 +2,7 @@
#include <daemon/BaseDaemon.h>
#include <Poco/Util/LayeredConfiguration.h>
#include <Poco/Util/Application.h>
#include <common/getFQDNOrHostName.h>
#include <base/getFQDNOrHostName.h>
#include <mutex>
#include <iomanip>

View File

@ -5,7 +5,7 @@
#include <Poco/Net/StreamSocket.h>
#include <Poco/Net/SocketStream.h>
#include <Poco/Util/Application.h>
#include <common/logger_useful.h>
#include <base/logger_useful.h>
/// пишет в Graphite данные в формате

Some files were not shown because too many files have changed in this diff Show More