thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-27 01:51:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'blessed/master' into pull_timeout

Browse Source
This commit is contained in:
Raúl Marín 2021-11-29 19:51:18 +01:00
commit a7ae715950
371 changed files with 2869 additions and 1733 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
.github/workflows/main.yml vendored
View File

@ -211,12 +211,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 0
BUILD_NAME: 'package_release'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -250,12 +250,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 8
BUILD_NAME: 'binary_release'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -289,12 +289,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 3
BUILD_NAME: 'package_asan'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -328,12 +328,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 4
BUILD_NAME: 'package_ubsan'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -367,12 +367,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 5
BUILD_NAME: 'package_tsan'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -406,12 +406,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 6
BUILD_NAME: 'package_msan'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -445,12 +445,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 7
BUILD_NAME: 'package_debug'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -486,13 +486,13 @@ jobs:
IMAGES_PATH: ${{runner.temp}}/images_path
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse special build check (actions)'
BUILD_NUMBER: 1
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NAME: 'binary_splitted'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:

36
.github/workflows/master.yml vendored
View File

@ -84,7 +84,7 @@ jobs:
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 compatibility_check.py 0
cd $REPO_COPY/tests/ci && python3 compatibility_check.py
- name: Cleanup
if: always()
run: |
@ -142,12 +142,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 0
BUILD_NAME: 'package_release'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -181,12 +181,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 8
BUILD_NAME: 'binary_release'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -219,12 +219,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 3
BUILD_NAME: 'package_asan'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -257,12 +257,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 4
BUILD_NAME: 'package_ubsan'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -295,12 +295,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 5
BUILD_NAME: 'package_tsan'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -333,12 +333,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 6
BUILD_NAME: 'package_msan'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -371,12 +371,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 7
BUILD_NAME: 'package_debug'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -412,13 +412,13 @@ jobs:
IMAGES_PATH: ${{runner.temp}}/images_path
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse special build check (actions)'
BUILD_NUMBER: 1
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NAME: 'binary_splitted'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:

26
.github/workflows/release_branches.yml vendored
View File

@ -41,7 +41,7 @@ jobs:
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 compatibility_check.py 0
cd $REPO_COPY/tests/ci && python3 compatibility_check.py
- name: Cleanup
if: always()
run: |
@ -72,12 +72,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 0
BUILD_NAME: 'package_release'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -110,12 +110,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 3
BUILD_NAME: 'package_asan'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -148,12 +148,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 4
BUILD_NAME: 'package_ubsan'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -186,12 +186,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 5
BUILD_NAME: 'package_tsan'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -224,12 +224,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 6
BUILD_NAME: 'package_msan'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -262,12 +262,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 7
BUILD_NAME: 'package_debug'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:

74
CMakeLists.txt
View File

@ -169,9 +169,7 @@ endif ()
include (cmake/check_flags.cmake)
include (cmake/add_warning.cmake)
if (NOT MSVC)
set (COMMON_WARNING_FLAGS "${COMMON_WARNING_FLAGS} -Wall") # -Werror and many more is also added inside cmake/warnings.cmake
endif ()
set (COMMON_WARNING_FLAGS "${COMMON_WARNING_FLAGS} -Wall") # -Werror and many more is also added inside cmake/warnings.cmake
if (COMPILER_CLANG)
# clang: warning: argument unused during compilation: '-specs=/usr/share/dpkg/no-pie-compile.specs' [-Wunused-command-line-argument]
@ -219,30 +217,6 @@ endif()
# Make sure the final executable has symbols exported
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -rdynamic")
find_program (OBJCOPY_PATH NAMES "llvm-objcopy" "llvm-objcopy-13" "llvm-objcopy-12" "llvm-objcopy-11" "llvm-objcopy-10" "llvm-objcopy-9" "llvm-objcopy-8" "objcopy")
if (NOT OBJCOPY_PATH AND OS_DARWIN)
find_program (BREW_PATH NAMES "brew")
if (BREW_PATH)
execute_process (COMMAND ${BREW_PATH} --prefix llvm ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE LLVM_PREFIX)
if (LLVM_PREFIX)
find_program (OBJCOPY_PATH NAMES "llvm-objcopy" PATHS "${LLVM_PREFIX}/bin" NO_DEFAULT_PATH)
endif ()
if (NOT OBJCOPY_PATH)
execute_process (COMMAND ${BREW_PATH} --prefix binutils ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE BINUTILS_PREFIX)
if (BINUTILS_PREFIX)
find_program (OBJCOPY_PATH NAMES "objcopy" PATHS "${BINUTILS_PREFIX}/bin" NO_DEFAULT_PATH)
endif ()
endif ()
endif ()
endif ()
if (OBJCOPY_PATH)
message (STATUS "Using objcopy: ${OBJCOPY_PATH}")
else ()
message (FATAL_ERROR "Cannot find objcopy.")
endif ()
if (OS_DARWIN)
# The `-all_load` flag forces loading of all symbols from all libraries,
# and leads to multiply-defined symbols. This flag allows force loading
@ -278,6 +252,13 @@ if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE")
endif ()
endif()
if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
set(USE_DEBUG_HELPERS ON)
else ()
set(USE_DEBUG_HELPERS ON)
endif()
option(USE_DEBUG_HELPERS "Enable debug helpers" ${USE_DEBUG_HELPERS})
# Create BuildID when using lld. For other linkers it is created by default.
if (LINKER_NAME MATCHES "lld$")
# SHA1 is not cryptographically secure but it is the best what lld is offering.
@ -312,6 +293,10 @@ include(cmake/cpu_features.cmake)
# Enable it explicitly.
set (COMPILER_FLAGS "${COMPILER_FLAGS} -fasynchronous-unwind-tables")
# Reproducible builds
set (COMPILER_FLAGS "${COMPILER_FLAGS} -ffile-prefix-map=${CMAKE_SOURCE_DIR}=.")
set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} -ffile-prefix-map=${CMAKE_SOURCE_DIR}=.")
if (${CMAKE_VERSION} VERSION_LESS "3.12.4")
# CMake < 3.12 doesn't support setting 20 as a C++ standard version.
# We will add C++ standard controlling flag in CMAKE_CXX_FLAGS manually for now.
@ -411,25 +396,6 @@ if (COMPILER_CLANG)
message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot enable ThinLTO")
endif ()
# Always prefer llvm tools when using clang. For instance, we cannot use GNU ar when llvm LTO is enabled
find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-13" "llvm-ar-12" "llvm-ar-11" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8")
if (LLVM_AR_PATH)
message(STATUS "Using llvm-ar: ${LLVM_AR_PATH}.")
set (CMAKE_AR ${LLVM_AR_PATH})
else ()
message(WARNING "Cannot find llvm-ar. System ar will be used instead. It does not work with ThinLTO.")
endif ()
find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-13" "llvm-ranlib-12" "llvm-ranlib-11" "llvm-ranlib-10" "llvm-ranlib-9" "llvm-ranlib-8")
if (LLVM_RANLIB_PATH)
message(STATUS "Using llvm-ranlib: ${LLVM_RANLIB_PATH}.")
set (CMAKE_RANLIB ${LLVM_RANLIB_PATH})
else ()
message(WARNING "Cannot find llvm-ranlib. System ranlib will be used instead. It does not work with ThinLTO.")
endif ()
elseif (ENABLE_THINLTO)
message (${RECONFIGURE_MESSAGE_LEVEL} "ThinLTO is only available with CLang")
endif ()
@ -483,19 +449,6 @@ else ()
set (CMAKE_POSITION_INDEPENDENT_CODE ON)
endif ()
# https://github.com/include-what-you-use/include-what-you-use
option (USE_INCLUDE_WHAT_YOU_USE "Automatically reduce unneeded includes in source code (external tool)" OFF)
if (USE_INCLUDE_WHAT_YOU_USE)
find_program(IWYU_PATH NAMES include-what-you-use iwyu)
if (NOT IWYU_PATH)
message(FATAL_ERROR "Could not find the program include-what-you-use")
endif()
if (${CMAKE_VERSION} VERSION_LESS "3.3.0")
message(FATAL_ERROR "include-what-you-use requires CMake version at least 3.3.")
endif()
endif ()
if (ENABLE_TESTS)
message (STATUS "Unit tests are enabled")
else()
@ -595,8 +548,6 @@ include (cmake/find/mysqlclient.cmake)
# When testing for memory leaks with Valgrind, don't link tcmalloc or jemalloc.
include (cmake/print_flags.cmake)
if (TARGET global-group)
install (EXPORT global DESTINATION cmake)
endif ()
@ -648,6 +599,7 @@ include_directories(${ConfigIncludePath})
# Add as many warnings as possible for our own code.
include (cmake/warnings.cmake)
include (cmake/print_flags.cmake)
add_subdirectory (base)
add_subdirectory (src)

3
base/base/CMakeLists.txt
View File

@ -29,7 +29,8 @@ elseif (ENABLE_READLINE)
endif ()
if (USE_DEBUG_HELPERS)
set (INCLUDE_DEBUG_HELPERS "-include \"${ClickHouse_SOURCE_DIR}/base/base/iostream_debug_helpers.h\"")
get_target_property(MAGIC_ENUM_INCLUDE_DIR magic_enum INTERFACE_INCLUDE_DIRECTORIES)
set (INCLUDE_DEBUG_HELPERS "-I\"${MAGIC_ENUM_INCLUDE_DIR}\" -include \"${ClickHouse_SOURCE_DIR}/base/base/iostream_debug_helpers.h\"")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${INCLUDE_DEBUG_HELPERS}")
endif ()

4
base/base/CachedFn.h
View File

@ -18,8 +18,8 @@ struct CachedFn
{
private:
using Traits = FnTraits<decltype(Func)>;
using DecayedArgs = TLMap<std::decay_t, typename Traits::Args>;
using Key = TLChangeRoot<std::tuple, DecayedArgs>;
using DecayedArgs = TypeListMap<std::decay_t, typename Traits::Args>;
using Key = TypeListChangeRoot<std::tuple, DecayedArgs>;
using Result = typename Traits::Ret;
std::map<Key, Result> cache; // Can't use hashmap as tuples are unhashable by default

4
base/base/FnTraits.h
View File

@ -1,6 +1,6 @@
#pragma once
#include "Typelist.h"
#include "TypeList.h"
namespace detail
{
@ -14,7 +14,7 @@ struct FnTraits<R(A...)>
static constexpr bool value = std::is_invocable_r_v<R, F, A...>;
using Ret = R;
using Args = Typelist<A...>;
using Args = TypeList<A...>;
};
template <class R, class ...A>

44
base/base/TypeList.h Normal file
View File

@ -0,0 +1,44 @@
#pragma once
#include <cstddef>
#include <type_traits>
#include <utility>
#include "defines.h"
#include "TypePair.h"
/// General-purpose typelist. Easy on compilation times as it does not use recursion.
template <typename ...Args>
struct TypeList { static constexpr size_t size = sizeof...(Args); };
namespace TypeListUtils /// In some contexts it's more handy to use functions instead of aliases
{
template <typename ...LArgs, typename ...RArgs>
constexpr TypeList<LArgs..., RArgs...> concat(TypeList<LArgs...>, TypeList<RArgs...>) { return {}; }
template <typename T, typename ...Args>
constexpr TypeList<T, Args...> prepend(TypeList<Args...>) { return {}; }
template <typename T, typename ...Args>
constexpr TypeList<Args..., T> append(TypeList<Args...>) { return {}; }
template <template <typename> typename F, typename ...Args>
constexpr TypeList<F<Args>...> map(TypeList<Args...>) { return {}; }
template <template <typename...> typename Root, typename ...Args>
constexpr Root<Args...> changeRoot(TypeList<Args...>) { return {}; }
template <typename F, typename ...Args>
constexpr void forEach(TypeList<Args...>, F && f) { (std::forward<F>(f)(Id<Args>{}), ...); }
}
template <typename TypeListLeft, typename TypeListRight>
using TypeListConcat = decltype(TypeListUtils::concat(TypeListLeft{}, TypeListRight{}));
template <typename T, typename List> using TypeListPrepend = decltype(TypeListUtils::prepend<T>(List{}));
template <typename T, typename List> using TypeListAppend = decltype(TypeListUtils::append<T>(List{}));
template <template <typename> typename F, typename List>
using TypeListMap = decltype(TypeListUtils::map<F>(List{}));
template <template <typename...> typename Root, typename List>
using TypeListChangeRoot = decltype(TypeListUtils::changeRoot<Root>(List{}));

21
base/base/TypeLists.h Normal file
View File

@ -0,0 +1,21 @@
#pragma once
#include "TypeList.h"
#include "extended_types.h"
#include "Decimal.h"
#include "UUID.h"
namespace DB
{
using TypeListNativeInt = TypeList<UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64>;
using TypeListFloat = TypeList<Float32, Float64>;
using TypeListNativeNumber = TypeListConcat<TypeListNativeInt, TypeListFloat>;
using TypeListWideInt = TypeList<UInt128, Int128, UInt256, Int256>;
using TypeListInt = TypeListConcat<TypeListNativeInt, TypeListWideInt>;
using TypeListIntAndFloat = TypeListConcat<TypeListInt, TypeListFloat>;
using TypeListDecimal = TypeList<Decimal32, Decimal64, Decimal128, Decimal256>;
using TypeListNumber = TypeListConcat<TypeListIntAndFloat, TypeListDecimal>;
using TypeListNumberWithUUID = TypeListAppend<UUID, TypeListNumber>;
}

4
base/base/TypePair.h
View File

@ -1,4 +1,4 @@
#pragma once
template <class T, class V> struct TypePair { };
template <class T> struct Id { };
template <typename T, typename V> struct TypePair {};
template <typename T> struct Id {};

44
base/base/Typelist.h
View File

@ -1,44 +0,0 @@
#pragma once
#include <cstddef>
#include <type_traits>
#include <utility>
#include "defines.h"
#include "TypePair.h"
/// General-purpose typelist. Easy on compilation times as it does not use recursion.
template <class ...Args>
struct Typelist { static constexpr size_t size = sizeof...(Args); };
namespace TLUtils /// In some contexts it's more handy to use functions instead of aliases
{
template <class ...LArgs, class ...RArgs>
constexpr Typelist<LArgs..., RArgs...> concat(Typelist<LArgs...>, Typelist<RArgs...>) { return {}; }
template <class T, class ...Args>
constexpr Typelist<T, Args...> prepend(Typelist<Args...>) { return {}; }
template <class T, class ...Args>
constexpr Typelist<Args..., T> append(Typelist<Args...>) { return {}; }
template <template<class> class F, class ...Args>
constexpr Typelist<F<Args>...> map(Typelist<Args...>) { return {}; }
template <template<class...> class Root, class ...Args>
constexpr Root<Args...> changeRoot(Typelist<Args...>) { return {}; }
template <class F, class ...Args>
constexpr void forEach(Typelist<Args...>, F && f) { (std::forward<F>(f)(Id<Args>{}), ...); }
}
template <class TLLeft, class TLRight>
using TLConcat = decltype(TLUtils::concat(TLLeft{}, TLRight{}));
template <class T, class Typelist> using TLPrepend = decltype(TLUtils::prepend<T>(Typelist{}));
template <class T, class Typelist> using TLAppend = decltype(TLUtils::append<T>(Typelist{}));
template <template<class> class F, class Typelist>
using TLMap = decltype(TLUtils::map<F>(Typelist{}));
template <template<class...> class Root, class Typelist>
using TLChangeRoot = decltype(TLUtils::changeRoot<Root>(Typelist{}));

18
base/base/Typelists.h
View File

@ -1,18 +0,0 @@
#pragma once
#include "Typelist.h"
#include "extended_types.h"
#include "Decimal.h"
#include "UUID.h"
namespace DB
{
using TLIntegral = Typelist<UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64>;
using TLExtendedIntegral = Typelist<UInt128, Int128, UInt256, Int256>;
using TLDecimals = Typelist<Decimal32, Decimal64, Decimal128, Decimal256>;
using TLIntegralWithExtended = TLConcat<TLIntegral, TLExtendedIntegral>;
using TLNumbers = TLConcat<TLIntegralWithExtended, TLDecimals>;
using TLNumbersWithUUID = TLAppend<UUID, TLNumbers>;
}

2
base/base/UUID.h
View File

@ -5,5 +5,5 @@
namespace DB
{
using UUID = StrongTypedef<UInt128, struct UUIDTag>;
using UUID = StrongTypedef<UInt128, struct UUIDTag>;
}

23
base/base/iostream_debug_helpers.h
View File

@ -6,6 +6,7 @@
#include <tuple>
#include <iomanip>
#include <iostream>
#include <magic_enum.hpp>
/** Usage:
*
@ -61,6 +62,13 @@ std::enable_if_t<priority == 2, Out> & dumpImpl(Out & out, T && x, std::decay_t<
}
template <int priority, typename Out, typename T>
std::enable_if_t<priority == 3 && std::is_enum_v<std::decay_t<T>>, Out> &
dumpImpl(Out & out, T && x)
{
return out << magic_enum::enum_name(x);
}
/// string and const char * - output not as container or pointer.
template <int priority, typename Out, typename T>
@ -131,15 +139,26 @@ Out & dumpValue(Out & out, T && x)
template <typename Out, typename T>
Out & dump(Out & out, const char * name, T && x)
{
// Dumping string literal, printing name and demangled type is irrelevant.
if constexpr (std::is_same_v<const char *, std::decay_t<std::remove_reference_t<T>>>)
{
const auto name_len = strlen(name);
const auto value_len = strlen(x);
// `name` is the same as quoted `x`
if (name_len > 2 && value_len > 0 && name[0] == '"' && name[name_len - 1] == '"'
&& strncmp(name + 1, x, std::min(value_len, name_len) - 1) == 0)
return out << x;
}
out << demangle(typeid(x).name()) << " " << name << " = ";
return dumpValue(out, x);
return dumpValue(out, x) << "; ";
}
#ifdef __clang__
#pragma clang diagnostic ignored "-Wgnu-zero-variadic-macro-arguments"
#endif
#define DUMPVAR(VAR) ::dump(std::cerr, #VAR, (VAR)); std::cerr << "; ";
#define DUMPVAR(VAR) ::dump(std::cerr, #VAR, (VAR));
#define DUMPHEAD std::cerr << __FILE__ << ':' << __LINE__ << " [ " << getThreadId() << " ] ";
#define DUMPTAIL std::cerr << '\n';

10
cmake/freebsd/toolchain-aarch64.cmake
View File

@ -7,13 +7,9 @@ set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/freebsd-aarc
set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it doesn't work in CMake
set (CMAKE_AR "/usr/bin/ar" CACHE FILEPATH "" FORCE)
set (CMAKE_RANLIB "/usr/bin/ranlib" CACHE FILEPATH "" FORCE)
set (LINKER_NAME "ld.lld" CACHE STRING "" FORCE)
set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld")
set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld")
# Will be changed later, but somehow needed to be set here.
set (CMAKE_AR "ar")
set (CMAKE_RANLIB "ranlib")
set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)

10
cmake/freebsd/toolchain-x86_64.cmake
View File

@ -7,13 +7,9 @@ set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/freebsd-x86_
set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it doesn't work in CMake
set (CMAKE_AR "/usr/bin/ar" CACHE FILEPATH "" FORCE)
set (CMAKE_RANLIB "/usr/bin/ranlib" CACHE FILEPATH "" FORCE)
set (LINKER_NAME "ld.lld" CACHE STRING "" FORCE)
set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld")
set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld")
# Will be changed later, but somehow needed to be set here.
set (CMAKE_AR "ar")
set (CMAKE_RANLIB "ranlib")
set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)

15
cmake/linux/toolchain-aarch64.cmake
View File

@ -6,25 +6,18 @@ set (CMAKE_C_COMPILER_TARGET "aarch64-linux-gnu")
set (CMAKE_CXX_COMPILER_TARGET "aarch64-linux-gnu")
set (CMAKE_ASM_COMPILER_TARGET "aarch64-linux-gnu")
# Will be changed later, but somehow needed to be set here.
set (CMAKE_AR "ar")
set (CMAKE_RANLIB "ranlib")
set (TOOLCHAIN_PATH "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-aarch64")
set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}/aarch64-linux-gnu/libc")
find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-13" "llvm-ar-12" "llvm-ar-11" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8")
find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-13" "llvm-ranlib-12" "llvm-ranlib-11" "llvm-ranlib-10" "llvm-ranlib-9")
set (CMAKE_AR "${LLVM_AR_PATH}" CACHE FILEPATH "" FORCE)
set (CMAKE_RANLIB "${LLVM_RANLIB_PATH}" CACHE FILEPATH "" FORCE)
set (CMAKE_C_FLAGS_INIT "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
set (CMAKE_CXX_FLAGS_INIT "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
set (CMAKE_ASM_FLAGS_INIT "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
set (LINKER_NAME "ld.lld" CACHE STRING "" FORCE)
set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld")
set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld")
set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)

15
cmake/linux/toolchain-ppc64le.cmake
View File

@ -6,25 +6,18 @@ set (CMAKE_C_COMPILER_TARGET "ppc64le-linux-gnu")
set (CMAKE_CXX_COMPILER_TARGET "ppc64le-linux-gnu")
set (CMAKE_ASM_COMPILER_TARGET "ppc64le-linux-gnu")
# Will be changed later, but somehow needed to be set here.
set (CMAKE_AR "ar")
set (CMAKE_RANLIB "ranlib")
set (TOOLCHAIN_PATH "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-powerpc64le")
set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}/powerpc64le-linux-gnu/libc")
find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-13" "llvm-ar-12" "llvm-ar-11" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8")
find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-13" "llvm-ranlib-12" "llvm-ranlib-11" "llvm-ranlib-10" "llvm-ranlib-9")
set (CMAKE_AR "${LLVM_AR_PATH}" CACHE FILEPATH "" FORCE)
set (CMAKE_RANLIB "${LLVM_RANLIB_PATH}" CACHE FILEPATH "" FORCE)
set (CMAKE_C_FLAGS_INIT "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
set (CMAKE_CXX_FLAGS_INIT "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
set (CMAKE_ASM_FLAGS_INIT "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
set (LINKER_NAME "ld.lld" CACHE STRING "" FORCE)
set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld")
set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld")
set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)

12
cmake/linux/toolchain-riscv64.cmake
View File

@ -6,22 +6,18 @@ set (CMAKE_C_COMPILER_TARGET "riscv64-linux-gnu")
set (CMAKE_CXX_COMPILER_TARGET "riscv64-linux-gnu")
set (CMAKE_ASM_COMPILER_TARGET "riscv64-linux-gnu")
# Will be changed later, but somehow needed to be set here.
set (CMAKE_AR "ar")
set (CMAKE_RANLIB "ranlib")
set (TOOLCHAIN_PATH "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-riscv64")
set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}")
find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-13" "llvm-ar-12" "llvm-ar-11" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8")
find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-13" "llvm-ranlib-12" "llvm-ranlib-11" "llvm-ranlib-10" "llvm-ranlib-9")
set (CMAKE_AR "${LLVM_AR_PATH}" CACHE FILEPATH "" FORCE)
set (CMAKE_RANLIB "${LLVM_RANLIB_PATH}" CACHE FILEPATH "" FORCE)
set (CMAKE_C_FLAGS_INIT "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
set (CMAKE_CXX_FLAGS_INIT "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
set (CMAKE_ASM_FLAGS_INIT "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
set (LINKER_NAME "riscv64-linux-gnu-ld.bfd" CACHE STRING "" FORCE)
set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=bfd")
set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=bfd")

15
cmake/linux/toolchain-x86_64-musl.cmake
View File

@ -6,25 +6,18 @@ set (CMAKE_C_COMPILER_TARGET "x86_64-linux-musl")
set (CMAKE_CXX_COMPILER_TARGET "x86_64-linux-musl")
set (CMAKE_ASM_COMPILER_TARGET "x86_64-linux-musl")
# Will be changed later, but somehow needed to be set here.
set (CMAKE_AR "ar")
set (CMAKE_RANLIB "ranlib")
set (TOOLCHAIN_PATH "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-x86_64-musl")
set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}")
find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-13" "llvm-ar-12" "llvm-ar-11" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8")
find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-13" "llvm-ranlib-12" "llvm-ranlib-11" "llvm-ranlib-10" "llvm-ranlib-9")
set (CMAKE_AR "${LLVM_AR_PATH}" CACHE FILEPATH "" FORCE)
set (CMAKE_RANLIB "${LLVM_RANLIB_PATH}" CACHE FILEPATH "" FORCE)
set (CMAKE_C_FLAGS_INIT "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
set (CMAKE_CXX_FLAGS_INIT "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
set (CMAKE_ASM_FLAGS_INIT "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
set (LINKER_NAME "ld.lld" CACHE STRING "" FORCE)
set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld")
set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld")
set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)

15
cmake/linux/toolchain-x86_64.cmake
View File

@ -6,25 +6,18 @@ set (CMAKE_C_COMPILER_TARGET "x86_64-linux-gnu")
set (CMAKE_CXX_COMPILER_TARGET "x86_64-linux-gnu")
set (CMAKE_ASM_COMPILER_TARGET "x86_64-linux-gnu")
# Will be changed later, but somehow needed to be set here.
set (CMAKE_AR "ar")
set (CMAKE_RANLIB "ranlib")
set (TOOLCHAIN_PATH "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-x86_64")
set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}/x86_64-linux-gnu/libc")
find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-13" "llvm-ar-12" "llvm-ar-11" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8")
find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-13" "llvm-ranlib-12" "llvm-ranlib-11" "llvm-ranlib-10" "llvm-ranlib-9")
set (CMAKE_AR "${LLVM_AR_PATH}" CACHE FILEPATH "" FORCE)
set (CMAKE_RANLIB "${LLVM_RANLIB_PATH}" CACHE FILEPATH "" FORCE)
set (CMAKE_C_FLAGS_INIT "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
set (CMAKE_CXX_FLAGS_INIT "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
set (CMAKE_ASM_FLAGS_INIT "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
set (LINKER_NAME "ld.lld" CACHE STRING "" FORCE)
set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld")
set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld")
set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)

6
cmake/print_flags.cmake
View File

@ -1,6 +1,12 @@
set (FULL_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE_UC}}")
set (FULL_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UC}}")
set (FULL_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${CMAKE_EXE_LINKER_FLAGS_${CMAKE_BUILD_TYPE_UC}}")
message (STATUS "compiler C = ${CMAKE_C_COMPILER} ${FULL_C_FLAGS}")
message (STATUS "compiler CXX = ${CMAKE_CXX_COMPILER} ${FULL_CXX_FLAGS}")
message (STATUS "LINKER_FLAGS = ${FULL_EXE_LINKER_FLAGS}")
# Reproducible builds
string (REPLACE "${CMAKE_SOURCE_DIR}" "." FULL_C_FLAGS_NORMALIZED "${FULL_C_FLAGS}")
string (REPLACE "${CMAKE_SOURCE_DIR}" "." FULL_CXX_FLAGS_NORMALIZED "${FULL_CXX_FLAGS}")
string (REPLACE "${CMAKE_SOURCE_DIR}" "." FULL_EXE_LINKER_FLAGS_NORMALIZED "${FULL_EXE_LINKER_FLAGS}")

9
cmake/sanitize.cmake
View File

@ -78,6 +78,15 @@ if (SANITIZE)
elseif (SANITIZE STREQUAL "undefined")
set (UBSAN_FLAGS "-fsanitize=undefined -fno-sanitize-recover=all -fno-sanitize=float-divide-by-zero")
if (ENABLE_FUZZING)
# Unsigned integer overflow is well defined behaviour from a perspective of C++ standard,
# compilers or CPU. We use in hash functions like SipHash and many other places in our codebase.
# This flag is needed only because fuzzers are run inside oss-fuzz infrastructure
# and they have a bunch of flags not halt the program if UIO happend and even to silence that warnings.
# But for unknown reason that flags don't work with ClickHouse or we don't understand how to properly use them,
# that's why we often receive reports about UIO. The simplest way to avoid this is just set this flag here.
set(UBSAN_FLAGS "${SAN_FLAGS} -fno-sanitize=unsigned-integer-overflow")
endif()
if (COMPILER_CLANG)
set (UBSAN_FLAGS "${UBSAN_FLAGS} -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/tests/ubsan_suppressions.txt")
else()

76
cmake/tools.cmake
View File

@ -1,3 +1,5 @@
# Compiler
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
set (COMPILER_GCC 1)
elseif (CMAKE_CXX_COMPILER_ID MATCHES "AppleClang")
@ -6,6 +8,8 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
set (COMPILER_CLANG 1)
endif ()
execute_process(COMMAND ${CMAKE_CXX_COMPILER} --version)
if (COMPILER_GCC)
# Require minimum version of gcc
set (GCC_MINIMUM_VERSION 11)
@ -44,8 +48,10 @@ else ()
message (WARNING "You are using an unsupported compiler. Compilation has only been tested with Clang and GCC.")
endif ()
STRING(REGEX MATCHALL "[0-9]+" COMPILER_VERSION_LIST ${CMAKE_CXX_COMPILER_VERSION})
LIST(GET COMPILER_VERSION_LIST 0 COMPILER_VERSION_MAJOR)
string (REGEX MATCHALL "[0-9]+" COMPILER_VERSION_LIST ${CMAKE_CXX_COMPILER_VERSION})
list (GET COMPILER_VERSION_LIST 0 COMPILER_VERSION_MAJOR)
# Linker
# Example values: `lld-10`, `gold`.
option (LINKER_NAME "Linker name or full path")
@ -93,3 +99,69 @@ if (LINKER_NAME)
message(STATUS "Using custom linker by name: ${LINKER_NAME}")
endif ()
# Archiver
if (COMPILER_GCC)
find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-13" "llvm-ar-12" "llvm-ar-11")
else ()
find_program (LLVM_AR_PATH NAMES "llvm-ar-${COMPILER_VERSION_MAJOR}" "llvm-ar")
endif ()
if (LLVM_AR_PATH)
set (CMAKE_AR "${LLVM_AR_PATH}")
endif ()
# Ranlib
if (COMPILER_GCC)
find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-13" "llvm-ranlib-12" "llvm-ranlib-11")
else ()
find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib-${COMPILER_VERSION_MAJOR}" "llvm-ranlib")
endif ()
if (LLVM_RANLIB_PATH)
set (CMAKE_RANLIB "${LLVM_RANLIB_PATH}")
endif ()
# Install Name Tool
if (COMPILER_GCC)
find_program (LLVM_INSTALL_NAME_TOOL_PATH NAMES "llvm-install-name-tool" "llvm-install-name-tool-13" "llvm-install-name-tool-12" "llvm-install-name-tool-11")
else ()
find_program (LLVM_INSTALL_NAME_TOOL_PATH NAMES "llvm-install-name-tool-${COMPILER_VERSION_MAJOR}" "llvm-install-name-tool")
endif ()
if (LLVM_INSTALL_NAME_TOOL_PATH)
set (CMAKE_INSTALL_NAME_TOOL "${LLVM_INSTALL_NAME_TOOL_PATH}")
endif ()
# Objcopy
if (COMPILER_GCC)
find_program (OBJCOPY_PATH NAMES "llvm-objcopy" "llvm-objcopy-13" "llvm-objcopy-12" "llvm-objcopy-11" "objcopy")
else ()
find_program (OBJCOPY_PATH NAMES "llvm-objcopy-${COMPILER_VERSION_MAJOR}" "llvm-objcopy" "objcopy")
endif ()
if (NOT OBJCOPY_PATH AND OS_DARWIN)
find_program (BREW_PATH NAMES "brew")
if (BREW_PATH)
execute_process (COMMAND ${BREW_PATH} --prefix llvm ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE LLVM_PREFIX)
if (LLVM_PREFIX)
find_program (OBJCOPY_PATH NAMES "llvm-objcopy" PATHS "${LLVM_PREFIX}/bin" NO_DEFAULT_PATH)
endif ()
if (NOT OBJCOPY_PATH)
execute_process (COMMAND ${BREW_PATH} --prefix binutils ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE BINUTILS_PREFIX)
if (BINUTILS_PREFIX)
find_program (OBJCOPY_PATH NAMES "objcopy" PATHS "${BINUTILS_PREFIX}/bin" NO_DEFAULT_PATH)
endif ()
endif ()
endif ()
endif ()
if (OBJCOPY_PATH)
message (STATUS "Using objcopy: ${OBJCOPY_PATH}")
else ()
message (FATAL_ERROR "Cannot find objcopy.")
endif ()

81
cmake/warnings.cmake
View File

@ -26,46 +26,6 @@ if (COMPILER_CLANG)
no_warning(zero-length-array)
no_warning(c11-extensions)
add_warning(comma)
add_warning(conditional-uninitialized)
add_warning(covered-switch-default)
add_warning(deprecated)
add_warning(embedded-directive)
add_warning(empty-init-stmt) # linux-only
add_warning(extra-semi-stmt) # linux-only
add_warning(extra-semi)
add_warning(gnu-case-range)
add_warning(inconsistent-missing-destructor-override)
add_warning(newline-eof)
add_warning(old-style-cast)
add_warning(range-loop-analysis)
add_warning(redundant-parens)
add_warning(reserved-id-macro)
add_warning(shadow-field)
add_warning(shadow-uncaptured-local)
add_warning(shadow)
add_warning(string-plus-int)
add_warning(undef)
add_warning(unreachable-code-return)
add_warning(unreachable-code)
add_warning(unused-exception-parameter)
add_warning(unused-macros)
add_warning(unused-member-function)
add_warning(unneeded-internal-declaration)
add_warning(implicit-int-float-conversion)
add_warning(no-delete-null-pointer-checks)
add_warning(anon-enum-enum-conversion)
add_warning(assign-enum)
add_warning(bitwise-op-parentheses)
add_warning(int-in-bool-context)
add_warning(sometimes-uninitialized)
add_warning(tautological-bitwise-compare)
# XXX: libstdc++ has some of these for 3way compare
if (USE_LIBCXX)
add_warning(zero-as-null-pointer-constant)
endif()
if (WEVERYTHING)
add_warning(everything)
no_warning(c++98-compat-pedantic)
@ -101,6 +61,47 @@ if (COMPILER_CLANG)
endif()
# TODO Enable conversion, sign-conversion, double-promotion warnings.
else ()
add_warning(comma)
add_warning(conditional-uninitialized)
add_warning(covered-switch-default)
add_warning(deprecated)
add_warning(embedded-directive)
add_warning(empty-init-stmt) # linux-only
add_warning(extra-semi-stmt) # linux-only
add_warning(extra-semi)
add_warning(gnu-case-range)
add_warning(inconsistent-missing-destructor-override)
add_warning(newline-eof)
add_warning(old-style-cast)
add_warning(range-loop-analysis)
add_warning(redundant-parens)
add_warning(reserved-id-macro)
add_warning(shadow-field)
add_warning(shadow-uncaptured-local)
add_warning(shadow)
add_warning(string-plus-int)
add_warning(undef)
add_warning(unreachable-code-return)
add_warning(unreachable-code)
add_warning(unused-exception-parameter)
add_warning(unused-macros)
add_warning(unused-member-function)
add_warning(unneeded-internal-declaration)
add_warning(implicit-int-float-conversion)
add_warning(no-delete-null-pointer-checks)
add_warning(anon-enum-enum-conversion)
add_warning(assign-enum)
add_warning(bitwise-op-parentheses)
add_warning(int-in-bool-context)
add_warning(sometimes-uninitialized)
add_warning(tautological-bitwise-compare)
# XXX: libstdc++ has some of these for 3way compare
if (USE_LIBCXX)
add_warning(zero-as-null-pointer-constant)
endif()
endif ()
elseif (COMPILER_GCC)
# Add compiler options only to c++ compiler

2
contrib/cctz-cmake/CMakeLists.txt
View File

@ -82,7 +82,7 @@ if (NOT EXTERNAL_CCTZ_LIBRARY_FOUND OR NOT EXTERNAL_CCTZ_LIBRARY_WORKS)
# each file in that dir (except of tab and localtime) store the info about timezone
execute_process(COMMAND
bash -c "cd ${TZDIR} && find * -type f -and ! -name '*.tab' -and ! -name 'localtime' | sort | paste -sd ';' -"
bash -c "cd ${TZDIR} && find * -type f -and ! -name '*.tab' -and ! -name 'localtime' | LC_ALL=C sort | paste -sd ';' -"
OUTPUT_STRIP_TRAILING_WHITESPACE
OUTPUT_VARIABLE TIMEZONES)

6
docker/builder/Dockerfile
View File

@ -1,8 +1,10 @@
FROM ubuntu:20.04
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13
# ARG for quick switch to a given ubuntu mirror
ARG apt_archive="http://archive.ubuntu.com"
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13
RUN apt-get update \
&& apt-get install ca-certificates lsb-release wget gnupg apt-transport-https \

6
docker/client/Dockerfile
View File

@ -1,10 +1,12 @@
FROM ubuntu:18.04
# ARG for quick switch to a given ubuntu mirror
ARG apt_archive="http://archive.ubuntu.com"
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
ARG repository="deb https://repo.clickhouse.com/deb/stable/ main/"
ARG version=21.12.1.*
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
RUN apt-get update \
&& apt-get install --yes --no-install-recommends \
apt-transport-https \

6
docker/docs/builder/Dockerfile
View File

@ -1,9 +1,11 @@
# docker build -t clickhouse/docs-build .
FROM ubuntu:20.04
ENV LANG=C.UTF-8
# ARG for quick switch to a given ubuntu mirror
ARG apt_archive="http://archive.ubuntu.com"
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
ENV LANG=C.UTF-8
RUN apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends \

6
docker/packager/binary/Dockerfile
View File

@ -1,9 +1,11 @@
# docker build -t clickhouse/binary-builder .
FROM ubuntu:20.04
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13
# ARG for quick switch to a given ubuntu mirror
ARG apt_archive="http://archive.ubuntu.com"
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13
RUN apt-get update \
&& apt-get install \

6
docker/packager/deb/Dockerfile
View File

@ -1,9 +1,11 @@
# docker build -t clickhouse/deb-builder .
FROM ubuntu:20.04
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13
# ARG for quick switch to a given ubuntu mirror
ARG apt_archive="http://archive.ubuntu.com"
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13
RUN apt-get update \
&& apt-get install ca-certificates lsb-release wget gnupg apt-transport-https \

7
docker/packager/packager
View File

@ -63,7 +63,7 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
PPC_SUFFIX = '-ppc64le'
result = []
cmake_flags = ['$CMAKE_FLAGS', '-DADD_GDB_INDEX_FOR_GOLD=1']
cmake_flags = ['$CMAKE_FLAGS']
is_clang = compiler.startswith(CLANG_PREFIX)
is_cross_darwin = compiler.endswith(DARWIN_SUFFIX)
@ -73,11 +73,6 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
is_cross_freebsd = compiler.endswith(FREEBSD_SUFFIX)
is_cross_compile = is_cross_darwin or is_cross_darwin_arm or is_cross_arm or is_cross_freebsd or is_cross_ppc
# Explicitly use LLD with Clang by default.
# Don't force linker for cross-compilation.
if is_clang and not is_cross_compile:
cmake_flags.append("-DLINKER_NAME=ld.lld")
if is_cross_darwin:
cc = compiler[:-len(DARWIN_SUFFIX)]
cmake_flags.append("-DCMAKE_AR:FILEPATH=/cctools/bin/x86_64-apple-darwin-ar")

6
docker/server/Dockerfile
View File

@ -1,5 +1,9 @@
FROM ubuntu:20.04
# ARG for quick switch to a given ubuntu mirror
ARG apt_archive="http://archive.ubuntu.com"
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
ARG repository="deb https://repo.clickhouse.com/deb/stable/ main/"
ARG version=21.12.1.*
ARG gosu_ver=1.10
@ -26,8 +30,6 @@ ARG DEBIAN_FRONTEND=noninteractive
# installed to prevent picking those uid / gid by some unrelated software.
# The same uid / gid (101) is used both for alpine and ubuntu.
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
RUN groupadd -r clickhouse --gid=101 \
&& useradd -r -g clickhouse --uid=101 --home-dir=/var/lib/clickhouse --shell=/bin/bash clickhouse \
&& apt-get update \

6
docker/test/base/Dockerfile
View File

@ -1,9 +1,11 @@
# docker build -t clickhouse/test-base .
FROM clickhouse/test-util
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13
# ARG for quick switch to a given ubuntu mirror
ARG apt_archive="http://archive.ubuntu.com"
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13
RUN apt-get update \
&& apt-get install ca-certificates lsb-release wget gnupg apt-transport-https \

4
docker/test/codebrowser/Dockerfile
View File

@ -2,7 +2,9 @@
# docker run --volume=path_to_repo:/repo_folder --volume=path_to_result:/test_output clickhouse/codebrowser
FROM clickhouse/binary-builder
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
# ARG for quick switch to a given ubuntu mirror
ARG apt_archive="http://archive.ubuntu.com"
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
RUN apt-get update && apt-get --yes --allow-unauthenticated install clang-9 libllvm9 libclang-9-dev

6
docker/test/fasttest/Dockerfile
View File

@ -1,9 +1,11 @@
# docker build -t clickhouse/fasttest .
FROM clickhouse/test-util
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13
# ARG for quick switch to a given ubuntu mirror
ARG apt_archive="http://archive.ubuntu.com"
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13
RUN apt-get update \
&& apt-get install ca-certificates lsb-release wget gnupg apt-transport-https \

6
docker/test/fuzzer/Dockerfile
View File

@ -1,12 +1,14 @@
# docker build -t clickhouse/fuzzer .
FROM clickhouse/test-base
# ARG for quick switch to a given ubuntu mirror
ARG apt_archive="http://archive.ubuntu.com"
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
ENV LANG=C.UTF-8
ENV TZ=Europe/Moscow
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
RUN apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends \
ca-certificates \

4
docker/test/integration/runner/Dockerfile
View File

@ -1,7 +1,9 @@
# docker build -t clickhouse/integration-tests-runner .
FROM ubuntu:20.04
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
# ARG for quick switch to a given ubuntu mirror
ARG apt_archive="http://archive.ubuntu.com"
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
RUN apt-get update \
&& env DEBIAN_FRONTEND=noninteractive apt-get install --yes \

6
docker/test/performance-comparison/Dockerfile
View File

@ -1,12 +1,14 @@
# docker build -t clickhouse/performance-comparison .
FROM ubuntu:18.04
# ARG for quick switch to a given ubuntu mirror
ARG apt_archive="http://archive.ubuntu.com"
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
ENV LANG=C.UTF-8
ENV TZ=Europe/Moscow
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
RUN apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends \
bash \

4
docker/test/sqlancer/Dockerfile
View File

@ -1,7 +1,9 @@
# docker build -t clickhouse/sqlancer-test .
FROM ubuntu:20.04
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
# ARG for quick switch to a given ubuntu mirror
ARG apt_archive="http://archive.ubuntu.com"
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
RUN apt-get update --yes && env DEBIAN_FRONTEND=noninteractive apt-get install wget unzip git default-jdk maven python3 --yes --no-install-recommends
RUN wget https://github.com/sqlancer/sqlancer/archive/master.zip -O /sqlancer.zip

4
docker/test/style/Dockerfile
View File

@ -1,7 +1,9 @@
# docker build -t clickhouse/style-test .
FROM ubuntu:20.04
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
# ARG for quick switch to a given ubuntu mirror
ARG apt_archive="http://archive.ubuntu.com"
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
shellcheck \

4
docker/test/testflows/runner/Dockerfile
View File

@ -1,7 +1,9 @@
# docker build -t clickhouse/testflows-runner .
FROM ubuntu:20.04
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
# ARG for quick switch to a given ubuntu mirror
ARG apt_archive="http://archive.ubuntu.com"
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
RUN apt-get update \
&& env DEBIAN_FRONTEND=noninteractive apt-get install --yes \

2
docs/en/engines/table-engines/integrations/kafka.md
View File

@ -192,6 +192,6 @@ Example:
**See Also**
- [Virtual columns](../../../engines/table-engines/index.md#table_engines-virtual_columns)
- [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size)
- [background_message_broker_schedule_pool_size](../../../operations/settings/settings.md#background_message_broker_schedule_pool_size)
[Original article](https://clickhouse.com/docs/en/engines/table-engines/integrations/kafka/) <!--hide-->

212
docs/en/operations/utilities/clickhouse-format.md
View File

@ -1,98 +1,114 @@
---
toc_priority: 65
toc_title: clickhouse-format
---
# clickhouse-format {#clickhouse-format}
Allows formatting input queries.
Keys:
- `--help` or`-h` — Produce help message.
- `--hilite` — Add syntax highlight with ANSI terminal escape sequences.
- `--oneline` — Format in single line.
- `--quiet` or `-q` — Just check syntax, no output on success.
- `--multiquery` or `-n` — Allow multiple queries in the same file.
- `--obfuscate` — Obfuscate instead of formatting.
- `--seed <string>` — Seed arbitrary string that determines the result of obfuscation.
- `--backslash` — Add a backslash at the end of each line of the formatted query. Can be useful when you copy a query from web or somewhere else with multiple lines, and want to execute it in command line.
## Examples {#examples}
1. Highlighting and single line:
```bash
$ clickhouse-format --oneline --hilite <<< "SELECT sum(number) FROM numbers(5);"
```
Result:
```sql
SELECT sum(number) FROM numbers(5)
```
2. Multiqueries:
```bash
$ clickhouse-format -n <<< "SELECT * FROM (SELECT 1 AS x UNION ALL SELECT 1 UNION DISTINCT SELECT 3);"
```
Result:
```text
SELECT *
FROM
(
SELECT 1 AS x
UNION ALL
SELECT 1
UNION DISTINCT
SELECT 3
)
;
```
3. Obfuscating:
```bash
$ clickhouse-format --seed Hello --obfuscate <<< "SELECT cost_first_screen BETWEEN a AND b, CASE WHEN x >= 123 THEN y ELSE NULL END;"
```
Result:
```text
SELECT treasury_mammoth_hazelnut BETWEEN nutmeg AND span, CASE WHEN chive >= 116 THEN switching ELSE ANYTHING END;
```
Same query and another seed string:
```bash
$ clickhouse-format --seed World --obfuscate <<< "SELECT cost_first_screen BETWEEN a AND b, CASE WHEN x >= 123 THEN y ELSE NULL END;"
```
Result:
```text
SELECT horse_tape_summer BETWEEN folklore AND moccasins, CASE WHEN intestine >= 116 THEN nonconformist ELSE FORESTRY END;
```
4. Adding backslash:
```bash
$ clickhouse-format --backslash <<< "SELECT * FROM (SELECT 1 AS x UNION ALL SELECT 1 UNION DISTINCT SELECT 3);"
```
Result:
```text
SELECT * \
FROM \
( \
SELECT 1 AS x \
UNION ALL \
SELECT 1 \
UNION DISTINCT \
SELECT 3 \
)
```
---
toc_priority: 65
toc_title: clickhouse-format
---
# clickhouse-format {#clickhouse-format}
Allows formatting input queries.
Keys:
- `--help` or`-h` — Produce help message.
- `--query` — Format queries of any length and complexity.
- `--hilite` — Add syntax highlight with ANSI terminal escape sequences.
- `--oneline` — Format in single line.
- `--quiet` or `-q` — Just check syntax, no output on success.
- `--multiquery` or `-n` — Allow multiple queries in the same file.
- `--obfuscate` — Obfuscate instead of formatting.
- `--seed <string>` — Seed arbitrary string that determines the result of obfuscation.
- `--backslash` — Add a backslash at the end of each line of the formatted query. Can be useful when you copy a query from web or somewhere else with multiple lines, and want to execute it in command line.
## Examples {#examples}
1. Formatting a query:
```bash
$ clickhouse-format --query "select number from numbers(10) where number%2 order by number desc;"
```
Result:
```text
SELECT number
FROM numbers(10)
WHERE number % 2
ORDER BY number DESC
```
2. Highlighting and single line:
```bash
$ clickhouse-format --oneline --hilite <<< "SELECT sum(number) FROM numbers(5);"
```
Result:
```sql
SELECT sum(number) FROM numbers(5)
```
3. Multiqueries:
```bash
$ clickhouse-format -n <<< "SELECT * FROM (SELECT 1 AS x UNION ALL SELECT 1 UNION DISTINCT SELECT 3);"
```
Result:
```text
SELECT *
FROM
(
SELECT 1 AS x
UNION ALL
SELECT 1
UNION DISTINCT
SELECT 3
)
;
```
4. Obfuscating:
```bash
$ clickhouse-format --seed Hello --obfuscate <<< "SELECT cost_first_screen BETWEEN a AND b, CASE WHEN x >= 123 THEN y ELSE NULL END;"
```
Result:
```text
SELECT treasury_mammoth_hazelnut BETWEEN nutmeg AND span, CASE WHEN chive >= 116 THEN switching ELSE ANYTHING END;
```
Same query and another seed string:
```bash
$ clickhouse-format --seed World --obfuscate <<< "SELECT cost_first_screen BETWEEN a AND b, CASE WHEN x >= 123 THEN y ELSE NULL END;"
```
Result:
```text
SELECT horse_tape_summer BETWEEN folklore AND moccasins, CASE WHEN intestine >= 116 THEN nonconformist ELSE FORESTRY END;
```
5. Adding backslash:
```bash
$ clickhouse-format --backslash <<< "SELECT * FROM (SELECT 1 AS x UNION ALL SELECT 1 UNION DISTINCT SELECT 3);"
```
Result:
```text
SELECT * \
FROM \
( \
SELECT 1 AS x \
UNION ALL \
SELECT 1 \
UNION DISTINCT \
SELECT 3 \
)
```

148
docs/en/sql-reference/aggregate-functions/reference/exponentialmovingaverage.md Normal file
View File

@ -0,0 +1,148 @@
---
toc_priority: 108
---
## exponentialMovingAverage {#exponential-moving-average}
Сalculates the exponential moving average of values for the determined time.
**Syntax**
```sql
exponentialMovingAverage(x)(value, timestamp)
```
Each `value` corresponds to the determinate `timestamp`. The half-life `x` is the time lag at which the exponential weights decay by one-half. The function returns a weighted average: the older the time point, the less weight the corresponding value is considered to be.
**Arguments**
- `value` — Value. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md).
- `timestamp` — Timestamp. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md).
**Parameters**
- `x` — Half-life period. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md).
**Returned values**
- Returnes an [exponentially smoothed moving average](https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average) of the values for the past `x` time at the latest point of time.
Type: [Float64](../../../sql-reference/data-types/float.md#float32-float64).
**Examples**
Input table:
``` text
┌──temperature─┬─timestamp──┐
│ 95 │ 1 │
│ 95 │ 2 │
│ 95 │ 3 │
│ 96 │ 4 │
│ 96 │ 5 │
│ 96 │ 6 │
│ 96 │ 7 │
│ 97 │ 8 │
│ 97 │ 9 │
│ 97 │ 10 │
│ 97 │ 11 │
│ 98 │ 12 │
│ 98 │ 13 │
│ 98 │ 14 │
│ 98 │ 15 │
│ 99 │ 16 │
│ 99 │ 17 │
│ 99 │ 18 │
│ 100 │ 19 │
│ 100 │ 20 │
└──────────────┴────────────┘
```
Query:
```sql
SELECT exponentialMovingAverage(5)(temperature, timestamp);
```
Result:
``` text
┌──exponentialMovingAverage(5)(temperature, timestamp)──┐
│ 92.25779635374204 │
└───────────────────────────────────────────────────────┘
```
Query:
```sql
SELECT
value,
time,
round(exp_smooth, 3),
bar(exp_smooth, 0, 1, 50) AS bar
FROM
(
SELECT
(number = 0) OR (number >= 25) AS value,
number AS time,
exponentialMovingAverage(10)(value, time) OVER (Rows BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS exp_smooth
FROM numbers(50)
)
```
Result:
``` text
┌─value─┬─time─┬─round(exp_smooth, 3)─┬─bar────────────────────────────────────────┐
│ 1 │ 0 │ 0.067 │ ███▎ │
│ 0 │ 1 │ 0.062 │ ███ │
│ 0 │ 2 │ 0.058 │ ██▊ │
│ 0 │ 3 │ 0.054 │ ██▋ │
│ 0 │ 4 │ 0.051 │ ██▌ │
│ 0 │ 5 │ 0.047 │ ██▎ │
│ 0 │ 6 │ 0.044 │ ██▏ │
│ 0 │ 7 │ 0.041 │ ██ │
│ 0 │ 8 │ 0.038 │ █▊ │
│ 0 │ 9 │ 0.036 │ █▋ │
│ 0 │ 10 │ 0.033 │ █▋ │
│ 0 │ 11 │ 0.031 │ █▌ │
│ 0 │ 12 │ 0.029 │ █▍ │
│ 0 │ 13 │ 0.027 │ █▎ │
│ 0 │ 14 │ 0.025 │ █▎ │
│ 0 │ 15 │ 0.024 │ █▏ │
│ 0 │ 16 │ 0.022 │ █ │
│ 0 │ 17 │ 0.021 │ █ │
│ 0 │ 18 │ 0.019 │ ▊ │
│ 0 │ 19 │ 0.018 │ ▊ │
│ 0 │ 20 │ 0.017 │ ▋ │
│ 0 │ 21 │ 0.016 │ ▋ │
│ 0 │ 22 │ 0.015 │ ▋ │
│ 0 │ 23 │ 0.014 │ ▋ │
│ 0 │ 24 │ 0.013 │ ▋ │
│ 1 │ 25 │ 0.079 │ ███▊ │
│ 1 │ 26 │ 0.14 │ ███████ │
│ 1 │ 27 │ 0.198 │ █████████▊ │
│ 1 │ 28 │ 0.252 │ ████████████▌ │
│ 1 │ 29 │ 0.302 │ ███████████████ │
│ 1 │ 30 │ 0.349 │ █████████████████▍ │
│ 1 │ 31 │ 0.392 │ ███████████████████▌ │
│ 1 │ 32 │ 0.433 │ █████████████████████▋ │
│ 1 │ 33 │ 0.471 │ ███████████████████████▌ │
│ 1 │ 34 │ 0.506 │ █████████████████████████▎ │
│ 1 │ 35 │ 0.539 │ ██████████████████████████▊ │
│ 1 │ 36 │ 0.57 │ ████████████████████████████▌ │
│ 1 │ 37 │ 0.599 │ █████████████████████████████▊ │
│ 1 │ 38 │ 0.626 │ ███████████████████████████████▎ │
│ 1 │ 39 │ 0.651 │ ████████████████████████████████▌ │
│ 1 │ 40 │ 0.674 │ █████████████████████████████████▋ │
│ 1 │ 41 │ 0.696 │ ██████████████████████████████████▋ │
│ 1 │ 42 │ 0.716 │ ███████████████████████████████████▋ │
│ 1 │ 43 │ 0.735 │ ████████████████████████████████████▋ │
│ 1 │ 44 │ 0.753 │ █████████████████████████████████████▋ │
│ 1 │ 45 │ 0.77 │ ██████████████████████████████████████▍ │
│ 1 │ 46 │ 0.785 │ ███████████████████████████████████████▎ │
│ 1 │ 47 │ 0.8 │ ███████████████████████████████████████▊ │
│ 1 │ 48 │ 0.813 │ ████████████████████████████████████████▋ │
│ 1 │ 49 │ 0.825 │ █████████████████████████████████████████▎│
└───────┴──────┴──────────────────────┴────────────────────────────────────────────┘
```

60
docs/en/sql-reference/functions/hash-functions.md
View File

@ -89,9 +89,39 @@ SELECT sipHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00
## sipHash128 {#hash_functions-siphash128}
Calculates SipHash from a string.
Accepts a String-type argument. Returns FixedString(16).
Differs from sipHash64 in that the final xor-folding state is only done up to 128 bits.
Produces a 128-bit [SipHash](https://131002.net/siphash/) hash value. Differs from [sipHash64](#hash_functions-siphash64) in that the final xor-folding state is done up to 128 bits.
**Syntax**
``` sql
sipHash128(par1,...)
```
**Arguments**
The function takes a variable number of input parameters. Arguments can be any of the [supported data types](../../sql-reference/data-types/index.md).
**Returned value**
A 128-bit `SipHash` hash value.
Type: [FixedString(16)](../../sql-reference/data-types/fixedstring.md).
**Example**
Query:
``` sql
SELECT hex(sipHash128('foo', '\x01', 3));
```
Result:
``` text
┌─hex(sipHash128('foo', '', 3))────┐
│ 9DE516A64A414D4B1B609415E4523F24 │
└──────────────────────────────────┘
```
## cityHash64 {#cityhash64}
@ -459,28 +489,36 @@ SELECT murmurHash3_32(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:
Produces a 128-bit [MurmurHash3](https://github.com/aappleby/smhasher) hash value.
**Syntax**
``` sql
murmurHash3_128( expr )
murmurHash3_128(expr)
```
**Arguments**
- `expr` — [Expressions](../../sql-reference/syntax.md#syntax-expressions) returning a [String](../../sql-reference/data-types/string.md)-type value.
- `expr` — A list of [expressions](../../sql-reference/syntax.md#syntax-expressions). [String](../../sql-reference/data-types/string.md).
**Returned Value**
**Returned value**
A [FixedString(16)](../../sql-reference/data-types/fixedstring.md) data type hash value.
A 128-bit `MurmurHash3` hash value.
Type: [FixedString(16)](../../sql-reference/data-types/fixedstring.md).
**Example**
Query:
``` sql
SELECT hex(murmurHash3_128('example_string')) AS MurmurHash3, toTypeName(MurmurHash3) AS type;
SELECT hex(murmurHash3_128('foo', 'foo', 'foo'));
```
Result:
``` text
┌─MurmurHash3──────────────────────┬─type───┐
│ 368A1A311CB7342253354B548E7E7E71 │ String │
└──────────────────────────────────┴────────┘
┌─hex(murmurHash3_128('foo', 'foo', 'foo'))─┐
│ F8F7AD9B6CD4CF117A71E277E2EC2931 │
└───────────────────────────────────────────┘
```
## xxHash32, xxHash64 {#hash-functions-xxhash32}

212
docs/ru/operations/utilities/clickhouse-format.md
View File

@ -1,98 +1,114 @@
---
toc_priority: 65
toc_title: clickhouse-format
---
# clickhouse-format {#clickhouse-format}
Позволяет форматировать входящие запросы.
Ключи:
- `--help` или`-h` — выводит описание ключей.
- `--hilite` — добавляет подсветку синтаксиса с экранированием символов.
- `--oneline` — форматирование в одну строку.
- `--quiet` или `-q` — проверяет синтаксис без вывода результата.
- `--multiquery` or `-n` — поддерживает несколько запросов в одной строке.
- `--obfuscate` — обфусцирует вместо форматирования.
- `--seed <строка>` — задает строку, которая определяет результат обфускации.
- `--backslash` — добавляет обратный слеш в конце каждой строки отформатированного запроса. Удобно использовать если многострочный запрос скопирован из интернета или другого источника и его нужно выполнить из командной строки.
## Примеры {#examples}
1. Подсветка синтаксиса и форматирование в одну строку:
```bash
$ clickhouse-format --oneline --hilite <<< "SELECT sum(number) FROM numbers(5);"
```
Результат:
```sql
SELECT sum(number) FROM numbers(5)
```
2. Несколько запросов в одной строке:
```bash
$ clickhouse-format -n <<< "SELECT * FROM (SELECT 1 AS x UNION ALL SELECT 1 UNION DISTINCT SELECT 3);"
```
Результат:
```text
SELECT *
FROM
(
SELECT 1 AS x
UNION ALL
SELECT 1
UNION DISTINCT
SELECT 3
)
;
```
3. Обфускация:
```bash
$ clickhouse-format --seed Hello --obfuscate <<< "SELECT cost_first_screen BETWEEN a AND b, CASE WHEN x >= 123 THEN y ELSE NULL END;"
```
Результат:
```text
SELECT treasury_mammoth_hazelnut BETWEEN nutmeg AND span, CASE WHEN chive >= 116 THEN switching ELSE ANYTHING END;
```
Тот же запрос с другой инициализацией обфускатора:
```bash
$ clickhouse-format --seed World --obfuscate <<< "SELECT cost_first_screen BETWEEN a AND b, CASE WHEN x >= 123 THEN y ELSE NULL END;"
```
Результат:
```text
SELECT horse_tape_summer BETWEEN folklore AND moccasins, CASE WHEN intestine >= 116 THEN nonconformist ELSE FORESTRY END;
```
4. Добавление обратного слеша:
```bash
$ clickhouse-format --backslash <<< "SELECT * FROM (SELECT 1 AS x UNION ALL SELECT 1 UNION DISTINCT SELECT 3);"
```
Результат:
```text
SELECT * \
FROM \
( \
SELECT 1 AS x \
UNION ALL \
SELECT 1 \
UNION DISTINCT \
SELECT 3 \
)
```
---
toc_priority: 65
toc_title: clickhouse-format
---
# clickhouse-format {#clickhouse-format}
Позволяет форматировать входящие запросы.
Ключи:
- `--help` или`-h` — выводит описание ключей.
- `--query` — форматирует запрос любой длины и сложности.
- `--hilite` — добавляет подсветку синтаксиса с экранированием символов.
- `--oneline` — форматирование в одну строку.
- `--quiet` или `-q` — проверяет синтаксис без вывода результата.
- `--multiquery` or `-n` — поддерживает несколько запросов в одной строке.
- `--obfuscate` — обфусцирует вместо форматирования.
- `--seed <строка>` — задает строку, которая определяет результат обфускации.
- `--backslash` — добавляет обратный слеш в конце каждой строки отформатированного запроса. Удобно использовать если многострочный запрос скопирован из интернета или другого источника и его нужно выполнить из командной строки.
## Примеры {#examples}
1. Форматирование запроса:
```bash
$ clickhouse-format --query "select number from numbers(10) where number%2 order by number desc;"
```
Результат:
```text
SELECT number
FROM numbers(10)
WHERE number % 2
ORDER BY number DESC
```
2. Подсветка синтаксиса и форматирование в одну строку:
```bash
$ clickhouse-format --oneline --hilite <<< "SELECT sum(number) FROM numbers(5);"
```
Результат:
```sql
SELECT sum(number) FROM numbers(5)
```
3. Несколько запросов в одной строке:
```bash
$ clickhouse-format -n <<< "SELECT * FROM (SELECT 1 AS x UNION ALL SELECT 1 UNION DISTINCT SELECT 3);"
```
Результат:
```text
SELECT *
FROM
(
SELECT 1 AS x
UNION ALL
SELECT 1
UNION DISTINCT
SELECT 3
)
;
```
4. Обфускация:
```bash
$ clickhouse-format --seed Hello --obfuscate <<< "SELECT cost_first_screen BETWEEN a AND b, CASE WHEN x >= 123 THEN y ELSE NULL END;"
```
Результат:
```text
SELECT treasury_mammoth_hazelnut BETWEEN nutmeg AND span, CASE WHEN chive >= 116 THEN switching ELSE ANYTHING END;
```
Тот же запрос с другой инициализацией обфускатора:
```bash
$ clickhouse-format --seed World --obfuscate <<< "SELECT cost_first_screen BETWEEN a AND b, CASE WHEN x >= 123 THEN y ELSE NULL END;"
```
Результат:
```text
SELECT horse_tape_summer BETWEEN folklore AND moccasins, CASE WHEN intestine >= 116 THEN nonconformist ELSE FORESTRY END;
```
5. Добавление обратного слеша:
```bash
$ clickhouse-format --backslash <<< "SELECT * FROM (SELECT 1 AS x UNION ALL SELECT 1 UNION DISTINCT SELECT 3);"
```
Результат:
```text
SELECT * \
FROM \
( \
SELECT 1 AS x \
UNION ALL \
SELECT 1 \
UNION DISTINCT \
SELECT 3 \
)
```

148
docs/ru/sql-reference/aggregate-functions/reference/exponentialmovingaverage.md Normal file
View File

@ -0,0 +1,148 @@
---
toc_priority: 108
---
## exponentialMovingAverage {#exponential-moving-average}
Вычисляет экспоненциальное скользящее среднее за определенный промежуток времени.
**Синтакис:**
```sql
exponentialMovingAverage(x)(value, timestamp)
```
Каждой точке `timestamp` на временном отрезке соответствует определенное значение `value`. Период полураспада — временной интервал `х`, через который вес значений уменьшается в 2 раза. Функция возвращает взвешенное среднее: чем старше временная точка, тем c меньшим весом считается соответствующее ей значение.
**Аргументы**
- `value` — входные значения. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) или [Decimal](../../../sql-reference/data-types/decimal.md).
- `timestamp` — параметр для упорядочивания значений. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) или [Decimal](../../../sql-reference/data-types/decimal.md).
**Параметры**
- `x` — период полураспада. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) или [Decimal](../../../sql-reference/data-types/decimal.md).
**Возвращаемые значения**
- Возвращает [экспоненциальное скользящее среднее](https://ru.wikipedia.org/wiki/Скользящая_средняя#Экспоненциально_взвешенное_скользящее_среднее) за прошедшее время `x` в последний момент времени.
Тип: [Float64](../../../sql-reference/data-types/float.md#float32-float64).
**Пример**
Исходная таблица:
``` text
┌──temperature─┬─timestamp──┐
│ 95 │ 1 │
│ 95 │ 2 │
│ 95 │ 3 │
│ 96 │ 4 │
│ 96 │ 5 │
│ 96 │ 6 │
│ 96 │ 7 │
│ 97 │ 8 │
│ 97 │ 9 │
│ 97 │ 10 │
│ 97 │ 11 │
│ 98 │ 12 │
│ 98 │ 13 │
│ 98 │ 14 │
│ 98 │ 15 │
│ 99 │ 16 │
│ 99 │ 17 │
│ 99 │ 18 │
│ 100 │ 19 │
│ 100 │ 20 │
└──────────────┴────────────┘
```
Запрос:
```sql
SELECT exponentialMovingAverage(5)(temperature, timestamp);
```
Результат:
``` text
┌──exponentialMovingAverage(5)(temperature, timestamp)──┐
│ 92.25779635374204 │
└───────────────────────────────────────────────────────┘
```
Запрос:
```sql
SELECT
value,
time,
round(exp_smooth, 3),
bar(exp_smooth, 0, 1, 50) AS bar
FROM
(
SELECT
(number = 0) OR (number >= 25) AS value,
number AS time,
exponentialMovingAverage(10)(value, time) OVER (Rows BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS exp_smooth
FROM numbers(50)
)
```
Результат:
``` text
┌─value─┬─time─┬─round(exp_smooth, 3)─┬─bar────────────────────────────────────────┐
│ 1 │ 0 │ 0.067 │ ███▎ │
│ 0 │ 1 │ 0.062 │ ███ │
│ 0 │ 2 │ 0.058 │ ██▊ │
│ 0 │ 3 │ 0.054 │ ██▋ │
│ 0 │ 4 │ 0.051 │ ██▌ │
│ 0 │ 5 │ 0.047 │ ██▎ │
│ 0 │ 6 │ 0.044 │ ██▏ │
│ 0 │ 7 │ 0.041 │ ██ │
│ 0 │ 8 │ 0.038 │ █▊ │
│ 0 │ 9 │ 0.036 │ █▋ │
│ 0 │ 10 │ 0.033 │ █▋ │
│ 0 │ 11 │ 0.031 │ █▌ │
│ 0 │ 12 │ 0.029 │ █▍ │
│ 0 │ 13 │ 0.027 │ █▎ │
│ 0 │ 14 │ 0.025 │ █▎ │
│ 0 │ 15 │ 0.024 │ █▏ │
│ 0 │ 16 │ 0.022 │ █ │
│ 0 │ 17 │ 0.021 │ █ │
│ 0 │ 18 │ 0.019 │ ▊ │
│ 0 │ 19 │ 0.018 │ ▊ │
│ 0 │ 20 │ 0.017 │ ▋ │
│ 0 │ 21 │ 0.016 │ ▋ │
│ 0 │ 22 │ 0.015 │ ▋ │
│ 0 │ 23 │ 0.014 │ ▋ │
│ 0 │ 24 │ 0.013 │ ▋ │
│ 1 │ 25 │ 0.079 │ ███▊ │
│ 1 │ 26 │ 0.14 │ ███████ │
│ 1 │ 27 │ 0.198 │ █████████▊ │
│ 1 │ 28 │ 0.252 │ ████████████▌ │
│ 1 │ 29 │ 0.302 │ ███████████████ │
│ 1 │ 30 │ 0.349 │ █████████████████▍ │
│ 1 │ 31 │ 0.392 │ ███████████████████▌ │
│ 1 │ 32 │ 0.433 │ █████████████████████▋ │
│ 1 │ 33 │ 0.471 │ ███████████████████████▌ │
│ 1 │ 34 │ 0.506 │ █████████████████████████▎ │
│ 1 │ 35 │ 0.539 │ ██████████████████████████▊ │
│ 1 │ 36 │ 0.57 │ ████████████████████████████▌ │
│ 1 │ 37 │ 0.599 │ █████████████████████████████▊ │
│ 1 │ 38 │ 0.626 │ ███████████████████████████████▎ │
│ 1 │ 39 │ 0.651 │ ████████████████████████████████▌ │
│ 1 │ 40 │ 0.674 │ █████████████████████████████████▋ │
│ 1 │ 41 │ 0.696 │ ██████████████████████████████████▋ │
│ 1 │ 42 │ 0.716 │ ███████████████████████████████████▋ │
│ 1 │ 43 │ 0.735 │ ████████████████████████████████████▋ │
│ 1 │ 44 │ 0.753 │ █████████████████████████████████████▋ │
│ 1 │ 45 │ 0.77 │ ██████████████████████████████████████▍ │
│ 1 │ 46 │ 0.785 │ ███████████████████████████████████████▎ │
│ 1 │ 47 │ 0.8 │ ███████████████████████████████████████▊ │
│ 1 │ 48 │ 0.813 │ ████████████████████████████████████████▋ │
│ 1 │ 49 │ 0.825 │ █████████████████████████████████████████▎│
└───────┴──────┴──────────────────────┴────────────────────────────────────────────┘
```

2
docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md
View File

@ -748,7 +748,7 @@ SOURCE(REDIS(
!!! info "Примечание"
Поля `column_family` или `where` не могут быть использованы вместе с полем `query`. Также обязательно должен быть один из источников данных: `column_family` или `query`.
### PosgreSQL {#dicts-external_dicts_dict_sources-postgresql}
### PostgreSQL {#dicts-external_dicts_dict_sources-postgresql}
Пример настройки:

60
docs/ru/sql-reference/functions/hash-functions.md
View File

@ -89,9 +89,39 @@ SELECT sipHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00
## sipHash128 {#hash_functions-siphash128}
Вычисляет SipHash от строки.
Принимает аргумент типа String. Возвращает FixedString(16).
Отличается от sipHash64 тем, что финальный xor-folding состояния делается только до 128 бит.
Генерирует 128-битное хеш-значение [SipHash](https://131002.net/siphash/). Отличается от [sipHash64](#hash_functions-siphash64) тем, что финальный xor-folding состояния делается до 128 бит.
**Синтаксис**
``` sql
sipHash128(par1,...)
```
**Аргументы**
Функция принимает переменное число входных параметров. Аргументы могут быть любого [поддерживаемого типа данных](../../sql-reference/functions/hash-functions.md).
**Возвращаемое значение**
128-битное хеш-значение `SipHash`.
Тип: [FixedString(16)](../../sql-reference/data-types/fixedstring.md).
**Пример**
Запрос:
``` sql
SELECT hex(sipHash128('foo', '\x01', 3));
```
Результат:
``` text
┌─hex(sipHash128('foo', '', 3))────┐
│ 9DE516A64A414D4B1B609415E4523F24 │
└──────────────────────────────────┘
```
## cityHash64 {#cityhash64}
@ -459,30 +489,38 @@ SELECT murmurHash3_32(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:
## murmurHash3_128 {#murmurhash3-128}
Генерирует значение [MurmurHash3](https://github.com/aappleby/smhasher).
Генерирует 128-битное хеш-значение [MurmurHash3](https://github.com/aappleby/smhasher).
**Синтаксис**
``` sql
murmurHash3_128( expr )
murmurHash3_128(expr)
```
**Аргументы**
- `expr` — [выражение](../syntax.md#syntax-expressions), возвращающее значение типа [String](../../sql-reference/functions/hash-functions.md).
- `expr` — список [выражений](../../sql-reference/syntax.md#syntax-expressions). [String](../../sql-reference/data-types/string.md).
**Возвращаемое значение**
Хэш-значение типа [FixedString(16)](../../sql-reference/functions/hash-functions.md).
128-битное значение хеш-значение `MurmurHash3`.
Тип: [FixedString(16)](../../sql-reference/data-types/fixedstring.md).
**Пример**
Запрос:
``` sql
SELECT hex(murmurHash3_128('example_string')) AS MurmurHash3, toTypeName(MurmurHash3) AS type;
SELECT hex(murmurHash3_128('foo', 'foo', 'foo'));
```
Результат:
``` text
┌─MurmurHash3──────────────────────┬─type───┐
│ 368A1A311CB7342253354B548E7E7E71 │ String │
└──────────────────────────────────┴────────┘
┌─hex(murmurHash3_128('foo', 'foo', 'foo'))─┐
│ F8F7AD9B6CD4CF117A71E277E2EC2931 │
└───────────────────────────────────────────┘
```
## xxHash32, xxHash64 {#hash-functions-xxhash32-xxhash64}

1
programs/copier/ClusterCopier.cpp
View File

@ -8,6 +8,7 @@
#include <Common/setThreadName.h>
#include <IO/ConnectionTimeoutsContext.h>
#include <Interpreters/InterpreterInsertQuery.h>
#include <Parsers/ASTFunction.h>
#include <Processors/Transforms/ExpressionTransform.h>
#include <QueryPipeline/QueryPipelineBuilder.h>
#include <QueryPipeline/Chain.h>

6
programs/copier/Internals.cpp
View File

@ -1,8 +1,10 @@
#include "Internals.h"
#include <Storages/MergeTree/MergeTreeData.h>
#include <Storages/extractKeyExpressionList.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTIdentifier.h>
#include <Processors/Executors/PullingPipelineExecutor.h>
#include <Processors/Transforms/SquashingChunksTransform.h>
#include <Storages/MergeTree/MergeTreeData.h>
#include <Storages/extractKeyExpressionList.h>
namespace DB
{

1
programs/copier/TaskTableAndShard.h
View File

@ -5,6 +5,7 @@
#include "ClusterPartition.h"
#include <Core/Defines.h>
#include <Parsers/ASTFunction.h>
#include <base/map.h>
#include <boost/algorithm/string/join.hpp>

2
programs/keeper/keeper_config.xml
View File

@ -49,7 +49,7 @@
<!-- Internal port and hostname -->
<hostname>localhost</hostname>
<port>44444</port>
<port>9234</port>
</server>
<!-- Add more servers here -->

2
programs/keeper/keeper_embedded.xml
View File

@ -14,7 +14,7 @@
<server>
<id>1</id>
<hostname>localhost</hostname>
<port>44444</port>
<port>9234</port>
</server>
</raft_configuration>
</keeper_server>

1
src/Access/AccessEntityIO.cpp
View File

@ -7,6 +7,7 @@
#include <Access/SettingsProfile.h>
#include <Access/User.h>
#include <Core/Defines.h>
#include <IO/WriteHelpers.h>
#include <Interpreters/Access/InterpreterCreateQuotaQuery.h>
#include <Interpreters/Access/InterpreterCreateRoleQuery.h>
#include <Interpreters/Access/InterpreterCreateRowPolicyQuery.h>

15
src/AggregateFunctions/AggregateFunctionCount.h
View File

@ -203,6 +203,21 @@ public:
data(place).count += !assert_cast<const ColumnNullable &>(*columns[0]).isNullAt(row_num);
}
void addBatchSinglePlace(
size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena *, ssize_t if_argument_pos) const override
{
auto & nc = assert_cast<const ColumnNullable &>(*columns[0]);
if (if_argument_pos >= 0)
{
const auto & flags = assert_cast<const ColumnUInt8 &>(*columns[if_argument_pos]).getData();
data(place).count += countBytesInFilterWithNull(flags, nc.getNullMapData().data());
}
else
{
data(place).count += batch_size - countBytesInFilter(nc.getNullMapData().data(), batch_size);
}
}
void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena *) const override
{
data(place).count += data(rhs).count;

2
src/AggregateFunctions/parseAggregateFunctionParameters.cpp
View File

@ -1,6 +1,6 @@
#include <AggregateFunctions/parseAggregateFunctionParameters.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ExpressionListParsers.h>
#include <Parsers/parseQuery.h>

4
src/AggregateFunctions/parseAggregateFunctionParameters.h
View File

@ -1,8 +1,8 @@
#pragma once
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTExpressionList.h>
#include <Interpreters/Context_fwd.h>
#include <Parsers/ASTExpressionList.h>
#include <Parsers/IAST_fwd.h>
namespace DB

7
src/Backups/renameInCreateQuery.cpp
View File

@ -1,12 +1,13 @@
#include <Backups/renameInCreateQuery.h>
#include <Backups/BackupRenamingConfig.h>
#include <Backups/renameInCreateQuery.h>
#include <Interpreters/InDepthNodeVisitor.h>
#include <Interpreters/evaluateConstantExpression.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTTablesInSelectQuery.h>
#include <TableFunctions/TableFunctionFactory.h>
#include <Interpreters/InDepthNodeVisitor.h>
#include <Interpreters/evaluateConstantExpression.h>
namespace DB

3
src/CMakeLists.txt
View File

@ -28,7 +28,8 @@ configure_file (Common/config_version.h.in ${CONFIG_VERSION})
configure_file (Core/config_core.h.in "${CMAKE_CURRENT_BINARY_DIR}/Core/include/config_core.h")
if (USE_DEBUG_HELPERS)
set (INCLUDE_DEBUG_HELPERS "-I\"${ClickHouse_SOURCE_DIR}/base\" -include \"${ClickHouse_SOURCE_DIR}/src/Core/iostream_debug_helpers.h\"")
get_target_property(MAGIC_ENUM_INCLUDE_DIR magic_enum INTERFACE_INCLUDE_DIRECTORIES)
set (INCLUDE_DEBUG_HELPERS "-I\"${ClickHouse_SOURCE_DIR}/base\" -I\"${MAGIC_ENUM_INCLUDE_DIR}\" -include \"${ClickHouse_SOURCE_DIR}/src/Core/iostream_debug_helpers.h\"")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${INCLUDE_DEBUG_HELPERS}")
endif ()

1
src/Client/ClientBase.cpp
View File

@ -40,7 +40,6 @@
#include <Parsers/ASTDropQuery.h>
#include <Parsers/ASTSetQuery.h>
#include <Parsers/ASTUseQuery.h>
#include <Parsers/ASTSelectQuery.h>
#include <Parsers/ASTSelectWithUnionQuery.h>
#include <Parsers/ASTQueryWithOutput.h>
#include <Parsers/ASTLiteral.h>

5
src/Common/ThreadStatus.h
View File

@ -165,7 +165,7 @@ protected:
std::function<void()> fatal_error_callback;
/// It is used to avoid enabling the query profiler when you have multiple ThreadStatus in the same thread
bool query_profiled_enabled = true;
bool query_profiler_enabled = true;
/// Requires access to query_id.
friend class MemoryTrackerThreadSwitcher;
@ -207,7 +207,8 @@ public:
void disableProfiling()
{
query_profiled_enabled = false;
assert(!query_profiler_real && !query_profiler_cpu);
query_profiler_enabled = false;
}
/// Starts new query and create new thread group for it, current thread becomes master thread of the query

14
src/Common/Visitor.h
View File

@ -1,7 +1,7 @@
#pragma once
#include <base/demangle.h>
#include <base/Typelist.h>
#include <base/TypeList.h>
#include <Common/Exception.h>
/* Generic utils which are intended for visitor pattern implementation.
@ -64,7 +64,7 @@ template <>
class Visitor<>
{
public:
using List = Typelist<>;
using List = TypeList<>;
protected:
~Visitor() = default;
@ -74,7 +74,7 @@ template <typename Type>
class Visitor<Type> : public Visitor<>
{
public:
using List = Typelist<Type>;
using List = TypeList<Type>;
virtual void visit(Type &) = 0;
@ -86,7 +86,7 @@ template <typename Type, typename ... Types>
class Visitor<Type, Types ...> : public Visitor<Types ...>
{
public:
using List = Typelist<Type, Types ...>;
using List = TypeList<Type, Types ...>;
using Visitor<Types ...>::visit;
virtual void visit(Type &) = 0;
@ -145,10 +145,10 @@ protected:
template <typename Derived, typename VisitorBase>
class VisitorImpl : public
TLChangeRoot<
TypeListChangeRoot<
VisitorImplHelper,
TLConcat<
Typelist<Derived, VisitorBase>,
TypeListConcat<
TypeList<Derived, VisitorBase>,
typename VisitorBase::List
>
>

4
src/Compression/CachedCompressedReadBuffer.cpp
View File

@ -87,7 +87,7 @@ CachedCompressedReadBuffer::CachedCompressedReadBuffer(
void CachedCompressedReadBuffer::seek(size_t offset_in_compressed_file, size_t offset_in_decompressed_block)
{
/// Nothing to do if we already at required position
if (file_pos == offset_in_compressed_file
if (!owned_cell && file_pos == offset_in_compressed_file
&& (offset() == offset_in_decompressed_block ||
nextimpl_working_buffer_offset == offset_in_decompressed_block))
return;
@ -106,6 +106,8 @@ void CachedCompressedReadBuffer::seek(size_t offset_in_compressed_file, size_t o
bytes += offset();
/// No data, everything discarded
pos = working_buffer.end();
owned_cell.reset();
/// Remember required offset in decompressed block which will be set in
/// the next ReadBuffer::next() call
nextimpl_working_buffer_offset = offset_in_decompressed_block;

2
src/Compression/CompressedReadBufferFromFile.h
View File

@ -49,7 +49,7 @@ public:
explicit CompressedReadBufferFromFile(std::unique_ptr<ReadBufferFromFileBase> buf, bool allow_different_codecs_ = false);
/// Seek is lazy in some sense. We move position in compressed file_in to offset_in_compressed_file, but don't
/// read data into working_buffer and don't shit our position to offset_in_decompressed_block. Instead
/// read data into working_buffer and don't shift our position to offset_in_decompressed_block. Instead
/// we store this offset inside nextimpl_working_buffer_offset.
void seek(size_t offset_in_compressed_file, size_t offset_in_decompressed_block);

2
src/Compression/CompressionCodecEncrypted.cpp
View File

@ -7,11 +7,9 @@
#include <Compression/CompressionCodecEncrypted.h>
#include <Poco/Logger.h>
#include <base/logger_useful.h>
#include <Common/ErrorCodes.h>
// This depends on BoringSSL-specific API, notably <openssl/aead.h>.
#if USE_SSL && USE_INTERNAL_SSL_LIBRARY
#include <Parsers/ASTLiteral.h>
#include <openssl/digest.h>
#include <openssl/err.h>
#include <boost/algorithm/hex.hpp>

1
src/Compression/CompressionCodecGorilla.cpp
View File

@ -7,7 +7,6 @@
#include <Compression/CompressionFactory.h>
#include <base/unaligned.h>
#include <Parsers/IAST_fwd.h>
#include <Parsers/ASTIdentifier.h>
#include <IO/WriteHelpers.h>
#include <IO/ReadBufferFromMemory.h>
#include <IO/BitHelpers.h>

1
src/Compression/CompressionCodecNone.cpp
View File

@ -1,7 +1,6 @@
#include <Compression/CompressionCodecNone.h>
#include <Compression/CompressionInfo.h>
#include <Compression/CompressionFactory.h>
#include <Parsers/ASTIdentifier.h>
namespace DB

1
src/Compression/CompressionCodecT64.cpp
View File

@ -5,7 +5,6 @@
#include <base/unaligned.h>
#include <Parsers/IAST.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTFunction.h>
#include <IO/WriteHelpers.h>
#include <Core/Types.h>

2
src/DataTypes/DataTypeAggregateFunction.cpp
View File

@ -15,8 +15,8 @@
#include <AggregateFunctions/AggregateFunctionFactory.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTIdentifier_fwd.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTIdentifier.h>
namespace DB

3
src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp
View File

@ -8,8 +8,9 @@
#include <AggregateFunctions/AggregateFunctionFactory.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTSelectWithUnionQuery.h>
#include <boost/algorithm/string/join.hpp>

6
src/DataTypes/DataTypeLowCardinality.cpp
View File

@ -6,7 +6,7 @@
#include <Common/assert_cast.h>
#include <Common/typeid_cast.h>
#include <Core/Field.h>
#include <base/Typelists.h>
#include <base/TypeLists.h>
#include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeDateTime.h>
#include <DataTypes/DataTypeFactory.h>
@ -53,7 +53,7 @@ namespace
{
}
template <class T>
template <typename T>
void operator()(Id<T>)
{
if (typeid_cast<const DataTypeNumber<T> *>(&keys_type))
@ -89,7 +89,7 @@ MutableColumnUniquePtr DataTypeLowCardinality::createColumnUniqueImpl(const IDat
else if (which.isInt() || which.isUInt() || which.isFloat())
{
MutableColumnUniquePtr column;
TLUtils::forEach(TLIntegral{}, CreateColumnVector(column, *type, creator));
TypeListUtils::forEach(TypeListIntAndFloat{}, CreateColumnVector(column, *type, creator));
if (!column)
throw Exception("Unexpected numeric type: " + type->getName(), ErrorCodes::LOGICAL_ERROR);

1
src/DataTypes/convertMySQLDataType.cpp
View File

@ -5,7 +5,6 @@
#include <Core/MultiEnum.h>
#include <Core/SettingsEnums.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/IAST.h>
#include "DataTypeDate.h"
#include "DataTypeDateTime.h"

9
src/Databases/DDLDependencyVisitor.cpp
View File

@ -1,10 +1,11 @@
#include <Databases/DDLDependencyVisitor.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTIdentifier.h>
#include <Dictionaries/getDictionaryConfigurationFromAST.h>
#include <Interpreters/Context.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTSelectWithUnionQuery.h>
#include <Poco/String.h>
namespace DB

2
src/Databases/DatabaseFactory.cpp
View File

@ -11,9 +11,7 @@
#include <Interpreters/evaluateConstantExpression.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/formatAST.h>
#include <Parsers/queryToString.h>
#include <Storages/ExternalDataSourceConfiguration.h>
#include <Common/Macros.h>

1
src/Databases/DatabaseMemory.cpp
View File

@ -5,6 +5,7 @@
#include <Databases/DDLDependencyVisitor.h>
#include <Interpreters/Context.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ASTFunction.h>
#include <Storages/IStorage.h>
#include <filesystem>

1
src/Databases/DatabaseOnDisk.cpp
View File

@ -7,6 +7,7 @@
#include <Interpreters/Context.h>
#include <Interpreters/InterpreterCreateQuery.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ParserCreateQuery.h>
#include <Parsers/formatAST.h>
#include <Parsers/parseQuery.h>

1
src/Databases/DatabaseReplicated.cpp
View File

@ -18,6 +18,7 @@
#include <base/getFQDNOrHostName.h>
#include <Parsers/ASTAlterQuery.h>
#include <Parsers/ASTDropQuery.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ParserCreateQuery.h>
#include <Parsers/parseQuery.h>
#include <Interpreters/InterpreterCreateQuery.h>

1
src/Databases/DatabasesCommon.cpp
View File

@ -2,6 +2,7 @@
#include <Interpreters/InterpreterCreateQuery.h>
#include <Interpreters/Context.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ASTSelectWithUnionQuery.h>
#include <Parsers/ParserCreateQuery.h>
#include <Parsers/formatAST.h>
#include <Storages/StorageDictionary.h>

1
src/Databases/DatabasesCommon.h
View File

@ -1,7 +1,6 @@
#pragma once
#include <base/types.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/IAST.h>
#include <Storages/IStorage_fwd.h>
#include <Databases/IDatabase.h>

2
src/Databases/PostgreSQL/DatabasePostgreSQL.cpp
View File

@ -1,5 +1,7 @@
#include <Databases/PostgreSQL/DatabasePostgreSQL.h>
#include <Parsers/ASTIdentifier.h>
#if USE_LIBPQXX
#include <DataTypes/DataTypeNullable.h>

1
src/Databases/SQLite/DatabaseSQLite.cpp
View File

@ -8,6 +8,7 @@
#include <Databases/SQLite/fetchSQLiteTableStructure.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ASTColumnDeclaration.h>
#include <Parsers/ASTFunction.h>
#include <Interpreters/Context.h>
#include <Storages/StorageSQLite.h>
#include <Databases/SQLite/SQLiteUtils.h>

1
src/Dictionaries/getDictionaryConfigurationFromAST.cpp
View File

@ -9,6 +9,7 @@
#include <Parsers/queryToString.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTLiteral.h>
#include <Core/Names.h>
#include <Common/FieldVisitorToString.h>
#include <Parsers/ASTFunctionWithKeyValueArguments.h>

2
src/Functions/EmptyImpl.h
View File

@ -58,7 +58,7 @@ struct EmptyImpl
static void uuid(const ColumnUUID::Container & container, size_t n, PaddedPODArray<UInt8> & res)
{
for (size_t i = 0; i < n; ++i)
res[i] = negative ^ (container.data()->toUnderType() == 0);
res[i] = negative ^ (container[i].toUnderType() == 0);
}
};

8
src/Functions/FunctionBinaryArithmetic.h
View File

@ -36,7 +36,7 @@
#include <Common/typeid_cast.h>
#include <Common/assert_cast.h>
#include <Common/FieldVisitorsAccurateComparison.h>
#include <base/Typelist.h>
#include <base/TypeList.h>
#include <base/map.h>
#include <Common/config.h>
@ -625,17 +625,17 @@ class FunctionBinaryArithmetic : public IFunction
static bool castType(const IDataType * type, auto && f)
{
using Types = Typelist<
using Types = TypeList<
DataTypeUInt8, DataTypeUInt16, DataTypeUInt32, DataTypeUInt64, DataTypeUInt128, DataTypeUInt256,
DataTypeInt8, DataTypeInt16, DataTypeInt32, DataTypeInt64, DataTypeInt128, DataTypeInt256,
DataTypeDecimal32, DataTypeDecimal64, DataTypeDecimal128, DataTypeDecimal256,
DataTypeDate, DataTypeDateTime,
DataTypeFixedString, DataTypeString>;
using Floats = Typelist<DataTypeFloat32, DataTypeFloat64>;
using Floats = TypeList<DataTypeFloat32, DataTypeFloat64>;
using ValidTypes = std::conditional_t<valid_on_float_arguments,
TLConcat<Types, Floats>,
TypeListConcat<Types, Floats>,
Types>;
return castTypeToEither(ValidTypes{}, type, std::forward<decltype(f)>(f));

78
src/Functions/FunctionsAES.h
View File

@ -279,37 +279,33 @@ private:
// That may lead later to reading unallocated data from underlying PaddedPODArray
// due to assumption that it is safe to read up to 15 bytes past end.
const auto pad_to_next_block = block_size == 1 ? 0 : 1;
for (size_t r = 0; r < input_rows_count; ++r)
for (size_t row_idx = 0; row_idx < input_rows_count; ++row_idx)
{
resulting_size += (input_column->getDataAt(r).size / block_size + pad_to_next_block) * block_size + 1;
resulting_size += (input_column->getDataAt(row_idx).size / block_size + pad_to_next_block) * block_size + 1;
if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM)
resulting_size += tag_size;
}
#if defined(MEMORY_SANITIZER)
encrypted_result_column_data.resize_fill(resulting_size, 0xFF);
#else
encrypted_result_column_data.resize(resulting_size);
#endif
}
auto * encrypted = encrypted_result_column_data.data();
KeyHolder<mode> key_holder;
for (size_t r = 0; r < input_rows_count; ++r)
for (size_t row_idx = 0; row_idx < input_rows_count; ++row_idx)
{
const auto key_value = key_holder.setKey(key_size, key_column->getDataAt(r));
const auto key_value = key_holder.setKey(key_size, key_column->getDataAt(row_idx));
auto iv_value = StringRef{};
if (iv_column)
{
iv_value = iv_column->getDataAt(r);
iv_value = iv_column->getDataAt(row_idx);
/// If the length is zero (empty string is passed) it should be treat as no IV.
if (iv_value.size == 0)
iv_value.data = nullptr;
}
const StringRef input_value = input_column->getDataAt(r);
const StringRef input_value = input_column->getDataAt(row_idx);
if constexpr (mode != CipherMode::MySQLCompatibility)
{
@ -348,7 +344,7 @@ private:
// 1.a.2 Set AAD
if (aad_column)
{
const auto aad_data = aad_column->getDataAt(r);
const auto aad_data = aad_column->getDataAt(row_idx);
int tmp_len = 0;
if (aad_data.size != 0 && EVP_EncryptUpdate(evp_ctx, nullptr, &tmp_len,
reinterpret_cast<const unsigned char *>(aad_data.data), aad_data.size) != 1)
@ -408,7 +404,7 @@ private:
};
/// AES_decrypt(string, key, block_mode[, init_vector])
/// decrypt(string, key, block_mode[, init_vector])
template <typename Impl>
class FunctionDecrypt : public IFunction
{
@ -471,7 +467,9 @@ private:
ColumnPtr result_column;
if (arguments.size() <= 3)
{
result_column = doDecrypt(evp_cipher, input_rows_count, input_column, key_column, nullptr, nullptr);
}
else
{
const auto iv_column = arguments[3].column;
@ -548,59 +546,58 @@ private:
{
size_t resulting_size = 0;
for (size_t r = 0; r < input_rows_count; ++r)
for (size_t row_idx = 0; row_idx < input_rows_count; ++row_idx)
{
size_t string_size = input_column->getDataAt(r).size;
size_t string_size = input_column->getDataAt(row_idx).size;
resulting_size += string_size + 1; /// With terminating zero.
if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM)
{
if (string_size < tag_size)
throw Exception("Encrypted data is smaller than the size of additional data for AEAD mode, cannot decrypt.",
ErrorCodes::BAD_ARGUMENTS);
if (string_size > 0)
{
if (string_size < tag_size)
throw Exception("Encrypted data is smaller than the size of additional data for AEAD mode, cannot decrypt.",
ErrorCodes::BAD_ARGUMENTS);
resulting_size -= tag_size;
resulting_size -= tag_size;
}
}
}
#if defined(MEMORY_SANITIZER)
// Pre-fill result column with values to prevent MSAN from dropping dead on
// aes-X-ecb mode with "WARNING: MemorySanitizer: use-of-uninitialized-value".
// This is most likely to be caused by the underlying assembler implementation:
// see crypto/aes/aesni-x86_64.s, function aesni_ecb_encrypt
// which msan seems to fail instrument correctly.
decrypted_result_column_data.resize_fill(resulting_size, 0xFF);
#else
decrypted_result_column_data.resize(resulting_size);
#endif
}
auto * decrypted = decrypted_result_column_data.data();
KeyHolder<mode> key_holder;
for (size_t r = 0; r < input_rows_count; ++r)
for (size_t row_idx = 0; row_idx < input_rows_count; ++row_idx)
{
// 0: prepare key if required
auto key_value = key_holder.setKey(key_size, key_column->getDataAt(r));
auto key_value = key_holder.setKey(key_size, key_column->getDataAt(row_idx));
auto iv_value = StringRef{};
if (iv_column)
{
iv_value = iv_column->getDataAt(r);
iv_value = iv_column->getDataAt(row_idx);
/// If the length is zero (empty string is passed) it should be treat as no IV.
if (iv_value.size == 0)
iv_value.data = nullptr;
}
auto input_value = input_column->getDataAt(r);
auto input_value = input_column->getDataAt(row_idx);
if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM)
{
// empty plaintext results in empty ciphertext + tag, means there should be at least tag_size bytes.
if (input_value.size < tag_size)
throw Exception(fmt::format("Encrypted data is too short: only {} bytes, "
"should contain at least {} bytes of a tag.",
input_value.size, block_size, tag_size), ErrorCodes::BAD_ARGUMENTS);
input_value.size -= tag_size;
if (input_value.size > 0)
{
// empty plaintext results in empty ciphertext + tag, means there should be at least tag_size bytes.
if (input_value.size < tag_size)
throw Exception(fmt::format("Encrypted data is too short: only {} bytes, "
"should contain at least {} bytes of a tag.",
input_value.size, block_size, tag_size), ErrorCodes::BAD_ARGUMENTS);
input_value.size -= tag_size;
}
}
if constexpr (mode != CipherMode::MySQLCompatibility)
@ -619,8 +616,9 @@ private:
}
}
// Avoid extra work on empty ciphertext/plaintext for some ciphers
if (!(input_value.size == 0 && block_size == 1 && mode != CipherMode::RFC5116_AEAD_AES_GCM))
/// Avoid extra work on empty ciphertext/plaintext. Always decrypt empty to empty.
/// This makes sense for default implementation for NULLs.
if (input_value.size > 0)
{
// 1: Init CTX
if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM)
@ -641,7 +639,7 @@ private:
// 1.a.2: Set AAD if present
if (aad_column)
{
StringRef aad_data = aad_column->getDataAt(r);
StringRef aad_data = aad_column->getDataAt(row_idx);
int tmp_len = 0;
if (aad_data.size != 0 && EVP_DecryptUpdate(evp_ctx, nullptr, &tmp_len,
reinterpret_cast<const unsigned char *>(aad_data.data), aad_data.size) != 1)

12
src/Functions/GatherUtils/ArraySinkVisitor.h
View File

@ -1,6 +1,6 @@
#pragma once
#include <Common/Visitor.h>
#include <base/Typelists.h>
#include <base/TypeLists.h>
namespace DB::GatherUtils
{
@ -14,12 +14,12 @@ struct GenericArraySink;
template <typename ArraySink>
struct NullableArraySink;
using NumericArraySinks = TLMap<NumericArraySink, TLNumbersWithUUID>;
using BasicArraySinks = TLAppend<GenericArraySink, NumericArraySinks>;
using NullableArraySinks = TLMap<NullableArraySink, BasicArraySinks>;
using TLArraySinks = TLConcat<BasicArraySinks, NullableArraySinks>;
using NumericArraySinks = TypeListMap<NumericArraySink, TypeListNumberWithUUID>;
using BasicArraySinks = TypeListAppend<GenericArraySink, NumericArraySinks>;
using NullableArraySinks = TypeListMap<NullableArraySink, BasicArraySinks>;
using TLArraySinks = TypeListConcat<BasicArraySinks, NullableArraySinks>;
class ArraySinkVisitor : public TLChangeRoot<Visitor, TLArraySinks>
class ArraySinkVisitor : public TypeListChangeRoot<Visitor, TLArraySinks>
{
protected:
~ArraySinkVisitor() = default;

8
src/Functions/GatherUtils/ArraySourceVisitor.h
View File

@ -1,6 +1,6 @@
#pragma once
#include <Common/Visitor.h>
#include <base/Typelists.h>
#include <base/TypeLists.h>
namespace DB::GatherUtils
{
@ -17,10 +17,10 @@ struct NullableArraySource;
template <typename Base>
struct ConstSource;
using NumericArraySources = TLMap<NumericArraySource, TLNumbersWithUUID>;
using BasicArraySources = TLAppend<GenericArraySource, NumericArraySources>;
using NumericArraySources = TypeListMap<NumericArraySource, TypeListNumberWithUUID>;
using BasicArraySources = TypeListAppend<GenericArraySource, NumericArraySources>;
class ArraySourceVisitor : public TLChangeRoot<Visitor, BasicArraySources>
class ArraySourceVisitor : public TypeListChangeRoot<Visitor, BasicArraySources>
{
protected:
~ArraySourceVisitor() = default;

16
src/Functions/GatherUtils/ValueSourceVisitor.h
View File

@ -1,6 +1,6 @@
#pragma once
#include <Common/Visitor.h>
#include <base/Typelists.h>
#include <base/TypeLists.h>
namespace DB::GatherUtils
{
@ -17,14 +17,14 @@ struct NullableValueSource;
template <typename Base>
struct ConstSource;
using NumericValueSources = TLMap<NumericValueSource, TLNumbersWithUUID>;
using BasicValueSources = TLAppend<GenericValueSource, NumericValueSources>;
using NullableValueSources = TLMap<NullableValueSource, BasicValueSources>;
using BasicAndNullableValueSources = TLConcat<BasicValueSources, NullableValueSources>;
using ConstValueSources = TLMap<ConstSource, BasicAndNullableValueSources>;
using TypeListValueSources = TLConcat<BasicAndNullableValueSources, ConstValueSources>;
using NumericValueSources = TypeListMap<NumericValueSource, TypeListNumberWithUUID>;
using BasicValueSources = TypeListAppend<GenericValueSource, NumericValueSources>;
using NullableValueSources = TypeListMap<NullableValueSource, BasicValueSources>;
using BasicAndNullableValueSources = TypeListConcat<BasicValueSources, NullableValueSources>;
using ConstValueSources = TypeListMap<ConstSource, BasicAndNullableValueSources>;
using TypeListValueSources = TypeListConcat<BasicAndNullableValueSources, ConstValueSources>;
class ValueSourceVisitor : public TLChangeRoot<Visitor, TypeListValueSources>
class ValueSourceVisitor : public TypeListChangeRoot<Visitor, TypeListValueSources>
{
protected:
~ValueSourceVisitor() = default;

4
src/Functions/GatherUtils/createArraySink.cpp
View File

@ -1,7 +1,7 @@
#include "GatherUtils.h"
#include "Sinks.h"
#include "Sources.h"
#include <base/Typelists.h>
#include <base/TypeLists.h>
namespace DB::GatherUtils
{
@ -55,7 +55,7 @@ struct ArraySinkCreator<>
std::unique_ptr<IArraySink> createArraySink(ColumnArray & col, size_t column_size)
{
using Creator = TLChangeRoot<ArraySinkCreator, TLNumbersWithUUID>;
using Creator = TypeListChangeRoot<ArraySinkCreator, TypeListNumberWithUUID>;
return Creator::create(col.getData(), col.getOffsets(), column_size);
}
}

4
src/Functions/GatherUtils/createArraySource.cpp
View File

@ -1,7 +1,7 @@
#include "GatherUtils.h"
#include "Sinks.h"
#include "Sources.h"
#include <base/Typelists.h>
#include <base/TypeLists.h>
namespace DB::GatherUtils
{
@ -58,7 +58,7 @@ struct ArraySourceCreator<>
std::unique_ptr<IArraySource> createArraySource(const ColumnArray & col, bool is_const, size_t total_rows)
{
using Creator = TLChangeRoot<ArraySourceCreator, TLNumbersWithUUID>;
using Creator = TypeListChangeRoot<ArraySourceCreator, TypeListNumberWithUUID>;
if (const auto * column_nullable = typeid_cast<const ColumnNullable *>(&col.getData()))
{
auto column = ColumnArray::create(column_nullable->getNestedColumnPtr(), col.getOffsetsPtr());

4
src/Functions/GatherUtils/createValueSource.cpp
View File

@ -1,7 +1,7 @@
#include "GatherUtils.h"
#include "Sinks.h"
#include "Sources.h"
#include <base/Typelists.h>
#include <base/TypeLists.h>
namespace DB::GatherUtils
{
@ -58,7 +58,7 @@ struct ValueSourceCreator<>
std::unique_ptr<IValueSource> createValueSource(const IColumn & col, bool is_const, size_t total_rows)
{
using Creator = TLChangeRoot<ValueSourceCreator, TLNumbersWithUUID>;
using Creator = TypeListChangeRoot<ValueSourceCreator, TypeListNumberWithUUID>;
if (const auto * column_nullable = typeid_cast<const ColumnNullable *>(&col))
{
return Creator::create(column_nullable->getNestedColumn(), &column_nullable->getNullMapData(), is_const, total_rows);

2
src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp
View File

@ -1,7 +1,7 @@
#include <Functions/JSONPath/ASTs/ASTJSONPathMemberAccess.h>
#include <Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTIdentifier_fwd.h>
#include <Parsers/ExpressionElementParsers.h>
#include <Parsers/Lexer.h>

6
src/Functions/PolygonUtils.h
View File

@ -2,7 +2,7 @@
#include <base/types.h>
#include <Core/Defines.h>
#include <base/Typelists.h>
#include <base/TypeLists.h>
#include <Columns/IColumn.h>
#include <Columns/ColumnVector.h>
#include <Common/typeid_cast.h>
@ -604,7 +604,7 @@ struct CallPointInPolygon<Type, Types ...>
template <typename PointInPolygonImpl>
static ColumnPtr call(const IColumn & x, const IColumn & y, PointInPolygonImpl && impl)
{
using Impl = TLChangeRoot<CallPointInPolygon, TLIntegral>;
using Impl = TypeListChangeRoot<CallPointInPolygon, TypeListIntAndFloat>;
if (auto column = typeid_cast<const ColumnVector<Type> *>(&x))
return Impl::template call<Type>(*column, y, impl);
return CallPointInPolygon<Types ...>::call(x, y, impl);
@ -630,7 +630,7 @@ struct CallPointInPolygon<>
template <typename PointInPolygonImpl>
NO_INLINE ColumnPtr pointInPolygon(const IColumn & x, const IColumn & y, PointInPolygonImpl && impl)
{
using Impl = TLChangeRoot<CallPointInPolygon, TLIntegral>;
using Impl = TypeListChangeRoot<CallPointInPolygon, TypeListIntAndFloat>;
return Impl::call(x, y, impl);
}

6
src/Functions/array/arrayIntersect.cpp
View File

@ -20,7 +20,7 @@
#include <Columns/ColumnTuple.h>
#include <Common/HashTable/ClearableHashMap.h>
#include <Common/assert_cast.h>
#include <base/Typelists.h>
#include <base/TypeLists.h>
#include <Interpreters/castColumn.h>
#include <base/range.h>
@ -403,8 +403,8 @@ ColumnPtr FunctionArrayIntersect::executeImpl(const ColumnsWithTypeAndName & arg
ColumnPtr result_column;
auto not_nullable_nested_return_type = removeNullable(nested_return_type);
TLUtils::forEach(TLIntegral{}, NumberExecutor(arrays, not_nullable_nested_return_type, result_column));
TLUtils::forEach(TLDecimals{}, DecimalExecutor(arrays, not_nullable_nested_return_type, result_column));
TypeListUtils::forEach(TypeListIntAndFloat{}, NumberExecutor(arrays, not_nullable_nested_return_type, result_column));
TypeListUtils::forEach(TypeListDecimal{}, DecimalExecutor(arrays, not_nullable_nested_return_type, result_column));
using DateMap = ClearableHashMapWithStackMemory<DataTypeDate::FieldType,
size_t, DefaultHash<DataTypeDate::FieldType>, INITIAL_SIZE_DEGREE>;

4
src/Functions/castTypeToEither.h
View File

@ -1,7 +1,7 @@
#pragma once
#include <Common/typeid_cast.h>
#include <base/Typelist.h>
#include <base/TypeList.h>
namespace DB
{
@ -13,7 +13,7 @@ static bool castTypeToEither(const T * type, F && f)
}
template <class ...Args>
constexpr bool castTypeToEither(Typelist<Args...>, const auto * type, auto && f)
constexpr bool castTypeToEither(TypeList<Args...>, const auto * type, auto && f)
{
return (
(typeid_cast<const Args *>(type) != nullptr

2
src/Interpreters/AggregateFunctionOfGroupByKeysVisitor.h
View File

@ -4,8 +4,6 @@
#include <IO/WriteHelpers.h>
#include <Interpreters/InDepthNodeVisitor.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTSelectQuery.h>
#include <Parsers/ASTSetQuery.h>
#include <Parsers/ASTTablesInSelectQuery.h>
#include <Parsers/IAST.h>

2
src/Interpreters/ApplyWithSubqueryVisitor.cpp
View File

@ -3,7 +3,9 @@
#include <Interpreters/StorageID.h>
#include <Interpreters/misc.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTSelectQuery.h>
#include <Parsers/ASTSelectWithUnionQuery.h>
#include <Parsers/ASTSubquery.h>
#include <Parsers/ASTTablesInSelectQuery.h>
#include <Parsers/ASTWithElement.h>

1
src/Interpreters/CollectJoinOnKeysVisitor.cpp
View File

@ -1,3 +1,4 @@
#include <Parsers/ASTIdentifier.h>
#include <Parsers/queryToString.h>
#include <Interpreters/CollectJoinOnKeysVisitor.h>

Some files were not shown because too many files have changed in this diff Show More