diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 803382ab467..d0a6189313d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -152,7 +152,7 @@ jobs: sudo rm -fr $TEMP_PATH mkdir -p $TEMP_PATH cp -r $GITHUB_WORKSPACE $TEMP_PATH - cd $REPO_COPY/tests/ci && python3 compatibility_check.py 0 + cd $REPO_COPY/tests/ci && python3 compatibility_check.py - name: Cleanup if: always() run: | @@ -518,6 +518,7 @@ jobs: - BuilderDebDebug - BuilderDebSplitted runs-on: [self-hosted, style-checker] + if: always() steps: - name: Download json reports uses: actions/download-artifact@v2 @@ -604,7 +605,9 @@ jobs: sudo rm -fr $TEMP_PATH FunctionalStatelessTestTsan: needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] + # tests can consume more than 60GB of memory, + # so use bigger server + runs-on: [self-hosted, stress-tester] steps: - name: Download json reports uses: actions/download-artifact@v2 @@ -1297,6 +1300,34 @@ jobs: docker kill $(docker ps -q) ||: docker rm -f $(docker ps -a -q) ||: sudo rm -fr $TEMP_PATH + IntegrationTestsFlakyCheck: + needs: [BuilderDebAsan] + runs-on: [self-hosted, stress-tester] + steps: + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{runner.temp}}/reports_dir + - name: Check out repository code + uses: actions/checkout@v2 + - name: Integration test + env: + TEMP_PATH: ${{runner.temp}}/integration_tests_asan_flaky_check + REPORTS_PATH: ${{runner.temp}}/reports_dir + CHECK_NAME: 'Integration tests flaky check (asan, actions)' + REPO_COPY: ${{runner.temp}}/integration_tests_asan_flaky_check/ClickHouse + run: | + sudo rm -fr $TEMP_PATH + mkdir -p $TEMP_PATH + cp -r $GITHUB_WORKSPACE $TEMP_PATH + cd $REPO_COPY/tests/ci + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker kill $(docker ps -q) ||: + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr $TEMP_PATH ############################################################################################# #################################### UNIT TESTS ############################################# ############################################################################################# @@ -1481,6 +1512,7 @@ jobs: - UnitTestsReleaseClang - SplitBuildSmokeTest - CompatibilityCheck + - IntegrationTestsFlakyCheck runs-on: [self-hosted, style-checker] steps: - name: Check out repository code diff --git a/CMakeLists.txt b/CMakeLists.txt index 1192c747347..b855213b365 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -169,9 +169,7 @@ endif () include (cmake/check_flags.cmake) include (cmake/add_warning.cmake) -if (NOT MSVC) - set (COMMON_WARNING_FLAGS "${COMMON_WARNING_FLAGS} -Wall") # -Werror and many more is also added inside cmake/warnings.cmake -endif () +set (COMMON_WARNING_FLAGS "${COMMON_WARNING_FLAGS} -Wall") # -Werror and many more is also added inside cmake/warnings.cmake if (COMPILER_CLANG) # clang: warning: argument unused during compilation: '-specs=/usr/share/dpkg/no-pie-compile.specs' [-Wunused-command-line-argument] @@ -312,6 +310,10 @@ include(cmake/cpu_features.cmake) # Enable it explicitly. set (COMPILER_FLAGS "${COMPILER_FLAGS} -fasynchronous-unwind-tables") +# Reproducible builds +set (COMPILER_FLAGS "${COMPILER_FLAGS} -ffile-prefix-map=${CMAKE_SOURCE_DIR}=.") +set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} -ffile-prefix-map=${CMAKE_SOURCE_DIR}=.") + if (${CMAKE_VERSION} VERSION_LESS "3.12.4") # CMake < 3.12 doesn't support setting 20 as a C++ standard version. # We will add C++ standard controlling flag in CMAKE_CXX_FLAGS manually for now. diff --git a/PreLoad.cmake b/PreLoad.cmake index 0a25a55e7bf..9fba896d72e 100644 --- a/PreLoad.cmake +++ b/PreLoad.cmake @@ -18,3 +18,26 @@ if (NOT DEFINED ENV{CLION_IDE} AND NOT DEFINED ENV{XCODE_IDE}) set(CMAKE_GENERATOR "Ninja" CACHE INTERNAL "" FORCE) endif () endif() + + +# Default toolchain - this is needed to avoid dependency on OS files. +execute_process(COMMAND uname -s OUTPUT_VARIABLE OS) +execute_process(COMMAND uname -m OUTPUT_VARIABLE ARCH) + +if (OS MATCHES "Linux" + AND NOT DEFINED CMAKE_TOOLCHAIN_FILE + AND NOT DISABLE_HERMETIC_BUILD + AND ($ENV{CC} MATCHES ".*clang.*" OR CMAKE_C_COMPILER MATCHES ".*clang.*") + AND (USE_STATIC_LIBRARIES OR NOT DEFINED USE_STATIC_LIBRARIES)) + + if (ARCH MATCHES "amd64|x86_64") + set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-x86_64.cmake" CACHE INTERNAL "" FORCE) + elseif (ARCH MATCHES "^(aarch64.*|AARCH64.*|arm64.*|ARM64.*)") + set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-aarch64.cmake" CACHE INTERNAL "" FORCE) + elseif (ARCH MATCHES "^(ppc64le.*|PPC64LE.*)") + set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-ppc64le.cmake" CACHE INTERNAL "" FORCE) + else () + message (FATAL_ERROR "Unsupported architecture: ${ARCH}") + endif () + +endif() diff --git a/SECURITY.md b/SECURITY.md index 6b82c97e7d9..1872d67a529 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -19,9 +19,11 @@ The following versions of ClickHouse server are currently being supported with s | 21.4 | :x: | | 21.5 | :x: | | 21.6 | :x: | -| 21.7 | ✅ | +| 21.7 | :x: | | 21.8 | ✅ | | 21.9 | ✅ | +| 21.10 | ✅ | +| 21.11 | ✅ | ## Reporting a Vulnerability diff --git a/base/base/CachedFn.h b/base/base/CachedFn.h index ed1dbdbd48c..19b2a8ce2c0 100644 --- a/base/base/CachedFn.h +++ b/base/base/CachedFn.h @@ -18,8 +18,8 @@ struct CachedFn { private: using Traits = FnTraits; - using DecayedArgs = TLMap; - using Key = TLChangeRoot; + using DecayedArgs = TypeListMap; + using Key = TypeListChangeRoot; using Result = typename Traits::Ret; std::map cache; // Can't use hashmap as tuples are unhashable by default diff --git a/base/base/FnTraits.h b/base/base/FnTraits.h index 6b8dc7cd0d2..78e1fa62ebb 100644 --- a/base/base/FnTraits.h +++ b/base/base/FnTraits.h @@ -1,6 +1,6 @@ #pragma once -#include "Typelist.h" +#include "TypeList.h" namespace detail { @@ -14,7 +14,7 @@ struct FnTraits static constexpr bool value = std::is_invocable_r_v; using Ret = R; - using Args = Typelist; + using Args = TypeList; }; template diff --git a/base/base/TypeList.h b/base/base/TypeList.h new file mode 100644 index 00000000000..244403b1c6b --- /dev/null +++ b/base/base/TypeList.h @@ -0,0 +1,44 @@ +#pragma once + +#include +#include +#include +#include "defines.h" +#include "TypePair.h" + +/// General-purpose typelist. Easy on compilation times as it does not use recursion. +template +struct TypeList { static constexpr size_t size = sizeof...(Args); }; + +namespace TypeListUtils /// In some contexts it's more handy to use functions instead of aliases +{ + template + constexpr TypeList concat(TypeList, TypeList) { return {}; } + + template + constexpr TypeList prepend(TypeList) { return {}; } + + template + constexpr TypeList append(TypeList) { return {}; } + + template