diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 5816a58081d..73cffc1d5d3 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -360,6 +360,52 @@ jobs: docker kill "$(docker ps -q)" ||: docker rm -f "$(docker ps -a -q)" ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderBinGCC: + needs: [DockerHubPush] + if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }} + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + CHECK_NAME=ClickHouse build check (actions) + BUILD_NAME=binary_gcc + EOF + - name: Download changed images + uses: actions/download-artifact@v2 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + with: + submodules: 'true' + fetch-depth: 0 # otherwise we will have no info about contributors + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$CHECK_NAME" "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v2 + with: + name: ${{ env.BUILD_NAME }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json + - name: Cleanup + if: always() + run: | + docker kill "$(docker ps -q)" ||: + docker rm -f "$(docker ps -a -q)" ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebAsan: needs: [DockerHubPush] runs-on: [self-hosted, builder] @@ -918,6 +964,7 @@ jobs: - BuilderDebRelease - BuilderDebAarch64 - BuilderBinRelease + - BuilderBinGCC - BuilderDebAsan - BuilderDebTsan - BuilderDebUBsan @@ -2608,6 +2655,40 @@ jobs: docker kill "$(docker ps -q)" ||: docker rm -f "$(docker ps -a -q)" ||: sudo rm -fr "$TEMP_PATH" + UnitTestsReleaseGCC: + needs: [BuilderBinGCC] + runs-on: [self-hosted, fuzzer-unit-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/unit_tests_asan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Unit tests (release-gcc, actions) + REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Unit test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 unit_tests_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker kill "$(docker ps -q)" ||: + docker rm -f "$(docker ps -a -q)" ||: + sudo rm -fr "$TEMP_PATH" UnitTestsTsan: needs: [BuilderDebTsan] runs-on: [self-hosted, fuzzer-unit-tester] diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index d50a2151f2f..87a31b9683c 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -370,6 +370,48 @@ jobs: docker kill "$(docker ps -q)" ||: docker rm -f "$(docker ps -a -q)" ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderBinGCC: + needs: [DockerHubPush, FastTest] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + CHECK_NAME=ClickHouse build check (actions) + BUILD_NAME=binary_gcc + EOF + - name: Download changed images + uses: actions/download-artifact@v2 + with: + name: changed_images + path: ${{ runner.temp }}/images_path + - name: Check out repository code + uses: actions/checkout@v2 + with: + submodules: 'true' + fetch-depth: 0 # otherwise we will have no info about contributors + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$CHECK_NAME" "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v2 + with: + name: ${{ env.BUILD_NAME }} + path: ${{ runner.temp }}/build_check/${{ env.BUILD_NAME }}.json + - name: Cleanup + if: always() + run: | + docker kill "$(docker ps -q)" ||: + docker rm -f "$(docker ps -a -q)" ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebAarch64: needs: [DockerHubPush, FastTest] runs-on: [self-hosted, builder] @@ -963,6 +1005,7 @@ jobs: - BuilderDebRelease - BuilderDebAarch64 - BuilderBinRelease + - BuilderBinGCC - BuilderDebAsan - BuilderDebTsan - BuilderDebUBsan @@ -2808,6 +2851,40 @@ jobs: docker kill "$(docker ps -q)" ||: docker rm -f "$(docker ps -a -q)" ||: sudo rm -fr "$TEMP_PATH" + UnitTestsReleaseGCC: + needs: [BuilderBinGCC] + runs-on: [self-hosted, fuzzer-unit-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/unit_tests_asan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Unit tests (release-gcc, actions) + REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Unit test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 unit_tests_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker kill "$(docker ps -q)" ||: + docker rm -f "$(docker ps -a -q)" ||: + sudo rm -fr "$TEMP_PATH" UnitTestsTsan: needs: [BuilderDebTsan] runs-on: [self-hosted, fuzzer-unit-tester] diff --git a/contrib/avro-cmake/CMakeLists.txt b/contrib/avro-cmake/CMakeLists.txt index d91ce40dd54..c5bda41782d 100644 --- a/contrib/avro-cmake/CMakeLists.txt +++ b/contrib/avro-cmake/CMakeLists.txt @@ -69,9 +69,10 @@ endif () target_compile_options(_avrocpp PRIVATE ${SUPPRESS_WARNINGS}) # create a symlink to include headers with +set(AVRO_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/include") ADD_CUSTOM_TARGET(avro_symlink_headers ALL - COMMAND ${CMAKE_COMMAND} -E make_directory "${AVROCPP_ROOT_DIR}/include" - COMMAND ${CMAKE_COMMAND} -E create_symlink "${AVROCPP_ROOT_DIR}/api" "${AVROCPP_ROOT_DIR}/include/avro" + COMMAND ${CMAKE_COMMAND} -E make_directory "${AVRO_INCLUDE_DIR}" + COMMAND ${CMAKE_COMMAND} -E create_symlink "${AVROCPP_ROOT_DIR}/api" "${AVRO_INCLUDE_DIR}/avro" ) add_dependencies(_avrocpp avro_symlink_headers) -target_include_directories(_avrocpp SYSTEM BEFORE PUBLIC "${AVROCPP_ROOT_DIR}/include") +target_include_directories(_avrocpp SYSTEM BEFORE PUBLIC "${AVRO_INCLUDE_DIR}") diff --git a/contrib/boost-cmake/CMakeLists.txt b/contrib/boost-cmake/CMakeLists.txt index 0215c68e683..3d66bc97971 100644 --- a/contrib/boost-cmake/CMakeLists.txt +++ b/contrib/boost-cmake/CMakeLists.txt @@ -27,7 +27,11 @@ target_include_directories (_boost_headers_only SYSTEM BEFORE INTERFACE ${LIBRAR # asio -target_compile_definitions (_boost_headers_only INTERFACE BOOST_ASIO_STANDALONE=1) +target_compile_definitions (_boost_headers_only INTERFACE + BOOST_ASIO_STANDALONE=1 + # Avoid using of deprecated in c++ > 17 std::result_of + BOOST_ASIO_HAS_STD_INVOKE_RESULT=1 +) # iostreams diff --git a/src/AggregateFunctions/AggregateFunctionSumMap.cpp b/src/AggregateFunctions/AggregateFunctionSumMap.cpp index 32c51d5f746..0b3a2067703 100644 --- a/src/AggregateFunctions/AggregateFunctionSumMap.cpp +++ b/src/AggregateFunctions/AggregateFunctionSumMap.cpp @@ -67,7 +67,7 @@ auto parseArguments(const std::string & name, const DataTypes & arguments) values_types.push_back(array_type->getNestedType()); } - return std::tuple{std::move(keys_type), std::move(values_types), tuple_argument}; + return std::tuple{std::move(keys_type), std::move(values_types), tuple_argument}; } // This function instantiates a particular overload of the sumMap family of diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b24181625d3..145015ad0f2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -573,10 +573,6 @@ if (ENABLE_TESTS) target_link_libraries(unit_tests_dbms PRIVATE ch_contrib::simdjson) endif() - if(TARGET ch_contrib::rapidjson) - target_include_directories(unit_tests_dbms PRIVATE ch_contrib::rapidjson) - endif() - if (TARGET ch_contrib::yaml_cpp) target_link_libraries(unit_tests_dbms PRIVATE ch_contrib::yaml_cpp) endif() diff --git a/src/Columns/ColumnObject.cpp b/src/Columns/ColumnObject.cpp index 54439edb93d..64c7a84c263 100644 --- a/src/Columns/ColumnObject.cpp +++ b/src/Columns/ColumnObject.cpp @@ -521,7 +521,7 @@ ColumnObject::ColumnObject(bool is_nullable_) { } -ColumnObject::ColumnObject(SubcolumnsTree && subcolumns_, bool is_nullable_) +ColumnObject::ColumnObject(Subcolumns && subcolumns_, bool is_nullable_) : is_nullable(is_nullable_) , subcolumns(std::move(subcolumns_)) , num_rows(subcolumns.empty() ? 0 : (*subcolumns.begin())->data.size()) @@ -696,7 +696,7 @@ const ColumnObject::Subcolumn & ColumnObject::getSubcolumn(const PathInData & ke ColumnObject::Subcolumn & ColumnObject::getSubcolumn(const PathInData & key) { if (const auto * node = subcolumns.findLeaf(key)) - return const_cast(node)->data; + return const_cast(node)->data; throw Exception(ErrorCodes::ILLEGAL_COLUMN, "There is no subcolumn {} in ColumnObject", key.getPath()); } @@ -794,7 +794,7 @@ bool ColumnObject::isFinalized() const void ColumnObject::finalize() { size_t old_size = size(); - SubcolumnsTree new_subcolumns; + Subcolumns new_subcolumns; for (auto && entry : subcolumns) { const auto & least_common_type = entry->data.getLeastCommonType(); diff --git a/src/Columns/ColumnObject.h b/src/Columns/ColumnObject.h index fe8763f3777..9d61b165042 100644 --- a/src/Columns/ColumnObject.h +++ b/src/Columns/ColumnObject.h @@ -138,20 +138,20 @@ public: size_t num_of_defaults_in_prefix = 0; }; - using SubcolumnsTree = SubcolumnsTree; + using Subcolumns = SubcolumnsTree; private: /// If true then all subcolumns are nullable. const bool is_nullable; - SubcolumnsTree subcolumns; + Subcolumns subcolumns; size_t num_rows; public: static constexpr auto COLUMN_NAME_DUMMY = "_dummy"; explicit ColumnObject(bool is_nullable_); - ColumnObject(SubcolumnsTree && subcolumns_, bool is_nullable_); + ColumnObject(Subcolumns && subcolumns_, bool is_nullable_); /// Checks that all subcolumns have consistent sizes. void checkConsistency() const; @@ -173,8 +173,8 @@ public: /// It cares about consistency of sizes of Nested arrays. void addNestedSubcolumn(const PathInData & key, const FieldInfo & field_info, size_t new_size); - const SubcolumnsTree & getSubcolumns() const { return subcolumns; } - SubcolumnsTree & getSubcolumns() { return subcolumns; } + const Subcolumns & getSubcolumns() const { return subcolumns; } + Subcolumns & getSubcolumns() { return subcolumns; } PathsInData getKeys() const; /// Finalizes all subcolumns. diff --git a/src/Common/FileSegment.cpp b/src/Common/FileSegment.cpp index ceb16249c66..ff4d114c71b 100644 --- a/src/Common/FileSegment.cpp +++ b/src/Common/FileSegment.cpp @@ -437,6 +437,7 @@ String FileSegment::stateToString(FileSegment::State state) case FileSegment::State::SKIP_CACHE: return "SKIP_CACHE"; } + __builtin_unreachable(); } String FileSegmentsHolder::toString() diff --git a/src/DataTypes/Serializations/SerializationObject.cpp b/src/DataTypes/Serializations/SerializationObject.cpp index 60b196092ed..64d79d107e1 100644 --- a/src/DataTypes/Serializations/SerializationObject.cpp +++ b/src/DataTypes/Serializations/SerializationObject.cpp @@ -63,12 +63,12 @@ private: size_t num_dimensions_to_keep; }; -using Node = typename ColumnObject::SubcolumnsTree::Node; +using Node = typename ColumnObject::Subcolumns::Node; /// Finds a subcolumn from the same Nested type as @entry and inserts /// an array with default values with consistent sizes as in Nested type. bool tryInsertDefaultFromNested( - const std::shared_ptr & entry, const ColumnObject::SubcolumnsTree & subcolumns) + const std::shared_ptr & entry, const ColumnObject::Subcolumns & subcolumns) { if (!entry->path.hasNested()) return false; @@ -198,7 +198,7 @@ void SerializationObject::deserializeWholeText(IColumn & column, ReadBuf template void SerializationObject::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const { - deserializeTextImpl(column, [&](String & s) { readEscapedStringInto(s, istr); }); + deserializeTextImpl(column, [&](String & s) { readEscapedString(s, istr); }); } template diff --git a/src/Disks/IO/CachedReadBufferFromRemoteFS.h b/src/Disks/IO/CachedReadBufferFromRemoteFS.h index 3d03debcd01..0e9dcc1033f 100644 --- a/src/Disks/IO/CachedReadBufferFromRemoteFS.h +++ b/src/Disks/IO/CachedReadBufferFromRemoteFS.h @@ -96,6 +96,7 @@ private: case ReadType::REMOTE_FS_READ_AND_PUT_IN_CACHE: return "REMOTE_FS_READ_AND_PUT_IN_CACHE"; } + __builtin_unreachable(); } size_t first_offset = 0; }; diff --git a/src/IO/ReadWriteBufferFromHTTP.h b/src/IO/ReadWriteBufferFromHTTP.h index 061dd772212..40a15ace4ad 100644 --- a/src/IO/ReadWriteBufferFromHTTP.h +++ b/src/IO/ReadWriteBufferFromHTTP.h @@ -663,7 +663,7 @@ public: Range range{from, to}; from = to; - return std::move(range); + return range; } private: diff --git a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp index 515ef6c3058..4c0aa20d795 100644 --- a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp +++ b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp @@ -150,12 +150,12 @@ static ColumnsDescription createColumnsDescription(const NamesAndTypesList & col ColumnsDescription columns_description; - for ( - auto [column_name_and_type, declare_column_ast] = std::tuple{columns_name_and_type.begin(), columns_definition->children.begin()}; - column_name_and_type != columns_name_and_type.end(); - column_name_and_type++, - declare_column_ast++ - ) + /// FIXME: we could write it like auto [a, b] = std::tuple(x, y), + /// but this produce endless recursion in gcc-11, and leads to SIGSEGV + /// (see git blame for details). + auto column_name_and_type = columns_name_and_type.begin(); + auto declare_column_ast = columns_definition->children.begin(); + for (; column_name_and_type != columns_name_and_type.end(); column_name_and_type++, declare_column_ast++) { const auto & declare_column = (*declare_column_ast)->as(); String comment; diff --git a/src/Interpreters/RowRefs.cpp b/src/Interpreters/RowRefs.cpp index 2b791f5a189..97feed54c08 100644 --- a/src/Interpreters/RowRefs.cpp +++ b/src/Interpreters/RowRefs.cpp @@ -192,7 +192,7 @@ private: using Result = Element; static TKey & extractKey(Element & elem) { return elem.value; } - static Element extractResult(Element & elem) { return elem; } + static Result extractResult(Element & elem) { return elem; } }; if constexpr (is_descending) diff --git a/src/Interpreters/threadPoolCallbackRunner.cpp b/src/Interpreters/threadPoolCallbackRunner.cpp index fadad235039..c580d352c49 100644 --- a/src/Interpreters/threadPoolCallbackRunner.cpp +++ b/src/Interpreters/threadPoolCallbackRunner.cpp @@ -25,7 +25,7 @@ CallbackRunner threadPoolCallbackRunner(ThreadPool & pool) /// Usually it could be ok, because thread pool task is executed before user-level memory tracker is destroyed. /// However, thread could stay alive inside the thread pool, and it's ThreadStatus as well. /// When, finally, we destroy the thread (and the ThreadStatus), - /// it can use memory tracker in the ~ThreadStatus in order to alloc/free untracked_memory,\ + /// it can use memory tracker in the ~ThreadStatus in order to alloc/free untracked_memory, /// and by this time user-level memory tracker may be already destroyed. /// /// As a work-around, reset memory tracker to total, which is always alive. diff --git a/src/Processors/Transforms/WindowTransform.cpp b/src/Processors/Transforms/WindowTransform.cpp index b81ed099915..3af72aff2cd 100644 --- a/src/Processors/Transforms/WindowTransform.cpp +++ b/src/Processors/Transforms/WindowTransform.cpp @@ -488,7 +488,7 @@ auto WindowTransform::moveRowNumberNoCheck(const RowNumber & _x, int64_t offset) } } - return std::tuple{x, offset}; + return std::tuple{x, offset}; } auto WindowTransform::moveRowNumber(const RowNumber & _x, int64_t offset) const @@ -505,7 +505,7 @@ auto WindowTransform::moveRowNumber(const RowNumber & _x, int64_t offset) const assert(oo == 0); #endif - return std::tuple{x, o}; + return std::tuple{x, o}; } diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 5c63b3f1ad1..7a183fd0bd6 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -206,6 +206,7 @@ CI_CONFIG = { "binary_freebsd", "binary_darwin_aarch64", "binary_ppc64le", + "binary_gcc", ], }, "tests_config": {