Build fixes (#3545)

* StorageSystemContributors clean

* Fix

* ARM fixes

* Fix arm compile

* fix

* Fix macos?

* Fix includes

* fix

* fix

* Try fix apple build part 1

* Fix identation

* Fix static libc++ in clang

* fix arm build

* better

* fix

* fix

*  better check-include

CMakeLists.txt

@@ -115,8 +115,8 @@ endif ()
 
 include (cmake/test_cpu.cmake)
 
-option (ARCHNATIVE "Enable -march=native compiler flag" OFF)
-if (ARCHNATIVE)
+option (ARCH_NATIVE "Enable -march=native compiler flag" OFF)
+if (ARCH_NATIVE)
     set (COMPILER_FLAGS                  "${COMPILER_FLAGS} -march=native")
 endif ()
 
@@ -155,6 +155,7 @@ if (MAKE_STATIC_LIBRARIES AND NOT APPLE AND NOT (COMPILER_CLANG AND OS_FREEBSD))
 endif ()
 
 set(THREADS_PREFER_PTHREAD_FLAG ON)
+find_package (Threads)
 
 include (cmake/test_compiler.cmake)
 

cmake/find_rdkafka.cmake

@@ -1,4 +1,6 @@
-option (ENABLE_RDKAFKA "Enable kafka" ON)
+if (NOT ARCH_ARM)
+    option (ENABLE_RDKAFKA "Enable kafka" ON)
+endif ()
 
 if (ENABLE_RDKAFKA)
 

cmake/test_compiler.cmake

@@ -14,6 +14,7 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
     endif ()
 
     set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG}")
+    set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} Threads::Threads)
 
     check_cxx_source_compiles("
         #include <iostream>
@@ -25,8 +26,6 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
 
 else ()
 
-    cmake_push_check_state ()
-
     set (TEST_FLAG "-no-pie")
     set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG}")
 
@@ -40,7 +39,6 @@ else ()
         set (FLAG_NO_PIE ${TEST_FLAG})
     endif ()
 
-
 endif ()
 
 cmake_pop_check_state ()

contrib/CMakeLists.txt

@@ -55,7 +55,7 @@ endif ()
 
 if (USE_INTERNAL_ZLIB_LIBRARY)
     set (ZLIB_COMPAT 1 CACHE INTERNAL "") # also enables WITH_GZFILEOP
-    set (WITH_NATIVE_INSTRUCTIONS ${ARCHNATIVE} CACHE INTERNAL "")
+    set (WITH_NATIVE_INSTRUCTIONS ${ARCH_NATIVE} CACHE INTERNAL "")
     if (OS_FREEBSD OR ARCH_I386)
         set (WITH_OPTIM 0 CACHE INTERNAL "") # Bug in assembler
     endif ()
@@ -164,6 +164,8 @@ if (USE_INTERNAL_LLVM_LIBRARY)
     if (APPLE)
         set (LINKER_SUPPORTS_COLOR_DIAGNOSTICS 0 CACHE INTERNAL "")
     endif ()
+    set (LLVM_ENABLE_EH 1 CACHE INTERNAL "")
+    set (LLVM_ENABLE_RTTI 1 CACHE INTERNAL "")
     add_subdirectory (llvm/llvm)
 endif ()
 

dbms/CMakeLists.txt

@@ -48,8 +48,6 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
     add_definitions ("-fno-tree-loop-distribute-patterns")
 endif ()
 
-find_package (Threads)
-
 add_subdirectory (src)
 
 set(dbms_headers)

dbms/programs/clang/Compiler-7.0.1

@@ -0,0 +1 @@
+Compiler-7.0.0

dbms/src/Common/HashTable/Hash.h

@@ -39,7 +39,7 @@ inline DB::UInt64 intHash64(DB::UInt64 x)
 #include <nmmintrin.h>
 #endif
 
-#if __aarch64__
+#if __aarch64__ && __ARM_FEATURE_CRC32
 #include <arm_acle.h>
 #include <arm_neon.h>
 #endif
@@ -48,8 +48,8 @@ inline DB::UInt64 intHashCRC32(DB::UInt64 x)
 {
 #if __SSE4_2__
     return _mm_crc32_u64(-1ULL, x);
-#elif __aarch64__
-    return __crc32cd(-1ULL, x);
+#elif __aarch64__ && __ARM_FEATURE_CRC32
+    return __crc32cd(-1U, x);
 #else
     /// On other platforms we do not have CRC32. NOTE This can be confusing.
     return intHash64(x);

dbms/src/Common/XDBCBridgeHelper.h

@@ -4,6 +4,7 @@
 #include <IO/ReadHelpers.h>
 #include <IO/ReadWriteBufferFromHTTP.h>
 #include <Interpreters/Context.h>
+#include <Parsers/IdentifierQuotingStyle.h>
 #include <Poco/File.h>
 #include <Poco/Logger.h>
 #include <Poco/Net/HTTPRequest.h>

dbms/src/Functions/DateTimeTransforms.h

@@ -1,3 +1,4 @@
+#pragma once
 #include <Core/Types.h>
 #include <Common/Exception.h>
 #include <common/DateLUTImpl.h>

dbms/src/Functions/EmptyImpl.h

@@ -1,3 +1,4 @@
+#pragma once
 #include <cstring>
 #include <Columns/ColumnString.h>
 #include <Functions/FunctionFactory.h>

dbms/src/Functions/FunctionDateOrDateTimeToSomething.h

@@ -1,9 +1,8 @@
 #include <DataTypes/DataTypeDate.h>
 #include <DataTypes/DataTypeDateTime.h>
-
 #include <Functions/IFunction.h>
 #include <Functions/extractTimeZoneFromFunctionArguments.h>
-
+#include <Functions/DateTimeTransforms.h>
 #include <IO/WriteHelpers.h>
 
 

dbms/src/Interpreters/ActionsVisitor.h

@@ -1,7 +1,6 @@
 #pragma once
 
 #include <Parsers/StringRange.h>
-
 #include <Interpreters/ExpressionActions.h>
 
 

dbms/src/Interpreters/ArrayJoinedColumnsVisitor.h

@@ -4,6 +4,7 @@
 #include <Common/typeid_cast.h>
 
 #include <Parsers/ASTSubquery.h>
+#include <Parsers/ASTSelectQuery.h>
 #include <Parsers/ASTTablesInSelectQuery.h>
 #include <Parsers/ASTIdentifier.h>
 

dbms/src/Interpreters/DatabaseAndTableWithAlias.h

@@ -3,6 +3,7 @@
 #include <memory>
 #include <optional>
 #include <Core/Types.h>
+#include <Parsers/ASTSelectQuery.h>
 
 namespace DB
 {

dbms/src/Interpreters/ExecuteScalarSubqueriesVisitor.h

@@ -1,16 +1,15 @@
 #pragma once
 
 #include <Common/typeid_cast.h>
+#include <Interpreters/Context.h>
 #include <Parsers/DumpASTNode.h>
+#include <Parsers/ASTSubquery.h>
+#include <Parsers/ASTTablesInSelectQuery.h>
+#include <Parsers/ASTFunction.h>
 
 namespace DB
 {
 
-class Context;
-class ASTSubquery;
-class ASTFunction;
-struct ASTTableExpression;
-
 /// Visitors consist of functions with unified interface 'void visit(Casted & x, ASTPtr & y)', there x is y, successfully casted to Casted.
 /// Both types and fuction could have const specifiers. The second argument is used by visitor to replaces AST node (y) if needed.
 

dbms/src/Interpreters/ExternalTablesVisitor.h

@@ -1,5 +1,10 @@
 #pragma once
 
+#include <Interpreters/Context.h>
+#include <Parsers/IAST.h>
+#include <Parsers/ASTIdentifier.h>
+#include <Common/typeid_cast.h>
+
 namespace DB
 {
 

dbms/src/Interpreters/GlobalSubqueriesVisitor.h

@@ -1,5 +1,21 @@
 #pragma once
 
+#include <Parsers/IAST.h>
+#include <Parsers/ASTSubquery.h>
+#include <Parsers/ASTFunction.h>
+#include <Parsers/ASTTablesInSelectQuery.h>
+#include <Parsers/ASTSelectQuery.h>
+#include <Parsers/ASTIdentifier.h>
+#include <Interpreters/Context.h>
+#include <Interpreters/ActionsVisitor.h>
+#include <Interpreters/interpretSubquery.h>
+#include <Common/typeid_cast.h>
+#include <Core/Block.h>
+#include <Core/NamesAndTypes.h>
+#include <Databases/IDatabase.h>
+#include <Storages/StorageMemory.h>
+#include <IO/WriteHelpers.h>
+
 namespace DB
 {
 

dbms/src/Interpreters/QueryAliasesVisitor.h

@@ -3,6 +3,7 @@
 #include <Common/typeid_cast.h>
 #include <Parsers/DumpASTNode.h>
 #include <unordered_map>
+#include <Common/typeid_cast.h>
 
 namespace DB
 {

dbms/src/Interpreters/RequiredSourceColumnsVisitor.h

@@ -1,11 +1,21 @@
 #pragma once
 
+#include <Core/Names.h>
+#include <Parsers/IAST.h>
+#include <Parsers/ASTIdentifier.h>
+#include <Parsers/ASTFunction.h>
+#include <Parsers/ASTSelectQuery.h>
+#include <Parsers/ASTTablesInSelectQuery.h>
+#include <DataTypes/NestedUtils.h>
+#include <Common/typeid_cast.h>
+
 namespace DB
 {
 
 namespace ErrorCodes
 {
     extern const int TYPE_MISMATCH;
+    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
 }
 
 /// Visitors consist of functions with unified interface 'void visit(Casted & x, ASTPtr & y)', there x is y, successfully casted to Casted.

dbms/src/Parsers/ASTEnumElement.h

@@ -1,6 +1,8 @@
 #pragma once
 
 #include <Parsers/IAST.h>
+#include <Core/Field.h>
+#include <Common/FieldVisitors.h>
 
 
 namespace DB

debian/.pbuilderrc

@@ -161,7 +161,15 @@ esac
 
 if [ "$ARCH" != arm64 ]; then
     case "$DIST" in
-    "cosmic" | "bionic" | "experimental" | "unstable" | "testing")
+    "unstable")
+        EXTRAPACKAGES+=" liblld-8-dev libclang-8-dev llvm-8-dev liblld-8 "
+        export CMAKE_FLAGS="-DLLVM_VERSION_POSTFIX=-8 $CMAKE_FLAGS"
+        ;;
+    "cosmic" | "experimental" | "testing")
+        EXTRAPACKAGES+=" liblld-7-dev libclang-7-dev llvm-7-dev liblld-7 "
+        export CMAKE_FLAGS="-DLLVM_VERSION_POSTFIX=-7 $CMAKE_FLAGS"
+        ;;
+    "bionic")
         EXTRAPACKAGES+=" liblld-6.0-dev libclang-6.0-dev liblld-6.0 "
         export CMAKE_FLAGS="-DLLVM_VERSION_POSTFIX=-6.0 $CMAKE_FLAGS"
         ;;

libs/libcommon/cmake/find_jemalloc.cmake

@@ -1,4 +1,4 @@
-if (OS_LINUX AND NOT SANITIZE)
+if (OS_LINUX AND NOT SANITIZE AND NOT ARCH_ARM)
     set(ENABLE_JEMALLOC_DEFAULT 1)
 else ()
     set(ENABLE_JEMALLOC_DEFAULT 0)

libs/libcommon/include/common/StringRef.h

@@ -271,7 +271,7 @@ struct CRC32Hash
     size_t operator() (StringRef /* x */) const
     {
        throw std::logic_error{"Not implemented CRC32Hash without SSE"};
-    };
+    }
 };
 
 struct StringRefHash : StringRefHash64 {};

utils/check-style/CMakeLists.txt

@@ -1,2 +1,2 @@
 add_test(NAME check-style COMMAND bash -c "${CMAKE_CURRENT_SOURCE_DIR}/check-style")
-add_test(NAME check-include COMMAND bash -c "env RUN_DIR=${CMAKE_CURRENT_SOURCE_DIR} ROOT_DIR=${ClickHouse_SOURCE_DIR} BUILD_DIR=${ClickHouse_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/check-include")
+add_test(NAME check-include COMMAND sh -c "env RUN_DIR=${CMAKE_CURRENT_SOURCE_DIR} ROOT_DIR=${ClickHouse_SOURCE_DIR} BUILD_DIR=${ClickHouse_BINARY_DIR} CXX=${CMAKE_CXX_COMPILER} ${CMAKE_CURRENT_SOURCE_DIR}/check-include-stat")

utils/check-style/check-include

@@ -6,7 +6,9 @@ set -e
 # Finds missing #include <...>
 # prints compile time, number of includes, use with sort: ./check_include.sh 2>&1 | sort -rk3
 # use with clang: CXX=`which clang++-7 clang++-7.0 clang++-6.0 clang++-5.0 | head -n1` ./check_include.sh
-RUN_DIR=${RUN_DIR=`pwd`}
+CUR_DIR=`dirname $0`
+CUR_DIR=`readlink -f $CUR_DIR`
+CUR_DIR="${CUR_DIR}/"
 BUILD_DIR=${BUILD_DIR:=./build}
 inc="-I. \
 -I./contrib/libdivide \
@@ -37,19 +39,22 @@ inc="-I. \
 -I./contrib/libbtrie/include \
 -I./contrib/libpcg-random/include \
 -I./contrib/capnproto/c++/src \
+-I./contrib/unixodbc/include \
+-I./contrib/unixodbc-cmake/linux_x86_64 \
 -I./libs/libmysqlxx/include \
 -I./libs/libcommon/include \
 -I${BUILD_DIR}/libs/libcommon/include \
 -I./libs/libpocoext/include \
 -I./libs/libzkutil/include \
 -I./libs/libdaemon/include \
--I./libs/libconsistent-hashing \
+-I./libs/consistent-hashing \
+-I./libs/consistent-hashing-sumbur \
 -I./dbms/src \
 -I${BUILD_DIR}/dbms/src"
 
 if [ -z $1 ]; then
-    cd ${ROOT_DIR=../..}
-    find dbms libs utils \( -name *.h -and -not -name *.inl.h \) -exec sh $RUN_DIR/`basename $0` {} \; ;
+    cd ${ROOT_DIR=${CUR_DIR}../..}
+    find dbms libs utils \( -name *.h -and -not -name *.inl.h \) -print0 | xargs -0 -n1 sh $CUR_DIR`basename $0`
 else
     echo -n "$1    "
     echo -n `grep "#include" $1| wc -l` "    "

utils/check-style/check-include-stat

@@ -0,0 +1,25 @@
+#!/bin/sh
+set -e
+
+CUR_DIR=`dirname $0`
+CUR_DIR=`readlink -f $CUR_DIR`
+CUR_DIR="${CUR_DIR}/"
+
+RESULT_FILE=${RESULT_FILE:=${CUR_DIR}check-include.log}
+finish() {
+    echo include check failed:
+    cat $RESULT_FILE
+}
+trap finish 0 1 3 6 15
+
+sh ${CUR_DIR}check-include > $RESULT_FILE 2>&1
+
+echo Results:
+echo Top by memory:
+cat $RESULT_FILE | sort -rk4 | head -n20
+
+echo Top by time:
+cat $RESULT_FILE | sort -rk3 | head -n20
+
+echo Top by includes:
+cat $RESULT_FILE | sort -rk2 | head -n20