Enable travis (#1772)

* Try travis * wip * Try travis * try * try * try * try * Revert "try" This reverts commit 7ecd73c2eb. * try unbundled * Try unbundled2 * Try undundled3 * Try unbundled4 * Try5 * try gcc7 * try6 * Try clang * Try min * try ccache * try ccache stats * no sudo * try clang * try ccache * reset changelog * Try fix clang * try limit time * Faster * try timeout * try faster * Try * try * try * try * try * try * ccache * Fix * try test * try test * Fix try_listen in some vm's * fix * test * fix * cache timeout * packages * fix * fix * fix * try emb compiler * Try emb compiler * Revert "try emb compiler" This reverts commit 471713cabe. * try * Revert "Try emb compiler" This reverts commit 95e632abf6. * Skip long * fix * fix * ccache * fix * debug * Fxi test * test fix * tes fix * Fix * Fix link * Fix odbc link * Fxi test * fix boost unbundled include * fix test * fix test * test fix * fix tests * tests * Test fail
2024-11-21 15:12:02 +00:00 · 2018-01-15 21:57:10 +03:00 · 2018-01-15 21:57:10 +03:00 · f6d8436f81
commit f6d8436f81
parent 07931cad69
46 changed files with 294 additions and 66 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -0,0 +1,66 @@
+language: cpp
+
+matrix:
+  include:
+    - os: linux
+
+      cache:
+        ccache: true
+        timeout: 1000
+
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+          packages: [ g++-7,            libicu-dev, libreadline-dev, libmysqlclient-dev, unixodbc-dev, libltdl-dev, libssl-dev, libboost-dev, zlib1g-dev, libdouble-conversion-dev, libzookeeper-mt-dev, libsparsehash-dev, librdkafka-dev, libcapnp-dev, libsparsehash-dev, libgoogle-perftools-dev, bash, expect, python, python-lxml, python-termcolor, curl, perl, sudo ]
+      env:
+        - MATRIX_EVAL="CC=gcc-7 && CXX=g++-7"
+
+      script:
+        - utils/travis/normal.sh
+
+
+    # We need to have gcc7 headers to compile c++17 code on clang
+    - os: linux
+
+      cache:
+        ccache: true
+        timeout: 1000
+
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+            - llvm-toolchain-trusty-5.0
+          packages: [ g++-7, clang-5.0, libicu-dev, libreadline-dev, libmysqlclient-dev, unixodbc-dev, libltdl-dev, libssl-dev, libboost-dev, zlib1g-dev, libdouble-conversion-dev, libzookeeper-mt-dev, libsparsehash-dev, librdkafka-dev, libcapnp-dev, libsparsehash-dev, libgoogle-perftools-dev, bash, expect, python, python-lxml, python-termcolor, curl, perl, sudo ]
+
+      env:
+        - MATRIX_EVAL="CC=clang-5.0 && CXX=clang++-5.0
+
+      before_script:
+        # fix gcc after clang
+        - export CC=clang-5.0
+        - export CXX=clang++-5.0
+
+      script:
+        - utils/travis/normal.sh
+
+
+    - os: linux
+
+      sudo: required
+
+      cache:
+        timeout: 1000
+        directories:
+        - /var/cache/pbuilder/ccache
+
+      addons:
+        apt:
+          packages: [ pbuilder, fakeroot, debhelper ]
+
+      script:
+        - utils/travis/pbuilder.sh
+
+before_install:
+    - eval "${MATRIX_EVAL}"
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -145,14 +145,14 @@ endif ()

 set (CMAKE_BUILD_COLOR_MAKEFILE          ON)
 set (CMAKE_CXX_FLAGS                     "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS} ${PLATFORM_EXTRA_CXX_FLAG} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${CXX_WARNING_FLAGS} ${GLIBC_COMPATIBILITY_COMPILE_FLAGS}")
-#set (CMAKE_CXX_FLAGS_RELEASE             "${CMAKE_CXX_FLAGS_RELEASE}")
-set (CMAKE_CXX_FLAGS_RELWITHDEBINFO      "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3")
-set (CMAKE_CXX_FLAGS_DEBUG               "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline")
+#set (CMAKE_CXX_FLAGS_RELEASE             "${CMAKE_CXX_FLAGS_RELEASE} ${CMAKE_CXX_FLAGS_ADD}")
+set (CMAKE_CXX_FLAGS_RELWITHDEBINFO      "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_CXX_FLAGS_ADD}")
+set (CMAKE_CXX_FLAGS_DEBUG               "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_CXX_FLAGS_ADD}")

-set (CMAKE_C_FLAGS                       "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${GLIBC_COMPATIBILITY_COMPILE_FLAGS}")
-#set (CMAKE_C_FLAGS_RELEASE               "${CMAKE_C_FLAGS_RELEASE}")
-set (CMAKE_C_FLAGS_RELWITHDEBINFO        "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3")
-set (CMAKE_C_FLAGS_DEBUG                 "${CMAKE_C_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline")
+set (CMAKE_C_FLAGS                       "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${GLIBC_COMPATIBILITY_COMPILE_FLAGS} ${CMAKE_C_FLAGS_ADD}")
+#set (CMAKE_C_FLAGS_RELEASE               "${CMAKE_C_FLAGS_RELEASE} ${CMAKE_C_FLAGS_ADD}")
+set (CMAKE_C_FLAGS_RELWITHDEBINFO        "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_C_FLAGS_ADD}")
+set (CMAKE_C_FLAGS_DEBUG                 "${CMAKE_C_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_C_FLAGS_ADD}")

 if (MAKE_STATIC_LIBRARIES AND NOT APPLE AND NOT (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_SYSTEM MATCHES "FreeBSD"))
    set (CMAKE_EXE_LINKER_FLAGS          "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++")
@ -265,6 +265,8 @@ include (cmake/find_boost.cmake)
 # openssl, zlib before poco
 include (cmake/find_zlib.cmake)
 include (cmake/find_zstd.cmake)
+include (cmake/find_ltdl.cmake) # for odbc
+include (contrib/poco/cmake/FindODBC.cmake) # for poco
 include (cmake/find_poco.cmake)
 include (cmake/find_lz4.cmake)
 include (cmake/find_sparsehash.cmake)
--- a/README.md
+++ b/README.md
@ -1,3 +1,5 @@
 ClickHouse is an open-source column-oriented database management system that allows generating analytical data reports in real time.

 [Read more...](https://clickhouse.yandex/)
+
+[![Build Status](https://travis-ci.org/yandex/ClickHouse.svg?branch=master)](https://travis-ci.org/yandex/ClickHouse)
--- a/cmake/Modules/FindPoco.cmake
+++ b/cmake/Modules/FindPoco.cmake
@ -191,6 +191,10 @@ foreach( component ${components} )
    endif()
 endforeach()

+if(Poco_DataODBC_LIBRARY)
+    list(APPEND Poco_DataODBC_LIBRARY ${ODBC_LIBRARIES} ${LTDL_LIBRARY})
+endif()
+
 if(DEFINED Poco_LIBRARIES)
    set(Poco_FOUND true)
 endif()
--- a/cmake/find_ltdl.cmake
+++ b/cmake/find_ltdl.cmake
@ -1,3 +1,3 @@
 set (LTDL_PATHS "/usr/local/opt/libtool/lib")
-find_library (LTDL_LIB ltdl PATHS ${LTDL_PATHS})
-message (STATUS "Using ltdl: ${LTDL_LIB}")
+find_library (LTDL_LIBRARY ltdl PATHS ${LTDL_PATHS})
+message (STATUS "Using ltdl: ${LTDL_LIBRARY}")
--- a/cmake/find_poco.cmake
+++ b/cmake/find_poco.cmake
@ -27,8 +27,6 @@ else ()
    set (POCO_STATIC ${MAKE_STATIC_LIBRARIES} CACHE BOOL "")
    set (POCO_VERBOSE_MESSAGES 1 CACHE BOOL "")

-    include (${ClickHouse_SOURCE_DIR}/cmake/find_ltdl.cmake)
-    include (${ClickHouse_SOURCE_DIR}/contrib/poco/cmake/FindODBC.cmake)

    # used in internal compiler
    list (APPEND Poco_INCLUDE_DIRS
@ -44,8 +42,7 @@ else ()

    if (ODBC_FOUND)
        set (Poco_DataODBC_FOUND 1)
-        set (Poco_DataODBC_LIBRARY PocoDataODBC)
-        list (APPEND Poco_DataODBC_LIBRARY ${LTDL_LIB})
+        set (Poco_DataODBC_LIBRARY PocoDataODBC ${ODBC_LIBRARIES} ${LTDL_LIBRARY})
        set (Poco_DataODBC_INCLUDE_DIRS "${ClickHouse_SOURCE_DIR}/contrib/poco/Data/ODBC/include/")
    endif ()

--- a/cmake/find_rdkafka.cmake
+++ b/cmake/find_rdkafka.cmake
@ -11,12 +11,13 @@ if (USE_INTERNAL_RDKAFKA_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contri
 endif ()

 if (NOT USE_INTERNAL_RDKAFKA_LIBRARY)
-    find_library (RDKAFKA_LIBRARY rdkafka)
+    find_library (RDKAFKA_LIB rdkafka)
    find_path (RDKAFKA_INCLUDE_DIR NAMES librdkafka/rdkafka.h PATHS ${RDKAFKA_INCLUDE_PATHS})
 endif ()

-if (RDKAFKA_LIBRARY AND RDKAFKA_INCLUDE_DIR)
+if (RDKAFKA_LIB AND RDKAFKA_INCLUDE_DIR)
    set (USE_RDKAFKA 1)
+    set (RDKAFKA_LIBRARY ${RDKAFKA_LIB} ${OPENSSL_LIBRARIES})
 elseif (NOT MISSING_INTERNAL_RDKAFKA_LIBRARY)
    set (USE_INTERNAL_RDKAFKA_LIBRARY 1)
    set (RDKAFKA_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/librdkafka/src")
--- a/dbms/CMakeLists.txt
+++ b/dbms/CMakeLists.txt
@ -150,7 +150,9 @@ target_link_libraries (clickhouse_common_io
    ${DOUBLE_CONVERSION_LIBRARIES}
    ${Poco_Net_LIBRARY}
    ${Poco_Data_LIBRARY}
+    ${ZLIB_LIBRARIES}
    ${EXECINFO_LIBRARY}
+    ${Boost_SYSTEM_LIBRARY}
 )

 target_link_libraries (dbms
@ -159,11 +161,9 @@ target_link_libraries (dbms
    ${MYSQLXX_LIBRARY}
    ${FARMHASH_LIBRARIES}
    ${METROHASH_LIBRARIES}
-    ${ZLIB_LIBRARIES}
    ${RE2_LIBRARY}
    ${RE2_ST_LIBRARY}
    ${OPENSSL_CRYPTO_LIBRARY}
-    ${Boost_SYSTEM_LIBRARY}
    ${BTRIE_LIBRARIES}
 )

@ -175,6 +175,10 @@ if (NOT USE_INTERNAL_ZOOKEEPER_LIBRARY)
    target_include_directories (clickhouse_common_io BEFORE PUBLIC ${ZOOKEEPER_INCLUDE_DIR})
 endif ()

+if (NOT USE_INTERNAL_BOOST_LIBRARY)
+    target_include_directories (clickhouse_common_io BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
+endif ()
+
 if (Poco_DataODBC_FOUND)
    target_link_libraries (dbms ${Poco_DataODBC_LIBRARY})
 endif()
--- a/dbms/src/Analyzers/tests/CMakeLists.txt
+++ b/dbms/src/Analyzers/tests/CMakeLists.txt
@ -9,7 +9,7 @@ target_link_libraries(analyze_columns dbms clickhouse_storages_system)

 add_executable(type_and_constant_inference type_and_constant_inference.cpp)
 target_link_libraries(type_and_constant_inference
-    clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions dbms)
+    clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions)

 add_executable(analyze_result_of_query analyze_result_of_query.cpp)
 target_link_libraries(analyze_result_of_query dbms clickhouse_storages_system)
--- a/dbms/src/Server/CMakeLists.txt
+++ b/dbms/src/Server/CMakeLists.txt
@ -100,7 +100,6 @@ else ()
        clickhouse-extract-from-config-lib
        clickhouse-compressor-lib
        clickhouse-format-lib
-        dbms
    )

    add_custom_target (clickhouse-server ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-server DEPENDS clickhouse)
--- a/dbms/src/Server/Server.cpp
+++ b/dbms/src/Server/Server.cpp
@ -451,7 +451,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
            }
            catch (const Poco::Net::NetException & e)
            {
-                if (try_listen && e.code() == POCO_EPROTONOSUPPORT)
+                if (try_listen && (e.code() == POCO_EPROTONOSUPPORT || e.code() == POCO_EADDRNOTAVAIL))
                    LOG_ERROR(log, "Listen [" << listen_host << "]: " << e.what() << ": " << e.message()
                        << "  If it is an IPv6 or IPv4 address and your host has disabled IPv6 or IPv4, then consider to "
                        "specify not disabled IPv4 or IPv6 address to listen in <listen_host> element of configuration "
--- a/dbms/tests/clickhouse-test
+++ b/dbms/tests/clickhouse-test
@ -156,7 +156,11 @@ def main(args):
                        print "{0:70}".format(name + ": "),
                        sys.stdout.flush()

-                        if not args.zookeeper and 'zookeeper' in name:
+                        if args.skip and any(s in name for s in args.skip):
+                            report_testcase.append(et.Element("skipped", attrib = {"message": "skip"}))
+                            print(MSG_SKIPPED + " - skip")
+                            skipped_total += 1
+                        elif not args.zookeeper and 'zookeeper' in name:
                            report_testcase.append(et.Element("skipped", attrib = {"message": "no zookeeper"}))
                            print(MSG_SKIPPED + " - no zookeeper")
                            skipped_total += 1
@ -164,6 +168,10 @@ def main(args):
                            report_testcase.append(et.Element("skipped", attrib = {"message": "no shard"}))
                            print(MSG_SKIPPED + " - no shard")
                            skipped_total += 1
+                        elif not args.no_long and 'long' in name:
+                            report_testcase.append(et.Element("skipped", attrib = {"message": "no long"}))
+                            print(MSG_SKIPPED + " - no long")
+                            skipped_total += 1
                        else:
                            disabled_file = os.path.join(suite_dir, name) + '.disabled'

@ -319,6 +327,8 @@ if __name__ == '__main__':
    parser.add_argument('--order', default = 'desc', help = 'Run order (asc, desc, random)')
    parser.add_argument('--testname', action = 'store_true', default = None, dest = 'testname', help = 'Make query with test name before test run')

+    parser.add_argument('--skip', nargs='+', help = "Skip these tests")
+    parser.add_argument('--no-long', action = 'store_false', default = None, dest = 'no_long', help = 'Do not run long tests')
    group = parser.add_mutually_exclusive_group(required = False)
    group.add_argument('--zookeeper', action = 'store_true', default = None, dest = 'zookeeper', help = 'Run zookeeper related tests')
    group.add_argument('--no-zookeeper', action = 'store_false', default = None, dest = 'zookeeper', help = 'Do not run zookeeper related tests')
@ -332,7 +342,7 @@ if __name__ == '__main__':
        args.queries = 'queries'
        if args.tmp is None:
            args.tmp = args.queries
-    else:
+    elif args.queries is None:
        args.queries = '/usr/share/clickhouse-test/queries'
        if args.tmp is None:
            args.tmp = '/tmp/clickhouse-test'
--- a/dbms/tests/client-test.xml
+++ b/dbms/tests/client-test.xml
@ -0,0 +1,3 @@
+<yandex>
+    <tcp_port>59000</tcp_port>
+</yandex>
--- a/dbms/tests/queries/0_stateless/00097_long_storage_buffer_race_condition.reference
+++ b/dbms/tests/queries/0_stateless/00097_long_storage_buffer_race_condition.reference
--- a/dbms/tests/queries/0_stateless/00097_long_storage_buffer_race_condition.sh
+++ b/dbms/tests/queries/0_stateless/00097_long_storage_buffer_race_condition.sh
--- a/dbms/tests/queries/0_stateless/00133_long_shard_memory_tracker_and_exception_safety.reference
+++ b/dbms/tests/queries/0_stateless/00133_long_shard_memory_tracker_and_exception_safety.reference
--- a/dbms/tests/queries/0_stateless/00133_long_shard_memory_tracker_and_exception_safety.sh
+++ b/dbms/tests/queries/0_stateless/00133_long_shard_memory_tracker_and_exception_safety.sh
--- a/dbms/tests/queries/0_stateless/00155_long_merges.reference
+++ b/dbms/tests/queries/0_stateless/00155_long_merges.reference
--- a/dbms/tests/queries/0_stateless/00155_long_merges.sh
+++ b/dbms/tests/queries/0_stateless/00155_long_merges.sh
--- a/dbms/tests/queries/0_stateless/00326_long_function_multi_if.reference
+++ b/dbms/tests/queries/0_stateless/00326_long_function_multi_if.reference
--- a/dbms/tests/queries/0_stateless/00326_long_function_multi_if.sql
+++ b/dbms/tests/queries/0_stateless/00326_long_function_multi_if.sql
--- a/dbms/tests/queries/0_stateless/00328_long_case_construction.reference
+++ b/dbms/tests/queries/0_stateless/00328_long_case_construction.reference
--- a/dbms/tests/queries/0_stateless/00328_long_case_construction.sql
+++ b/dbms/tests/queries/0_stateless/00328_long_case_construction.sql
--- a/dbms/tests/queries/0_stateless/00411_accurate_number_comparison.python
+++ b/dbms/tests/queries/0_stateless/00411_accurate_number_comparison.python
--- a/dbms/tests/queries/0_stateless/00411_accurate_number_comparison.sh
+++ b/dbms/tests/queries/0_stateless/00411_accurate_number_comparison.sh
@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+. $CURDIR/../shell_config.sh
+
+# We should have correct env vars from shell_config.sh to run this test
+
+python $CURDIR/00411_accurate_number_comparison.python
--- a/dbms/tests/queries/0_stateless/00429_long_http_bufferization.reference
+++ b/dbms/tests/queries/0_stateless/00429_long_http_bufferization.reference
--- a/dbms/tests/queries/0_stateless/00429_long_http_bufferization.sh
+++ b/dbms/tests/queries/0_stateless/00429_long_http_bufferization.sh
--- a/dbms/tests/queries/0_stateless/00463_long_sessions_in_http_interface.reference
+++ b/dbms/tests/queries/0_stateless/00463_long_sessions_in_http_interface.reference
--- a/dbms/tests/queries/0_stateless/00463_long_sessions_in_http_interface.sh
+++ b/dbms/tests/queries/0_stateless/00463_long_sessions_in_http_interface.sh
--- a/dbms/tests/queries/0_stateless/00532_pager_sigpipe.tcl
+++ b/dbms/tests/queries/0_stateless/00532_pager_sigpipe.tcl
@ -2,7 +2,7 @@ set timeout -1

 log_user 0

-spawn clickhouse-client --pager=head
+spawn sh -c $::env(CLICKHOUSE_CLIENT) --pager=head

 expect ":) "

--- a/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments.reference
+++ b/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments.reference
--- a/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments.sh
+++ b/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments.sh
--- a/dbms/tests/server-test.xml
+++ b/dbms/tests/server-test.xml
@ -0,0 +1,49 @@
+<?xml version="1.0"?>
+<yandex>
+    <logger>
+        <level>trace</level>
+        <log>/tmp/clickhouse/log/clickhouse-server.log</log>
+        <errorlog>/tmp/clickhouse/log/clickhouse-server.err.log</errorlog>
+        <size>10M</size>
+        <count>1</count>
+    </logger>
+    <http_port>58123</http_port>
+    <tcp_port>59000</tcp_port>
+    <interserver_http_port>59009</interserver_http_port>
+    <keep_alive_timeout>3</keep_alive_timeout>
+    <path>/tmp/clickhouse/data/</path>
+    <tmp_path>/tmp/clickhouse/tmp/</tmp_path>
+    <users_config>users.xml</users_config>
+    <mark_cache_size>5368709120</mark_cache_size>
+    <default_profile>default</default_profile>
+    <default_database>default</default_database>
+    <timezone>Europe/Moscow</timezone>
+    <remote_servers incl="clickhouse_remote_servers" >
+        <!-- Test only shard config for testing distributed storage -->
+        <test_shard_localhost>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>59000</port>
+                </replica>
+            </shard>
+        </test_shard_localhost>
+    </remote_servers>
+    <zookeeper incl="zookeeper-servers" optional="true" />
+    <macros incl="macros" optional="true" />
+    <builtin_dictionaries_reload_interval>3600</builtin_dictionaries_reload_interval>
+    <max_session_timeout>3600</max_session_timeout>
+    <default_session_timeout>60</default_session_timeout>
+    <query_log>
+        <database>system</database>
+        <table>query_log</table>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </query_log>
+    <dictionaries_config>*_dictionary.xml</dictionaries_config>
+    <compression incl="clickhouse_compression">
+    </compression>
+    <distributed_ddl>
+        <path>/clickhouse/task_queue/ddl</path>
+    </distributed_ddl>
+    <format_schema_path>/tmp/clickhouse/data/format_schemas/</format_schema_path>
+</yandex>
--- a/dbms/tests/server_wrapper.sh
+++ b/dbms/tests/server_wrapper.sh
@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+ROOTDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && cd ../.. && pwd)
+DATADIR=${DATADIR:=/tmp/clickhouse}
+LOGDIR=${LOGDIR:=$DATADIR/log}
+
+rm -rf $DATADIR
+
+mkdir -p $LOGDIR
+
+# Start a local clickhouse server which will be used to run tests
+#PATH=$PATH:$ROOTDIR/build${BUILD_TYPE}/dbms/src/Server \
+  $ROOTDIR/build${BUILD_TYPE}/dbms/src/Server/clickhouse-server --config-file=$CURDIR/server-test.xml > $LOGDIR/stdout 2>&1 &
+CH_PID=$!
+sleep 3
+
+# Define needed stuff to kill test clickhouse server after tests completion
+function finish {
+  kill $CH_PID || true
+  wait
+  tail -n 50 $LOGDIR/stdout
+  rm -rf $DATADIR
+}
+trap finish EXIT
+
+# Do tests
+export CLICKHOUSE_CONFIG=${CLICKHOUSE_CONFIG:=$CURDIR/server-test.xml}
+#cd $CURDIR
+if [ -n "$*" ]; then
+    $*
+else
+    $ROOTDIR/build${BUILD_TYPE}/dbms/src/Server/clickhouse-client --config $CURDIR/client-test.xml -q 'SELECT * from system.build_options;'
+    PATH=$PATH:$ROOTDIR/build${BUILD_TYPE}/dbms/src/Server \
+      $CURDIR/clickhouse-test --binary $ROOTDIR/build${BUILD_TYPE}/dbms/src/Server/clickhouse --clientconfig $CURDIR/client-test.xml --tmp $DATADIR/tmp --queries $CURDIR/queries $TEST_OPT
+fi
--- a/dbms/tests/users.xml
+++ b/dbms/tests/users.xml
@ -0,0 +1 @@
+../src/Server/users.xml
--- a/debian/.pbuilderrc
+++ b/debian/.pbuilderrc
@ -24,11 +24,11 @@
 # sudo DIST=testing      ARCH=i386 pbuilder create --configfile debian/.pbuilderrc && DIST=testing      ARCH=i386 pdebuild --configfile debian/.pbuilderrc
 # sudo DIST=experimental ARCH=i386 pbuilder create --configfile debian/.pbuilderrc && DIST=experimental ARCH=i386 pdebuild --configfile debian/.pbuilderrc
 # use only clang:
-# env DEB_CC=clang-5.0 DEB_CXX=clang++-5.0 env EXTRAPACKAGES="clang-5.0" DIST=artful pdebuild --configfile debian/.pbuilderrc
+# env DEB_CC=clang-5.0 DEB_CXX=clang++-5.0 EXTRAPACKAGES="clang-5.0" DIST=artful pdebuild --configfile debian/.pbuilderrc
 # clang+asan:
-# env DEB_CC=clang-5.0 DEB_CXX=clang++-5.0 env EXTRAPACKAGES="clang-5.0 libc++abi-dev libc++-dev" CMAKE_FLAGS="-DENABLE_TCMALLOC=0 -DENABLE_UNWIND=0 -DCMAKE_BUILD_TYPE=Asan" DIST=artful pdebuild --configfile debian/.pbuilderrc
+# env DEB_CC=clang-5.0 DEB_CXX=clang++-5.0 EXTRAPACKAGES="clang-5.0 libc++abi-dev libc++-dev" CMAKE_FLAGS="-DENABLE_TCMALLOC=0 -DENABLE_UNWIND=0 -DCMAKE_BUILD_TYPE=Asan" DIST=artful pdebuild --configfile debian/.pbuilderrc
 # clang+tsan:
-# env DEB_CC=clang-5.0 DEB_CXX=clang++-5.0 env EXTRAPACKAGES="clang-5.0 libc++abi-dev libc++-dev" CMAKE_FLAGS="-DCMAKE_BUILD_TYPE=Tsan" DIST=artful pdebuild --configfile debian/.pbuilderrc
+# env DEB_CC=clang-5.0 DEB_CXX=clang++-5.0 EXTRAPACKAGES="clang-5.0 libc++abi-dev libc++-dev" CMAKE_FLAGS="-DCMAKE_BUILD_TYPE=Tsan" DIST=artful pdebuild --configfile debian/.pbuilderrc
 # without sse for old systems and some VM:
 # env DH_VERBOSE=1 CMAKE_FLAGS="-DHAVE_SSE41=0 -DHAVE_SSE42=0 -DHAVE_POPCNT=0 -DHAVE_SSE2_INTRIN=0 -DSSE2FLAG=' ' -DHAVE_SSE42_INTRIN=0 -DSSE4FLAG=' ' -DHAVE_PCLMULQDQ_INTRIN=0 -DPCLMULFLAG=' '" DIST=artful pdebuild --configfile debian/.pbuilderrc

@ -131,7 +131,7 @@ case "$DIST" in
        # ccache broken
        ;;
    *)
-        CCACHEDIR=/var/cache/pbuilder/ccache
+        CCACHEDIR=${CCACHEDIR:="/var/cache/pbuilder/ccache"}
        ;;
 esac

@ -157,15 +157,17 @@ case "$ARCH" in
        ;;
 esac

+# will test symbols
+EXTRAPACKAGES+=" gdb "

 export CCACHE_PREFIX=

 export DEB_BUILD_OPTIONS=parallel=`nproc`

 # Floating bug with permissions:
-sudo mkdir -p /var/cache/pbuilder/ccache
-sudo chmod -R a+rwx /var/cache/pbuilder/ccache
-# chown -R 1234:1234 /var/cache/pbuilder/ccache
+sudo mkdir -p $CCACHEDIR
+sudo chmod -R a+rwx $CCACHEDIR
+# chown -R $BUILDUSERID:$BUILDUSERID $CCACHEDIR

 # echo "DEBOOTSTRAPOPTS = ${DEBOOTSTRAPOPTS[@]}"

--- a/debian/control
+++ b/debian/control
@ -5,6 +5,7 @@ Maintainer: Alexey Milovidov <milovidov@yandex-team.ru>
 Build-Depends: debhelper (>= 9),
               cmake3 | cmake,
               gcc-7, g++-7,
+               libc6-dev,
               libmariadbclient-dev | default-libmysqlclient-dev | libmysqlclient-dev,
               libicu-dev,
               libltdl-dev,
--- a/debian/pbuilder-hooks/A00ccache
+++ b/debian/pbuilder-hooks/A00ccache
@ -1,3 +1,9 @@
 #!/bin/sh

-ccache -M 32G
+mkdir -p $CCACHEDIR
+chown -R $BUILDUSERID:$BUILDUSERID $CCACHEDIR
+chmod -R a+rwx $CCACHEDIR
+
+CCACHE_SIZE=${CCACHE_SIZE:=32G}
+ccache --show-stats
+ccache -M $CCACHE_SIZE
--- a/debian/pbuilder-hooks/B90test-server
+++ b/debian/pbuilder-hooks/B90test-server
@ -20,13 +20,20 @@ dpkg-reconfigure -f noninteractive tzdata
 service clickhouse-server start
 sleep 3

+# TODO: remove me or make only on error:
+tail -n100 /var/log/clickhouse-server/*

 clickhouse-client -q "SELECT * from system.build_options;"
 clickhouse-client -q "SELECT toDateTime(1);"


+TEST_TRUE=${TEST_TRUE:=true}
+
 # --no-shard because default server listen only :: and 127.0.0.1
-[ -n "$TEST_RUN" ] && clickhouse-test --no-shard --queries /usr/share/clickhouse-test/queries --tmp /tmp/clickhouse-test/ || true
+[ -n "$TEST_RUN" ] && clickhouse-test --no-shard --queries /usr/share/clickhouse-test/queries --tmp /tmp/clickhouse-test/ $TEST_OPT || $TEST_TRUE


 service clickhouse-server stop
+
+# Test debug symbols
+gdb -ex quit --args /usr/bin/clickhouse-server
--- a/debian/pbuilder-hooks/C99kill-make
+++ b/debian/pbuilder-hooks/C99kill-make
@ -0,0 +1,5 @@
+#!/bin/sh
+
+# Try stop parallel build after timeout
+
+killall make gcc gcc-7 g++-7 clang-5.0 clang++-5.0 || true
--- a/debian/source/options
+++ b/debian/source/options
@ -1,3 +1,4 @@
 tar-ignore
-tar-ignore="ClickHouse/contrib/poco/openssl/*"
-tar-ignore="ClickHouse/build*"
+tar-ignore="contrib/poco/openssl/*"
+tar-ignore="build"
+tar-ignore="build_*"
--- a/debian/tests_wrapper.sh
+++ b/debian/tests_wrapper.sh
@ -1,30 +0,0 @@
-#!/bin/bash
-
-set -o errexit
-set -o pipefail
-
-ln -sf clickhouse build/dbms/src/Server/clickhouse-server
-ln -sf clickhouse build/dbms/src/Server/clickhouse-client
-ln -sf clickhouse build/dbms/src/Server/clickhouse-local
-# Start a local clickhouse server which will be used to run tests
-PWD=$(pwd)
-echo ${PWD}
-PATH=$PATH:./build/dbms/src/Server \
-  ./build/dbms/src/Server/clickhouse-server --config-file=./debian/clickhouse-server-config-tests.xml 2>/dev/null &
-CH_PID=$!
-
-# Define needed stuff to kill test clickhouse server after tests completion
-function finish {
-  kill $CH_PID
-  wait
-  rm -rf /tmp/clickhouse
-  rm -f build/dbms/src/Server/clickhouse-local
-  rm -f build/dbms/src/Server/clickhouse-client
-  rm -f build/dbms/src/Server/clickhouse-server
-}
-trap finish EXIT
-
-# Do tests
-cd dbms/tests
-PATH=$PATH:../../build/dbms/src/Server \
-  ./clickhouse-test -c ../../build/dbms/src/Server/clickhouse-client
--- a/2
+++ b/2
@ -88,5 +88,5 @@ else
    if [[ -n "$FORCE_PBUILDER_CREATE" || ! -e "$BASETGZ" ]] ; then
        sudo --preserve-env pbuilder create --configfile $CURDIR/debian/.pbuilderrc
    fi
-    pdebuild --configfile $CURDIR/debian/.pbuilderrc
+    pdebuild --configfile $CURDIR/debian/.pbuilderrc -- $PBUILDER_OPT
 fi
--- a/utils/build/build_debian.sh
+++ b/utils/build/build_debian.sh
@ -8,7 +8,7 @@
 # install compiler and libs
 sudo apt install -y git bash cmake gcc-7 g++-7 libicu-dev libreadline-dev libmysqlclient-dev unixodbc-dev libltdl-dev libssl-dev
 # for -DUNBUNDLED=1 mode:
-#sudo apt install -y libboost-dev zlib1g-dev liblz4-dev libdouble-conversion-dev libzstd-dev libre2-dev libzookeeper-mt-dev libsparsehash-dev librdkafka-dev libcapnp-dev libpoco-dev libsparsehash-dev libgoogle-perftools-dev libunwind-dev libzstd-dev
+#sudo apt install -y libboost-dev zlib1g-dev liblz4-dev libdouble-conversion-dev libzstd-dev libre2-dev libzookeeper-mt-dev libsparsehash-dev librdkafka-dev libcapnp-dev libpoco-dev libsparsehash-dev libgoogle-perftools-dev libunwind-dev

 # install testing only stuff if you want:
 sudo apt install -y python python-lxml python-termcolor curl perl
--- a/utils/travis/normal.sh
+++ b/utils/travis/normal.sh
@ -0,0 +1,28 @@
+#!/bin/sh
+
+# Manual run:
+# env CXX=g++-7 CC=gcc-7 utils/travis/normal.sh
+# env CXX=clang++-5.0 CC=clang-5.0 utils/travis/normal.sh
+
+set -e
+set -x
+
+TEST_TRUE=${TEST_TRUE:=false}
+
+ccache -s
+ccache -M 6G
+mkdir -p build
+cd build
+cmake .. -DCMAKE_CXX_COMPILER=`which $CXX` -DCMAKE_C_COMPILER=`which $CC` \
+    `# Does not optimize to speedup build` \
+    -DCMAKE_C_FLAGS_ADD=-O0 -DCMAKE_CXX_FLAGS_ADD=-O0 \
+    -DCMAKE_C_COMPILER_LAUNCHER=/usr/bin/ccache -DCMAKE_CXX_COMPILER_LAUNCHER=/usr/bin/ccache \
+    `# Use all possible contrib libs from system` \
+    -DUNBUNDLED=1 \
+    `# Disable all features` \
+    -DENABLE_CAPNP=0 -DENABLE_RDKAFKA=0 -DUSE_EMBEDDED_COMPILER=0 -DENABLE_TCMALLOC=0 -DENABLE_UNWIND=0 -DENABLE_MYSQL=0 \
+    && make -j `nproc || grep -c ^processor /proc/cpuinfo` clickhouse-bundle \
+    `# Skip tests:` \
+    `# 00281 requires internal compiler` \
+    `# 00428 requires sudo (not all vms allow this)` \
+    && ( ( cd .. && env TEST_OPT="--no-long --no-shard --skip 00281 00428" bash -x dbms/tests/server_wrapper.sh ) || $TEST_TRUE )
--- a/utils/travis/pbuilder.sh
+++ b/utils/travis/pbuilder.sh
@ -0,0 +1,23 @@
+#!/bin/sh
+
+set -e
+set -x
+
+env TEST_RUN=1 \
+    `# Skip tests:` \
+    `# 00281 requires internal compiler` \
+    `# 00416 requires patched poco from contrib/` \
+    TEST_OPT="--no-long --skip 00281 00416" \
+    TEST_TRUE=false \
+    `# travisci will not upload ccache cache after timeout (48min), use our less timeout` \
+    PBUILDER_OPT="--timeout 35m" \
+    `# clang faster than gcc` \
+    DEB_CC=clang-5.0 DEB_CXX=clang++-5.0 \
+    CCACHE_SIZE=6G CCACHEDIR=$HOME/.ccache \
+    `# Disable all features` \
+    CMAKE_FLAGS="-DCMAKE_BUILD_TYPE=Debug -DUNBUNDLED=1 -DENABLE_UNWIND=0 -DENABLE_MYSQL=0 -DENABLE_CAPNP=0 -DENABLE_RDKAFKA=0 -DUSE_EMBEDDED_COMPILER=0" \
+    `# Use all possible contrib libs from system` \
+    `# psmisc - killall` \
+    `# gdb - symbol test in pbuilder` \
+    EXTRAPACKAGES="psmisc gdb clang-5.0 libc++abi-dev libc++-dev libboost-dev libboost-program-options-dev zlib1g-dev liblz4-dev libdouble-conversion-dev libzookeeper-mt-dev libsparsehash-dev librdkafka-dev libpoco-dev libsparsehash-dev libgoogle-perftools-dev libzstd-dev libre2-dev" \
+    ./release --pbuilder