Merge branch 'master' into clang-compliant

cmake/find_capnp.cmake

@@ -1,7 +1,6 @@
 option (ENABLE_CAPNP "Enable Cap'n Proto" ${NOT_MSVC})
 
 if (ENABLE_CAPNP)
-
     # cmake 3.5.1 bug:
     # capnproto uses this cmake feature:
     # target_compile_features(kj PUBLIC cxx_constexpr)
@@ -42,7 +41,6 @@ if (ENABLE_CAPNP)
         set (CAPNP_LIBRARY capnpc)
         set (USE_CAPNP 1)
     endif ()
-
 endif ()
 
 if (USE_CAPNP)

cmake/find_poco.cmake

@@ -27,7 +27,6 @@ endif ()
 
 if (Poco_INCLUDE_DIRS AND Poco_Foundation_LIBRARY)
 elseif (NOT MISSING_INTERNAL_POCO_LIBRARY)
-
     set (USE_INTERNAL_POCO_LIBRARY 1)
 
     set (ENABLE_ZIP 0 CACHE BOOL "")

dbms/CMakeLists.txt

@@ -102,7 +102,7 @@ if (USE_EMBEDDED_COMPILER)
     llvm_libs_all(REQUIRED_LLVM_LIBRARIES)
 
     target_link_libraries (dbms ${REQUIRED_LLVM_LIBRARIES})
-    target_include_directories (dbms BEFORE PUBLIC ${LLVM_INCLUDE_DIRS})
+    target_include_directories (dbms SYSTEM BEFORE PUBLIC ${LLVM_INCLUDE_DIRS})
 endif ()
 
 
@@ -159,32 +159,32 @@ target_link_libraries (dbms
 )
 
 if (NOT USE_INTERNAL_RE2_LIBRARY)
-    target_include_directories (dbms BEFORE PRIVATE ${RE2_INCLUDE_DIR})
+    target_include_directories (dbms SYSTEM BEFORE PRIVATE ${RE2_INCLUDE_DIR})
 endif ()
 
 if (NOT USE_INTERNAL_BOOST_LIBRARY)
-    target_include_directories (clickhouse_common_io BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
+    target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
 endif ()
 
 if (USE_POCO_SQLODBC)
     target_link_libraries (clickhouse_common_io ${Poco_SQL_LIBRARY})
     target_link_libraries (dbms ${Poco_SQLODBC_LIBRARY} ${Poco_SQL_LIBRARY})
     if (NOT USE_INTERNAL_POCO_LIBRARY)
-        target_include_directories (clickhouse_common_io PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQL_INCLUDE_DIRS})
+        target_include_directories (clickhouse_common_io SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQL_INCLUDE_DIRS})
-        target_include_directories (dbms PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIRS} PUBLIC ${Poco_SQL_INCLUDE_DIRS})
+        target_include_directories (dbms SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIRS} PUBLIC ${Poco_SQL_INCLUDE_DIRS})
     endif()
 endif()
 
 if (Poco_Data_FOUND AND NOT USE_INTERNAL_POCO_LIBRARY)
-    target_include_directories (clickhouse_common_io PRIVATE ${Poco_Data_INCLUDE_DIRS})
+    target_include_directories (clickhouse_common_io SYSTEM PRIVATE ${Poco_Data_INCLUDE_DIRS})
-    target_include_directories (dbms PRIVATE ${Poco_Data_INCLUDE_DIRS})
+    target_include_directories (dbms SYSTEM PRIVATE ${Poco_Data_INCLUDE_DIRS})
 endif()
 
 if (USE_POCO_DATAODBC)
     target_link_libraries (clickhouse_common_io ${Poco_Data_LIBRARY})
     target_link_libraries (dbms ${Poco_DataODBC_LIBRARY})
     if (NOT USE_INTERNAL_POCO_LIBRARY)
-        target_include_directories (dbms PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIRS})
+        target_include_directories (dbms SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIRS})
     endif()
 endif()
 
@@ -200,20 +200,20 @@ target_link_libraries (dbms ${Poco_Foundation_LIBRARY})
 
 if (USE_ICU)
     target_link_libraries (dbms ${ICU_LIBS})
-    target_include_directories (dbms PRIVATE ${ICU_INCLUDE_DIR})
+    target_include_directories (dbms SYSTEM PRIVATE ${ICU_INCLUDE_DIR})
 endif ()
 
 if (USE_CAPNP)
     target_link_libraries (dbms ${CAPNP_LIBRARY})
     if (NOT USE_INTERNAL_CAPNP_LIBRARY)
-        target_include_directories (dbms BEFORE PRIVATE ${CAPNP_INCLUDE_DIR})
+        target_include_directories (dbms SYSTEM BEFORE PRIVATE ${CAPNP_INCLUDE_DIR})
     endif ()
 endif ()
 
 if (USE_RDKAFKA)
     target_link_libraries (dbms ${RDKAFKA_LIBRARY})
     if (NOT USE_INTERNAL_RDKAFKA_LIBRARY)
-        target_include_directories (dbms BEFORE PRIVATE ${RDKAFKA_INCLUDE_DIR})
+        target_include_directories (dbms SYSTEM BEFORE PRIVATE ${RDKAFKA_INCLUDE_DIR})
     endif ()
 endif ()
 
@@ -221,21 +221,21 @@ target_link_libraries (dbms
     Threads::Threads
 )
 
-target_include_directories (dbms BEFORE PRIVATE ${DIVIDE_INCLUDE_DIR})
+target_include_directories (dbms SYSTEM BEFORE PRIVATE ${DIVIDE_INCLUDE_DIR})
-target_include_directories (dbms BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
+target_include_directories (dbms SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
 
 if (NOT USE_INTERNAL_LZ4_LIBRARY)
-    target_include_directories (dbms BEFORE PRIVATE ${LZ4_INCLUDE_DIR})
+    target_include_directories (dbms SYSTEM BEFORE PRIVATE ${LZ4_INCLUDE_DIR})
 endif ()
 if (NOT USE_INTERNAL_ZSTD_LIBRARY)
-    target_include_directories (dbms BEFORE PRIVATE ${ZSTD_INCLUDE_DIR})
+    target_include_directories (dbms SYSTEM BEFORE PRIVATE ${ZSTD_INCLUDE_DIR})
 endif ()
 
 target_include_directories (dbms PUBLIC ${DBMS_INCLUDE_DIR})
 target_include_directories (clickhouse_common_io PUBLIC ${DBMS_INCLUDE_DIR})
-target_include_directories (clickhouse_common_io PUBLIC ${PCG_RANDOM_INCLUDE_DIR})
+target_include_directories (clickhouse_common_io SYSTEM PUBLIC ${PCG_RANDOM_INCLUDE_DIR})
-target_include_directories (clickhouse_common_io PUBLIC ${Poco_DataODBC_INCLUDE_DIRS})
+target_include_directories (clickhouse_common_io SYSTEM PUBLIC ${Poco_DataODBC_INCLUDE_DIRS})
-target_include_directories (clickhouse_common_io BEFORE PUBLIC ${DOUBLE_CONVERSION_INCLUDE_DIR})
+target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${DOUBLE_CONVERSION_INCLUDE_DIR})
 
 # also for copy_headers.sh:
 target_include_directories (clickhouse_common_io BEFORE PRIVATE ${COMMON_INCLUDE_DIR})
@@ -252,7 +252,7 @@ if (ENABLE_TESTS)
         add_subdirectory(${ClickHouse_SOURCE_DIR}/contrib/googletest/googletest ${CMAKE_CURRENT_BINARY_DIR}/googletest)
         # avoid problems with <regexp.h>
         target_compile_definitions (gtest INTERFACE GTEST_HAS_POSIX_RE=0)
-        target_include_directories (gtest INTERFACE ${ClickHouse_SOURCE_DIR}/contrib/googletest/include)
+        target_include_directories (gtest SYSTEM INTERFACE ${ClickHouse_SOURCE_DIR}/contrib/googletest/include)
     endif ()
 
     macro(grep_gtest_sources BASE_DIR DST_VAR)

dbms/programs/CMakeLists.txt

@@ -15,7 +15,6 @@ option (ENABLE_CLICKHOUSE_FORMAT "Enable format" ${ENABLE_CLICKHOUSE_ALL})
 
 configure_file (config_tools.h.in ${CMAKE_CURRENT_BINARY_DIR}/config_tools.h)
 
-add_subdirectory (clang)
 add_subdirectory (server)
 add_subdirectory (client)
 add_subdirectory (local)
@@ -25,6 +24,7 @@ add_subdirectory (extract-from-config)
 add_subdirectory (compressor)
 add_subdirectory (copier)
 add_subdirectory (format)
+add_subdirectory (clang)
 
 if (CLICKHOUSE_SPLIT_BINARY)
     set (CLICKHOUSE_ALL_TARGETS clickhouse-server clickhouse-client clickhouse-local clickhouse-benchmark clickhouse-performance-test
@@ -139,16 +139,6 @@ else ()
 
 endif ()
 
-if (USE_EMBEDDED_COMPILER)
+if (USE_EMBEDDED_COMPILER AND ENABLE_CLICKHOUSE_SERVER)
-    add_custom_target(copy-headers ALL env CLANG=${CMAKE_CURRENT_BINARY_DIR}/clickhouse-clang BUILD_PATH=${ClickHouse_BINARY_DIR} DESTDIR=${ClickHouse_SOURCE_DIR} ${ClickHouse_SOURCE_DIR}/copy_headers.sh ${ClickHouse_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/headers DEPENDS clickhouse-clang WORKING_DIRECTORY ${ClickHouse_SOURCE_DIR} SOURCES ${ClickHouse_SOURCE_DIR}/copy_headers.sh)
-    install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/headers DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/clickhouse COMPONENT clickhouse)
-
-    if (USE_INTERNAL_LLVM_LIBRARY)
-        set(CLANG_HEADERS_DIR "${ClickHouse_SOURCE_DIR}/contrib/llvm/clang/lib/Headers")
-        set(CLANG_HEADERS_DEST "${CMAKE_CURRENT_BINARY_DIR}/headers/usr/local/lib/clang/${LLVM_VERSION}/include") # original: ${LLVM_LIBRARY_OUTPUT_INTDIR}/clang/${CLANG_VERSION}/include
-        add_custom_target(copy-headers-clang ALL ${CMAKE_COMMAND} -E make_directory ${CLANG_HEADERS_DEST} && ${CMAKE_COMMAND} -E copy_if_different ${CLANG_HEADERS_DIR}/* ${CLANG_HEADERS_DEST} )
-        add_dependencies(copy-headers copy-headers-clang)
-    endif ()
-
     add_dependencies(clickhouse-server copy-headers)
 endif ()

dbms/programs/benchmark/CMakeLists.txt

@@ -1,6 +1,6 @@
 add_library (clickhouse-benchmark-lib ${SPLIT_SHARED} Benchmark.cpp)
 target_link_libraries (clickhouse-benchmark-lib clickhouse-client-lib clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
-target_include_directories (clickhouse-benchmark-lib PRIVATE ${PCG_RANDOM_INCLUDE_DIR})
+target_include_directories (clickhouse-benchmark-lib SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR})
 
 if (CLICKHOUSE_SPLIT_BINARY)
     add_executable (clickhouse-benchmark clickhouse-benchmark.cpp)

dbms/programs/clang/CMakeLists.txt

@@ -13,3 +13,18 @@ if (CLICKHOUSE_SPLIT_BINARY)
         set_target_properties(clickhouse-clang clickhouse-lld PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..)
     endif ()
 endif ()
+
+set(TMP_HEADERS_DIR "${CMAKE_CURRENT_BINARY_DIR}/headers")
+# Make and install empty dir for debian package if compiler disabled
+add_custom_target(make-headers-directory ALL COMMAND ${CMAKE_COMMAND} -E make_directory ${TMP_HEADERS_DIR})
+install(DIRECTORY ${TMP_HEADERS_DIR} DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/clickhouse COMPONENT clickhouse)
+if (USE_EMBEDDED_COMPILER)
+    add_custom_target(copy-headers ALL env CLANG=${CMAKE_CURRENT_BINARY_DIR}/../clickhouse-clang BUILD_PATH=${ClickHouse_BINARY_DIR} DESTDIR=${ClickHouse_SOURCE_DIR} ${ClickHouse_SOURCE_DIR}/copy_headers.sh ${ClickHouse_SOURCE_DIR} ${TMP_HEADERS_DIR} DEPENDS clickhouse-clang WORKING_DIRECTORY ${ClickHouse_SOURCE_DIR} SOURCES ${ClickHouse_SOURCE_DIR}/copy_headers.sh)
+
+    if (USE_INTERNAL_LLVM_LIBRARY)
+        set(CLANG_HEADERS_DIR "${ClickHouse_SOURCE_DIR}/contrib/llvm/clang/lib/Headers")
+        set(CLANG_HEADERS_DEST "${TMP_HEADERS_DIR}/usr/local/lib/clang/${LLVM_VERSION}/include") # original: ${LLVM_LIBRARY_OUTPUT_INTDIR}/clang/${CLANG_VERSION}/include
+        add_custom_target(copy-headers-clang ALL ${CMAKE_COMMAND} -E make_directory ${CLANG_HEADERS_DEST} && ${CMAKE_COMMAND} -E copy_if_different ${CLANG_HEADERS_DIR}/* ${CLANG_HEADERS_DEST} )
+        add_dependencies(copy-headers copy-headers-clang)
+    endif ()
+endif ()

dbms/programs/clang/Compiler-5.0.0/CMakeLists.txt

@@ -12,7 +12,7 @@ llvm_libs_all(REQUIRED_LLVM_LIBRARIES)
 
 message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}")
 
-target_include_directories(clickhouse-compiler-lib PRIVATE ${LLVM_INCLUDE_DIRS})
+target_include_directories(clickhouse-compiler-lib SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS})
 
 # This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory.
 

dbms/programs/clang/Compiler-6.0.0/CMakeLists.txt

@@ -12,7 +12,7 @@ llvm_libs_all(REQUIRED_LLVM_LIBRARIES)
 
 message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}")
 
-target_include_directories(clickhouse-compiler-lib PRIVATE ${LLVM_INCLUDE_DIRS})
+target_include_directories(clickhouse-compiler-lib SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS})
 
 # This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory.
 

dbms/programs/clang/Compiler-7.0.0/CMakeLists.txt

@@ -12,7 +12,7 @@ llvm_libs_all(REQUIRED_LLVM_LIBRARIES)
 
 message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}")
 
-target_include_directories(clickhouse-compiler-lib PRIVATE ${LLVM_INCLUDE_DIRS})
+target_include_directories(clickhouse-compiler-lib SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS})
 
 # This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory.
 

dbms/programs/client/CMakeLists.txt

@@ -1,6 +1,6 @@
 add_library (clickhouse-client-lib Client.cpp)
 target_link_libraries (clickhouse-client-lib clickhouse_functions clickhouse_aggregate_functions ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY})
-target_include_directories (clickhouse-client-lib PRIVATE ${READLINE_INCLUDE_DIR})
+target_include_directories (clickhouse-client-lib SYSTEM PRIVATE ${READLINE_INCLUDE_DIR})
 
 if (CLICKHOUSE_SPLIT_BINARY)
     add_executable (clickhouse-client clickhouse-client.cpp)

dbms/programs/main.cpp

@@ -34,7 +34,7 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv);
 #if ENABLE_CLICKHOUSE_PERFORMANCE
 int mainEntryClickHousePerformanceTest(int argc, char ** argv);
 #endif
-#if ENABLE_CLICKHOUSE_extract-from-config
+#if ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG
 int mainEntryClickHouseExtractFromConfig(int argc, char ** argv);
 #endif
 #if ENABLE_CLICKHOUSE_COMPRESSOR
@@ -76,9 +76,13 @@ std::pair<const char *, MainFunc> clickhouse_applications[] =
 #if ENABLE_CLICKHOUSE_PERFORMANCE
     {"performance-test", mainEntryClickHousePerformanceTest},
 #endif
-#if ENABLE_CLICKHOUSE_TOOLS
+#if ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG
     {"extract-from-config", mainEntryClickHouseExtractFromConfig},
+#endif
+#if ENABLE_CLICKHOUSE_COMPRESSOR
     {"compressor", mainEntryClickHouseCompressor},
+#endif
+#if ENABLE_CLICKHOUSE_FORMAT
     {"format", mainEntryClickHouseFormat},
 #endif
 #if ENABLE_CLICKHOUSE_COPIER

dbms/programs/performance-test/CMakeLists.txt

@@ -1,6 +1,6 @@
 add_library (clickhouse-performance-test-lib ${SPLIT_SHARED} PerformanceTest.cpp)
 target_link_libraries (clickhouse-performance-test-lib clickhouse_common_io dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
-target_include_directories (clickhouse-performance-test-lib PRIVATE ${PCG_RANDOM_INCLUDE_DIR})
+target_include_directories (clickhouse-performance-test-lib SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR})
 
 if (CLICKHOUSE_SPLIT_BINARY)
     add_executable (clickhouse-performance-test clickhouse-performance-test.cpp)

dbms/programs/server/CMakeLists.txt

@@ -20,9 +20,10 @@ if (CLICKHOUSE_SPLIT_BINARY)
     install (TARGETS clickhouse-server ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
 endif ()
 
-
+if (NOT APPLE AND NOT ARCH_FREEBSD)
-set (GLIBC_MAX_REQUIRED 2.4)
+    set (GLIBC_MAX_REQUIRED 2.4)
-add_test(NAME GLIBC_required_version COMMAND bash -c "readelf -s ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-server | grep '@GLIBC' | grep -oP 'GLIBC_[\\d\\.]+' | sort | uniq | sort -r | perl -lnE 'exit 1 if $_ gt q{GLIBC_${GLIBC_MAX_REQUIRED}}'")
+    add_test(NAME GLIBC_required_version COMMAND bash -c "readelf -s ${CMAKE_CURRENT_BINARY_DIR}/../clickhouse-server | grep '@GLIBC' | grep -oP 'GLIBC_[\\d\\.]+' | sort | uniq | sort -r | perl -lnE 'exit 1 if $_ gt q{GLIBC_${GLIBC_MAX_REQUIRED}}'")
+endif ()
 
 install (
     FILES config.xml users.xml

dbms/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h

@@ -204,6 +204,7 @@ public:
             case StatisticsFunctionKind::covarSamp: return "covarSamp";
             case StatisticsFunctionKind::corr: return "corr";
         }
+        __builtin_unreachable();
     }
 
     DataTypePtr getReturnType() const override

dbms/src/Client/Connection.cpp

@@ -18,6 +18,7 @@
 #include <Common/NetException.h>
 #include <Common/CurrentMetrics.h>
 #include <Common/DNSResolver.h>
+#include <Common/StringUtils/StringUtils.h>
 #include <Interpreters/ClientInfo.h>
 
 #include <Common/config.h>
@@ -42,6 +43,7 @@ namespace ErrorCodes
     extern const int UNEXPECTED_PACKET_FROM_SERVER;
     extern const int UNKNOWN_PACKET_FROM_SERVER;
     extern const int SUPPORT_IS_DISABLED;
+    extern const int BAD_ARGUMENTS;
 }
 
 
@@ -121,7 +123,27 @@ void Connection::disconnect()
 
 void Connection::sendHello()
 {
-    //LOG_TRACE(log_wrapper.get(), "Sending hello");
+    /** Disallow control characters in user controlled parameters
+      *  to mitigate the possibility of SSRF.
+      * The user may do server side requests with 'remote' table function.
+      * Malicious user with full r/w access to ClickHouse
+      *  may use 'remote' table function to forge requests
+      *  to another services in the network other than ClickHouse (examples: SMTP).
+      * Limiting number of possible characters in user-controlled part of handshake
+      *  will mitigate this possibility but doesn't solve it completely.
+      */
+    auto has_control_character = [](const std::string & s)
+    {
+        for (auto c : s)
+            if (isControlASCII(c))
+                return true;
+        return false;
+    };
+
+    if (has_control_character(default_database)
+        || has_control_character(user)
+        || has_control_character(password))
+        throw Exception("Parameters 'default_database', 'user' and 'password' must not contain ASCII control characters", ErrorCodes::BAD_ARGUMENTS);
 
     writeVarUInt(Protocol::Client::Hello, *out);
     writeStringBinary((DBMS_NAME " ") + client_name, *out);

dbms/src/Common/StringUtils/StringUtils.h

@@ -107,6 +107,11 @@ inline bool isWhitespaceASCII(char c)
     return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v';
 }
 
+inline bool isControlASCII(char c)
+{
+    return c >= 0 && c <= 31;
+}
+
 /// Works assuming isAlphaASCII.
 inline char toLowerIfAlphaASCII(char c)
 {

dbms/src/Core/tests/CMakeLists.txt

@@ -3,7 +3,7 @@ target_link_libraries (exception clickhouse_common_io)
 
 add_executable (string_pool string_pool.cpp)
 target_link_libraries (string_pool clickhouse_common_io)
-target_include_directories (string_pool BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
+target_include_directories (string_pool SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
 
 add_executable (field field.cpp)
 target_link_libraries (field dbms)

dbms/src/DataStreams/FormatFactory.cpp

@@ -119,7 +119,6 @@ BlockInputStreamPtr FormatFactory::getInput(const String & name, ReadBuffer & bu
         || name == "PrettyCompactNoEscapes"
         || name == "PrettySpaceNoEscapes"
         || name == "Vertical"
-        || name == "VerticalRaw"
         || name == "Null"
         || name == "JSON"
         || name == "JSONCompact"
@@ -180,9 +179,6 @@ static BlockOutputStreamPtr getOutputImpl(const String & name, WriteBuffer & buf
     else if (name == "Vertical")
         return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<VerticalRowOutputStream>(
             buf, sample, settings.output_format_pretty_max_rows), sample);
-    else if (name == "VerticalRaw")
-        return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<VerticalRawRowOutputStream>(
-            buf, sample, settings.output_format_pretty_max_rows), sample);
     else if (name == "Values")
         return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<ValuesRowOutputStream>(buf), sample);
     else if (name == "JSON")

dbms/src/DataStreams/PrettyBlockOutputStream.cpp

@@ -55,7 +55,7 @@ void PrettyBlockOutputStream::calculateWidths(const Block & block, WidthsPerColu
         {
             {
                 WriteBufferFromString out(serialized_value);
-                elem.type->serializeTextEscaped(*elem.column, j, out);
+                elem.type->serializeText(*elem.column, j, out);
             }
 
             widths[i][j] = UTF8::countCodePoints(reinterpret_cast<const UInt8 *>(serialized_value.data()), serialized_value.size());
@@ -64,13 +64,7 @@ void PrettyBlockOutputStream::calculateWidths(const Block & block, WidthsPerColu
 
         /// And also calculate widths for names of columns.
         {
-            /// We need to obtain length in escaped form.
+            name_widths[i] = UTF8::countCodePoints(reinterpret_cast<const UInt8 *>(elem.name.data()), elem.name.size());
-            {
-                WriteBufferFromString out(serialized_value);
-                writeEscapedString(elem.name, out);
-            }
-
-            name_widths[i] = UTF8::countCodePoints(reinterpret_cast<const UInt8 *>(serialized_value.data()), serialized_value.size());
             max_widths[i] = std::max(max_widths[i], name_widths[i]);
         }
     }
@@ -151,11 +145,11 @@ void PrettyBlockOutputStream::write(const Block & block)
             for (size_t k = 0; k < max_widths[i] - name_widths[i]; ++k)
                 writeChar(' ', ostr);
 
-            writeEscapedString(col.name, ostr);
+            writeString(col.name, ostr);
         }
         else
         {
-            writeEscapedString(col.name, ostr);
+            writeString(col.name, ostr);
 
             for (size_t k = 0; k < max_widths[i] - name_widths[i]; ++k)
                 writeChar(' ', ostr);
@@ -203,11 +197,11 @@ void PrettyBlockOutputStream::writeValueWithPadding(const ColumnWithTypeAndName
     if (elem.type->shouldAlignRightInPrettyFormats())
     {
         writePadding();
-        elem.type->serializeTextEscaped(*elem.column.get(), row_num, ostr);
+        elem.type->serializeText(*elem.column.get(), row_num, ostr);
     }
     else
     {
-        elem.type->serializeTextEscaped(*elem.column.get(), row_num, ostr);
+        elem.type->serializeText(*elem.column.get(), row_num, ostr);
         writePadding();
     }
 }

dbms/src/DataStreams/PrettyCompactBlockOutputStream.cpp

@@ -34,7 +34,7 @@ void PrettyCompactBlockOutputStream::writeHeader(
 
             if (!no_escapes)
                 writeCString("\033[1m", ostr);
-            writeEscapedString(col.name, ostr);
+            writeString(col.name, ostr);
             if (!no_escapes)
                 writeCString("\033[0m", ostr);
         }
@@ -42,7 +42,7 @@ void PrettyCompactBlockOutputStream::writeHeader(
         {
             if (!no_escapes)
                 writeCString("\033[1m", ostr);
-            writeEscapedString(col.name, ostr);
+            writeString(col.name, ostr);
             if (!no_escapes)
                 writeCString("\033[0m", ostr);
 

dbms/src/DataStreams/PrettySpaceBlockOutputStream.cpp

@@ -44,7 +44,7 @@ void PrettySpaceBlockOutputStream::write(const Block & block)
 
             if (!no_escapes)
                 writeCString("\033[1m", ostr);
-            writeEscapedString(col.name, ostr);
+            writeString(col.name, ostr);
             if (!no_escapes)
                 writeCString("\033[0m", ostr);
         }
@@ -52,7 +52,7 @@ void PrettySpaceBlockOutputStream::write(const Block & block)
         {
             if (!no_escapes)
                 writeCString("\033[1m", ostr);
-            writeEscapedString(col.name, ostr);
+            writeString(col.name, ostr);
             if (!no_escapes)
                 writeCString("\033[0m", ostr);
 

dbms/src/DataStreams/VerticalRowOutputStream.cpp

@@ -26,13 +26,7 @@ VerticalRowOutputStream::VerticalRowOutputStream(
         /// Note that number of code points is just a rough approximation of visible string width.
         const String & name = sample.getByPosition(i).name;
 
-        {
+        name_widths[i] = UTF8::countCodePoints(reinterpret_cast<const UInt8 *>(name.data()), name.size());
-            /// We need to obtain length in escaped form.
-            WriteBufferFromString out(serialized_value);
-            writeEscapedString(name, out);
-        }
-
-        name_widths[i] = UTF8::countCodePoints(reinterpret_cast<const UInt8 *>(serialized_value.data()), serialized_value.size());
 
         if (name_widths[i] > max_name_width)
             max_name_width = name_widths[i];
@@ -42,7 +36,7 @@ VerticalRowOutputStream::VerticalRowOutputStream(
     for (size_t i = 0; i < columns; ++i)
     {
         WriteBufferFromString out(names_and_paddings[i]);
-        writeEscapedString(sample.getByPosition(i).name, out);
+        writeString(sample.getByPosition(i).name, out);
         writeCString(": ", out);
     }
 
@@ -71,11 +65,6 @@ void VerticalRowOutputStream::writeField(const IColumn & column, const IDataType
 
 
 void VerticalRowOutputStream::writeValue(const IColumn & column, const IDataType & type, size_t row_num) const
-{
-    type.serializeTextEscaped(column, row_num, ostr);
-}
-
-void VerticalRawRowOutputStream::writeValue(const IColumn & column, const IDataType & type, size_t row_num) const
 {
     type.serializeText(column, row_num, ostr);
 }

dbms/src/DataStreams/VerticalRowOutputStream.h

@@ -51,17 +51,5 @@ protected:
     Block extremes;
 };
 
-
-/** Same but values are printed without escaping.
-  */
-class VerticalRawRowOutputStream final : public VerticalRowOutputStream
-{
-public:
-    using VerticalRowOutputStream::VerticalRowOutputStream;
-
-protected:
-    void writeValue(const IColumn & column, const IDataType & type, size_t row_num) const override;
-};
-
 }
 

dbms/src/DataStreams/tests/CMakeLists.txt

@@ -1,5 +1,3 @@
-include_directories (${CMAKE_CURRENT_BINARY_DIR})
-
 set(SRCS )
 
 add_executable (tab_separated_streams tab_separated_streams.cpp ${SRCS})

dbms/src/DataTypes/DataTypeNullable.cpp

@@ -242,8 +242,14 @@ void DataTypeNullable::serializeText(const IColumn & column, size_t row_num, Wri
 {
     const ColumnNullable & col = static_cast<const ColumnNullable &>(column);
 
+    /// In simple text format (like 'Pretty' format) (these formats are suitable only for output and cannot be parsed back),
+    ///  data is printed without escaping.
+    /// It makes theoretically impossible to distinguish between NULL and some string value, regardless on how do we print NULL.
+    /// For this reason, we output NULL in a bit strange way.
+    /// This assumes UTF-8 and proper font support. This is Ok, because Pretty formats are "presentational", not for data exchange.
+
     if (col.isNullAt(row_num))
-        writeCString("NULL", ostr);
+        writeCString("ᴺᵁᴸᴸ", ostr);
     else
         nested_data_type->serializeText(col.getNestedColumn(), row_num, ostr);
 }

dbms/src/DataTypes/tests/CMakeLists.txt

@@ -1,5 +1,3 @@
-include_directories (${CMAKE_CURRENT_BINARY_DIR})
-
 set(SRCS )
 
 add_executable (data_types_number_fixed data_types_number_fixed.cpp ${SRCS})

dbms/src/Functions/CMakeLists.txt

@@ -88,9 +88,9 @@ add_library(clickhouse_functions ${clickhouse_functions_sources})
 
 target_link_libraries(clickhouse_functions PUBLIC dbms PRIVATE libconsistent-hashing ${FARMHASH_LIBRARIES} ${METROHASH_LIBRARIES})
 
-target_include_directories (clickhouse_functions BEFORE PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/libfarmhash)
+target_include_directories (clickhouse_functions SYSTEM BEFORE PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/libfarmhash)
-target_include_directories (clickhouse_functions BEFORE PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/libmetrohash/src)
+target_include_directories (clickhouse_functions SYSTEM BEFORE PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/libmetrohash/src)
-target_include_directories (clickhouse_functions BEFORE PUBLIC ${DIVIDE_INCLUDE_DIR})
+target_include_directories (clickhouse_functions SYSTEM BEFORE PUBLIC ${DIVIDE_INCLUDE_DIR})
 
 if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" OR CMAKE_BUILD_TYPE_UC STREQUAL "MINSIZEREL")
     # Won't generate debug info for files with heavy template instantiation to achieve faster linking and lower size.
@@ -99,11 +99,11 @@ endif ()
 
 if (USE_ICU)
     #target_link_libraries (clickhouse_functions ${ICU_LIBS})
-    target_include_directories (clickhouse_functions PRIVATE ${ICU_INCLUDE_DIR})
+    target_include_directories (clickhouse_functions SYSTEM PRIVATE ${ICU_INCLUDE_DIR})
 endif ()
 
 if (USE_VECTORCLASS)
-    target_include_directories (clickhouse_functions BEFORE PUBLIC ${VECTORCLASS_INCLUDE_DIR})
+    target_include_directories (clickhouse_functions SYSTEM BEFORE PUBLIC ${VECTORCLASS_INCLUDE_DIR})
 endif ()
 
 if (ENABLE_TESTS)
@@ -111,5 +111,5 @@ if (ENABLE_TESTS)
 endif ()
 
 if (USE_EMBEDDED_COMPILER)
-    target_include_directories (clickhouse_functions BEFORE PUBLIC ${LLVM_INCLUDE_DIRS})
+    target_include_directories (clickhouse_functions SYSTEM BEFORE PUBLIC ${LLVM_INCLUDE_DIRS})
 endif ()

dbms/src/Functions/FunctionsMiscellaneous.cpp

@@ -1688,7 +1688,7 @@ void FunctionVisibleWidth::executeImpl(Block & block, const ColumnNumbers & argu
     {
         {
             WriteBufferFromString out(tmp);
-            src.type->serializeTextEscaped(*src.column, i, out);
+            src.type->serializeText(*src.column, i, out);
         }
 
         res_data[i] = UTF8::countCodePoints(reinterpret_cast<const UInt8 *>(tmp.data()), tmp.size());

dbms/src/Functions/FunctionsRound.h

@@ -219,6 +219,8 @@ struct IntegerRoundingComputation
                     x = -x;
                 return x;
             }
+            default:
+                __builtin_unreachable();
         }
     }
 
@@ -232,6 +234,8 @@ struct IntegerRoundingComputation
                 return x;
             case ScaleMode::Negative:
                 return computeImpl(x, scale);
+            default:
+                __builtin_unreachable();
         }
     }
 

dbms/src/Functions/FunctionsString.cpp

@@ -1084,6 +1084,125 @@ private:
 };
 
 
+struct NameStartsWith
+{
+    static constexpr auto name = "startsWith";
+};
+struct NameEndsWith
+{
+    static constexpr auto name = "endsWith";
+};
+
+template <typename Name>
+class FunctionStartsEndsWith : public IFunction
+{
+public:
+    static constexpr auto name = Name::name;
+    static FunctionPtr create(const Context &)
+    {
+        return std::make_shared<FunctionStartsEndsWith>();
+    }
+
+    String getName() const override
+    {
+        return name;
+    }
+
+    size_t getNumberOfArguments() const override
+    {
+        return 2;
+    }
+
+    bool useDefaultImplementationForConstants() const override
+    {
+        return true;
+    }
+
+    DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
+    {
+        if (!arguments[0]->isStringOrFixedString())
+            throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+        if (!arguments[1]->isStringOrFixedString())
+            throw Exception("Illegal type " + arguments[1]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+        return std::make_shared<DataTypeNumber<UInt8>>();
+    }
+
+    void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override
+    {
+        const IColumn * haystack_column = block.getByPosition(arguments[0]).column.get();
+        const IColumn * needle_column = block.getByPosition(arguments[1]).column.get();
+
+        auto col_res = ColumnVector<UInt8>::create();
+        typename ColumnVector<UInt8>::Container & vec_res = col_res->getData();
+
+        vec_res.resize(input_rows_count);
+
+        if (const ColumnString * haystack = checkAndGetColumn<ColumnString>(haystack_column))
+            dispatch<StringSource>(StringSource(*haystack), needle_column, vec_res);
+        else if (const ColumnFixedString * haystack = checkAndGetColumn<ColumnFixedString>(haystack_column))
+            dispatch<FixedStringSource>(FixedStringSource(*haystack), needle_column, vec_res);
+        else if (const ColumnConst * haystack = checkAndGetColumnConst<ColumnString>(haystack_column))
+            dispatch<ConstSource<StringSource>>(ConstSource<StringSource>(*haystack), needle_column, vec_res);
+        else if (const ColumnConst * haystack = checkAndGetColumnConst<ColumnFixedString>(haystack_column))
+            dispatch<ConstSource<FixedStringSource>>(ConstSource<FixedStringSource>(*haystack), needle_column, vec_res);
+        else
+            throw Exception("Illegal combination of columns as arguments of function " + getName(), ErrorCodes::ILLEGAL_COLUMN);
+
+        block.getByPosition(result).column = std::move(col_res);
+    }
+
+private:
+    template <typename HaystackSource>
+    void dispatch(HaystackSource haystack_source, const IColumn * needle_column, PaddedPODArray<UInt8> & res_data) const
+    {
+        if (const ColumnString * needle = checkAndGetColumn<ColumnString>(needle_column))
+            execute<HaystackSource, StringSource>(haystack_source, StringSource(*needle), res_data);
+        else if (const ColumnFixedString * needle = checkAndGetColumn<ColumnFixedString>(needle_column))
+            execute<HaystackSource, FixedStringSource>(haystack_source, FixedStringSource(*needle), res_data);
+        else if (const ColumnConst * needle = checkAndGetColumnConst<ColumnString>(needle_column))
+            execute<HaystackSource, ConstSource<StringSource>>(haystack_source, ConstSource<StringSource>(*needle), res_data);
+        else if (const ColumnConst * needle = checkAndGetColumnConst<ColumnFixedString>(needle_column))
+            execute<HaystackSource, ConstSource<FixedStringSource>>(haystack_source, ConstSource<FixedStringSource>(*needle), res_data);
+        else
+            throw Exception("Illegal combination of columns as arguments of function " + getName(), ErrorCodes::ILLEGAL_COLUMN);
+    }
+
+    template <typename HaystackSource, typename NeedleSource>
+    static void execute(HaystackSource haystack_source, NeedleSource needle_source, PaddedPODArray<UInt8> & res_data)
+    {
+        size_t row_num = 0;
+
+        while (!haystack_source.isEnd())
+        {
+            auto haystack = haystack_source.getWhole();
+            auto needle = needle_source.getWhole();
+
+            if (needle.size > haystack.size)
+            {
+                res_data[row_num] = false;
+            }
+            else
+            {
+                if constexpr (std::is_same_v<Name, NameStartsWith>)
+                {
+                    res_data[row_num] = StringRef(haystack.data, needle.size) == StringRef(needle.data, needle.size);
+                }
+                else    /// endsWith
+                {
+                    res_data[row_num] = StringRef(haystack.data + haystack.size - needle.size, needle.size) == StringRef(needle.data, needle.size);
+                }
+            }
+
+            haystack_source.next();
+            needle_source.next();
+            ++row_num;
+        }
+    }
+};
+
+
 struct NameEmpty
 {
     static constexpr auto name = "empty";
@@ -1121,6 +1240,7 @@ struct NameConcatAssumeInjective
     static constexpr auto name = "concatAssumeInjective";
 };
 
+
 using FunctionEmpty = FunctionStringOrArrayToT<EmptyImpl<false>, NameEmpty, UInt8>;
 using FunctionNotEmpty = FunctionStringOrArrayToT<EmptyImpl<true>, NameNotEmpty, UInt8>;
 using FunctionLength = FunctionStringOrArrayToT<LengthImpl, NameLength, UInt64>;
@@ -1130,6 +1250,8 @@ using FunctionUpper = FunctionStringToString<LowerUpperImpl<'a', 'z'>, NameUpper
 using FunctionReverseUTF8 = FunctionStringToString<ReverseUTF8Impl, NameReverseUTF8, true>;
 using FunctionConcat = ConcatImpl<NameConcat, false>;
 using FunctionConcatAssumeInjective = ConcatImpl<NameConcatAssumeInjective, true>;
+using FunctionStartsWith = FunctionStartsEndsWith<NameStartsWith>;
+using FunctionEndsWith = FunctionStartsEndsWith<NameEndsWith>;
 
 
 void registerFunctionsString(FunctionFactory & factory)
@@ -1149,5 +1271,7 @@ void registerFunctionsString(FunctionFactory & factory)
     factory.registerFunction<FunctionSubstring>();
     factory.registerFunction<FunctionSubstringUTF8>();
     factory.registerFunction<FunctionAppendTrailingCharIfAbsent>();
+    factory.registerFunction<FunctionStartsWith>();
+    factory.registerFunction<FunctionEndsWith>();
 }
 }

dbms/src/Functions/tests/CMakeLists.txt

@@ -1,4 +1,2 @@
-include_directories (${CMAKE_CURRENT_BINARY_DIR})
-
 add_executable (number_traits number_traits.cpp)
 target_link_libraries (number_traits dbms)

dbms/src/IO/tests/CMakeLists.txt

@@ -1,5 +1,3 @@
-include_directories (${CMAKE_CURRENT_BINARY_DIR})
-
 add_executable (read_buffer read_buffer.cpp)
 target_link_libraries (read_buffer clickhouse_common_io)
 

dbms/src/Interpreters/Context.cpp

@@ -355,7 +355,7 @@ std::shared_ptr<Context> Context::acquireSession(const String & session_id, std:
         if (session_check)
             throw Exception("Session not found.", ErrorCodes::SESSION_NOT_FOUND);
 
-        auto new_session = std::make_shared<Context>(*global_context);
+        auto new_session = std::make_shared<Context>(*this);
 
         new_session->scheduleCloseSession(key, timeout);
 

dbms/src/Interpreters/Context.h

@@ -6,6 +6,7 @@
 #include <memory>
 #include <mutex>
 #include <thread>
+#include <atomic>
 
 #include <common/MultiVersion.h>
 #include <Core/Types.h>

dbms/src/Interpreters/ExpressionAnalyzer.h

@@ -38,6 +38,8 @@ class ASTFunction;
 class ASTExpressionList;
 class ASTSelectQuery;
 
+struct ProjectionManipulatorBase;
+using ProjectionManipulatorPtr = std::shared_ptr<ProjectionManipulatorBase>;
 
 /** Information on what to do when executing a subquery in the [GLOBAL] IN/JOIN section.
   */

dbms/src/Interpreters/InterpreterShowTablesQuery.cpp

@@ -1,17 +1,14 @@
 #include <IO/ReadBufferFromString.h>
-
 #include <Parsers/ASTShowTablesQuery.h>
 #include <Parsers/ASTIdentifier.h>
-
 #include <Interpreters/Context.h>
 #include <Interpreters/executeQuery.h>
 #include <Interpreters/InterpreterShowTablesQuery.h>
-
 #include <Common/typeid_cast.h>
-
 #include <iomanip>
 #include <sstream>
 
+
 namespace DB
 {
 

dbms/src/Interpreters/tests/CMakeLists.txt

@@ -11,7 +11,7 @@ add_executable (aggregate aggregate.cpp)
 target_link_libraries (aggregate dbms)
 
 add_executable (hash_map hash_map.cpp)
-target_include_directories (hash_map BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
+target_include_directories (hash_map SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
 target_link_libraries (hash_map dbms)
 
 add_executable (hash_map3 hash_map3.cpp)
@@ -26,15 +26,15 @@ target_link_libraries (hash_map_string_2 dbms)
 
 add_executable (hash_map_string_3 hash_map_string_3.cpp)
 target_link_libraries (hash_map_string_3 dbms ${FARMHASH_LIBRARIES} ${METROHASH_LIBRARIES})
-target_include_directories (hash_map_string_3 BEFORE PRIVATE ${ClickHouse_SOURCE_DIR}/contrib/libfarmhash)
+target_include_directories (hash_map_string_3 SYSTEM BEFORE PRIVATE ${ClickHouse_SOURCE_DIR}/contrib/libfarmhash)
-target_include_directories (hash_map_string_3 BEFORE PRIVATE ${ClickHouse_SOURCE_DIR}/contrib/libmetrohash/src)
+target_include_directories (hash_map_string_3 SYSTEM BEFORE PRIVATE ${ClickHouse_SOURCE_DIR}/contrib/libmetrohash/src)
 
 add_executable (hash_map_string_small hash_map_string_small.cpp)
-target_include_directories (hash_map_string_small BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
+target_include_directories (hash_map_string_small SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
 target_link_libraries (hash_map_string_small dbms)
 
 add_executable (two_level_hash_map two_level_hash_map.cpp)
-target_include_directories (two_level_hash_map BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
+target_include_directories (two_level_hash_map SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
 target_link_libraries (two_level_hash_map dbms)
 
 add_executable (compiler_test compiler_test.cpp)

dbms/src/Parsers/tests/CMakeLists.txt

@@ -1,5 +1,3 @@
-include_directories (${CMAKE_CURRENT_BINARY_DIR})
-
 set(SRCS )
 
 add_executable (lexer lexer.cpp ${SRCS})

dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp

@@ -22,11 +22,11 @@ MergeTreeBaseBlockInputStream::MergeTreeBaseBlockInputStream(
     MergeTreeData & storage,
     const ExpressionActionsPtr & prewhere_actions,
     const String & prewhere_column_name,
-    size_t max_block_size_rows,
+    UInt64 max_block_size_rows,
-    size_t preferred_block_size_bytes,
+    UInt64 preferred_block_size_bytes,
-    size_t preferred_max_column_in_block_size_bytes,
+    UInt64 preferred_max_column_in_block_size_bytes,
-    size_t min_bytes_to_use_direct_io,
+    UInt64 min_bytes_to_use_direct_io,
-    size_t max_read_buffer_size,
+    UInt64 max_read_buffer_size,
     bool use_uncompressed_cache,
     bool save_marks_in_cache,
     const Names & virt_column_names)
@@ -89,7 +89,7 @@ Block MergeTreeBaseBlockInputStream::readFromPart()
 
         /// Calculates number of rows will be read using preferred_block_size_bytes.
         /// Can't be less than index_granularity.
-        size_t rows_to_read = task.size_predictor->estimateNumRows(preferred_block_size_bytes);
+        UInt64 rows_to_read = task.size_predictor->estimateNumRows(preferred_block_size_bytes);
         if (!rows_to_read)
             return rows_to_read;
         rows_to_read = std::max(index_granularity, rows_to_read);
@@ -97,21 +97,21 @@ Block MergeTreeBaseBlockInputStream::readFromPart()
         if (preferred_max_column_in_block_size_bytes)
         {
             /// Calculates number of rows will be read using preferred_max_column_in_block_size_bytes.
-            size_t rows_to_read_for_max_size_column
+            UInt64 rows_to_read_for_max_size_column
                 = task.size_predictor->estimateNumRowsForMaxSizeColumn(preferred_max_column_in_block_size_bytes);
             double filtration_ratio = std::max(min_filtration_ratio, 1.0 - task.size_predictor->filtered_rows_ratio);
             auto rows_to_read_for_max_size_column_with_filtration
-                = static_cast<size_t>(rows_to_read_for_max_size_column / filtration_ratio);
+                = static_cast<UInt64>(rows_to_read_for_max_size_column / filtration_ratio);
 
             /// If preferred_max_column_in_block_size_bytes is used, number of rows to read can be less than index_granularity.
             rows_to_read = std::min(rows_to_read, rows_to_read_for_max_size_column_with_filtration);
         }
 
-        size_t unread_rows_in_current_granule = reader.numPendingRowsInCurrentGranule();
+        UInt64 unread_rows_in_current_granule = reader.numPendingRowsInCurrentGranule();
         if (unread_rows_in_current_granule >= rows_to_read)
             return rows_to_read;
 
-        size_t granule_to_read = (rows_to_read + reader.numReadRowsInCurrentGranule() + index_granularity / 2) / index_granularity;
+        UInt64 granule_to_read = (rows_to_read + reader.numReadRowsInCurrentGranule() + index_granularity / 2) / index_granularity;
         return index_granularity * granule_to_read - reader.numReadRowsInCurrentGranule();
     };
 
@@ -146,8 +146,8 @@ Block MergeTreeBaseBlockInputStream::readFromPart()
         }
     }
 
-    size_t recommended_rows = estimateNumRows(*task, task->range_reader);
+    UInt64 recommended_rows = estimateNumRows(*task, task->range_reader);
-    size_t rows_to_read = std::max(static_cast<decltype(max_block_size_rows)>(1), std::min(max_block_size_rows, recommended_rows));
+    UInt64 rows_to_read = std::max(UInt64(1), std::min(max_block_size_rows, recommended_rows));
 
     auto read_result = task->range_reader.read(rows_to_read, task->mark_ranges);
 
@@ -155,7 +155,7 @@ Block MergeTreeBaseBlockInputStream::readFromPart()
     if (read_result.block.rows() == 0)
         read_result.block.clear();
 
-    size_t num_filtered_rows = read_result.numReadRows() - read_result.block.rows();
+    UInt64 num_filtered_rows = read_result.numReadRows() - read_result.block.rows();
 
     progressImpl({ read_result.numReadRows(), read_result.numBytesRead() });
 

dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.h

@@ -20,11 +20,11 @@ public:
         MergeTreeData & storage,
         const ExpressionActionsPtr & prewhere_actions,
         const String & prewhere_column,
-        size_t max_block_size_rows,
+        UInt64 max_block_size_rows,
-        size_t preferred_block_size_bytes,
+        UInt64 preferred_block_size_bytes,
-        size_t preferred_max_column_in_block_size_bytes,
+        UInt64 preferred_max_column_in_block_size_bytes,
-        size_t min_bytes_to_use_direct_io,
+        UInt64 min_bytes_to_use_direct_io,
-        size_t max_read_buffer_size,
+        UInt64 max_read_buffer_size,
         bool use_uncompressed_cache,
         bool save_marks_in_cache = true,
         const Names & virt_column_names = {});
@@ -50,12 +50,12 @@ protected:
     ExpressionActionsPtr prewhere_actions;
     String prewhere_column_name;
 
-    size_t max_block_size_rows;
+    UInt64 max_block_size_rows;
-    size_t preferred_block_size_bytes;
+    UInt64 preferred_block_size_bytes;
-    size_t preferred_max_column_in_block_size_bytes;
+    UInt64 preferred_max_column_in_block_size_bytes;
 
-    size_t min_bytes_to_use_direct_io;
+    UInt64 min_bytes_to_use_direct_io;
-    size_t max_read_buffer_size;
+    UInt64 max_read_buffer_size;
 
     bool use_uncompressed_cache;
     bool save_marks_in_cache;
@@ -71,7 +71,7 @@ protected:
     MergeTreeReaderPtr reader;
     MergeTreeReaderPtr pre_reader;
 
-    size_t max_block_size_marks;
+    UInt64 max_block_size_marks;
 };
 
 }

dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp

@@ -977,11 +977,11 @@ ReplicatedMergeTreeQueue::Status ReplicatedMergeTreeQueue::getStatus() const
 
     res.inserts_in_queue = 0;
     res.merges_in_queue = 0;
-    res.mutations_in_queue = 0;
+    res.part_mutations_in_queue = 0;
     res.queue_oldest_time = 0;
     res.inserts_oldest_time = 0;
     res.merges_oldest_time = 0;
-    res.mutations_oldest_time = 0;
+    res.part_mutations_oldest_time = 0;
 
     for (const LogEntryPtr & entry : queue)
     {
@@ -1012,11 +1012,11 @@ ReplicatedMergeTreeQueue::Status ReplicatedMergeTreeQueue::getStatus() const
 
         if (entry->type == LogEntry::MUTATE_PART)
         {
-            ++res.mutations_in_queue;
+            ++res.part_mutations_in_queue;
 
-            if (entry->create_time && (!res.mutations_oldest_time || entry->create_time < res.mutations_oldest_time))
+            if (entry->create_time && (!res.part_mutations_oldest_time || entry->create_time < res.part_mutations_oldest_time))
             {
-                res.mutations_oldest_time = entry->create_time;
+                res.part_mutations_oldest_time = entry->create_time;
                 res.oldest_part_to_mutate_to = entry->new_part_name;
             }
         }

dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h

@@ -284,11 +284,11 @@ public:
         UInt32 queue_size;
         UInt32 inserts_in_queue;
         UInt32 merges_in_queue;
-        UInt32 mutations_in_queue;
+        UInt32 part_mutations_in_queue;
         UInt32 queue_oldest_time;
         UInt32 inserts_oldest_time;
         UInt32 merges_oldest_time;
-        UInt32 mutations_oldest_time;
+        UInt32 part_mutations_oldest_time;
         String oldest_part_to_get;
         String oldest_part_to_merge_to;
         String oldest_part_to_mutate_to;

dbms/src/Storages/MutationCommands.cpp

@@ -36,7 +36,7 @@ void MutationCommand::writeText(WriteBuffer & out) const
         {
             std::stringstream ss;
             formatAST(*predicate, ss, /* hilite = */ false, /* one_line = */ true);
-            out << "predicate: " << ss.str() << "\n";
+            out << "predicate: " << escape << ss.str() << "\n";
             break;
         }
         default:
@@ -54,7 +54,7 @@ void MutationCommand::readText(ReadBuffer & in)
         type = DELETE;
 
         String predicate_str;
-        in >> "predicate: " >> predicate_str >> "\n";
+        in >> "predicate: " >> escape >> predicate_str >> "\n";
         ParserExpressionWithOptionalAlias p_expr(false);
         predicate = parseQuery(
             p_expr, predicate_str.data(), predicate_str.data() + predicate_str.length(), "mutation predicate", 0);

dbms/src/Storages/System/StorageSystemReplicas.cpp

@@ -33,11 +33,14 @@ StorageSystemReplicas::StorageSystemReplicas(const std::string & name_)
         { "queue_size",                           std::make_shared<DataTypeUInt32>()   },
         { "inserts_in_queue",                     std::make_shared<DataTypeUInt32>()   },
         { "merges_in_queue",                      std::make_shared<DataTypeUInt32>()   },
+        { "part_mutations_in_queue",              std::make_shared<DataTypeUInt32>()   },
         { "queue_oldest_time",                    std::make_shared<DataTypeDateTime>() },
         { "inserts_oldest_time",                  std::make_shared<DataTypeDateTime>() },
         { "merges_oldest_time",                   std::make_shared<DataTypeDateTime>() },
+        { "part_mutations_oldest_time",           std::make_shared<DataTypeDateTime>() },
         { "oldest_part_to_get",                   std::make_shared<DataTypeString>()   },
         { "oldest_part_to_merge_to",              std::make_shared<DataTypeString>()   },
+        { "oldest_part_to_mutate_to",             std::make_shared<DataTypeString>()   },
         { "log_max_index",                        std::make_shared<DataTypeUInt64>()   },
         { "log_pointer",                          std::make_shared<DataTypeUInt64>()   },
         { "last_queue_update",                    std::make_shared<DataTypeDateTime>() },
@@ -140,11 +143,14 @@ BlockInputStreams StorageSystemReplicas::read(
         res_columns[col_num++]->insert(UInt64(status.queue.queue_size));
         res_columns[col_num++]->insert(UInt64(status.queue.inserts_in_queue));
         res_columns[col_num++]->insert(UInt64(status.queue.merges_in_queue));
+        res_columns[col_num++]->insert(UInt64(status.queue.part_mutations_in_queue));
         res_columns[col_num++]->insert(UInt64(status.queue.queue_oldest_time));
         res_columns[col_num++]->insert(UInt64(status.queue.inserts_oldest_time));
         res_columns[col_num++]->insert(UInt64(status.queue.merges_oldest_time));
+        res_columns[col_num++]->insert(UInt64(status.queue.part_mutations_oldest_time));
         res_columns[col_num++]->insert(status.queue.oldest_part_to_get);
         res_columns[col_num++]->insert(status.queue.oldest_part_to_merge_to);
+        res_columns[col_num++]->insert(status.queue.oldest_part_to_mutate_to);
         res_columns[col_num++]->insert(status.log_max_index);
         res_columns[col_num++]->insert(status.log_pointer);
         res_columns[col_num++]->insert(UInt64(status.queue.last_queue_update));

dbms/src/Storages/tests/CMakeLists.txt

@@ -1,5 +1,3 @@
-include_directories (${CMAKE_CURRENT_BINARY_DIR})
-
 add_executable (system_numbers system_numbers.cpp)
 target_link_libraries (system_numbers dbms clickhouse_storages_system)
 

dbms/src/TableFunctions/CMakeLists.txt

@@ -9,10 +9,10 @@ target_link_libraries(clickhouse_table_functions clickhouse_storages_system dbms
 
 if (USE_POCO_SQLODBC)
     target_link_libraries (clickhouse_table_functions ${Poco_SQLODBC_LIBRARY})
-    target_include_directories (clickhouse_table_functions PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIRS})
+    target_include_directories (clickhouse_table_functions SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIRS})
 endif ()
 
 if (USE_POCO_DATAODBC)
     target_link_libraries (clickhouse_table_functions ${Poco_DataODBC_LIBRARY})
-    target_include_directories (clickhouse_table_functions PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIRS})
+    target_include_directories (clickhouse_table_functions SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIRS})
 endif ()

dbms/tests/clickhouse-test-server

@@ -42,9 +42,8 @@ export CLICKHOUSE_PORT_INTERSERVER=${CLICKHOUSE_PORT_INTERSERVER:=`$CLICKHOUSE_E
 export CLICKHOUSE_PORT_TCP_SECURE=${CLICKHOUSE_PORT_TCP_SECURE:=`$CLICKHOUSE_EXTRACT_CONFIG --key=tcp_port_secure`}
 export CLICKHOUSE_PORT_HTTPS=${CLICKHOUSE_PORT_HTTPS:=`$CLICKHOUSE_EXTRACT_CONFIG --key=https_port`}
 
-
+rm -rf $DATA_DIR || true
-rm -rf $DATA_DIR
+mkdir -p $LOG_DIR || true
-mkdir -p $LOG_DIR
 
 DHPARAM=`$CLICKHOUSE_EXTRACT_CONFIG --key=openSSL.server.dhParamsFile`
 PRIVATEKEY=`${BIN_DIR}clickhouse-extract-from-config --config=$CLICKHOUSE_CONFIG  --key=openSSL.server.privateKeyFile`
@@ -78,8 +77,6 @@ $GDB ${BIN_DIR}clickhouse-server --config-file=$CLICKHOUSE_CONFIG -- \
 CH_PID=$!
 sleep 3
 
-
-
 if [ "$GDB" ]; then
     # Long symbols read
     sleep 40

dbms/tests/queries/0_stateless/00231_format_vertical_raw.reference

@@ -1,7 +1,4 @@
 Row 1:
 ──────
-x: a\tb\nc\td
-Row 1:
-──────
 x: a	b
 c	d

dbms/tests/queries/0_stateless/00231_format_vertical_raw.sql

@@ -1,2 +1 @@
 SELECT 'a\tb\nc\td' AS x FORMAT Vertical;
-SELECT 'a\tb\nc\td' AS x FORMAT VerticalRaw;

dbms/tests/queries/0_stateless/00298_enum_width_and_cast.reference

@@ -3,15 +3,15 @@
 └───────┴───┘
 ┌─x─────┬─y─┐
 │ Hello │ 0 │
-│ \\    │ 0 │
+│ \     │ 0 │
 └───────┴───┘
 ┌─x─────┬─y─┐
 │ Hello │ 0 │
-│ \\    │ 0 │
+│ \     │ 0 │
-│ \t\\t │ 0 │
+│ 	\t   │ 0 │
 └───────┴───┘
 ┌─x─────┬─y─┬─toInt8(x)─┬─s─────┬─casted─┐
 │ Hello │ 0 │      -100 │ Hello │ Hello  │
-│ \\    │ 0 │         0 │ \\    │ \\     │
+│ \     │ 0 │         0 │ \     │ \      │
-│ \t\\t │ 0 │       111 │ \t\\t │ \t\\t  │
+│ 	\t   │ 0 │       111 │ 	\t   │ 	\t    │
 └───────┴───┴───────────┴───────┴────────┘

dbms/tests/queries/0_stateless/00405_pretty_formats.reference

@@ -1,13 +1,13 @@
 ┏━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
 ┃ hello ┃ world ┃ tuple   ┃ sometimes_nulls ┃
 ┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
-│     0 │ 0     │ (0,'0') │              \N │
+│     0 │ 0     │ (0,'0') │            ᴺᵁᴸᴸ │
 ├───────┼───────┼─────────┼─────────────────┤
 │     1 │ 1     │ (1,'1') │               1 │
 ├───────┼───────┼─────────┼─────────────────┤
 │     2 │ 2     │ (2,'2') │               2 │
 ├───────┼───────┼─────────┼─────────────────┤
-│     3 │ 3     │ (3,'3') │              \N │
+│     3 │ 3     │ (3,'3') │            ᴺᵁᴸᴸ │
 ├───────┼───────┼─────────┼─────────────────┤
 │     4 │ 4     │ (4,'4') │               1 │
 └───────┴───────┴─────────┴─────────────────┘
@@ -16,64 +16,64 @@
 ┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
 │     5 │ 5     │ (5,'5') │               2 │
 ├───────┼───────┼─────────┼─────────────────┤
-│     6 │ 6     │ (6,'6') │              \N │
+│     6 │ 6     │ (6,'6') │            ᴺᵁᴸᴸ │
 ├───────┼───────┼─────────┼─────────────────┤
 │     7 │ 7     │ (7,'7') │               1 │
 ├───────┼───────┼─────────┼─────────────────┤
 │     8 │ 8     │ (8,'8') │               2 │
 ├───────┼───────┼─────────┼─────────────────┤
-│     9 │ 9     │ (9,'9') │              \N │
+│     9 │ 9     │ (9,'9') │            ᴺᵁᴸᴸ │
 └───────┴───────┴─────────┴─────────────────┘
 ┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐
-│     0 │ 0     │ (0,'0') │              \N │
+│     0 │ 0     │ (0,'0') │            ᴺᵁᴸᴸ │
 │     1 │ 1     │ (1,'1') │               1 │
 │     2 │ 2     │ (2,'2') │               2 │
-│     3 │ 3     │ (3,'3') │              \N │
+│     3 │ 3     │ (3,'3') │            ᴺᵁᴸᴸ │
 │     4 │ 4     │ (4,'4') │               1 │
 └───────┴───────┴─────────┴─────────────────┘
 ┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐
 │     5 │ 5     │ (5,'5') │               2 │
-│     6 │ 6     │ (6,'6') │              \N │
+│     6 │ 6     │ (6,'6') │            ᴺᵁᴸᴸ │
 │     7 │ 7     │ (7,'7') │               1 │
 │     8 │ 8     │ (8,'8') │               2 │
-│     9 │ 9     │ (9,'9') │              \N │
+│     9 │ 9     │ (9,'9') │            ᴺᵁᴸᴸ │
 └───────┴───────┴─────────┴─────────────────┘
 hello   world   tuple     sometimes_nulls
 
-    0   0       (0,'0')                \N
+    0   0       (0,'0')              ᴺᵁᴸᴸ
     1   1       (1,'1')                 1
     2   2       (2,'2')                 2
-    3   3       (3,'3')                \N
+    3   3       (3,'3')              ᴺᵁᴸᴸ
     4   4       (4,'4')                 1
 hello   world   tuple     sometimes_nulls
 
     5   5       (5,'5')                 2
-    6   6       (6,'6')                \N
+    6   6       (6,'6')              ᴺᵁᴸᴸ
     7   7       (7,'7')                 1
     8   8       (8,'8')                 2
-    9   9       (9,'9')                \N
+    9   9       (9,'9')              ᴺᵁᴸᴸ
 ┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐
-│     0 │ 0     │ (0,'0') │              \N │
+│     0 │ 0     │ (0,'0') │            ᴺᵁᴸᴸ │
 │     1 │ 1     │ (1,'1') │               1 │
 │     2 │ 2     │ (2,'2') │               2 │
-│     3 │ 3     │ (3,'3') │              \N │
+│     3 │ 3     │ (3,'3') │            ᴺᵁᴸᴸ │
 │     4 │ 4     │ (4,'4') │               1 │
 │     5 │ 5     │ (5,'5') │               2 │
-│     6 │ 6     │ (6,'6') │              \N │
+│     6 │ 6     │ (6,'6') │            ᴺᵁᴸᴸ │
 │     7 │ 7     │ (7,'7') │               1 │
 │     8 │ 8     │ (8,'8') │               2 │
-│     9 │ 9     │ (9,'9') │              \N │
+│     9 │ 9     │ (9,'9') │            ᴺᵁᴸᴸ │
 └───────┴───────┴─────────┴─────────────────┘
 ┏━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
 ┃ hello ┃ world ┃ tuple   ┃ sometimes_nulls ┃
 ┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
-│     0 │ 0     │ (0,'0') │              \N │
+│     0 │ 0     │ (0,'0') │            ᴺᵁᴸᴸ │
 ├───────┼───────┼─────────┼─────────────────┤
 │     1 │ 1     │ (1,'1') │               1 │
 ├───────┼───────┼─────────┼─────────────────┤
 │     2 │ 2     │ (2,'2') │               2 │
 ├───────┼───────┼─────────┼─────────────────┤
-│     3 │ 3     │ (3,'3') │              \N │
+│     3 │ 3     │ (3,'3') │            ᴺᵁᴸᴸ │
 ├───────┼───────┼─────────┼─────────────────┤
 │     4 │ 4     │ (4,'4') │               1 │
 └───────┴───────┴─────────┴─────────────────┘
@@ -82,52 +82,52 @@
 ┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
 │     5 │ 5     │ (5,'5') │               2 │
 ├───────┼───────┼─────────┼─────────────────┤
-│     6 │ 6     │ (6,'6') │              \N │
+│     6 │ 6     │ (6,'6') │            ᴺᵁᴸᴸ │
 ├───────┼───────┼─────────┼─────────────────┤
 │     7 │ 7     │ (7,'7') │               1 │
 ├───────┼───────┼─────────┼─────────────────┤
 │     8 │ 8     │ (8,'8') │               2 │
 ├───────┼───────┼─────────┼─────────────────┤
-│     9 │ 9     │ (9,'9') │              \N │
+│     9 │ 9     │ (9,'9') │            ᴺᵁᴸᴸ │
 └───────┴───────┴─────────┴─────────────────┘
 ┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐
-│     0 │ 0     │ (0,'0') │              \N │
+│     0 │ 0     │ (0,'0') │            ᴺᵁᴸᴸ │
 │     1 │ 1     │ (1,'1') │               1 │
 │     2 │ 2     │ (2,'2') │               2 │
-│     3 │ 3     │ (3,'3') │              \N │
+│     3 │ 3     │ (3,'3') │            ᴺᵁᴸᴸ │
 │     4 │ 4     │ (4,'4') │               1 │
 └───────┴───────┴─────────┴─────────────────┘
 ┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐
 │     5 │ 5     │ (5,'5') │               2 │
-│     6 │ 6     │ (6,'6') │              \N │
+│     6 │ 6     │ (6,'6') │            ᴺᵁᴸᴸ │
 │     7 │ 7     │ (7,'7') │               1 │
 │     8 │ 8     │ (8,'8') │               2 │
-│     9 │ 9     │ (9,'9') │              \N │
+│     9 │ 9     │ (9,'9') │            ᴺᵁᴸᴸ │
 └───────┴───────┴─────────┴─────────────────┘
 hello   world   tuple     sometimes_nulls
 
-    0   0       (0,'0')                \N
+    0   0       (0,'0')              ᴺᵁᴸᴸ
     1   1       (1,'1')                 1
     2   2       (2,'2')                 2
-    3   3       (3,'3')                \N
+    3   3       (3,'3')              ᴺᵁᴸᴸ
     4   4       (4,'4')                 1
 hello   world   tuple     sometimes_nulls
 
     5   5       (5,'5')                 2
-    6   6       (6,'6')                \N
+    6   6       (6,'6')              ᴺᵁᴸᴸ
     7   7       (7,'7')                 1
     8   8       (8,'8')                 2
-    9   9       (9,'9')                \N
+    9   9       (9,'9')              ᴺᵁᴸᴸ
 ┏━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
 ┃ hello ┃ world ┃ tuple   ┃ sometimes_nulls ┃
 ┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
-│     0 │ 0     │ (0,'0') │              \N │
+│     0 │ 0     │ (0,'0') │            ᴺᵁᴸᴸ │
 ├───────┼───────┼─────────┼─────────────────┤
 │     1 │ 1     │ (1,'1') │               1 │
 ├───────┼───────┼─────────┼─────────────────┤
 │     2 │ 2     │ (2,'2') │               2 │
 ├───────┼───────┼─────────┼─────────────────┤
-│     3 │ 3     │ (3,'3') │              \N │
+│     3 │ 3     │ (3,'3') │            ᴺᵁᴸᴸ │
 ├───────┼───────┼─────────┼─────────────────┤
 │     4 │ 4     │ (4,'4') │               1 │
 └───────┴───────┴─────────┴─────────────────┘
@@ -138,10 +138,10 @@ hello   world   tuple     sometimes_nulls
 └───────┴───────┴─────────┴─────────────────┘
   Showed first 6.
 ┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐
-│     0 │ 0     │ (0,'0') │              \N │
+│     0 │ 0     │ (0,'0') │            ᴺᵁᴸᴸ │
 │     1 │ 1     │ (1,'1') │               1 │
 │     2 │ 2     │ (2,'2') │               2 │
-│     3 │ 3     │ (3,'3') │              \N │
+│     3 │ 3     │ (3,'3') │            ᴺᵁᴸᴸ │
 │     4 │ 4     │ (4,'4') │               1 │
 └───────┴───────┴─────────┴─────────────────┘
 ┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐
@@ -150,10 +150,10 @@ hello   world   tuple     sometimes_nulls
   Showed first 6.
 hello   world   tuple     sometimes_nulls
 
-    0   0       (0,'0')                \N
+    0   0       (0,'0')              ᴺᵁᴸᴸ
     1   1       (1,'1')                 1
     2   2       (2,'2')                 2
-    3   3       (3,'3')                \N
+    3   3       (3,'3')              ᴺᵁᴸᴸ
     4   4       (4,'4')                 1
 hello   world   tuple     sometimes_nulls
 
@@ -161,10 +161,10 @@ hello   world   tuple     sometimes_nulls
 
 Showed first 6.
 ┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐
-│     0 │ 0     │ (0,'0') │              \N │
+│     0 │ 0     │ (0,'0') │            ᴺᵁᴸᴸ │
 │     1 │ 1     │ (1,'1') │               1 │
 │     2 │ 2     │ (2,'2') │               2 │
-│     3 │ 3     │ (3,'3') │              \N │
+│     3 │ 3     │ (3,'3') │            ᴺᵁᴸᴸ │
 │     4 │ 4     │ (4,'4') │               1 │
 │     5 │ 5     │ (5,'5') │               2 │
 └───────┴───────┴─────────┴─────────────────┘
@@ -172,13 +172,13 @@ Showed first 6.
 ┏━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
 ┃ hello ┃ world ┃ tuple   ┃ sometimes_nulls ┃
 ┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
-│     0 │ 0     │ (0,'0') │              \N │
+│     0 │ 0     │ (0,'0') │            ᴺᵁᴸᴸ │
 ├───────┼───────┼─────────┼─────────────────┤
 │     1 │ 1     │ (1,'1') │               1 │
 ├───────┼───────┼─────────┼─────────────────┤
 │     2 │ 2     │ (2,'2') │               2 │
 ├───────┼───────┼─────────┼─────────────────┤
-│     3 │ 3     │ (3,'3') │              \N │
+│     3 │ 3     │ (3,'3') │            ᴺᵁᴸᴸ │
 ├───────┼───────┼─────────┼─────────────────┤
 │     4 │ 4     │ (4,'4') │               1 │
 └───────┴───────┴─────────┴─────────────────┘
@@ -189,10 +189,10 @@ Showed first 6.
 └───────┴───────┴─────────┴─────────────────┘
   Showed first 6.
 ┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐
-│     0 │ 0     │ (0,'0') │              \N │
+│     0 │ 0     │ (0,'0') │            ᴺᵁᴸᴸ │
 │     1 │ 1     │ (1,'1') │               1 │
 │     2 │ 2     │ (2,'2') │               2 │
-│     3 │ 3     │ (3,'3') │              \N │
+│     3 │ 3     │ (3,'3') │            ᴺᵁᴸᴸ │
 │     4 │ 4     │ (4,'4') │               1 │
 └───────┴───────┴─────────┴─────────────────┘
 ┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐
@@ -201,10 +201,10 @@ Showed first 6.
   Showed first 6.
 hello   world   tuple     sometimes_nulls
 
-    0   0       (0,'0')                \N
+    0   0       (0,'0')              ᴺᵁᴸᴸ
     1   1       (1,'1')                 1
     2   2       (2,'2')                 2
-    3   3       (3,'3')                \N
+    3   3       (3,'3')              ᴺᵁᴸᴸ
     4   4       (4,'4')                 1
 hello   world   tuple     sometimes_nulls
 

dbms/tests/queries/0_stateless/00406_tuples_with_nulls.reference

@@ -10,6 +10,6 @@
 │ (8,2,NULL)    │
 │ (9,NULL,'1')  │
 └───────────────┘
-┌─x──┬─y──────┐
+┌─x────┬─y──────┐
-│ \N │ (NULL) │
+│ ᴺᵁᴸᴸ │ (NULL) │
-└────┴────────┘
+└──────┴────────┘

dbms/tests/queries/0_stateless/00460_vertical_and_totals_extremes.reference

@@ -131,157 +131,6 @@ count(): 20
 Showed first 4.
 
 
-Totals:
-───────
-k:       0
-count(): 100
-
-Min:
-────
-k:       0
-count(): 20
-
-Max:
-────
-k:       4
-count(): 20
-Row 1:
-──────
-k:       0
-count(): 20
-
-Row 2:
-──────
-k:       1
-count(): 20
-
-Row 3:
-──────
-k:       2
-count(): 20
-
-Row 4:
-──────
-k:       3
-count(): 20
-
-Showed first 4.
-
-
-Totals:
-───────
-k:       0
-count(): 100
-
-Min:
-────
-k:       0
-count(): 20
-
-Max:
-────
-k:       4
-count(): 20
-Row 1:
-──────
-k:       0
-count(): 20
-
-Row 2:
-──────
-k:       1
-count(): 20
-
-Row 3:
-──────
-k:       2
-count(): 20
-
-Row 4:
-──────
-k:       3
-count(): 20
-
-Showed first 4.
-
-
-Totals:
-───────
-k:       0
-count(): 100
-
-Min:
-────
-k:       0
-count(): 20
-
-Max:
-────
-k:       4
-count(): 20
-Row 1:
-──────
-k:       0
-count(): 20
-
-Row 2:
-──────
-k:       1
-count(): 20
-
-Row 3:
-──────
-k:       2
-count(): 20
-
-Row 4:
-──────
-k:       3
-count(): 20
-
-Row 5:
-──────
-k:       4
-count(): 20
-
-
-Totals:
-───────
-k:       0
-count(): 100
-
-Min:
-────
-k:       0
-count(): 20
-
-Max:
-────
-k:       4
-count(): 20
-Row 1:
-──────
-k:       0
-count(): 20
-
-Row 2:
-──────
-k:       1
-count(): 20
-
-Row 3:
-──────
-k:       2
-count(): 20
-
-Row 4:
-──────
-k:       3
-count(): 20
-
-Showed first 4.
-
-
 Totals:
 ───────
 k:       0

dbms/tests/queries/0_stateless/00460_vertical_and_totals_extremes.sql

@@ -8,15 +8,3 @@ SELECT k, count() FROM (SELECT number % 5 AS k FROM system.numbers LIMIT 100) GR
 
 SET output_format_pretty_max_rows = 4;
 SELECT k, count() FROM (SELECT number % 5 AS k FROM system.numbers LIMIT 100) GROUP BY k WITH TOTALS ORDER BY k FORMAT Vertical;
-
-
-SELECT k, count() FROM (SELECT number % 5 AS k FROM system.numbers LIMIT 100) GROUP BY k WITH TOTALS ORDER BY k FORMAT VerticalRaw;
-
-SET extremes = 1;
-SELECT k, count() FROM (SELECT number % 5 AS k FROM system.numbers LIMIT 100) GROUP BY k WITH TOTALS ORDER BY k FORMAT VerticalRaw;
-
-SET output_format_pretty_max_rows = 5;
-SELECT k, count() FROM (SELECT number % 5 AS k FROM system.numbers LIMIT 100) GROUP BY k WITH TOTALS ORDER BY k FORMAT VerticalRaw;
-
-SET output_format_pretty_max_rows = 4;
-SELECT k, count() FROM (SELECT number % 5 AS k FROM system.numbers LIMIT 100) GROUP BY k WITH TOTALS ORDER BY k FORMAT VerticalRaw;

dbms/tests/queries/0_stateless/00476_pretty_formats_and_widths.reference

@@ -57,12 +57,12 @@
 │  100000000 │ 100000000  │
 │ 1000000000 │ 1000000000 │
 └────────────┴────────────┘
-┏━━━━━━━━━━━━━━━━━━┓
+┏━━━━━━━━━━┓
-┃ \'\\\\\\\'\\\'\' ┃
+┃ '\\\'\'' ┃
-┡━━━━━━━━━━━━━━━━━━┩
+┡━━━━━━━━━━┩
-│ \\\'\'           │
+│ \''      │
-└──────────────────┘
+└──────────┘
 Row 1:
 ──────
-\'\\\\\\\'\\\'\': \\\'\'
+'\\\'\'': \''
-1:                1
+1:        1

dbms/tests/queries/0_stateless/00569_parse_date_time_best_effort.reference

@@ -19,7 +19,7 @@ s                                                   a                     b
 19700102010203Z                   1970-01-02 01:02:03   1970-01-02 01:02:03
 1970/01/02 010203Z                1970-01-02 01:02:03   1970-01-02 01:02:03
 20                                2000-01-20 00:00:00   2000-01-20 00:00:00
-201                                                \N   0000-00-00 00:00:00
+201                                              ᴺᵁᴸᴸ   0000-00-00 00:00:00
 20160101                          2016-01-01 00:00:00   2016-01-01 00:00:00
 2016-01-01                        2016-01-01 00:00:00   2016-01-01 00:00:00
 201601-01                         2016-01-01 01:00:00   2016-01-01 01:00:00
@@ -35,7 +35,7 @@ s                                                   a                     b
 2017/01/01                        2017-01-01 00:00:00   2017-01-01 00:00:00
 201701 02 010203 UTC+0300         2017-01-01 22:02:03   2017-01-01 22:02:03
 2017-01-02 03:04:05               2017-01-02 03:04:05   2017-01-02 03:04:05
-2017-01-0203:04:05                                 \N   0000-00-00 00:00:00
+2017-01-0203:04:05                               ᴺᵁᴸᴸ   0000-00-00 00:00:00
 2017-01-02 03:04:05+0             2017-01-02 03:04:05   2017-01-02 03:04:05
 2017-01-02 03:04:05+00            2017-01-02 03:04:05   2017-01-02 03:04:05
 2017-01-02 03:04:05+0000          2017-01-02 03:04:05   2017-01-02 03:04:05
@@ -65,7 +65,7 @@ s                                                   a                     b
 2017 25 1:2:3                     0000-00-00 00:00:00   0000-00-00 00:00:00
 2017 25 Apr 1:2:3                 2017-04-01 01:02:03   2017-04-01 01:02:03
 2017 Apr 01 11:22:33              2017-04-01 11:22:33   2017-04-01 11:22:33
-2017 Apr 02 01/02/03 UTC+0300                      \N   0000-00-00 00:00:00
+2017 Apr 02 01/02/03 UTC+0300                    ᴺᵁᴸᴸ   0000-00-00 00:00:00
 2017 Apr 02 010203 UTC+0300       2017-04-01 22:02:03   2017-04-01 22:02:03
 2017 Apr 02 01:2:3 UTC+0300       2017-04-01 22:02:03   2017-04-01 22:02:03
 2017 Apr 02 1:02:3                2017-04-02 01:02:03   2017-04-02 01:02:03
@@ -91,14 +91,14 @@ s                                                   a                     b
 25 Jan 2017 1:2:3 Z               2017-01-25 01:02:03   2017-01-25 01:02:03
 25 Jan 2017 1:2:3 Z +0300         2017-01-24 22:02:03   2017-01-24 22:02:03
 25 Jan 2017 1:2:3 Z+03:00         2017-01-24 22:02:03   2017-01-24 22:02:03
-25 Jan 2017 1:2:3 Z +0300 OM                       \N   0000-00-00 00:00:00
+25 Jan 2017 1:2:3 Z +0300 OM                     ᴺᵁᴸᴸ   0000-00-00 00:00:00
 25 Jan 2017 1:2:3 Z +03:00 PM     2017-01-25 10:02:03   2017-01-25 10:02:03
 25 Jan 2017 1:2:3 Z +0300 PM      2017-01-25 10:02:03   2017-01-25 10:02:03
 25 Jan 2017 1:2:3 Z+03:00 PM      2017-01-25 10:02:03   2017-01-25 10:02:03
 25 Jan 2017 1:2:3 Z +03:30 PM     2017-01-25 09:32:03   2017-01-25 09:32:03
-25 Jan 2017 1:2:3Z Mo                              \N   0000-00-00 00:00:00
+25 Jan 2017 1:2:3Z Mo                            ᴺᵁᴸᴸ   0000-00-00 00:00:00
 25 Jan 2017 1:2:3Z Mon            2017-01-25 01:02:03   2017-01-25 01:02:03
-25 Jan 2017 1:2:3Z Moo                             \N   0000-00-00 00:00:00
+25 Jan 2017 1:2:3Z Moo                           ᴺᵁᴸᴸ   0000-00-00 00:00:00
 25 Jan 2017 1:2:3 Z PM            2017-01-25 13:02:03   2017-01-25 13:02:03
 25 Jan 2017 1:2:3Z PM             2017-01-25 13:02:03   2017-01-25 13:02:03
 25 Jan 2017 1:2:3 Z PM +03:00     2017-01-25 10:02:03   2017-01-25 10:02:03

dbms/tests/queries/0_stateless/00637_sessions_in_http_interface_and_settings.reference

@@ -0,0 +1 @@
+readonly	1	1

dbms/tests/queries/0_stateless/00637_sessions_in_http_interface_and_settings.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+. $CURDIR/../shell_config.sh
+
+user=readonly
+address=${CLICKHOUSE_HOST}
+port=${CLICKHOUSE_PORT_HTTP}
+url="${CLICKHOUSE_PORT_HTTP_PROTO}://readonly@$address:$port/?session_id=test"
+select="SELECT name, value, changed FROM system.settings WHERE name = 'readonly'"
+
+${CLICKHOUSE_CURL} -sS $url --data-binary "$select"

dbms/tests/queries/0_stateless/00638_remote_ssrf.reference

@@ -0,0 +1 @@
+must not contain ASCII control characters

dbms/tests/queries/0_stateless/00638_remote_ssrf.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+. $CURDIR/../shell_config.sh
+
+user=readonly
+address=${CLICKHOUSE_HOST}
+port=${CLICKHOUSE_PORT_HTTP}
+url="${CLICKHOUSE_PORT_HTTP_PROTO}://$address:$port/"
+
+# Port is arbitary
+
+nc -l -p 61845 -q 0 > /dev/null &
+
+${CLICKHOUSE_CURL} -sS $url --data-binary "SELECT * FROM remote('localhost:61845', system.one, 'user', 'password')" > /dev/null 2>&1
+
+wait
+
+nc -l -p 61846 -q 0 > /dev/null &
+
+${CLICKHOUSE_CURL} -sS $url --data-binary "SELECT * FROM remote('localhost:61846', system.one, 'user', 'passw
+ord')" 2>&1 | grep -o 'must not contain ASCII control characters'
+
+wait

dbms/tests/queries/0_stateless/00639_startsWith.reference

@@ -0,0 +1,16 @@
+0
+0
+1
+1
+0
+0
+1
+1
+1
+1
+1
+0
+1
+3
+2
+0

dbms/tests/queries/0_stateless/00639_startsWith.sql

@@ -0,0 +1,17 @@
+USE test;
+
+SELECT startsWith(s, 'He') FROM (SELECT arrayJoin(['', 'H', 'He', 'Hellow', '3434', 'fffffffffdHe']) AS s);
+SELECT startsWith(s, '') FROM (SELECT arrayJoin(['', 'h', 'hi']) AS s);
+SELECT startsWith('123', '123');
+SELECT startsWith('123', '12');
+SELECT startsWith('123', '1234');
+SELECT startsWith('123', '');
+
+DROP TABLE IF EXISTS startsWith_test;
+CREATE TABLE startsWith_test(S1 String, S2 String, S3 FixedString(2)) ENGINE=Memory;
+INSERT INTO startsWith_test values ('11', '22', '33'), ('a', 'a', 'bb'), ('abc', 'ab', '23');
+
+SELECT COUNT() FROM startsWith_test WHERE startsWith(S1, S1);
+SELECT COUNT() FROM startsWith_test WHERE startsWith(S1, S2);
+SELECT COUNT() FROM startsWith_test WHERE startsWith(S2, S3);
+DROP TABLE startsWith_test;

dbms/tests/queries/0_stateless/00640_endsWith.reference

@@ -0,0 +1,16 @@
+0
+0
+1
+1
+0
+0
+1
+1
+1
+1
+1
+0
+1
+3
+2
+0

dbms/tests/queries/0_stateless/00640_endsWith.sql

@@ -0,0 +1,17 @@
+USE test;
+
+SELECT endsWith(s, 'ow') FROM (SELECT arrayJoin(['', 'o', 'ow', 'Hellow', '3434', 'owfffffffdHe']) AS s);
+SELECT endsWith(s, '') FROM (SELECT arrayJoin(['', 'h', 'hi']) AS s);
+SELECT endsWith('123', '3');
+SELECT endsWith('123', '23');
+SELECT endsWith('123', '32');
+SELECT endsWith('123', '');
+
+DROP TABLE IF EXISTS endsWith_test;
+CREATE TABLE endsWith_test(S1 String, S2 String, S3 FixedString(2)) ENGINE=Memory;
+INSERT INTO endsWith_test values ('11', '22', '33'), ('a', 'a', 'bb'), ('abc', 'bc', '23');
+
+SELECT COUNT() FROM endsWith_test WHERE endsWith(S1, S1);
+SELECT COUNT() FROM endsWith_test WHERE endsWith(S1, S2);
+SELECT COUNT() FROM endsWith_test WHERE endsWith(S2, S3);
+DROP TABLE endsWith_test;

dbms/tests/users.xml

@@ -1 +1 @@
-../src/Server/users.xml
+../programs/server/users.xml

debian/control

@@ -62,7 +62,7 @@ Description: debugging symbols for clickhouse-common-static
 Package: clickhouse-test
 Priority: optional
 Architecture: all
-Depends: ${shlibs:Depends}, ${misc:Depends}, clickhouse-client, bash, expect, python, python-lxml, python-termcolor, python-requests, curl, perl, sudo, openssl
+Depends: ${shlibs:Depends}, ${misc:Depends}, clickhouse-client, bash, expect, python, python-lxml, python-termcolor, python-requests, curl, perl, sudo, openssl, netcat-openbsd
 Description: Clickhouse tests
 
 

debian/rules

@@ -99,11 +99,6 @@ override_dh_install:
 	mkdir -p $(DESTDIR)/etc/systemd/system/
 	cp debian/clickhouse-server.service $(DESTDIR)/etc/systemd/system/
 
-	#TODO: use from cmake:
-	# In case building clickhouse-server, adding to package binary of clang, ld and header files - for dynamic compilation.
-	#mkdir -p $(DESTDIR)/usr/share/clickhouse/headers
-	#CLANG=$(DESTDIR)/usr/bin/clickhouse-clang DESTDIR=$(DESTDIR) ./copy_headers.sh . $(DESTDIR)/usr/share/clickhouse/headers
-
 	# fake metrika files when private dir is empty
 	mkdir -p $(DESTDIR)/etc/clickhouse-server/metrika
 	touch $(DESTDIR)/etc/clickhouse-server/metrika/config.xml

docker/client/Dockerfile

@@ -9,7 +9,7 @@ RUN apt-get update && \
     apt-key adv --keyserver keyserver.ubuntu.com --recv E0C56BD4 && \
     echo $repository | tee /etc/apt/sources.list.d/clickhouse.list && \
     apt-get update && \
-    env DEBIAN_FRONTEND=noninteractive apt-get install --allow-unauthenticated -y clickhouse-client=$version locales tzdata && \
+    env DEBIAN_FRONTEND=noninteractive apt-get install --allow-unauthenticated -y clickhouse-client=$version clickhouse-common-static=$version locales tzdata && \
     rm -rf /var/lib/apt/lists/* /var/cache/debconf && \
     apt-get clean
 

docker/server/Dockerfile

@@ -9,7 +9,7 @@ RUN apt-get update && \
     apt-key adv --keyserver keyserver.ubuntu.com --recv E0C56BD4 && \
     echo $repository | tee /etc/apt/sources.list.d/clickhouse.list && \
     apt-get update && \
-    env DEBIAN_FRONTEND=noninteractive apt-get install --allow-unauthenticated -y "clickhouse-server=$version" libgcc-7-dev && \
+    env DEBIAN_FRONTEND=noninteractive apt-get install --allow-unauthenticated -y clickhouse-server=$version clickhouse-common-static=$version libgcc-7-dev && \
     rm -rf /var/lib/apt/lists/* /var/cache/debconf && \
     apt-get clean
 

libs/libmysqlxx/CMakeLists.txt

@@ -33,7 +33,7 @@ target_include_directories (mysqlxx PUBLIC include)
 set(PLATFORM_LIBRARIES ${CMAKE_DL_LIBS})
 
 if (USE_MYSQL)
-    target_include_directories (mysqlxx PRIVATE ${MYSQL_INCLUDE_DIR})
+    target_include_directories (mysqlxx SYSTEM PRIVATE ${MYSQL_INCLUDE_DIR})
 endif ()
 
 if (USE_STATIC_LIBRARIES AND STATIC_MYSQLCLIENT_LIB)
@@ -49,7 +49,7 @@ if (USE_STATIC_LIBRARIES AND STATIC_MYSQLCLIENT_LIB)
 endif ()
 
 target_link_libraries (mysqlxx common ${MYSQLCLIENT_LIBRARIES} ${OPENSSL_LIBRARIES} ${ZLIB_LIBRARIES} ${PLATFORM_LIBRARIES})
-target_include_directories (mysqlxx PRIVATE ${OPENSSL_INCLUDE_DIR})
+target_include_directories (mysqlxx SYSTEM PRIVATE ${OPENSSL_INCLUDE_DIR})
 
 if (ENABLE_TESTS)
     add_subdirectory (src/tests)

libs/libpocoext/CMakeLists.txt

@@ -8,14 +8,14 @@ add_library (pocoext ${SPLIT_SHARED}
     include/Poco/Ext/SessionPoolHelpers.h)
 
 if (Poco_Data_FOUND)
-    target_include_directories (pocoext PRIVATE ${Poco_Data_INCLUDE_DIRS})
+    target_include_directories (pocoext SYSTEM PRIVATE ${Poco_Data_INCLUDE_DIRS})
     target_link_libraries(pocoext ${Poco_Data_LIBRARY})
 endif()
 
 target_include_directories (pocoext PUBLIC include PRIVATE ${COMMON_INCLUDE_DIR})
 
 if (NOT USE_INTERNAL_POCO_LIBRARY)
-    target_include_directories (pocoext BEFORE PUBLIC ${Poco_INCLUDE_DIRS})
+    target_include_directories (pocoext SYSTEM BEFORE PUBLIC ${Poco_INCLUDE_DIRS})
 endif ()
 
 target_link_libraries(pocoext ${Poco_Util_LIBRARY} ${Poco_Net_LIBRARY} ${Poco_XML_LIBRARY} ${Poco_Foundation_LIBRARY})

release

@@ -53,7 +53,7 @@ do
         shift
     elif [[ $1 == '--fast' ]]; then
         # Wrong but fast pbuilder mode: create base package with all depends
-        EXTRAPACKAGES="$EXTRAPACKAGES debhelper cmake ninja-build gcc-7 g++-7 libc6-dev libmariadbclient-dev libicu-dev libltdl-dev libreadline-dev libssl-dev unixodbc-dev psmisc bash expect python python-lxml python-termcolor python-requests curl perl sudo openssl"
+        EXTRAPACKAGES="$EXTRAPACKAGES debhelper cmake ninja-build gcc-7 g++-7 libc6-dev libmariadbclient-dev libicu-dev libltdl-dev libreadline-dev libssl-dev unixodbc-dev psmisc bash expect python python-lxml python-termcolor python-requests curl perl sudo openssl netcat-openbsd"
         shift
     else
         echo "Unknown option $1"

utils/check_include.sh

@@ -3,6 +3,7 @@
 # Small .h isolated compile checker
 # Finds missing #include <...>
 # prints compile time, number of includes, use with sort: ./check_include.sh 2>&1 | sort -rk3
+# use with clang: CXX=`which clang++-7 clang++-7.0 clang++-6.0 clang++-5.0 | head -n1` ./check_include.sh
 pwd=`pwd`
 BUILD_DIR=${BUILD_DIR:=./build}
 inc="-I. \
@@ -28,6 +29,8 @@ inc="-I. \
 -I./contrib/poco/Foundation/include \
 -I./contrib/boost/libs/*/include \
 -I./contrib/boost \
+-I./contrib/llvm/llvm/include \
+-I${BUILD_DIR}/contrib/llvm/llvm/include \
 -I./contrib/libbtrie/include \
 -I./contrib/libpcg-random/include \
 -I./libs/libmysqlxx/include \