diff --git a/cmake/find_capnp.cmake b/cmake/find_capnp.cmake index 22301acc3ee..03ecadda6a1 100644 --- a/cmake/find_capnp.cmake +++ b/cmake/find_capnp.cmake @@ -1,7 +1,6 @@ option (ENABLE_CAPNP "Enable Cap'n Proto" ${NOT_MSVC}) if (ENABLE_CAPNP) - # cmake 3.5.1 bug: # capnproto uses this cmake feature: # target_compile_features(kj PUBLIC cxx_constexpr) @@ -42,7 +41,6 @@ if (ENABLE_CAPNP) set (CAPNP_LIBRARY capnpc) set (USE_CAPNP 1) endif () - endif () if (USE_CAPNP) diff --git a/cmake/find_poco.cmake b/cmake/find_poco.cmake index 6349e6e59e6..947d31951c9 100644 --- a/cmake/find_poco.cmake +++ b/cmake/find_poco.cmake @@ -27,7 +27,6 @@ endif () if (Poco_INCLUDE_DIRS AND Poco_Foundation_LIBRARY) elseif (NOT MISSING_INTERNAL_POCO_LIBRARY) - set (USE_INTERNAL_POCO_LIBRARY 1) set (ENABLE_ZIP 0 CACHE BOOL "") diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index 337899b70f0..19e59f78677 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -102,7 +102,7 @@ if (USE_EMBEDDED_COMPILER) llvm_libs_all(REQUIRED_LLVM_LIBRARIES) target_link_libraries (dbms ${REQUIRED_LLVM_LIBRARIES}) - target_include_directories (dbms BEFORE PUBLIC ${LLVM_INCLUDE_DIRS}) + target_include_directories (dbms SYSTEM BEFORE PUBLIC ${LLVM_INCLUDE_DIRS}) endif () @@ -159,32 +159,32 @@ target_link_libraries (dbms ) if (NOT USE_INTERNAL_RE2_LIBRARY) - target_include_directories (dbms BEFORE PRIVATE ${RE2_INCLUDE_DIR}) + target_include_directories (dbms SYSTEM BEFORE PRIVATE ${RE2_INCLUDE_DIR}) endif () if (NOT USE_INTERNAL_BOOST_LIBRARY) - target_include_directories (clickhouse_common_io BEFORE PUBLIC ${Boost_INCLUDE_DIRS}) + target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS}) endif () if (USE_POCO_SQLODBC) target_link_libraries (clickhouse_common_io ${Poco_SQL_LIBRARY}) target_link_libraries (dbms ${Poco_SQLODBC_LIBRARY} ${Poco_SQL_LIBRARY}) if (NOT USE_INTERNAL_POCO_LIBRARY) - target_include_directories (clickhouse_common_io PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQL_INCLUDE_DIRS}) - target_include_directories (dbms PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIRS} PUBLIC ${Poco_SQL_INCLUDE_DIRS}) + target_include_directories (clickhouse_common_io SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQL_INCLUDE_DIRS}) + target_include_directories (dbms SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIRS} PUBLIC ${Poco_SQL_INCLUDE_DIRS}) endif() endif() if (Poco_Data_FOUND AND NOT USE_INTERNAL_POCO_LIBRARY) - target_include_directories (clickhouse_common_io PRIVATE ${Poco_Data_INCLUDE_DIRS}) - target_include_directories (dbms PRIVATE ${Poco_Data_INCLUDE_DIRS}) + target_include_directories (clickhouse_common_io SYSTEM PRIVATE ${Poco_Data_INCLUDE_DIRS}) + target_include_directories (dbms SYSTEM PRIVATE ${Poco_Data_INCLUDE_DIRS}) endif() if (USE_POCO_DATAODBC) target_link_libraries (clickhouse_common_io ${Poco_Data_LIBRARY}) target_link_libraries (dbms ${Poco_DataODBC_LIBRARY}) if (NOT USE_INTERNAL_POCO_LIBRARY) - target_include_directories (dbms PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIRS}) + target_include_directories (dbms SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIRS}) endif() endif() @@ -200,20 +200,20 @@ target_link_libraries (dbms ${Poco_Foundation_LIBRARY}) if (USE_ICU) target_link_libraries (dbms ${ICU_LIBS}) - target_include_directories (dbms PRIVATE ${ICU_INCLUDE_DIR}) + target_include_directories (dbms SYSTEM PRIVATE ${ICU_INCLUDE_DIR}) endif () if (USE_CAPNP) target_link_libraries (dbms ${CAPNP_LIBRARY}) if (NOT USE_INTERNAL_CAPNP_LIBRARY) - target_include_directories (dbms BEFORE PRIVATE ${CAPNP_INCLUDE_DIR}) + target_include_directories (dbms SYSTEM BEFORE PRIVATE ${CAPNP_INCLUDE_DIR}) endif () endif () if (USE_RDKAFKA) target_link_libraries (dbms ${RDKAFKA_LIBRARY}) if (NOT USE_INTERNAL_RDKAFKA_LIBRARY) - target_include_directories (dbms BEFORE PRIVATE ${RDKAFKA_INCLUDE_DIR}) + target_include_directories (dbms SYSTEM BEFORE PRIVATE ${RDKAFKA_INCLUDE_DIR}) endif () endif () @@ -221,21 +221,21 @@ target_link_libraries (dbms Threads::Threads ) -target_include_directories (dbms BEFORE PRIVATE ${DIVIDE_INCLUDE_DIR}) -target_include_directories (dbms BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) +target_include_directories (dbms SYSTEM BEFORE PRIVATE ${DIVIDE_INCLUDE_DIR}) +target_include_directories (dbms SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) if (NOT USE_INTERNAL_LZ4_LIBRARY) - target_include_directories (dbms BEFORE PRIVATE ${LZ4_INCLUDE_DIR}) + target_include_directories (dbms SYSTEM BEFORE PRIVATE ${LZ4_INCLUDE_DIR}) endif () if (NOT USE_INTERNAL_ZSTD_LIBRARY) - target_include_directories (dbms BEFORE PRIVATE ${ZSTD_INCLUDE_DIR}) + target_include_directories (dbms SYSTEM BEFORE PRIVATE ${ZSTD_INCLUDE_DIR}) endif () target_include_directories (dbms PUBLIC ${DBMS_INCLUDE_DIR}) target_include_directories (clickhouse_common_io PUBLIC ${DBMS_INCLUDE_DIR}) -target_include_directories (clickhouse_common_io PUBLIC ${PCG_RANDOM_INCLUDE_DIR}) -target_include_directories (clickhouse_common_io PUBLIC ${Poco_DataODBC_INCLUDE_DIRS}) -target_include_directories (clickhouse_common_io BEFORE PUBLIC ${DOUBLE_CONVERSION_INCLUDE_DIR}) +target_include_directories (clickhouse_common_io SYSTEM PUBLIC ${PCG_RANDOM_INCLUDE_DIR}) +target_include_directories (clickhouse_common_io SYSTEM PUBLIC ${Poco_DataODBC_INCLUDE_DIRS}) +target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${DOUBLE_CONVERSION_INCLUDE_DIR}) # also for copy_headers.sh: target_include_directories (clickhouse_common_io BEFORE PRIVATE ${COMMON_INCLUDE_DIR}) @@ -252,7 +252,7 @@ if (ENABLE_TESTS) add_subdirectory(${ClickHouse_SOURCE_DIR}/contrib/googletest/googletest ${CMAKE_CURRENT_BINARY_DIR}/googletest) # avoid problems with target_compile_definitions (gtest INTERFACE GTEST_HAS_POSIX_RE=0) - target_include_directories (gtest INTERFACE ${ClickHouse_SOURCE_DIR}/contrib/googletest/include) + target_include_directories (gtest SYSTEM INTERFACE ${ClickHouse_SOURCE_DIR}/contrib/googletest/include) endif () macro(grep_gtest_sources BASE_DIR DST_VAR) diff --git a/dbms/programs/CMakeLists.txt b/dbms/programs/CMakeLists.txt index cb01b57bb56..775013c7402 100644 --- a/dbms/programs/CMakeLists.txt +++ b/dbms/programs/CMakeLists.txt @@ -15,7 +15,6 @@ option (ENABLE_CLICKHOUSE_FORMAT "Enable format" ${ENABLE_CLICKHOUSE_ALL}) configure_file (config_tools.h.in ${CMAKE_CURRENT_BINARY_DIR}/config_tools.h) -add_subdirectory (clang) add_subdirectory (server) add_subdirectory (client) add_subdirectory (local) @@ -25,6 +24,7 @@ add_subdirectory (extract-from-config) add_subdirectory (compressor) add_subdirectory (copier) add_subdirectory (format) +add_subdirectory (clang) if (CLICKHOUSE_SPLIT_BINARY) set (CLICKHOUSE_ALL_TARGETS clickhouse-server clickhouse-client clickhouse-local clickhouse-benchmark clickhouse-performance-test @@ -139,16 +139,6 @@ else () endif () -if (USE_EMBEDDED_COMPILER) - add_custom_target(copy-headers ALL env CLANG=${CMAKE_CURRENT_BINARY_DIR}/clickhouse-clang BUILD_PATH=${ClickHouse_BINARY_DIR} DESTDIR=${ClickHouse_SOURCE_DIR} ${ClickHouse_SOURCE_DIR}/copy_headers.sh ${ClickHouse_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/headers DEPENDS clickhouse-clang WORKING_DIRECTORY ${ClickHouse_SOURCE_DIR} SOURCES ${ClickHouse_SOURCE_DIR}/copy_headers.sh) - install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/headers DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/clickhouse COMPONENT clickhouse) - - if (USE_INTERNAL_LLVM_LIBRARY) - set(CLANG_HEADERS_DIR "${ClickHouse_SOURCE_DIR}/contrib/llvm/clang/lib/Headers") - set(CLANG_HEADERS_DEST "${CMAKE_CURRENT_BINARY_DIR}/headers/usr/local/lib/clang/${LLVM_VERSION}/include") # original: ${LLVM_LIBRARY_OUTPUT_INTDIR}/clang/${CLANG_VERSION}/include - add_custom_target(copy-headers-clang ALL ${CMAKE_COMMAND} -E make_directory ${CLANG_HEADERS_DEST} && ${CMAKE_COMMAND} -E copy_if_different ${CLANG_HEADERS_DIR}/* ${CLANG_HEADERS_DEST} ) - add_dependencies(copy-headers copy-headers-clang) - endif () - +if (USE_EMBEDDED_COMPILER AND ENABLE_CLICKHOUSE_SERVER) add_dependencies(clickhouse-server copy-headers) endif () diff --git a/dbms/programs/benchmark/CMakeLists.txt b/dbms/programs/benchmark/CMakeLists.txt index 219812a8ade..454954d7a6c 100644 --- a/dbms/programs/benchmark/CMakeLists.txt +++ b/dbms/programs/benchmark/CMakeLists.txt @@ -1,6 +1,6 @@ add_library (clickhouse-benchmark-lib ${SPLIT_SHARED} Benchmark.cpp) target_link_libraries (clickhouse-benchmark-lib clickhouse-client-lib clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY}) -target_include_directories (clickhouse-benchmark-lib PRIVATE ${PCG_RANDOM_INCLUDE_DIR}) +target_include_directories (clickhouse-benchmark-lib SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR}) if (CLICKHOUSE_SPLIT_BINARY) add_executable (clickhouse-benchmark clickhouse-benchmark.cpp) diff --git a/dbms/programs/clang/CMakeLists.txt b/dbms/programs/clang/CMakeLists.txt index bc3222c49fd..ab5934e071c 100644 --- a/dbms/programs/clang/CMakeLists.txt +++ b/dbms/programs/clang/CMakeLists.txt @@ -13,3 +13,18 @@ if (CLICKHOUSE_SPLIT_BINARY) set_target_properties(clickhouse-clang clickhouse-lld PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..) endif () endif () + +set(TMP_HEADERS_DIR "${CMAKE_CURRENT_BINARY_DIR}/headers") +# Make and install empty dir for debian package if compiler disabled +add_custom_target(make-headers-directory ALL COMMAND ${CMAKE_COMMAND} -E make_directory ${TMP_HEADERS_DIR}) +install(DIRECTORY ${TMP_HEADERS_DIR} DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/clickhouse COMPONENT clickhouse) +if (USE_EMBEDDED_COMPILER) + add_custom_target(copy-headers ALL env CLANG=${CMAKE_CURRENT_BINARY_DIR}/../clickhouse-clang BUILD_PATH=${ClickHouse_BINARY_DIR} DESTDIR=${ClickHouse_SOURCE_DIR} ${ClickHouse_SOURCE_DIR}/copy_headers.sh ${ClickHouse_SOURCE_DIR} ${TMP_HEADERS_DIR} DEPENDS clickhouse-clang WORKING_DIRECTORY ${ClickHouse_SOURCE_DIR} SOURCES ${ClickHouse_SOURCE_DIR}/copy_headers.sh) + + if (USE_INTERNAL_LLVM_LIBRARY) + set(CLANG_HEADERS_DIR "${ClickHouse_SOURCE_DIR}/contrib/llvm/clang/lib/Headers") + set(CLANG_HEADERS_DEST "${TMP_HEADERS_DIR}/usr/local/lib/clang/${LLVM_VERSION}/include") # original: ${LLVM_LIBRARY_OUTPUT_INTDIR}/clang/${CLANG_VERSION}/include + add_custom_target(copy-headers-clang ALL ${CMAKE_COMMAND} -E make_directory ${CLANG_HEADERS_DEST} && ${CMAKE_COMMAND} -E copy_if_different ${CLANG_HEADERS_DIR}/* ${CLANG_HEADERS_DEST} ) + add_dependencies(copy-headers copy-headers-clang) + endif () +endif () diff --git a/dbms/programs/clang/Compiler-5.0.0/CMakeLists.txt b/dbms/programs/clang/Compiler-5.0.0/CMakeLists.txt index 5ccedebb142..e07a570a434 100644 --- a/dbms/programs/clang/Compiler-5.0.0/CMakeLists.txt +++ b/dbms/programs/clang/Compiler-5.0.0/CMakeLists.txt @@ -12,7 +12,7 @@ llvm_libs_all(REQUIRED_LLVM_LIBRARIES) message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}") -target_include_directories(clickhouse-compiler-lib PRIVATE ${LLVM_INCLUDE_DIRS}) +target_include_directories(clickhouse-compiler-lib SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS}) # This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory. diff --git a/dbms/programs/clang/Compiler-6.0.0/CMakeLists.txt b/dbms/programs/clang/Compiler-6.0.0/CMakeLists.txt index fc9bafafdee..4f5e703bd63 100644 --- a/dbms/programs/clang/Compiler-6.0.0/CMakeLists.txt +++ b/dbms/programs/clang/Compiler-6.0.0/CMakeLists.txt @@ -12,7 +12,7 @@ llvm_libs_all(REQUIRED_LLVM_LIBRARIES) message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}") -target_include_directories(clickhouse-compiler-lib PRIVATE ${LLVM_INCLUDE_DIRS}) +target_include_directories(clickhouse-compiler-lib SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS}) # This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory. diff --git a/dbms/programs/clang/Compiler-7.0.0/CMakeLists.txt b/dbms/programs/clang/Compiler-7.0.0/CMakeLists.txt index b581b226e23..f46e8ef0dc1 100644 --- a/dbms/programs/clang/Compiler-7.0.0/CMakeLists.txt +++ b/dbms/programs/clang/Compiler-7.0.0/CMakeLists.txt @@ -12,7 +12,7 @@ llvm_libs_all(REQUIRED_LLVM_LIBRARIES) message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}") -target_include_directories(clickhouse-compiler-lib PRIVATE ${LLVM_INCLUDE_DIRS}) +target_include_directories(clickhouse-compiler-lib SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS}) # This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory. diff --git a/dbms/programs/client/CMakeLists.txt b/dbms/programs/client/CMakeLists.txt index 840b52ac4e1..89c34669a6b 100644 --- a/dbms/programs/client/CMakeLists.txt +++ b/dbms/programs/client/CMakeLists.txt @@ -1,6 +1,6 @@ add_library (clickhouse-client-lib Client.cpp) target_link_libraries (clickhouse-client-lib clickhouse_functions clickhouse_aggregate_functions ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY}) -target_include_directories (clickhouse-client-lib PRIVATE ${READLINE_INCLUDE_DIR}) +target_include_directories (clickhouse-client-lib SYSTEM PRIVATE ${READLINE_INCLUDE_DIR}) if (CLICKHOUSE_SPLIT_BINARY) add_executable (clickhouse-client clickhouse-client.cpp) diff --git a/dbms/programs/main.cpp b/dbms/programs/main.cpp index a5e7aec0736..f46e6f682a9 100644 --- a/dbms/programs/main.cpp +++ b/dbms/programs/main.cpp @@ -34,7 +34,7 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv); #if ENABLE_CLICKHOUSE_PERFORMANCE int mainEntryClickHousePerformanceTest(int argc, char ** argv); #endif -#if ENABLE_CLICKHOUSE_extract-from-config +#if ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG int mainEntryClickHouseExtractFromConfig(int argc, char ** argv); #endif #if ENABLE_CLICKHOUSE_COMPRESSOR @@ -76,9 +76,13 @@ std::pair clickhouse_applications[] = #if ENABLE_CLICKHOUSE_PERFORMANCE {"performance-test", mainEntryClickHousePerformanceTest}, #endif -#if ENABLE_CLICKHOUSE_TOOLS +#if ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG {"extract-from-config", mainEntryClickHouseExtractFromConfig}, +#endif +#if ENABLE_CLICKHOUSE_COMPRESSOR {"compressor", mainEntryClickHouseCompressor}, +#endif +#if ENABLE_CLICKHOUSE_FORMAT {"format", mainEntryClickHouseFormat}, #endif #if ENABLE_CLICKHOUSE_COPIER diff --git a/dbms/programs/performance-test/CMakeLists.txt b/dbms/programs/performance-test/CMakeLists.txt index 322d681128f..25c963c7a1e 100644 --- a/dbms/programs/performance-test/CMakeLists.txt +++ b/dbms/programs/performance-test/CMakeLists.txt @@ -1,6 +1,6 @@ add_library (clickhouse-performance-test-lib ${SPLIT_SHARED} PerformanceTest.cpp) target_link_libraries (clickhouse-performance-test-lib clickhouse_common_io dbms ${Boost_PROGRAM_OPTIONS_LIBRARY}) -target_include_directories (clickhouse-performance-test-lib PRIVATE ${PCG_RANDOM_INCLUDE_DIR}) +target_include_directories (clickhouse-performance-test-lib SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR}) if (CLICKHOUSE_SPLIT_BINARY) add_executable (clickhouse-performance-test clickhouse-performance-test.cpp) diff --git a/dbms/programs/server/CMakeLists.txt b/dbms/programs/server/CMakeLists.txt index 1cf249479d1..945f12051a6 100644 --- a/dbms/programs/server/CMakeLists.txt +++ b/dbms/programs/server/CMakeLists.txt @@ -20,9 +20,10 @@ if (CLICKHOUSE_SPLIT_BINARY) install (TARGETS clickhouse-server ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) endif () - -set (GLIBC_MAX_REQUIRED 2.4) -add_test(NAME GLIBC_required_version COMMAND bash -c "readelf -s ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-server | grep '@GLIBC' | grep -oP 'GLIBC_[\\d\\.]+' | sort | uniq | sort -r | perl -lnE 'exit 1 if $_ gt q{GLIBC_${GLIBC_MAX_REQUIRED}}'") +if (NOT APPLE AND NOT ARCH_FREEBSD) + set (GLIBC_MAX_REQUIRED 2.4) + add_test(NAME GLIBC_required_version COMMAND bash -c "readelf -s ${CMAKE_CURRENT_BINARY_DIR}/../clickhouse-server | grep '@GLIBC' | grep -oP 'GLIBC_[\\d\\.]+' | sort | uniq | sort -r | perl -lnE 'exit 1 if $_ gt q{GLIBC_${GLIBC_MAX_REQUIRED}}'") +endif () install ( FILES config.xml users.xml diff --git a/dbms/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h b/dbms/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h index 98350e3eba9..e864d1cf64a 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h @@ -204,6 +204,7 @@ public: case StatisticsFunctionKind::covarSamp: return "covarSamp"; case StatisticsFunctionKind::corr: return "corr"; } + __builtin_unreachable(); } DataTypePtr getReturnType() const override diff --git a/dbms/src/Client/Connection.cpp b/dbms/src/Client/Connection.cpp index 9cd6e29986d..c461b4cafde 100644 --- a/dbms/src/Client/Connection.cpp +++ b/dbms/src/Client/Connection.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -42,6 +43,7 @@ namespace ErrorCodes extern const int UNEXPECTED_PACKET_FROM_SERVER; extern const int UNKNOWN_PACKET_FROM_SERVER; extern const int SUPPORT_IS_DISABLED; + extern const int BAD_ARGUMENTS; } @@ -121,7 +123,27 @@ void Connection::disconnect() void Connection::sendHello() { - //LOG_TRACE(log_wrapper.get(), "Sending hello"); + /** Disallow control characters in user controlled parameters + * to mitigate the possibility of SSRF. + * The user may do server side requests with 'remote' table function. + * Malicious user with full r/w access to ClickHouse + * may use 'remote' table function to forge requests + * to another services in the network other than ClickHouse (examples: SMTP). + * Limiting number of possible characters in user-controlled part of handshake + * will mitigate this possibility but doesn't solve it completely. + */ + auto has_control_character = [](const std::string & s) + { + for (auto c : s) + if (isControlASCII(c)) + return true; + return false; + }; + + if (has_control_character(default_database) + || has_control_character(user) + || has_control_character(password)) + throw Exception("Parameters 'default_database', 'user' and 'password' must not contain ASCII control characters", ErrorCodes::BAD_ARGUMENTS); writeVarUInt(Protocol::Client::Hello, *out); writeStringBinary((DBMS_NAME " ") + client_name, *out); diff --git a/dbms/src/Common/StringUtils/StringUtils.h b/dbms/src/Common/StringUtils/StringUtils.h index a94699206d9..8043b88a89e 100644 --- a/dbms/src/Common/StringUtils/StringUtils.h +++ b/dbms/src/Common/StringUtils/StringUtils.h @@ -107,6 +107,11 @@ inline bool isWhitespaceASCII(char c) return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'; } +inline bool isControlASCII(char c) +{ + return c >= 0 && c <= 31; +} + /// Works assuming isAlphaASCII. inline char toLowerIfAlphaASCII(char c) { diff --git a/dbms/src/Core/tests/CMakeLists.txt b/dbms/src/Core/tests/CMakeLists.txt index f8cfe2eee2c..26e89341bbb 100644 --- a/dbms/src/Core/tests/CMakeLists.txt +++ b/dbms/src/Core/tests/CMakeLists.txt @@ -3,7 +3,7 @@ target_link_libraries (exception clickhouse_common_io) add_executable (string_pool string_pool.cpp) target_link_libraries (string_pool clickhouse_common_io) -target_include_directories (string_pool BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) +target_include_directories (string_pool SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) add_executable (field field.cpp) target_link_libraries (field dbms) diff --git a/dbms/src/DataStreams/FormatFactory.cpp b/dbms/src/DataStreams/FormatFactory.cpp index 73c7dbdaa3b..d17176d5b2b 100644 --- a/dbms/src/DataStreams/FormatFactory.cpp +++ b/dbms/src/DataStreams/FormatFactory.cpp @@ -119,7 +119,6 @@ BlockInputStreamPtr FormatFactory::getInput(const String & name, ReadBuffer & bu || name == "PrettyCompactNoEscapes" || name == "PrettySpaceNoEscapes" || name == "Vertical" - || name == "VerticalRaw" || name == "Null" || name == "JSON" || name == "JSONCompact" @@ -180,9 +179,6 @@ static BlockOutputStreamPtr getOutputImpl(const String & name, WriteBuffer & buf else if (name == "Vertical") return std::make_shared(std::make_shared( buf, sample, settings.output_format_pretty_max_rows), sample); - else if (name == "VerticalRaw") - return std::make_shared(std::make_shared( - buf, sample, settings.output_format_pretty_max_rows), sample); else if (name == "Values") return std::make_shared(std::make_shared(buf), sample); else if (name == "JSON") diff --git a/dbms/src/DataStreams/PrettyBlockOutputStream.cpp b/dbms/src/DataStreams/PrettyBlockOutputStream.cpp index 2bb919d4806..94c643b46c0 100644 --- a/dbms/src/DataStreams/PrettyBlockOutputStream.cpp +++ b/dbms/src/DataStreams/PrettyBlockOutputStream.cpp @@ -55,7 +55,7 @@ void PrettyBlockOutputStream::calculateWidths(const Block & block, WidthsPerColu { { WriteBufferFromString out(serialized_value); - elem.type->serializeTextEscaped(*elem.column, j, out); + elem.type->serializeText(*elem.column, j, out); } widths[i][j] = UTF8::countCodePoints(reinterpret_cast(serialized_value.data()), serialized_value.size()); @@ -64,13 +64,7 @@ void PrettyBlockOutputStream::calculateWidths(const Block & block, WidthsPerColu /// And also calculate widths for names of columns. { - /// We need to obtain length in escaped form. - { - WriteBufferFromString out(serialized_value); - writeEscapedString(elem.name, out); - } - - name_widths[i] = UTF8::countCodePoints(reinterpret_cast(serialized_value.data()), serialized_value.size()); + name_widths[i] = UTF8::countCodePoints(reinterpret_cast(elem.name.data()), elem.name.size()); max_widths[i] = std::max(max_widths[i], name_widths[i]); } } @@ -151,11 +145,11 @@ void PrettyBlockOutputStream::write(const Block & block) for (size_t k = 0; k < max_widths[i] - name_widths[i]; ++k) writeChar(' ', ostr); - writeEscapedString(col.name, ostr); + writeString(col.name, ostr); } else { - writeEscapedString(col.name, ostr); + writeString(col.name, ostr); for (size_t k = 0; k < max_widths[i] - name_widths[i]; ++k) writeChar(' ', ostr); @@ -203,11 +197,11 @@ void PrettyBlockOutputStream::writeValueWithPadding(const ColumnWithTypeAndName if (elem.type->shouldAlignRightInPrettyFormats()) { writePadding(); - elem.type->serializeTextEscaped(*elem.column.get(), row_num, ostr); + elem.type->serializeText(*elem.column.get(), row_num, ostr); } else { - elem.type->serializeTextEscaped(*elem.column.get(), row_num, ostr); + elem.type->serializeText(*elem.column.get(), row_num, ostr); writePadding(); } } diff --git a/dbms/src/DataStreams/PrettyCompactBlockOutputStream.cpp b/dbms/src/DataStreams/PrettyCompactBlockOutputStream.cpp index e6854d055a3..c9e91dd98eb 100644 --- a/dbms/src/DataStreams/PrettyCompactBlockOutputStream.cpp +++ b/dbms/src/DataStreams/PrettyCompactBlockOutputStream.cpp @@ -34,7 +34,7 @@ void PrettyCompactBlockOutputStream::writeHeader( if (!no_escapes) writeCString("\033[1m", ostr); - writeEscapedString(col.name, ostr); + writeString(col.name, ostr); if (!no_escapes) writeCString("\033[0m", ostr); } @@ -42,7 +42,7 @@ void PrettyCompactBlockOutputStream::writeHeader( { if (!no_escapes) writeCString("\033[1m", ostr); - writeEscapedString(col.name, ostr); + writeString(col.name, ostr); if (!no_escapes) writeCString("\033[0m", ostr); diff --git a/dbms/src/DataStreams/PrettySpaceBlockOutputStream.cpp b/dbms/src/DataStreams/PrettySpaceBlockOutputStream.cpp index db88f685861..fa3923f78a3 100644 --- a/dbms/src/DataStreams/PrettySpaceBlockOutputStream.cpp +++ b/dbms/src/DataStreams/PrettySpaceBlockOutputStream.cpp @@ -44,7 +44,7 @@ void PrettySpaceBlockOutputStream::write(const Block & block) if (!no_escapes) writeCString("\033[1m", ostr); - writeEscapedString(col.name, ostr); + writeString(col.name, ostr); if (!no_escapes) writeCString("\033[0m", ostr); } @@ -52,7 +52,7 @@ void PrettySpaceBlockOutputStream::write(const Block & block) { if (!no_escapes) writeCString("\033[1m", ostr); - writeEscapedString(col.name, ostr); + writeString(col.name, ostr); if (!no_escapes) writeCString("\033[0m", ostr); diff --git a/dbms/src/DataStreams/VerticalRowOutputStream.cpp b/dbms/src/DataStreams/VerticalRowOutputStream.cpp index 4c6e1d861c5..7ae5278adac 100644 --- a/dbms/src/DataStreams/VerticalRowOutputStream.cpp +++ b/dbms/src/DataStreams/VerticalRowOutputStream.cpp @@ -26,13 +26,7 @@ VerticalRowOutputStream::VerticalRowOutputStream( /// Note that number of code points is just a rough approximation of visible string width. const String & name = sample.getByPosition(i).name; - { - /// We need to obtain length in escaped form. - WriteBufferFromString out(serialized_value); - writeEscapedString(name, out); - } - - name_widths[i] = UTF8::countCodePoints(reinterpret_cast(serialized_value.data()), serialized_value.size()); + name_widths[i] = UTF8::countCodePoints(reinterpret_cast(name.data()), name.size()); if (name_widths[i] > max_name_width) max_name_width = name_widths[i]; @@ -42,7 +36,7 @@ VerticalRowOutputStream::VerticalRowOutputStream( for (size_t i = 0; i < columns; ++i) { WriteBufferFromString out(names_and_paddings[i]); - writeEscapedString(sample.getByPosition(i).name, out); + writeString(sample.getByPosition(i).name, out); writeCString(": ", out); } @@ -71,11 +65,6 @@ void VerticalRowOutputStream::writeField(const IColumn & column, const IDataType void VerticalRowOutputStream::writeValue(const IColumn & column, const IDataType & type, size_t row_num) const -{ - type.serializeTextEscaped(column, row_num, ostr); -} - -void VerticalRawRowOutputStream::writeValue(const IColumn & column, const IDataType & type, size_t row_num) const { type.serializeText(column, row_num, ostr); } diff --git a/dbms/src/DataStreams/VerticalRowOutputStream.h b/dbms/src/DataStreams/VerticalRowOutputStream.h index 4dbd6e44c6d..6edada2e92e 100644 --- a/dbms/src/DataStreams/VerticalRowOutputStream.h +++ b/dbms/src/DataStreams/VerticalRowOutputStream.h @@ -51,17 +51,5 @@ protected: Block extremes; }; - -/** Same but values are printed without escaping. - */ -class VerticalRawRowOutputStream final : public VerticalRowOutputStream -{ -public: - using VerticalRowOutputStream::VerticalRowOutputStream; - -protected: - void writeValue(const IColumn & column, const IDataType & type, size_t row_num) const override; -}; - } diff --git a/dbms/src/DataStreams/tests/CMakeLists.txt b/dbms/src/DataStreams/tests/CMakeLists.txt index 8c3acb1332e..03f6d4f0d56 100644 --- a/dbms/src/DataStreams/tests/CMakeLists.txt +++ b/dbms/src/DataStreams/tests/CMakeLists.txt @@ -1,5 +1,3 @@ -include_directories (${CMAKE_CURRENT_BINARY_DIR}) - set(SRCS ) add_executable (tab_separated_streams tab_separated_streams.cpp ${SRCS}) diff --git a/dbms/src/DataTypes/DataTypeNullable.cpp b/dbms/src/DataTypes/DataTypeNullable.cpp index 751f32eb462..68a78acb856 100644 --- a/dbms/src/DataTypes/DataTypeNullable.cpp +++ b/dbms/src/DataTypes/DataTypeNullable.cpp @@ -242,8 +242,14 @@ void DataTypeNullable::serializeText(const IColumn & column, size_t row_num, Wri { const ColumnNullable & col = static_cast(column); + /// In simple text format (like 'Pretty' format) (these formats are suitable only for output and cannot be parsed back), + /// data is printed without escaping. + /// It makes theoretically impossible to distinguish between NULL and some string value, regardless on how do we print NULL. + /// For this reason, we output NULL in a bit strange way. + /// This assumes UTF-8 and proper font support. This is Ok, because Pretty formats are "presentational", not for data exchange. + if (col.isNullAt(row_num)) - writeCString("NULL", ostr); + writeCString("ᴺᵁᴸᴸ", ostr); else nested_data_type->serializeText(col.getNestedColumn(), row_num, ostr); } diff --git a/dbms/src/DataTypes/tests/CMakeLists.txt b/dbms/src/DataTypes/tests/CMakeLists.txt index 6a20692bca0..6186c7dfef4 100644 --- a/dbms/src/DataTypes/tests/CMakeLists.txt +++ b/dbms/src/DataTypes/tests/CMakeLists.txt @@ -1,5 +1,3 @@ -include_directories (${CMAKE_CURRENT_BINARY_DIR}) - set(SRCS ) add_executable (data_types_number_fixed data_types_number_fixed.cpp ${SRCS}) diff --git a/dbms/src/Functions/CMakeLists.txt b/dbms/src/Functions/CMakeLists.txt index 2306f0c109d..2f9907f2247 100644 --- a/dbms/src/Functions/CMakeLists.txt +++ b/dbms/src/Functions/CMakeLists.txt @@ -88,9 +88,9 @@ add_library(clickhouse_functions ${clickhouse_functions_sources}) target_link_libraries(clickhouse_functions PUBLIC dbms PRIVATE libconsistent-hashing ${FARMHASH_LIBRARIES} ${METROHASH_LIBRARIES}) -target_include_directories (clickhouse_functions BEFORE PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/libfarmhash) -target_include_directories (clickhouse_functions BEFORE PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/libmetrohash/src) -target_include_directories (clickhouse_functions BEFORE PUBLIC ${DIVIDE_INCLUDE_DIR}) +target_include_directories (clickhouse_functions SYSTEM BEFORE PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/libfarmhash) +target_include_directories (clickhouse_functions SYSTEM BEFORE PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/libmetrohash/src) +target_include_directories (clickhouse_functions SYSTEM BEFORE PUBLIC ${DIVIDE_INCLUDE_DIR}) if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" OR CMAKE_BUILD_TYPE_UC STREQUAL "MINSIZEREL") # Won't generate debug info for files with heavy template instantiation to achieve faster linking and lower size. @@ -99,11 +99,11 @@ endif () if (USE_ICU) #target_link_libraries (clickhouse_functions ${ICU_LIBS}) - target_include_directories (clickhouse_functions PRIVATE ${ICU_INCLUDE_DIR}) + target_include_directories (clickhouse_functions SYSTEM PRIVATE ${ICU_INCLUDE_DIR}) endif () if (USE_VECTORCLASS) - target_include_directories (clickhouse_functions BEFORE PUBLIC ${VECTORCLASS_INCLUDE_DIR}) + target_include_directories (clickhouse_functions SYSTEM BEFORE PUBLIC ${VECTORCLASS_INCLUDE_DIR}) endif () if (ENABLE_TESTS) @@ -111,5 +111,5 @@ if (ENABLE_TESTS) endif () if (USE_EMBEDDED_COMPILER) - target_include_directories (clickhouse_functions BEFORE PUBLIC ${LLVM_INCLUDE_DIRS}) + target_include_directories (clickhouse_functions SYSTEM BEFORE PUBLIC ${LLVM_INCLUDE_DIRS}) endif () diff --git a/dbms/src/Functions/FunctionsMiscellaneous.cpp b/dbms/src/Functions/FunctionsMiscellaneous.cpp index d9695111a0f..42db6ab7618 100644 --- a/dbms/src/Functions/FunctionsMiscellaneous.cpp +++ b/dbms/src/Functions/FunctionsMiscellaneous.cpp @@ -1688,7 +1688,7 @@ void FunctionVisibleWidth::executeImpl(Block & block, const ColumnNumbers & argu { { WriteBufferFromString out(tmp); - src.type->serializeTextEscaped(*src.column, i, out); + src.type->serializeText(*src.column, i, out); } res_data[i] = UTF8::countCodePoints(reinterpret_cast(tmp.data()), tmp.size()); diff --git a/dbms/src/Functions/FunctionsRound.h b/dbms/src/Functions/FunctionsRound.h index 89fad9fbf1c..35cfa56426d 100644 --- a/dbms/src/Functions/FunctionsRound.h +++ b/dbms/src/Functions/FunctionsRound.h @@ -219,6 +219,8 @@ struct IntegerRoundingComputation x = -x; return x; } + default: + __builtin_unreachable(); } } @@ -232,6 +234,8 @@ struct IntegerRoundingComputation return x; case ScaleMode::Negative: return computeImpl(x, scale); + default: + __builtin_unreachable(); } } diff --git a/dbms/src/Functions/FunctionsString.cpp b/dbms/src/Functions/FunctionsString.cpp index 4c1a38142c6..26022837d3b 100644 --- a/dbms/src/Functions/FunctionsString.cpp +++ b/dbms/src/Functions/FunctionsString.cpp @@ -1084,6 +1084,125 @@ private: }; +struct NameStartsWith +{ + static constexpr auto name = "startsWith"; +}; +struct NameEndsWith +{ + static constexpr auto name = "endsWith"; +}; + +template +class FunctionStartsEndsWith : public IFunction +{ +public: + static constexpr auto name = Name::name; + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } + + String getName() const override + { + return name; + } + + size_t getNumberOfArguments() const override + { + return 2; + } + + bool useDefaultImplementationForConstants() const override + { + return true; + } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + if (!arguments[0]->isStringOrFixedString()) + throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + if (!arguments[1]->isStringOrFixedString()) + throw Exception("Illegal type " + arguments[1]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + return std::make_shared>(); + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + { + const IColumn * haystack_column = block.getByPosition(arguments[0]).column.get(); + const IColumn * needle_column = block.getByPosition(arguments[1]).column.get(); + + auto col_res = ColumnVector::create(); + typename ColumnVector::Container & vec_res = col_res->getData(); + + vec_res.resize(input_rows_count); + + if (const ColumnString * haystack = checkAndGetColumn(haystack_column)) + dispatch(StringSource(*haystack), needle_column, vec_res); + else if (const ColumnFixedString * haystack = checkAndGetColumn(haystack_column)) + dispatch(FixedStringSource(*haystack), needle_column, vec_res); + else if (const ColumnConst * haystack = checkAndGetColumnConst(haystack_column)) + dispatch>(ConstSource(*haystack), needle_column, vec_res); + else if (const ColumnConst * haystack = checkAndGetColumnConst(haystack_column)) + dispatch>(ConstSource(*haystack), needle_column, vec_res); + else + throw Exception("Illegal combination of columns as arguments of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); + + block.getByPosition(result).column = std::move(col_res); + } + +private: + template + void dispatch(HaystackSource haystack_source, const IColumn * needle_column, PaddedPODArray & res_data) const + { + if (const ColumnString * needle = checkAndGetColumn(needle_column)) + execute(haystack_source, StringSource(*needle), res_data); + else if (const ColumnFixedString * needle = checkAndGetColumn(needle_column)) + execute(haystack_source, FixedStringSource(*needle), res_data); + else if (const ColumnConst * needle = checkAndGetColumnConst(needle_column)) + execute>(haystack_source, ConstSource(*needle), res_data); + else if (const ColumnConst * needle = checkAndGetColumnConst(needle_column)) + execute>(haystack_source, ConstSource(*needle), res_data); + else + throw Exception("Illegal combination of columns as arguments of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); + } + + template + static void execute(HaystackSource haystack_source, NeedleSource needle_source, PaddedPODArray & res_data) + { + size_t row_num = 0; + + while (!haystack_source.isEnd()) + { + auto haystack = haystack_source.getWhole(); + auto needle = needle_source.getWhole(); + + if (needle.size > haystack.size) + { + res_data[row_num] = false; + } + else + { + if constexpr (std::is_same_v) + { + res_data[row_num] = StringRef(haystack.data, needle.size) == StringRef(needle.data, needle.size); + } + else /// endsWith + { + res_data[row_num] = StringRef(haystack.data + haystack.size - needle.size, needle.size) == StringRef(needle.data, needle.size); + } + } + + haystack_source.next(); + needle_source.next(); + ++row_num; + } + } +}; + + struct NameEmpty { static constexpr auto name = "empty"; @@ -1121,6 +1240,7 @@ struct NameConcatAssumeInjective static constexpr auto name = "concatAssumeInjective"; }; + using FunctionEmpty = FunctionStringOrArrayToT, NameEmpty, UInt8>; using FunctionNotEmpty = FunctionStringOrArrayToT, NameNotEmpty, UInt8>; using FunctionLength = FunctionStringOrArrayToT; @@ -1130,6 +1250,8 @@ using FunctionUpper = FunctionStringToString, NameUpper using FunctionReverseUTF8 = FunctionStringToString; using FunctionConcat = ConcatImpl; using FunctionConcatAssumeInjective = ConcatImpl; +using FunctionStartsWith = FunctionStartsEndsWith; +using FunctionEndsWith = FunctionStartsEndsWith; void registerFunctionsString(FunctionFactory & factory) @@ -1149,5 +1271,7 @@ void registerFunctionsString(FunctionFactory & factory) factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); } } diff --git a/dbms/src/Functions/tests/CMakeLists.txt b/dbms/src/Functions/tests/CMakeLists.txt index 64998be158e..b0fd9d1642d 100644 --- a/dbms/src/Functions/tests/CMakeLists.txt +++ b/dbms/src/Functions/tests/CMakeLists.txt @@ -1,4 +1,2 @@ -include_directories (${CMAKE_CURRENT_BINARY_DIR}) - add_executable (number_traits number_traits.cpp) target_link_libraries (number_traits dbms) diff --git a/dbms/src/IO/tests/CMakeLists.txt b/dbms/src/IO/tests/CMakeLists.txt index 1bda663e847..7b427a6ae00 100644 --- a/dbms/src/IO/tests/CMakeLists.txt +++ b/dbms/src/IO/tests/CMakeLists.txt @@ -1,5 +1,3 @@ -include_directories (${CMAKE_CURRENT_BINARY_DIR}) - add_executable (read_buffer read_buffer.cpp) target_link_libraries (read_buffer clickhouse_common_io) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 5605c215dea..b502532b224 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -355,7 +355,7 @@ std::shared_ptr Context::acquireSession(const String & session_id, std: if (session_check) throw Exception("Session not found.", ErrorCodes::SESSION_NOT_FOUND); - auto new_session = std::make_shared(*global_context); + auto new_session = std::make_shared(*this); new_session->scheduleCloseSession(key, timeout); diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index f730a508e36..cd5628698d6 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.h b/dbms/src/Interpreters/ExpressionAnalyzer.h index 5e5a836e266..91904767445 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.h +++ b/dbms/src/Interpreters/ExpressionAnalyzer.h @@ -38,6 +38,8 @@ class ASTFunction; class ASTExpressionList; class ASTSelectQuery; +struct ProjectionManipulatorBase; +using ProjectionManipulatorPtr = std::shared_ptr; /** Information on what to do when executing a subquery in the [GLOBAL] IN/JOIN section. */ diff --git a/dbms/src/Interpreters/InterpreterShowTablesQuery.cpp b/dbms/src/Interpreters/InterpreterShowTablesQuery.cpp index 17576d2257c..df4332c5b16 100644 --- a/dbms/src/Interpreters/InterpreterShowTablesQuery.cpp +++ b/dbms/src/Interpreters/InterpreterShowTablesQuery.cpp @@ -1,17 +1,14 @@ #include - #include #include - #include #include #include - #include - #include #include + namespace DB { diff --git a/dbms/src/Interpreters/tests/CMakeLists.txt b/dbms/src/Interpreters/tests/CMakeLists.txt index ad3a24d8608..c88ce789c81 100644 --- a/dbms/src/Interpreters/tests/CMakeLists.txt +++ b/dbms/src/Interpreters/tests/CMakeLists.txt @@ -11,7 +11,7 @@ add_executable (aggregate aggregate.cpp) target_link_libraries (aggregate dbms) add_executable (hash_map hash_map.cpp) -target_include_directories (hash_map BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) +target_include_directories (hash_map SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) target_link_libraries (hash_map dbms) add_executable (hash_map3 hash_map3.cpp) @@ -26,15 +26,15 @@ target_link_libraries (hash_map_string_2 dbms) add_executable (hash_map_string_3 hash_map_string_3.cpp) target_link_libraries (hash_map_string_3 dbms ${FARMHASH_LIBRARIES} ${METROHASH_LIBRARIES}) -target_include_directories (hash_map_string_3 BEFORE PRIVATE ${ClickHouse_SOURCE_DIR}/contrib/libfarmhash) -target_include_directories (hash_map_string_3 BEFORE PRIVATE ${ClickHouse_SOURCE_DIR}/contrib/libmetrohash/src) +target_include_directories (hash_map_string_3 SYSTEM BEFORE PRIVATE ${ClickHouse_SOURCE_DIR}/contrib/libfarmhash) +target_include_directories (hash_map_string_3 SYSTEM BEFORE PRIVATE ${ClickHouse_SOURCE_DIR}/contrib/libmetrohash/src) add_executable (hash_map_string_small hash_map_string_small.cpp) -target_include_directories (hash_map_string_small BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) +target_include_directories (hash_map_string_small SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) target_link_libraries (hash_map_string_small dbms) add_executable (two_level_hash_map two_level_hash_map.cpp) -target_include_directories (two_level_hash_map BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) +target_include_directories (two_level_hash_map SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) target_link_libraries (two_level_hash_map dbms) add_executable (compiler_test compiler_test.cpp) diff --git a/dbms/src/Parsers/tests/CMakeLists.txt b/dbms/src/Parsers/tests/CMakeLists.txt index 54a77b0cf7d..4f87424bf89 100644 --- a/dbms/src/Parsers/tests/CMakeLists.txt +++ b/dbms/src/Parsers/tests/CMakeLists.txt @@ -1,5 +1,3 @@ -include_directories (${CMAKE_CURRENT_BINARY_DIR}) - set(SRCS ) add_executable (lexer lexer.cpp ${SRCS}) diff --git a/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp index fe9cdfad26a..12e6ead5325 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp @@ -22,11 +22,11 @@ MergeTreeBaseBlockInputStream::MergeTreeBaseBlockInputStream( MergeTreeData & storage, const ExpressionActionsPtr & prewhere_actions, const String & prewhere_column_name, - size_t max_block_size_rows, - size_t preferred_block_size_bytes, - size_t preferred_max_column_in_block_size_bytes, - size_t min_bytes_to_use_direct_io, - size_t max_read_buffer_size, + UInt64 max_block_size_rows, + UInt64 preferred_block_size_bytes, + UInt64 preferred_max_column_in_block_size_bytes, + UInt64 min_bytes_to_use_direct_io, + UInt64 max_read_buffer_size, bool use_uncompressed_cache, bool save_marks_in_cache, const Names & virt_column_names) @@ -89,7 +89,7 @@ Block MergeTreeBaseBlockInputStream::readFromPart() /// Calculates number of rows will be read using preferred_block_size_bytes. /// Can't be less than index_granularity. - size_t rows_to_read = task.size_predictor->estimateNumRows(preferred_block_size_bytes); + UInt64 rows_to_read = task.size_predictor->estimateNumRows(preferred_block_size_bytes); if (!rows_to_read) return rows_to_read; rows_to_read = std::max(index_granularity, rows_to_read); @@ -97,21 +97,21 @@ Block MergeTreeBaseBlockInputStream::readFromPart() if (preferred_max_column_in_block_size_bytes) { /// Calculates number of rows will be read using preferred_max_column_in_block_size_bytes. - size_t rows_to_read_for_max_size_column + UInt64 rows_to_read_for_max_size_column = task.size_predictor->estimateNumRowsForMaxSizeColumn(preferred_max_column_in_block_size_bytes); double filtration_ratio = std::max(min_filtration_ratio, 1.0 - task.size_predictor->filtered_rows_ratio); auto rows_to_read_for_max_size_column_with_filtration - = static_cast(rows_to_read_for_max_size_column / filtration_ratio); + = static_cast(rows_to_read_for_max_size_column / filtration_ratio); /// If preferred_max_column_in_block_size_bytes is used, number of rows to read can be less than index_granularity. rows_to_read = std::min(rows_to_read, rows_to_read_for_max_size_column_with_filtration); } - size_t unread_rows_in_current_granule = reader.numPendingRowsInCurrentGranule(); + UInt64 unread_rows_in_current_granule = reader.numPendingRowsInCurrentGranule(); if (unread_rows_in_current_granule >= rows_to_read) return rows_to_read; - size_t granule_to_read = (rows_to_read + reader.numReadRowsInCurrentGranule() + index_granularity / 2) / index_granularity; + UInt64 granule_to_read = (rows_to_read + reader.numReadRowsInCurrentGranule() + index_granularity / 2) / index_granularity; return index_granularity * granule_to_read - reader.numReadRowsInCurrentGranule(); }; @@ -146,8 +146,8 @@ Block MergeTreeBaseBlockInputStream::readFromPart() } } - size_t recommended_rows = estimateNumRows(*task, task->range_reader); - size_t rows_to_read = std::max(static_cast(1), std::min(max_block_size_rows, recommended_rows)); + UInt64 recommended_rows = estimateNumRows(*task, task->range_reader); + UInt64 rows_to_read = std::max(UInt64(1), std::min(max_block_size_rows, recommended_rows)); auto read_result = task->range_reader.read(rows_to_read, task->mark_ranges); @@ -155,7 +155,7 @@ Block MergeTreeBaseBlockInputStream::readFromPart() if (read_result.block.rows() == 0) read_result.block.clear(); - size_t num_filtered_rows = read_result.numReadRows() - read_result.block.rows(); + UInt64 num_filtered_rows = read_result.numReadRows() - read_result.block.rows(); progressImpl({ read_result.numReadRows(), read_result.numBytesRead() }); diff --git a/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.h b/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.h index 6586027f9b6..f5e8e586e5d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.h +++ b/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.h @@ -20,11 +20,11 @@ public: MergeTreeData & storage, const ExpressionActionsPtr & prewhere_actions, const String & prewhere_column, - size_t max_block_size_rows, - size_t preferred_block_size_bytes, - size_t preferred_max_column_in_block_size_bytes, - size_t min_bytes_to_use_direct_io, - size_t max_read_buffer_size, + UInt64 max_block_size_rows, + UInt64 preferred_block_size_bytes, + UInt64 preferred_max_column_in_block_size_bytes, + UInt64 min_bytes_to_use_direct_io, + UInt64 max_read_buffer_size, bool use_uncompressed_cache, bool save_marks_in_cache = true, const Names & virt_column_names = {}); @@ -50,12 +50,12 @@ protected: ExpressionActionsPtr prewhere_actions; String prewhere_column_name; - size_t max_block_size_rows; - size_t preferred_block_size_bytes; - size_t preferred_max_column_in_block_size_bytes; + UInt64 max_block_size_rows; + UInt64 preferred_block_size_bytes; + UInt64 preferred_max_column_in_block_size_bytes; - size_t min_bytes_to_use_direct_io; - size_t max_read_buffer_size; + UInt64 min_bytes_to_use_direct_io; + UInt64 max_read_buffer_size; bool use_uncompressed_cache; bool save_marks_in_cache; @@ -71,7 +71,7 @@ protected: MergeTreeReaderPtr reader; MergeTreeReaderPtr pre_reader; - size_t max_block_size_marks; + UInt64 max_block_size_marks; }; } diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 848d3108e40..b2dccae82bc 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -977,11 +977,11 @@ ReplicatedMergeTreeQueue::Status ReplicatedMergeTreeQueue::getStatus() const res.inserts_in_queue = 0; res.merges_in_queue = 0; - res.mutations_in_queue = 0; + res.part_mutations_in_queue = 0; res.queue_oldest_time = 0; res.inserts_oldest_time = 0; res.merges_oldest_time = 0; - res.mutations_oldest_time = 0; + res.part_mutations_oldest_time = 0; for (const LogEntryPtr & entry : queue) { @@ -1012,11 +1012,11 @@ ReplicatedMergeTreeQueue::Status ReplicatedMergeTreeQueue::getStatus() const if (entry->type == LogEntry::MUTATE_PART) { - ++res.mutations_in_queue; + ++res.part_mutations_in_queue; - if (entry->create_time && (!res.mutations_oldest_time || entry->create_time < res.mutations_oldest_time)) + if (entry->create_time && (!res.part_mutations_oldest_time || entry->create_time < res.part_mutations_oldest_time)) { - res.mutations_oldest_time = entry->create_time; + res.part_mutations_oldest_time = entry->create_time; res.oldest_part_to_mutate_to = entry->new_part_name; } } diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index 623c981db5d..13ed28f2b35 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -284,11 +284,11 @@ public: UInt32 queue_size; UInt32 inserts_in_queue; UInt32 merges_in_queue; - UInt32 mutations_in_queue; + UInt32 part_mutations_in_queue; UInt32 queue_oldest_time; UInt32 inserts_oldest_time; UInt32 merges_oldest_time; - UInt32 mutations_oldest_time; + UInt32 part_mutations_oldest_time; String oldest_part_to_get; String oldest_part_to_merge_to; String oldest_part_to_mutate_to; diff --git a/dbms/src/Storages/MutationCommands.cpp b/dbms/src/Storages/MutationCommands.cpp index f53cc74731b..0119be9132c 100644 --- a/dbms/src/Storages/MutationCommands.cpp +++ b/dbms/src/Storages/MutationCommands.cpp @@ -36,7 +36,7 @@ void MutationCommand::writeText(WriteBuffer & out) const { std::stringstream ss; formatAST(*predicate, ss, /* hilite = */ false, /* one_line = */ true); - out << "predicate: " << ss.str() << "\n"; + out << "predicate: " << escape << ss.str() << "\n"; break; } default: @@ -54,7 +54,7 @@ void MutationCommand::readText(ReadBuffer & in) type = DELETE; String predicate_str; - in >> "predicate: " >> predicate_str >> "\n"; + in >> "predicate: " >> escape >> predicate_str >> "\n"; ParserExpressionWithOptionalAlias p_expr(false); predicate = parseQuery( p_expr, predicate_str.data(), predicate_str.data() + predicate_str.length(), "mutation predicate", 0); diff --git a/dbms/src/Storages/System/StorageSystemReplicas.cpp b/dbms/src/Storages/System/StorageSystemReplicas.cpp index ea2c8b1cde2..e73f349aec6 100644 --- a/dbms/src/Storages/System/StorageSystemReplicas.cpp +++ b/dbms/src/Storages/System/StorageSystemReplicas.cpp @@ -33,11 +33,14 @@ StorageSystemReplicas::StorageSystemReplicas(const std::string & name_) { "queue_size", std::make_shared() }, { "inserts_in_queue", std::make_shared() }, { "merges_in_queue", std::make_shared() }, + { "part_mutations_in_queue", std::make_shared() }, { "queue_oldest_time", std::make_shared() }, { "inserts_oldest_time", std::make_shared() }, { "merges_oldest_time", std::make_shared() }, + { "part_mutations_oldest_time", std::make_shared() }, { "oldest_part_to_get", std::make_shared() }, { "oldest_part_to_merge_to", std::make_shared() }, + { "oldest_part_to_mutate_to", std::make_shared() }, { "log_max_index", std::make_shared() }, { "log_pointer", std::make_shared() }, { "last_queue_update", std::make_shared() }, @@ -140,11 +143,14 @@ BlockInputStreams StorageSystemReplicas::read( res_columns[col_num++]->insert(UInt64(status.queue.queue_size)); res_columns[col_num++]->insert(UInt64(status.queue.inserts_in_queue)); res_columns[col_num++]->insert(UInt64(status.queue.merges_in_queue)); + res_columns[col_num++]->insert(UInt64(status.queue.part_mutations_in_queue)); res_columns[col_num++]->insert(UInt64(status.queue.queue_oldest_time)); res_columns[col_num++]->insert(UInt64(status.queue.inserts_oldest_time)); res_columns[col_num++]->insert(UInt64(status.queue.merges_oldest_time)); + res_columns[col_num++]->insert(UInt64(status.queue.part_mutations_oldest_time)); res_columns[col_num++]->insert(status.queue.oldest_part_to_get); res_columns[col_num++]->insert(status.queue.oldest_part_to_merge_to); + res_columns[col_num++]->insert(status.queue.oldest_part_to_mutate_to); res_columns[col_num++]->insert(status.log_max_index); res_columns[col_num++]->insert(status.log_pointer); res_columns[col_num++]->insert(UInt64(status.queue.last_queue_update)); diff --git a/dbms/src/Storages/tests/CMakeLists.txt b/dbms/src/Storages/tests/CMakeLists.txt index 568d2fd80d5..e07ac810fee 100644 --- a/dbms/src/Storages/tests/CMakeLists.txt +++ b/dbms/src/Storages/tests/CMakeLists.txt @@ -1,5 +1,3 @@ -include_directories (${CMAKE_CURRENT_BINARY_DIR}) - add_executable (system_numbers system_numbers.cpp) target_link_libraries (system_numbers dbms clickhouse_storages_system) diff --git a/dbms/src/TableFunctions/CMakeLists.txt b/dbms/src/TableFunctions/CMakeLists.txt index b4ba1191ba0..d61802020a4 100644 --- a/dbms/src/TableFunctions/CMakeLists.txt +++ b/dbms/src/TableFunctions/CMakeLists.txt @@ -9,10 +9,10 @@ target_link_libraries(clickhouse_table_functions clickhouse_storages_system dbms if (USE_POCO_SQLODBC) target_link_libraries (clickhouse_table_functions ${Poco_SQLODBC_LIBRARY}) - target_include_directories (clickhouse_table_functions PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIRS}) + target_include_directories (clickhouse_table_functions SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIRS}) endif () if (USE_POCO_DATAODBC) target_link_libraries (clickhouse_table_functions ${Poco_DataODBC_LIBRARY}) - target_include_directories (clickhouse_table_functions PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIRS}) + target_include_directories (clickhouse_table_functions SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIRS}) endif () diff --git a/dbms/tests/clickhouse-test-server b/dbms/tests/clickhouse-test-server index ba96506a140..37c5999fd9d 100755 --- a/dbms/tests/clickhouse-test-server +++ b/dbms/tests/clickhouse-test-server @@ -42,9 +42,8 @@ export CLICKHOUSE_PORT_INTERSERVER=${CLICKHOUSE_PORT_INTERSERVER:=`$CLICKHOUSE_E export CLICKHOUSE_PORT_TCP_SECURE=${CLICKHOUSE_PORT_TCP_SECURE:=`$CLICKHOUSE_EXTRACT_CONFIG --key=tcp_port_secure`} export CLICKHOUSE_PORT_HTTPS=${CLICKHOUSE_PORT_HTTPS:=`$CLICKHOUSE_EXTRACT_CONFIG --key=https_port`} - -rm -rf $DATA_DIR -mkdir -p $LOG_DIR +rm -rf $DATA_DIR || true +mkdir -p $LOG_DIR || true DHPARAM=`$CLICKHOUSE_EXTRACT_CONFIG --key=openSSL.server.dhParamsFile` PRIVATEKEY=`${BIN_DIR}clickhouse-extract-from-config --config=$CLICKHOUSE_CONFIG --key=openSSL.server.privateKeyFile` @@ -78,8 +77,6 @@ $GDB ${BIN_DIR}clickhouse-server --config-file=$CLICKHOUSE_CONFIG -- \ CH_PID=$! sleep 3 - - if [ "$GDB" ]; then # Long symbols read sleep 40 diff --git a/dbms/tests/queries/0_stateless/00231_format_vertical_raw.reference b/dbms/tests/queries/0_stateless/00231_format_vertical_raw.reference index b4381ee2432..76c7dc4f8d1 100644 --- a/dbms/tests/queries/0_stateless/00231_format_vertical_raw.reference +++ b/dbms/tests/queries/0_stateless/00231_format_vertical_raw.reference @@ -1,7 +1,4 @@ Row 1: ────── -x: a\tb\nc\td -Row 1: -────── x: a b c d diff --git a/dbms/tests/queries/0_stateless/00231_format_vertical_raw.sql b/dbms/tests/queries/0_stateless/00231_format_vertical_raw.sql index 43209c771e6..48f8c447703 100644 --- a/dbms/tests/queries/0_stateless/00231_format_vertical_raw.sql +++ b/dbms/tests/queries/0_stateless/00231_format_vertical_raw.sql @@ -1,2 +1 @@ SELECT 'a\tb\nc\td' AS x FORMAT Vertical; -SELECT 'a\tb\nc\td' AS x FORMAT VerticalRaw; diff --git a/dbms/tests/queries/0_stateless/00298_enum_width_and_cast.reference b/dbms/tests/queries/0_stateless/00298_enum_width_and_cast.reference index 55bf1846eeb..8ce3d6a62e2 100644 --- a/dbms/tests/queries/0_stateless/00298_enum_width_and_cast.reference +++ b/dbms/tests/queries/0_stateless/00298_enum_width_and_cast.reference @@ -3,15 +3,15 @@ └───────┴───┘ ┌─x─────┬─y─┐ │ Hello │ 0 │ -│ \\ │ 0 │ +│ \ │ 0 │ └───────┴───┘ ┌─x─────┬─y─┐ │ Hello │ 0 │ -│ \\ │ 0 │ -│ \t\\t │ 0 │ +│ \ │ 0 │ +│ \t │ 0 │ └───────┴───┘ ┌─x─────┬─y─┬─toInt8(x)─┬─s─────┬─casted─┐ │ Hello │ 0 │ -100 │ Hello │ Hello │ -│ \\ │ 0 │ 0 │ \\ │ \\ │ -│ \t\\t │ 0 │ 111 │ \t\\t │ \t\\t │ +│ \ │ 0 │ 0 │ \ │ \ │ +│ \t │ 0 │ 111 │ \t │ \t │ └───────┴───┴───────────┴───────┴────────┘ diff --git a/dbms/tests/queries/0_stateless/00405_pretty_formats.reference b/dbms/tests/queries/0_stateless/00405_pretty_formats.reference index 69081e6e56b..07dfd76339b 100644 --- a/dbms/tests/queries/0_stateless/00405_pretty_formats.reference +++ b/dbms/tests/queries/0_stateless/00405_pretty_formats.reference @@ -1,13 +1,13 @@ ┏━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓ ┃ hello ┃ world ┃ tuple  ┃ sometimes_nulls ┃ ┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩ -│ 0 │ 0 │ (0,'0') │ \N │ +│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │ ├───────┼───────┼─────────┼─────────────────┤ │ 1 │ 1 │ (1,'1') │ 1 │ ├───────┼───────┼─────────┼─────────────────┤ │ 2 │ 2 │ (2,'2') │ 2 │ ├───────┼───────┼─────────┼─────────────────┤ -│ 3 │ 3 │ (3,'3') │ \N │ +│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │ ├───────┼───────┼─────────┼─────────────────┤ │ 4 │ 4 │ (4,'4') │ 1 │ └───────┴───────┴─────────┴─────────────────┘ @@ -16,64 +16,64 @@ ┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩ │ 5 │ 5 │ (5,'5') │ 2 │ ├───────┼───────┼─────────┼─────────────────┤ -│ 6 │ 6 │ (6,'6') │ \N │ +│ 6 │ 6 │ (6,'6') │ ᴺᵁᴸᴸ │ ├───────┼───────┼─────────┼─────────────────┤ │ 7 │ 7 │ (7,'7') │ 1 │ ├───────┼───────┼─────────┼─────────────────┤ │ 8 │ 8 │ (8,'8') │ 2 │ ├───────┼───────┼─────────┼─────────────────┤ -│ 9 │ 9 │ (9,'9') │ \N │ +│ 9 │ 9 │ (9,'9') │ ᴺᵁᴸᴸ │ └───────┴───────┴─────────┴─────────────────┘ ┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ -│ 0 │ 0 │ (0,'0') │ \N │ +│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │ │ 1 │ 1 │ (1,'1') │ 1 │ │ 2 │ 2 │ (2,'2') │ 2 │ -│ 3 │ 3 │ (3,'3') │ \N │ +│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │ │ 4 │ 4 │ (4,'4') │ 1 │ └───────┴───────┴─────────┴─────────────────┘ ┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ │ 5 │ 5 │ (5,'5') │ 2 │ -│ 6 │ 6 │ (6,'6') │ \N │ +│ 6 │ 6 │ (6,'6') │ ᴺᵁᴸᴸ │ │ 7 │ 7 │ (7,'7') │ 1 │ │ 8 │ 8 │ (8,'8') │ 2 │ -│ 9 │ 9 │ (9,'9') │ \N │ +│ 9 │ 9 │ (9,'9') │ ᴺᵁᴸᴸ │ └───────┴───────┴─────────┴─────────────────┘ hello world tuple sometimes_nulls - 0 0 (0,'0') \N + 0 0 (0,'0') ᴺᵁᴸᴸ 1 1 (1,'1') 1 2 2 (2,'2') 2 - 3 3 (3,'3') \N + 3 3 (3,'3') ᴺᵁᴸᴸ 4 4 (4,'4') 1 hello world tuple sometimes_nulls 5 5 (5,'5') 2 - 6 6 (6,'6') \N + 6 6 (6,'6') ᴺᵁᴸᴸ 7 7 (7,'7') 1 8 8 (8,'8') 2 - 9 9 (9,'9') \N + 9 9 (9,'9') ᴺᵁᴸᴸ ┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ -│ 0 │ 0 │ (0,'0') │ \N │ +│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │ │ 1 │ 1 │ (1,'1') │ 1 │ │ 2 │ 2 │ (2,'2') │ 2 │ -│ 3 │ 3 │ (3,'3') │ \N │ +│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │ │ 4 │ 4 │ (4,'4') │ 1 │ │ 5 │ 5 │ (5,'5') │ 2 │ -│ 6 │ 6 │ (6,'6') │ \N │ +│ 6 │ 6 │ (6,'6') │ ᴺᵁᴸᴸ │ │ 7 │ 7 │ (7,'7') │ 1 │ │ 8 │ 8 │ (8,'8') │ 2 │ -│ 9 │ 9 │ (9,'9') │ \N │ +│ 9 │ 9 │ (9,'9') │ ᴺᵁᴸᴸ │ └───────┴───────┴─────────┴─────────────────┘ ┏━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓ ┃ hello ┃ world ┃ tuple ┃ sometimes_nulls ┃ ┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩ -│ 0 │ 0 │ (0,'0') │ \N │ +│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │ ├───────┼───────┼─────────┼─────────────────┤ │ 1 │ 1 │ (1,'1') │ 1 │ ├───────┼───────┼─────────┼─────────────────┤ │ 2 │ 2 │ (2,'2') │ 2 │ ├───────┼───────┼─────────┼─────────────────┤ -│ 3 │ 3 │ (3,'3') │ \N │ +│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │ ├───────┼───────┼─────────┼─────────────────┤ │ 4 │ 4 │ (4,'4') │ 1 │ └───────┴───────┴─────────┴─────────────────┘ @@ -82,52 +82,52 @@ ┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩ │ 5 │ 5 │ (5,'5') │ 2 │ ├───────┼───────┼─────────┼─────────────────┤ -│ 6 │ 6 │ (6,'6') │ \N │ +│ 6 │ 6 │ (6,'6') │ ᴺᵁᴸᴸ │ ├───────┼───────┼─────────┼─────────────────┤ │ 7 │ 7 │ (7,'7') │ 1 │ ├───────┼───────┼─────────┼─────────────────┤ │ 8 │ 8 │ (8,'8') │ 2 │ ├───────┼───────┼─────────┼─────────────────┤ -│ 9 │ 9 │ (9,'9') │ \N │ +│ 9 │ 9 │ (9,'9') │ ᴺᵁᴸᴸ │ └───────┴───────┴─────────┴─────────────────┘ ┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ -│ 0 │ 0 │ (0,'0') │ \N │ +│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │ │ 1 │ 1 │ (1,'1') │ 1 │ │ 2 │ 2 │ (2,'2') │ 2 │ -│ 3 │ 3 │ (3,'3') │ \N │ +│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │ │ 4 │ 4 │ (4,'4') │ 1 │ └───────┴───────┴─────────┴─────────────────┘ ┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ │ 5 │ 5 │ (5,'5') │ 2 │ -│ 6 │ 6 │ (6,'6') │ \N │ +│ 6 │ 6 │ (6,'6') │ ᴺᵁᴸᴸ │ │ 7 │ 7 │ (7,'7') │ 1 │ │ 8 │ 8 │ (8,'8') │ 2 │ -│ 9 │ 9 │ (9,'9') │ \N │ +│ 9 │ 9 │ (9,'9') │ ᴺᵁᴸᴸ │ └───────┴───────┴─────────┴─────────────────┘ hello world tuple sometimes_nulls - 0 0 (0,'0') \N + 0 0 (0,'0') ᴺᵁᴸᴸ 1 1 (1,'1') 1 2 2 (2,'2') 2 - 3 3 (3,'3') \N + 3 3 (3,'3') ᴺᵁᴸᴸ 4 4 (4,'4') 1 hello world tuple sometimes_nulls 5 5 (5,'5') 2 - 6 6 (6,'6') \N + 6 6 (6,'6') ᴺᵁᴸᴸ 7 7 (7,'7') 1 8 8 (8,'8') 2 - 9 9 (9,'9') \N + 9 9 (9,'9') ᴺᵁᴸᴸ ┏━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓ ┃ hello ┃ world ┃ tuple  ┃ sometimes_nulls ┃ ┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩ -│ 0 │ 0 │ (0,'0') │ \N │ +│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │ ├───────┼───────┼─────────┼─────────────────┤ │ 1 │ 1 │ (1,'1') │ 1 │ ├───────┼───────┼─────────┼─────────────────┤ │ 2 │ 2 │ (2,'2') │ 2 │ ├───────┼───────┼─────────┼─────────────────┤ -│ 3 │ 3 │ (3,'3') │ \N │ +│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │ ├───────┼───────┼─────────┼─────────────────┤ │ 4 │ 4 │ (4,'4') │ 1 │ └───────┴───────┴─────────┴─────────────────┘ @@ -138,10 +138,10 @@ hello world tuple sometimes_nulls └───────┴───────┴─────────┴─────────────────┘ Showed first 6. ┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ -│ 0 │ 0 │ (0,'0') │ \N │ +│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │ │ 1 │ 1 │ (1,'1') │ 1 │ │ 2 │ 2 │ (2,'2') │ 2 │ -│ 3 │ 3 │ (3,'3') │ \N │ +│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │ │ 4 │ 4 │ (4,'4') │ 1 │ └───────┴───────┴─────────┴─────────────────┘ ┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ @@ -150,10 +150,10 @@ hello world tuple sometimes_nulls Showed first 6. hello world tuple sometimes_nulls - 0 0 (0,'0') \N + 0 0 (0,'0') ᴺᵁᴸᴸ 1 1 (1,'1') 1 2 2 (2,'2') 2 - 3 3 (3,'3') \N + 3 3 (3,'3') ᴺᵁᴸᴸ 4 4 (4,'4') 1 hello world tuple sometimes_nulls @@ -161,10 +161,10 @@ hello world tuple sometimes_nulls Showed first 6. ┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ -│ 0 │ 0 │ (0,'0') │ \N │ +│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │ │ 1 │ 1 │ (1,'1') │ 1 │ │ 2 │ 2 │ (2,'2') │ 2 │ -│ 3 │ 3 │ (3,'3') │ \N │ +│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │ │ 4 │ 4 │ (4,'4') │ 1 │ │ 5 │ 5 │ (5,'5') │ 2 │ └───────┴───────┴─────────┴─────────────────┘ @@ -172,13 +172,13 @@ Showed first 6. ┏━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓ ┃ hello ┃ world ┃ tuple ┃ sometimes_nulls ┃ ┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩ -│ 0 │ 0 │ (0,'0') │ \N │ +│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │ ├───────┼───────┼─────────┼─────────────────┤ │ 1 │ 1 │ (1,'1') │ 1 │ ├───────┼───────┼─────────┼─────────────────┤ │ 2 │ 2 │ (2,'2') │ 2 │ ├───────┼───────┼─────────┼─────────────────┤ -│ 3 │ 3 │ (3,'3') │ \N │ +│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │ ├───────┼───────┼─────────┼─────────────────┤ │ 4 │ 4 │ (4,'4') │ 1 │ └───────┴───────┴─────────┴─────────────────┘ @@ -189,10 +189,10 @@ Showed first 6. └───────┴───────┴─────────┴─────────────────┘ Showed first 6. ┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ -│ 0 │ 0 │ (0,'0') │ \N │ +│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │ │ 1 │ 1 │ (1,'1') │ 1 │ │ 2 │ 2 │ (2,'2') │ 2 │ -│ 3 │ 3 │ (3,'3') │ \N │ +│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │ │ 4 │ 4 │ (4,'4') │ 1 │ └───────┴───────┴─────────┴─────────────────┘ ┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ @@ -201,10 +201,10 @@ Showed first 6. Showed first 6. hello world tuple sometimes_nulls - 0 0 (0,'0') \N + 0 0 (0,'0') ᴺᵁᴸᴸ 1 1 (1,'1') 1 2 2 (2,'2') 2 - 3 3 (3,'3') \N + 3 3 (3,'3') ᴺᵁᴸᴸ 4 4 (4,'4') 1 hello world tuple sometimes_nulls diff --git a/dbms/tests/queries/0_stateless/00406_tuples_with_nulls.reference b/dbms/tests/queries/0_stateless/00406_tuples_with_nulls.reference index 27da6935e26..c383c208b64 100644 --- a/dbms/tests/queries/0_stateless/00406_tuples_with_nulls.reference +++ b/dbms/tests/queries/0_stateless/00406_tuples_with_nulls.reference @@ -10,6 +10,6 @@ │ (8,2,NULL) │ │ (9,NULL,'1') │ └───────────────┘ -┌─x──┬─y──────┐ -│ \N │ (NULL) │ -└────┴────────┘ +┌─x────┬─y──────┐ +│ ᴺᵁᴸᴸ │ (NULL) │ +└──────┴────────┘ diff --git a/dbms/tests/queries/0_stateless/00460_vertical_and_totals_extremes.reference b/dbms/tests/queries/0_stateless/00460_vertical_and_totals_extremes.reference index e36a0b01701..026f27fc7a9 100644 --- a/dbms/tests/queries/0_stateless/00460_vertical_and_totals_extremes.reference +++ b/dbms/tests/queries/0_stateless/00460_vertical_and_totals_extremes.reference @@ -131,157 +131,6 @@ count(): 20 Showed first 4. -Totals: -─────── -k: 0 -count(): 100 - -Min: -──── -k: 0 -count(): 20 - -Max: -──── -k: 4 -count(): 20 -Row 1: -────── -k: 0 -count(): 20 - -Row 2: -────── -k: 1 -count(): 20 - -Row 3: -────── -k: 2 -count(): 20 - -Row 4: -────── -k: 3 -count(): 20 - -Showed first 4. - - -Totals: -─────── -k: 0 -count(): 100 - -Min: -──── -k: 0 -count(): 20 - -Max: -──── -k: 4 -count(): 20 -Row 1: -────── -k: 0 -count(): 20 - -Row 2: -────── -k: 1 -count(): 20 - -Row 3: -────── -k: 2 -count(): 20 - -Row 4: -────── -k: 3 -count(): 20 - -Showed first 4. - - -Totals: -─────── -k: 0 -count(): 100 - -Min: -──── -k: 0 -count(): 20 - -Max: -──── -k: 4 -count(): 20 -Row 1: -────── -k: 0 -count(): 20 - -Row 2: -────── -k: 1 -count(): 20 - -Row 3: -────── -k: 2 -count(): 20 - -Row 4: -────── -k: 3 -count(): 20 - -Row 5: -────── -k: 4 -count(): 20 - - -Totals: -─────── -k: 0 -count(): 100 - -Min: -──── -k: 0 -count(): 20 - -Max: -──── -k: 4 -count(): 20 -Row 1: -────── -k: 0 -count(): 20 - -Row 2: -────── -k: 1 -count(): 20 - -Row 3: -────── -k: 2 -count(): 20 - -Row 4: -────── -k: 3 -count(): 20 - -Showed first 4. - - Totals: ─────── k: 0 diff --git a/dbms/tests/queries/0_stateless/00460_vertical_and_totals_extremes.sql b/dbms/tests/queries/0_stateless/00460_vertical_and_totals_extremes.sql index 137304ce713..26b8d14d754 100644 --- a/dbms/tests/queries/0_stateless/00460_vertical_and_totals_extremes.sql +++ b/dbms/tests/queries/0_stateless/00460_vertical_and_totals_extremes.sql @@ -8,15 +8,3 @@ SELECT k, count() FROM (SELECT number % 5 AS k FROM system.numbers LIMIT 100) GR SET output_format_pretty_max_rows = 4; SELECT k, count() FROM (SELECT number % 5 AS k FROM system.numbers LIMIT 100) GROUP BY k WITH TOTALS ORDER BY k FORMAT Vertical; - - -SELECT k, count() FROM (SELECT number % 5 AS k FROM system.numbers LIMIT 100) GROUP BY k WITH TOTALS ORDER BY k FORMAT VerticalRaw; - -SET extremes = 1; -SELECT k, count() FROM (SELECT number % 5 AS k FROM system.numbers LIMIT 100) GROUP BY k WITH TOTALS ORDER BY k FORMAT VerticalRaw; - -SET output_format_pretty_max_rows = 5; -SELECT k, count() FROM (SELECT number % 5 AS k FROM system.numbers LIMIT 100) GROUP BY k WITH TOTALS ORDER BY k FORMAT VerticalRaw; - -SET output_format_pretty_max_rows = 4; -SELECT k, count() FROM (SELECT number % 5 AS k FROM system.numbers LIMIT 100) GROUP BY k WITH TOTALS ORDER BY k FORMAT VerticalRaw; diff --git a/dbms/tests/queries/0_stateless/00476_pretty_formats_and_widths.reference b/dbms/tests/queries/0_stateless/00476_pretty_formats_and_widths.reference index 7ddf9a5b007..c97991ae0e9 100644 --- a/dbms/tests/queries/0_stateless/00476_pretty_formats_and_widths.reference +++ b/dbms/tests/queries/0_stateless/00476_pretty_formats_and_widths.reference @@ -57,12 +57,12 @@ │ 100000000 │ 100000000 │ │ 1000000000 │ 1000000000 │ └────────────┴────────────┘ -┏━━━━━━━━━━━━━━━━━━┓ -┃ \'\\\\\\\'\\\'\' ┃ -┡━━━━━━━━━━━━━━━━━━┩ -│ \\\'\' │ -└──────────────────┘ +┏━━━━━━━━━━┓ +┃ '\\\'\'' ┃ +┡━━━━━━━━━━┩ +│ \'' │ +└──────────┘ Row 1: ────── -\'\\\\\\\'\\\'\': \\\'\' -1: 1 +'\\\'\'': \'' +1: 1 diff --git a/dbms/tests/queries/0_stateless/00569_parse_date_time_best_effort.reference b/dbms/tests/queries/0_stateless/00569_parse_date_time_best_effort.reference index 38264a337f0..4b209d6a90f 100644 --- a/dbms/tests/queries/0_stateless/00569_parse_date_time_best_effort.reference +++ b/dbms/tests/queries/0_stateless/00569_parse_date_time_best_effort.reference @@ -19,7 +19,7 @@ s a b 19700102010203Z 1970-01-02 01:02:03 1970-01-02 01:02:03 1970/01/02 010203Z 1970-01-02 01:02:03 1970-01-02 01:02:03 20 2000-01-20 00:00:00 2000-01-20 00:00:00 -201 \N 0000-00-00 00:00:00 +201 ᴺᵁᴸᴸ 0000-00-00 00:00:00 20160101 2016-01-01 00:00:00 2016-01-01 00:00:00 2016-01-01 2016-01-01 00:00:00 2016-01-01 00:00:00 201601-01 2016-01-01 01:00:00 2016-01-01 01:00:00 @@ -35,7 +35,7 @@ s a b 2017/01/01 2017-01-01 00:00:00 2017-01-01 00:00:00 201701 02 010203 UTC+0300 2017-01-01 22:02:03 2017-01-01 22:02:03 2017-01-02 03:04:05 2017-01-02 03:04:05 2017-01-02 03:04:05 -2017-01-0203:04:05 \N 0000-00-00 00:00:00 +2017-01-0203:04:05 ᴺᵁᴸᴸ 0000-00-00 00:00:00 2017-01-02 03:04:05+0 2017-01-02 03:04:05 2017-01-02 03:04:05 2017-01-02 03:04:05+00 2017-01-02 03:04:05 2017-01-02 03:04:05 2017-01-02 03:04:05+0000 2017-01-02 03:04:05 2017-01-02 03:04:05 @@ -65,7 +65,7 @@ s a b 2017 25 1:2:3 0000-00-00 00:00:00 0000-00-00 00:00:00 2017 25 Apr 1:2:3 2017-04-01 01:02:03 2017-04-01 01:02:03 2017 Apr 01 11:22:33 2017-04-01 11:22:33 2017-04-01 11:22:33 -2017 Apr 02 01/02/03 UTC+0300 \N 0000-00-00 00:00:00 +2017 Apr 02 01/02/03 UTC+0300 ᴺᵁᴸᴸ 0000-00-00 00:00:00 2017 Apr 02 010203 UTC+0300 2017-04-01 22:02:03 2017-04-01 22:02:03 2017 Apr 02 01:2:3 UTC+0300 2017-04-01 22:02:03 2017-04-01 22:02:03 2017 Apr 02 1:02:3 2017-04-02 01:02:03 2017-04-02 01:02:03 @@ -91,14 +91,14 @@ s a b 25 Jan 2017 1:2:3 Z 2017-01-25 01:02:03 2017-01-25 01:02:03 25 Jan 2017 1:2:3 Z +0300 2017-01-24 22:02:03 2017-01-24 22:02:03 25 Jan 2017 1:2:3 Z+03:00 2017-01-24 22:02:03 2017-01-24 22:02:03 -25 Jan 2017 1:2:3 Z +0300 OM \N 0000-00-00 00:00:00 +25 Jan 2017 1:2:3 Z +0300 OM ᴺᵁᴸᴸ 0000-00-00 00:00:00 25 Jan 2017 1:2:3 Z +03:00 PM 2017-01-25 10:02:03 2017-01-25 10:02:03 25 Jan 2017 1:2:3 Z +0300 PM 2017-01-25 10:02:03 2017-01-25 10:02:03 25 Jan 2017 1:2:3 Z+03:00 PM 2017-01-25 10:02:03 2017-01-25 10:02:03 25 Jan 2017 1:2:3 Z +03:30 PM 2017-01-25 09:32:03 2017-01-25 09:32:03 -25 Jan 2017 1:2:3Z Mo \N 0000-00-00 00:00:00 +25 Jan 2017 1:2:3Z Mo ᴺᵁᴸᴸ 0000-00-00 00:00:00 25 Jan 2017 1:2:3Z Mon 2017-01-25 01:02:03 2017-01-25 01:02:03 -25 Jan 2017 1:2:3Z Moo \N 0000-00-00 00:00:00 +25 Jan 2017 1:2:3Z Moo ᴺᵁᴸᴸ 0000-00-00 00:00:00 25 Jan 2017 1:2:3 Z PM 2017-01-25 13:02:03 2017-01-25 13:02:03 25 Jan 2017 1:2:3Z PM 2017-01-25 13:02:03 2017-01-25 13:02:03 25 Jan 2017 1:2:3 Z PM +03:00 2017-01-25 10:02:03 2017-01-25 10:02:03 diff --git a/dbms/tests/queries/0_stateless/00637_sessions_in_http_interface_and_settings.reference b/dbms/tests/queries/0_stateless/00637_sessions_in_http_interface_and_settings.reference new file mode 100644 index 00000000000..423ed399bac --- /dev/null +++ b/dbms/tests/queries/0_stateless/00637_sessions_in_http_interface_and_settings.reference @@ -0,0 +1 @@ +readonly 1 1 diff --git a/dbms/tests/queries/0_stateless/00637_sessions_in_http_interface_and_settings.sh b/dbms/tests/queries/0_stateless/00637_sessions_in_http_interface_and_settings.sh new file mode 100755 index 00000000000..522e3e40737 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00637_sessions_in_http_interface_and_settings.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +user=readonly +address=${CLICKHOUSE_HOST} +port=${CLICKHOUSE_PORT_HTTP} +url="${CLICKHOUSE_PORT_HTTP_PROTO}://readonly@$address:$port/?session_id=test" +select="SELECT name, value, changed FROM system.settings WHERE name = 'readonly'" + +${CLICKHOUSE_CURL} -sS $url --data-binary "$select" diff --git a/dbms/tests/queries/0_stateless/00638_remote_ssrf.reference b/dbms/tests/queries/0_stateless/00638_remote_ssrf.reference new file mode 100644 index 00000000000..877ff071fc3 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00638_remote_ssrf.reference @@ -0,0 +1 @@ +must not contain ASCII control characters diff --git a/dbms/tests/queries/0_stateless/00638_remote_ssrf.sh b/dbms/tests/queries/0_stateless/00638_remote_ssrf.sh new file mode 100755 index 00000000000..b998424b530 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00638_remote_ssrf.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +user=readonly +address=${CLICKHOUSE_HOST} +port=${CLICKHOUSE_PORT_HTTP} +url="${CLICKHOUSE_PORT_HTTP_PROTO}://$address:$port/" + +# Port is arbitary + +nc -l -p 61845 -q 0 > /dev/null & + +${CLICKHOUSE_CURL} -sS $url --data-binary "SELECT * FROM remote('localhost:61845', system.one, 'user', 'password')" > /dev/null 2>&1 + +wait + +nc -l -p 61846 -q 0 > /dev/null & + +${CLICKHOUSE_CURL} -sS $url --data-binary "SELECT * FROM remote('localhost:61846', system.one, 'user', 'passw +ord')" 2>&1 | grep -o 'must not contain ASCII control characters' + +wait diff --git a/dbms/tests/queries/0_stateless/00639_startsWith.reference b/dbms/tests/queries/0_stateless/00639_startsWith.reference new file mode 100644 index 00000000000..faa1c0241e4 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00639_startsWith.reference @@ -0,0 +1,16 @@ +0 +0 +1 +1 +0 +0 +1 +1 +1 +1 +1 +0 +1 +3 +2 +0 diff --git a/dbms/tests/queries/0_stateless/00639_startsWith.sql b/dbms/tests/queries/0_stateless/00639_startsWith.sql new file mode 100644 index 00000000000..422332a3eb4 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00639_startsWith.sql @@ -0,0 +1,17 @@ +USE test; + +SELECT startsWith(s, 'He') FROM (SELECT arrayJoin(['', 'H', 'He', 'Hellow', '3434', 'fffffffffdHe']) AS s); +SELECT startsWith(s, '') FROM (SELECT arrayJoin(['', 'h', 'hi']) AS s); +SELECT startsWith('123', '123'); +SELECT startsWith('123', '12'); +SELECT startsWith('123', '1234'); +SELECT startsWith('123', ''); + +DROP TABLE IF EXISTS startsWith_test; +CREATE TABLE startsWith_test(S1 String, S2 String, S3 FixedString(2)) ENGINE=Memory; +INSERT INTO startsWith_test values ('11', '22', '33'), ('a', 'a', 'bb'), ('abc', 'ab', '23'); + +SELECT COUNT() FROM startsWith_test WHERE startsWith(S1, S1); +SELECT COUNT() FROM startsWith_test WHERE startsWith(S1, S2); +SELECT COUNT() FROM startsWith_test WHERE startsWith(S2, S3); +DROP TABLE startsWith_test; diff --git a/dbms/tests/queries/0_stateless/00640_endsWith.reference b/dbms/tests/queries/0_stateless/00640_endsWith.reference new file mode 100644 index 00000000000..faa1c0241e4 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00640_endsWith.reference @@ -0,0 +1,16 @@ +0 +0 +1 +1 +0 +0 +1 +1 +1 +1 +1 +0 +1 +3 +2 +0 diff --git a/dbms/tests/queries/0_stateless/00640_endsWith.sql b/dbms/tests/queries/0_stateless/00640_endsWith.sql new file mode 100644 index 00000000000..c42a4618739 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00640_endsWith.sql @@ -0,0 +1,17 @@ +USE test; + +SELECT endsWith(s, 'ow') FROM (SELECT arrayJoin(['', 'o', 'ow', 'Hellow', '3434', 'owfffffffdHe']) AS s); +SELECT endsWith(s, '') FROM (SELECT arrayJoin(['', 'h', 'hi']) AS s); +SELECT endsWith('123', '3'); +SELECT endsWith('123', '23'); +SELECT endsWith('123', '32'); +SELECT endsWith('123', ''); + +DROP TABLE IF EXISTS endsWith_test; +CREATE TABLE endsWith_test(S1 String, S2 String, S3 FixedString(2)) ENGINE=Memory; +INSERT INTO endsWith_test values ('11', '22', '33'), ('a', 'a', 'bb'), ('abc', 'bc', '23'); + +SELECT COUNT() FROM endsWith_test WHERE endsWith(S1, S1); +SELECT COUNT() FROM endsWith_test WHERE endsWith(S1, S2); +SELECT COUNT() FROM endsWith_test WHERE endsWith(S2, S3); +DROP TABLE endsWith_test; diff --git a/dbms/tests/users.xml b/dbms/tests/users.xml index db0109cd456..32a83e5806f 120000 --- a/dbms/tests/users.xml +++ b/dbms/tests/users.xml @@ -1 +1 @@ -../src/Server/users.xml \ No newline at end of file +../programs/server/users.xml \ No newline at end of file diff --git a/debian/control b/debian/control index f3aa1be41ca..7834efea266 100644 --- a/debian/control +++ b/debian/control @@ -62,7 +62,7 @@ Description: debugging symbols for clickhouse-common-static Package: clickhouse-test Priority: optional Architecture: all -Depends: ${shlibs:Depends}, ${misc:Depends}, clickhouse-client, bash, expect, python, python-lxml, python-termcolor, python-requests, curl, perl, sudo, openssl +Depends: ${shlibs:Depends}, ${misc:Depends}, clickhouse-client, bash, expect, python, python-lxml, python-termcolor, python-requests, curl, perl, sudo, openssl, netcat-openbsd Description: Clickhouse tests diff --git a/debian/rules b/debian/rules index 5848fd4b453..4533dd9ca3d 100755 --- a/debian/rules +++ b/debian/rules @@ -99,11 +99,6 @@ override_dh_install: mkdir -p $(DESTDIR)/etc/systemd/system/ cp debian/clickhouse-server.service $(DESTDIR)/etc/systemd/system/ - #TODO: use from cmake: - # In case building clickhouse-server, adding to package binary of clang, ld and header files - for dynamic compilation. - #mkdir -p $(DESTDIR)/usr/share/clickhouse/headers - #CLANG=$(DESTDIR)/usr/bin/clickhouse-clang DESTDIR=$(DESTDIR) ./copy_headers.sh . $(DESTDIR)/usr/share/clickhouse/headers - # fake metrika files when private dir is empty mkdir -p $(DESTDIR)/etc/clickhouse-server/metrika touch $(DESTDIR)/etc/clickhouse-server/metrika/config.xml diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile index 7b4d8932c80..3393dd0e697 100644 --- a/docker/client/Dockerfile +++ b/docker/client/Dockerfile @@ -9,7 +9,7 @@ RUN apt-get update && \ apt-key adv --keyserver keyserver.ubuntu.com --recv E0C56BD4 && \ echo $repository | tee /etc/apt/sources.list.d/clickhouse.list && \ apt-get update && \ - env DEBIAN_FRONTEND=noninteractive apt-get install --allow-unauthenticated -y clickhouse-client=$version locales tzdata && \ + env DEBIAN_FRONTEND=noninteractive apt-get install --allow-unauthenticated -y clickhouse-client=$version clickhouse-common-static=$version locales tzdata && \ rm -rf /var/lib/apt/lists/* /var/cache/debconf && \ apt-get clean diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index a85c6da5f8c..e1ba52b0989 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -9,7 +9,7 @@ RUN apt-get update && \ apt-key adv --keyserver keyserver.ubuntu.com --recv E0C56BD4 && \ echo $repository | tee /etc/apt/sources.list.d/clickhouse.list && \ apt-get update && \ - env DEBIAN_FRONTEND=noninteractive apt-get install --allow-unauthenticated -y "clickhouse-server=$version" libgcc-7-dev && \ + env DEBIAN_FRONTEND=noninteractive apt-get install --allow-unauthenticated -y clickhouse-server=$version clickhouse-common-static=$version libgcc-7-dev && \ rm -rf /var/lib/apt/lists/* /var/cache/debconf && \ apt-get clean diff --git a/libs/libmysqlxx/CMakeLists.txt b/libs/libmysqlxx/CMakeLists.txt index e3c9e836c22..d7a608994b4 100644 --- a/libs/libmysqlxx/CMakeLists.txt +++ b/libs/libmysqlxx/CMakeLists.txt @@ -33,7 +33,7 @@ target_include_directories (mysqlxx PUBLIC include) set(PLATFORM_LIBRARIES ${CMAKE_DL_LIBS}) if (USE_MYSQL) - target_include_directories (mysqlxx PRIVATE ${MYSQL_INCLUDE_DIR}) + target_include_directories (mysqlxx SYSTEM PRIVATE ${MYSQL_INCLUDE_DIR}) endif () if (USE_STATIC_LIBRARIES AND STATIC_MYSQLCLIENT_LIB) @@ -49,7 +49,7 @@ if (USE_STATIC_LIBRARIES AND STATIC_MYSQLCLIENT_LIB) endif () target_link_libraries (mysqlxx common ${MYSQLCLIENT_LIBRARIES} ${OPENSSL_LIBRARIES} ${ZLIB_LIBRARIES} ${PLATFORM_LIBRARIES}) -target_include_directories (mysqlxx PRIVATE ${OPENSSL_INCLUDE_DIR}) +target_include_directories (mysqlxx SYSTEM PRIVATE ${OPENSSL_INCLUDE_DIR}) if (ENABLE_TESTS) add_subdirectory (src/tests) diff --git a/libs/libpocoext/CMakeLists.txt b/libs/libpocoext/CMakeLists.txt index 6b2c09b4bf1..4a244c37d3c 100644 --- a/libs/libpocoext/CMakeLists.txt +++ b/libs/libpocoext/CMakeLists.txt @@ -8,14 +8,14 @@ add_library (pocoext ${SPLIT_SHARED} include/Poco/Ext/SessionPoolHelpers.h) if (Poco_Data_FOUND) - target_include_directories (pocoext PRIVATE ${Poco_Data_INCLUDE_DIRS}) + target_include_directories (pocoext SYSTEM PRIVATE ${Poco_Data_INCLUDE_DIRS}) target_link_libraries(pocoext ${Poco_Data_LIBRARY}) endif() target_include_directories (pocoext PUBLIC include PRIVATE ${COMMON_INCLUDE_DIR}) if (NOT USE_INTERNAL_POCO_LIBRARY) - target_include_directories (pocoext BEFORE PUBLIC ${Poco_INCLUDE_DIRS}) + target_include_directories (pocoext SYSTEM BEFORE PUBLIC ${Poco_INCLUDE_DIRS}) endif () target_link_libraries(pocoext ${Poco_Util_LIBRARY} ${Poco_Net_LIBRARY} ${Poco_XML_LIBRARY} ${Poco_Foundation_LIBRARY}) diff --git a/release b/release index b918227837d..5623df2a712 100755 --- a/release +++ b/release @@ -53,7 +53,7 @@ do shift elif [[ $1 == '--fast' ]]; then # Wrong but fast pbuilder mode: create base package with all depends - EXTRAPACKAGES="$EXTRAPACKAGES debhelper cmake ninja-build gcc-7 g++-7 libc6-dev libmariadbclient-dev libicu-dev libltdl-dev libreadline-dev libssl-dev unixodbc-dev psmisc bash expect python python-lxml python-termcolor python-requests curl perl sudo openssl" + EXTRAPACKAGES="$EXTRAPACKAGES debhelper cmake ninja-build gcc-7 g++-7 libc6-dev libmariadbclient-dev libicu-dev libltdl-dev libreadline-dev libssl-dev unixodbc-dev psmisc bash expect python python-lxml python-termcolor python-requests curl perl sudo openssl netcat-openbsd" shift else echo "Unknown option $1" diff --git a/utils/check_include.sh b/utils/check_include.sh index 474f08e1a44..3e3ef44aaf1 100755 --- a/utils/check_include.sh +++ b/utils/check_include.sh @@ -3,6 +3,7 @@ # Small .h isolated compile checker # Finds missing #include <...> # prints compile time, number of includes, use with sort: ./check_include.sh 2>&1 | sort -rk3 +# use with clang: CXX=`which clang++-7 clang++-7.0 clang++-6.0 clang++-5.0 | head -n1` ./check_include.sh pwd=`pwd` BUILD_DIR=${BUILD_DIR:=./build} inc="-I. \ @@ -28,6 +29,8 @@ inc="-I. \ -I./contrib/poco/Foundation/include \ -I./contrib/boost/libs/*/include \ -I./contrib/boost \ +-I./contrib/llvm/llvm/include \ +-I${BUILD_DIR}/contrib/llvm/llvm/include \ -I./contrib/libbtrie/include \ -I./contrib/libpcg-random/include \ -I./libs/libmysqlxx/include \