thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-30 03:22:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into clang-compliant

Browse Source
This commit is contained in:
alexey-milovidov 2018-06-07 17:43:20 +03:00 committed by GitHub
commit 587189b211
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
73 changed files with 468 additions and 406 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmake/find_capnp.cmake
View File

@ -1,7 +1,6 @@
option (ENABLE_CAPNP "Enable Cap'n Proto" ${NOT_MSVC})
if (ENABLE_CAPNP)
# cmake 3.5.1 bug:
# capnproto uses this cmake feature:
# target_compile_features(kj PUBLIC cxx_constexpr)
@ -42,7 +41,6 @@ if (ENABLE_CAPNP)
set (CAPNP_LIBRARY capnpc)
set (USE_CAPNP 1)
endif ()
endif ()
if (USE_CAPNP)

1
cmake/find_poco.cmake
View File

@ -27,7 +27,6 @@ endif ()
if (Poco_INCLUDE_DIRS AND Poco_Foundation_LIBRARY)
elseif (NOT MISSING_INTERNAL_POCO_LIBRARY)
set (USE_INTERNAL_POCO_LIBRARY 1)
set (ENABLE_ZIP 0 CACHE BOOL "")

38
dbms/CMakeLists.txt
View File

@ -102,7 +102,7 @@ if (USE_EMBEDDED_COMPILER)
llvm_libs_all(REQUIRED_LLVM_LIBRARIES)
target_link_libraries (dbms ${REQUIRED_LLVM_LIBRARIES})
target_include_directories (dbms BEFORE PUBLIC ${LLVM_INCLUDE_DIRS})
target_include_directories (dbms SYSTEM BEFORE PUBLIC ${LLVM_INCLUDE_DIRS})
endif ()
@ -159,32 +159,32 @@ target_link_libraries (dbms
)
if (NOT USE_INTERNAL_RE2_LIBRARY)
target_include_directories (dbms BEFORE PRIVATE ${RE2_INCLUDE_DIR})
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${RE2_INCLUDE_DIR})
endif ()
if (NOT USE_INTERNAL_BOOST_LIBRARY)
target_include_directories (clickhouse_common_io BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
endif ()
if (USE_POCO_SQLODBC)
target_link_libraries (clickhouse_common_io ${Poco_SQL_LIBRARY})
target_link_libraries (dbms ${Poco_SQLODBC_LIBRARY} ${Poco_SQL_LIBRARY})
if (NOT USE_INTERNAL_POCO_LIBRARY)
target_include_directories (clickhouse_common_io PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQL_INCLUDE_DIRS})
target_include_directories (dbms PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIRS} PUBLIC ${Poco_SQL_INCLUDE_DIRS})
target_include_directories (clickhouse_common_io SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQL_INCLUDE_DIRS})
target_include_directories (dbms SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIRS} PUBLIC ${Poco_SQL_INCLUDE_DIRS})
endif()
endif()
if (Poco_Data_FOUND AND NOT USE_INTERNAL_POCO_LIBRARY)
target_include_directories (clickhouse_common_io PRIVATE ${Poco_Data_INCLUDE_DIRS})
target_include_directories (dbms PRIVATE ${Poco_Data_INCLUDE_DIRS})
target_include_directories (clickhouse_common_io SYSTEM PRIVATE ${Poco_Data_INCLUDE_DIRS})
target_include_directories (dbms SYSTEM PRIVATE ${Poco_Data_INCLUDE_DIRS})
endif()
if (USE_POCO_DATAODBC)
target_link_libraries (clickhouse_common_io ${Poco_Data_LIBRARY})
target_link_libraries (dbms ${Poco_DataODBC_LIBRARY})
if (NOT USE_INTERNAL_POCO_LIBRARY)
target_include_directories (dbms PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIRS})
target_include_directories (dbms SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIRS})
endif()
endif()
@ -200,20 +200,20 @@ target_link_libraries (dbms ${Poco_Foundation_LIBRARY})
if (USE_ICU)
target_link_libraries (dbms ${ICU_LIBS})
target_include_directories (dbms PRIVATE ${ICU_INCLUDE_DIR})
target_include_directories (dbms SYSTEM PRIVATE ${ICU_INCLUDE_DIR})
endif ()
if (USE_CAPNP)
target_link_libraries (dbms ${CAPNP_LIBRARY})
if (NOT USE_INTERNAL_CAPNP_LIBRARY)
target_include_directories (dbms BEFORE PRIVATE ${CAPNP_INCLUDE_DIR})
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${CAPNP_INCLUDE_DIR})
endif ()
endif ()
if (USE_RDKAFKA)
target_link_libraries (dbms ${RDKAFKA_LIBRARY})
if (NOT USE_INTERNAL_RDKAFKA_LIBRARY)
target_include_directories (dbms BEFORE PRIVATE ${RDKAFKA_INCLUDE_DIR})
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${RDKAFKA_INCLUDE_DIR})
endif ()
endif ()
@ -221,21 +221,21 @@ target_link_libraries (dbms
Threads::Threads
)
target_include_directories (dbms BEFORE PRIVATE ${DIVIDE_INCLUDE_DIR})
target_include_directories (dbms BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${DIVIDE_INCLUDE_DIR})
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
if (NOT USE_INTERNAL_LZ4_LIBRARY)
target_include_directories (dbms BEFORE PRIVATE ${LZ4_INCLUDE_DIR})
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${LZ4_INCLUDE_DIR})
endif ()
if (NOT USE_INTERNAL_ZSTD_LIBRARY)
target_include_directories (dbms BEFORE PRIVATE ${ZSTD_INCLUDE_DIR})
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${ZSTD_INCLUDE_DIR})
endif ()
target_include_directories (dbms PUBLIC ${DBMS_INCLUDE_DIR})
target_include_directories (clickhouse_common_io PUBLIC ${DBMS_INCLUDE_DIR})
target_include_directories (clickhouse_common_io PUBLIC ${PCG_RANDOM_INCLUDE_DIR})
target_include_directories (clickhouse_common_io PUBLIC ${Poco_DataODBC_INCLUDE_DIRS})
target_include_directories (clickhouse_common_io BEFORE PUBLIC ${DOUBLE_CONVERSION_INCLUDE_DIR})
target_include_directories (clickhouse_common_io SYSTEM PUBLIC ${PCG_RANDOM_INCLUDE_DIR})
target_include_directories (clickhouse_common_io SYSTEM PUBLIC ${Poco_DataODBC_INCLUDE_DIRS})
target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${DOUBLE_CONVERSION_INCLUDE_DIR})
# also for copy_headers.sh:
target_include_directories (clickhouse_common_io BEFORE PRIVATE ${COMMON_INCLUDE_DIR})
@ -252,7 +252,7 @@ if (ENABLE_TESTS)
add_subdirectory(${ClickHouse_SOURCE_DIR}/contrib/googletest/googletest ${CMAKE_CURRENT_BINARY_DIR}/googletest)
# avoid problems with <regexp.h>
target_compile_definitions (gtest INTERFACE GTEST_HAS_POSIX_RE=0)
target_include_directories (gtest INTERFACE ${ClickHouse_SOURCE_DIR}/contrib/googletest/include)
target_include_directories (gtest SYSTEM INTERFACE ${ClickHouse_SOURCE_DIR}/contrib/googletest/include)
endif ()
macro(grep_gtest_sources BASE_DIR DST_VAR)

14
dbms/programs/CMakeLists.txt
View File

@ -15,7 +15,6 @@ option (ENABLE_CLICKHOUSE_FORMAT "Enable format" ${ENABLE_CLICKHOUSE_ALL})
configure_file (config_tools.h.in ${CMAKE_CURRENT_BINARY_DIR}/config_tools.h)
add_subdirectory (clang)
add_subdirectory (server)
add_subdirectory (client)
add_subdirectory (local)
@ -25,6 +24,7 @@ add_subdirectory (extract-from-config)
add_subdirectory (compressor)
add_subdirectory (copier)
add_subdirectory (format)
add_subdirectory (clang)
if (CLICKHOUSE_SPLIT_BINARY)
set (CLICKHOUSE_ALL_TARGETS clickhouse-server clickhouse-client clickhouse-local clickhouse-benchmark clickhouse-performance-test
@ -139,16 +139,6 @@ else ()
endif ()
if (USE_EMBEDDED_COMPILER)
add_custom_target(copy-headers ALL env CLANG=${CMAKE_CURRENT_BINARY_DIR}/clickhouse-clang BUILD_PATH=${ClickHouse_BINARY_DIR} DESTDIR=${ClickHouse_SOURCE_DIR} ${ClickHouse_SOURCE_DIR}/copy_headers.sh ${ClickHouse_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/headers DEPENDS clickhouse-clang WORKING_DIRECTORY ${ClickHouse_SOURCE_DIR} SOURCES ${ClickHouse_SOURCE_DIR}/copy_headers.sh)
install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/headers DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/clickhouse COMPONENT clickhouse)
if (USE_INTERNAL_LLVM_LIBRARY)
set(CLANG_HEADERS_DIR "${ClickHouse_SOURCE_DIR}/contrib/llvm/clang/lib/Headers")
set(CLANG_HEADERS_DEST "${CMAKE_CURRENT_BINARY_DIR}/headers/usr/local/lib/clang/${LLVM_VERSION}/include") # original: ${LLVM_LIBRARY_OUTPUT_INTDIR}/clang/${CLANG_VERSION}/include
add_custom_target(copy-headers-clang ALL ${CMAKE_COMMAND} -E make_directory ${CLANG_HEADERS_DEST} && ${CMAKE_COMMAND} -E copy_if_different ${CLANG_HEADERS_DIR}/* ${CLANG_HEADERS_DEST} )
add_dependencies(copy-headers copy-headers-clang)
endif ()
if (USE_EMBEDDED_COMPILER AND ENABLE_CLICKHOUSE_SERVER)
add_dependencies(clickhouse-server copy-headers)
endif ()

2
dbms/programs/benchmark/CMakeLists.txt
View File

@ -1,6 +1,6 @@
add_library (clickhouse-benchmark-lib ${SPLIT_SHARED} Benchmark.cpp)
target_link_libraries (clickhouse-benchmark-lib clickhouse-client-lib clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_include_directories (clickhouse-benchmark-lib PRIVATE ${PCG_RANDOM_INCLUDE_DIR})
target_include_directories (clickhouse-benchmark-lib SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR})
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-benchmark clickhouse-benchmark.cpp)

15
dbms/programs/clang/CMakeLists.txt
View File

@ -13,3 +13,18 @@ if (CLICKHOUSE_SPLIT_BINARY)
set_target_properties(clickhouse-clang clickhouse-lld PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..)
endif ()
endif ()
set(TMP_HEADERS_DIR "${CMAKE_CURRENT_BINARY_DIR}/headers")
# Make and install empty dir for debian package if compiler disabled
add_custom_target(make-headers-directory ALL COMMAND ${CMAKE_COMMAND} -E make_directory ${TMP_HEADERS_DIR})
install(DIRECTORY ${TMP_HEADERS_DIR} DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/clickhouse COMPONENT clickhouse)
if (USE_EMBEDDED_COMPILER)
add_custom_target(copy-headers ALL env CLANG=${CMAKE_CURRENT_BINARY_DIR}/../clickhouse-clang BUILD_PATH=${ClickHouse_BINARY_DIR} DESTDIR=${ClickHouse_SOURCE_DIR} ${ClickHouse_SOURCE_DIR}/copy_headers.sh ${ClickHouse_SOURCE_DIR} ${TMP_HEADERS_DIR} DEPENDS clickhouse-clang WORKING_DIRECTORY ${ClickHouse_SOURCE_DIR} SOURCES ${ClickHouse_SOURCE_DIR}/copy_headers.sh)
if (USE_INTERNAL_LLVM_LIBRARY)
set(CLANG_HEADERS_DIR "${ClickHouse_SOURCE_DIR}/contrib/llvm/clang/lib/Headers")
set(CLANG_HEADERS_DEST "${TMP_HEADERS_DIR}/usr/local/lib/clang/${LLVM_VERSION}/include") # original: ${LLVM_LIBRARY_OUTPUT_INTDIR}/clang/${CLANG_VERSION}/include
add_custom_target(copy-headers-clang ALL ${CMAKE_COMMAND} -E make_directory ${CLANG_HEADERS_DEST} && ${CMAKE_COMMAND} -E copy_if_different ${CLANG_HEADERS_DIR}/* ${CLANG_HEADERS_DEST} )
add_dependencies(copy-headers copy-headers-clang)
endif ()
endif ()

2
dbms/programs/clang/Compiler-5.0.0/CMakeLists.txt
View File

@ -12,7 +12,7 @@ llvm_libs_all(REQUIRED_LLVM_LIBRARIES)
message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}")
target_include_directories(clickhouse-compiler-lib PRIVATE ${LLVM_INCLUDE_DIRS})
target_include_directories(clickhouse-compiler-lib SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS})
# This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory.

2
dbms/programs/clang/Compiler-6.0.0/CMakeLists.txt
View File

@ -12,7 +12,7 @@ llvm_libs_all(REQUIRED_LLVM_LIBRARIES)
message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}")
target_include_directories(clickhouse-compiler-lib PRIVATE ${LLVM_INCLUDE_DIRS})
target_include_directories(clickhouse-compiler-lib SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS})
# This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory.

2
dbms/programs/clang/Compiler-7.0.0/CMakeLists.txt
View File

@ -12,7 +12,7 @@ llvm_libs_all(REQUIRED_LLVM_LIBRARIES)
message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}")
target_include_directories(clickhouse-compiler-lib PRIVATE ${LLVM_INCLUDE_DIRS})
target_include_directories(clickhouse-compiler-lib SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS})
# This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory.

2
dbms/programs/client/CMakeLists.txt
View File

@ -1,6 +1,6 @@
add_library (clickhouse-client-lib Client.cpp)
target_link_libraries (clickhouse-client-lib clickhouse_functions clickhouse_aggregate_functions ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_include_directories (clickhouse-client-lib PRIVATE ${READLINE_INCLUDE_DIR})
target_include_directories (clickhouse-client-lib SYSTEM PRIVATE ${READLINE_INCLUDE_DIR})
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-client clickhouse-client.cpp)

8
dbms/programs/main.cpp
View File

@ -34,7 +34,7 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv);
#if ENABLE_CLICKHOUSE_PERFORMANCE
int mainEntryClickHousePerformanceTest(int argc, char ** argv);
#endif
#if ENABLE_CLICKHOUSE_extract-from-config
#if ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG
int mainEntryClickHouseExtractFromConfig(int argc, char ** argv);
#endif
#if ENABLE_CLICKHOUSE_COMPRESSOR
@ -76,9 +76,13 @@ std::pair<const char *, MainFunc> clickhouse_applications[] =
#if ENABLE_CLICKHOUSE_PERFORMANCE
{"performance-test", mainEntryClickHousePerformanceTest},
#endif
#if ENABLE_CLICKHOUSE_TOOLS
#if ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG
{"extract-from-config", mainEntryClickHouseExtractFromConfig},
#endif
#if ENABLE_CLICKHOUSE_COMPRESSOR
{"compressor", mainEntryClickHouseCompressor},
#endif
#if ENABLE_CLICKHOUSE_FORMAT
{"format", mainEntryClickHouseFormat},
#endif
#if ENABLE_CLICKHOUSE_COPIER

2
dbms/programs/performance-test/CMakeLists.txt
View File

@ -1,6 +1,6 @@
add_library (clickhouse-performance-test-lib ${SPLIT_SHARED} PerformanceTest.cpp)
target_link_libraries (clickhouse-performance-test-lib clickhouse_common_io dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_include_directories (clickhouse-performance-test-lib PRIVATE ${PCG_RANDOM_INCLUDE_DIR})
target_include_directories (clickhouse-performance-test-lib SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR})
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-performance-test clickhouse-performance-test.cpp)

7
dbms/programs/server/CMakeLists.txt
View File

@ -20,9 +20,10 @@ if (CLICKHOUSE_SPLIT_BINARY)
install (TARGETS clickhouse-server ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
endif ()
set (GLIBC_MAX_REQUIRED 2.4)
add_test(NAME GLIBC_required_version COMMAND bash -c "readelf -s ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-server | grep '@GLIBC' | grep -oP 'GLIBC_[\\d\\.]+' | sort | uniq | sort -r | perl -lnE 'exit 1 if $_ gt q{GLIBC_${GLIBC_MAX_REQUIRED}}'")
if (NOT APPLE AND NOT ARCH_FREEBSD)
set (GLIBC_MAX_REQUIRED 2.4)
add_test(NAME GLIBC_required_version COMMAND bash -c "readelf -s ${CMAKE_CURRENT_BINARY_DIR}/../clickhouse-server | grep '@GLIBC' | grep -oP 'GLIBC_[\\d\\.]+' | sort | uniq | sort -r | perl -lnE 'exit 1 if $_ gt q{GLIBC_${GLIBC_MAX_REQUIRED}}'")
endif ()
install (
FILES config.xml users.xml

1
dbms/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h
View File

@ -204,6 +204,7 @@ public:
case StatisticsFunctionKind::covarSamp: return "covarSamp";
case StatisticsFunctionKind::corr: return "corr";
}
__builtin_unreachable();
}
DataTypePtr getReturnType() const override

24
dbms/src/Client/Connection.cpp
View File

@ -18,6 +18,7 @@
#include <Common/NetException.h>
#include <Common/CurrentMetrics.h>
#include <Common/DNSResolver.h>
#include <Common/StringUtils/StringUtils.h>
#include <Interpreters/ClientInfo.h>
#include <Common/config.h>
@ -42,6 +43,7 @@ namespace ErrorCodes
extern const int UNEXPECTED_PACKET_FROM_SERVER;
extern const int UNKNOWN_PACKET_FROM_SERVER;
extern const int SUPPORT_IS_DISABLED;
extern const int BAD_ARGUMENTS;
}
@ -121,7 +123,27 @@ void Connection::disconnect()
void Connection::sendHello()
{
//LOG_TRACE(log_wrapper.get(), "Sending hello");
/** Disallow control characters in user controlled parameters
* to mitigate the possibility of SSRF.
* The user may do server side requests with 'remote' table function.
* Malicious user with full r/w access to ClickHouse
* may use 'remote' table function to forge requests
* to another services in the network other than ClickHouse (examples: SMTP).
* Limiting number of possible characters in user-controlled part of handshake
* will mitigate this possibility but doesn't solve it completely.
*/
auto has_control_character = [](const std::string & s)
{
for (auto c : s)
if (isControlASCII(c))
return true;
return false;
};
if (has_control_character(default_database)
|| has_control_character(user)
|| has_control_character(password))
throw Exception("Parameters 'default_database', 'user' and 'password' must not contain ASCII control characters", ErrorCodes::BAD_ARGUMENTS);
writeVarUInt(Protocol::Client::Hello, *out);
writeStringBinary((DBMS_NAME " ") + client_name, *out);

5
dbms/src/Common/StringUtils/StringUtils.h
View File

@ -107,6 +107,11 @@ inline bool isWhitespaceASCII(char c)
return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v';
}
inline bool isControlASCII(char c)
{
return c >= 0 && c <= 31;
}
/// Works assuming isAlphaASCII.
inline char toLowerIfAlphaASCII(char c)
{

2
dbms/src/Core/tests/CMakeLists.txt
View File

@ -3,7 +3,7 @@ target_link_libraries (exception clickhouse_common_io)
add_executable (string_pool string_pool.cpp)
target_link_libraries (string_pool clickhouse_common_io)
target_include_directories (string_pool BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
target_include_directories (string_pool SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
add_executable (field field.cpp)
target_link_libraries (field dbms)

4
dbms/src/DataStreams/FormatFactory.cpp
View File

@ -119,7 +119,6 @@ BlockInputStreamPtr FormatFactory::getInput(const String & name, ReadBuffer & bu
|| name == "PrettyCompactNoEscapes"
|| name == "PrettySpaceNoEscapes"
|| name == "Vertical"
|| name == "VerticalRaw"
|| name == "Null"
|| name == "JSON"
|| name == "JSONCompact"
@ -180,9 +179,6 @@ static BlockOutputStreamPtr getOutputImpl(const String & name, WriteBuffer & buf
else if (name == "Vertical")
return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<VerticalRowOutputStream>(
buf, sample, settings.output_format_pretty_max_rows), sample);
else if (name == "VerticalRaw")
return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<VerticalRawRowOutputStream>(
buf, sample, settings.output_format_pretty_max_rows), sample);
else if (name == "Values")
return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<ValuesRowOutputStream>(buf), sample);
else if (name == "JSON")

18
dbms/src/DataStreams/PrettyBlockOutputStream.cpp
View File

@ -55,7 +55,7 @@ void PrettyBlockOutputStream::calculateWidths(const Block & block, WidthsPerColu
{
{
WriteBufferFromString out(serialized_value);
elem.type->serializeTextEscaped(*elem.column, j, out);
elem.type->serializeText(*elem.column, j, out);
}
widths[i][j] = UTF8::countCodePoints(reinterpret_cast<const UInt8 *>(serialized_value.data()), serialized_value.size());
@ -64,13 +64,7 @@ void PrettyBlockOutputStream::calculateWidths(const Block & block, WidthsPerColu
/// And also calculate widths for names of columns.
{
/// We need to obtain length in escaped form.
{
WriteBufferFromString out(serialized_value);
writeEscapedString(elem.name, out);
}
name_widths[i] = UTF8::countCodePoints(reinterpret_cast<const UInt8 *>(serialized_value.data()), serialized_value.size());
name_widths[i] = UTF8::countCodePoints(reinterpret_cast<const UInt8 *>(elem.name.data()), elem.name.size());
max_widths[i] = std::max(max_widths[i], name_widths[i]);
}
}
@ -151,11 +145,11 @@ void PrettyBlockOutputStream::write(const Block & block)
for (size_t k = 0; k < max_widths[i] - name_widths[i]; ++k)
writeChar(' ', ostr);
writeEscapedString(col.name, ostr);
writeString(col.name, ostr);
}
else
{
writeEscapedString(col.name, ostr);
writeString(col.name, ostr);
for (size_t k = 0; k < max_widths[i] - name_widths[i]; ++k)
writeChar(' ', ostr);
@ -203,11 +197,11 @@ void PrettyBlockOutputStream::writeValueWithPadding(const ColumnWithTypeAndName
if (elem.type->shouldAlignRightInPrettyFormats())
{
writePadding();
elem.type->serializeTextEscaped(*elem.column.get(), row_num, ostr);
elem.type->serializeText(*elem.column.get(), row_num, ostr);
}
else
{
elem.type->serializeTextEscaped(*elem.column.get(), row_num, ostr);
elem.type->serializeText(*elem.column.get(), row_num, ostr);
writePadding();
}
}

4
dbms/src/DataStreams/PrettyCompactBlockOutputStream.cpp
View File

@ -34,7 +34,7 @@ void PrettyCompactBlockOutputStream::writeHeader(
if (!no_escapes)
writeCString("\033[1m", ostr);
writeEscapedString(col.name, ostr);
writeString(col.name, ostr);
if (!no_escapes)
writeCString("\033[0m", ostr);
}
@ -42,7 +42,7 @@ void PrettyCompactBlockOutputStream::writeHeader(
{
if (!no_escapes)
writeCString("\033[1m", ostr);
writeEscapedString(col.name, ostr);
writeString(col.name, ostr);
if (!no_escapes)
writeCString("\033[0m", ostr);

4
dbms/src/DataStreams/PrettySpaceBlockOutputStream.cpp
View File

@ -44,7 +44,7 @@ void PrettySpaceBlockOutputStream::write(const Block & block)
if (!no_escapes)
writeCString("\033[1m", ostr);
writeEscapedString(col.name, ostr);
writeString(col.name, ostr);
if (!no_escapes)
writeCString("\033[0m", ostr);
}
@ -52,7 +52,7 @@ void PrettySpaceBlockOutputStream::write(const Block & block)
{
if (!no_escapes)
writeCString("\033[1m", ostr);
writeEscapedString(col.name, ostr);
writeString(col.name, ostr);
if (!no_escapes)
writeCString("\033[0m", ostr);

15
dbms/src/DataStreams/VerticalRowOutputStream.cpp
View File

@ -26,13 +26,7 @@ VerticalRowOutputStream::VerticalRowOutputStream(
/// Note that number of code points is just a rough approximation of visible string width.
const String & name = sample.getByPosition(i).name;
{
/// We need to obtain length in escaped form.
WriteBufferFromString out(serialized_value);
writeEscapedString(name, out);
}
name_widths[i] = UTF8::countCodePoints(reinterpret_cast<const UInt8 *>(serialized_value.data()), serialized_value.size());
name_widths[i] = UTF8::countCodePoints(reinterpret_cast<const UInt8 *>(name.data()), name.size());
if (name_widths[i] > max_name_width)
max_name_width = name_widths[i];
@ -42,7 +36,7 @@ VerticalRowOutputStream::VerticalRowOutputStream(
for (size_t i = 0; i < columns; ++i)
{
WriteBufferFromString out(names_and_paddings[i]);
writeEscapedString(sample.getByPosition(i).name, out);
writeString(sample.getByPosition(i).name, out);
writeCString(": ", out);
}
@ -71,11 +65,6 @@ void VerticalRowOutputStream::writeField(const IColumn & column, const IDataType
void VerticalRowOutputStream::writeValue(const IColumn & column, const IDataType & type, size_t row_num) const
{
type.serializeTextEscaped(column, row_num, ostr);
}
void VerticalRawRowOutputStream::writeValue(const IColumn & column, const IDataType & type, size_t row_num) const
{
type.serializeText(column, row_num, ostr);
}

12
dbms/src/DataStreams/VerticalRowOutputStream.h
View File

@ -51,17 +51,5 @@ protected:
Block extremes;
};
/** Same but values are printed without escaping.
*/
class VerticalRawRowOutputStream final : public VerticalRowOutputStream
{
public:
using VerticalRowOutputStream::VerticalRowOutputStream;
protected:
void writeValue(const IColumn & column, const IDataType & type, size_t row_num) const override;
};
}

2
dbms/src/DataStreams/tests/CMakeLists.txt
View File

@ -1,5 +1,3 @@
include_directories (${CMAKE_CURRENT_BINARY_DIR})
set(SRCS )
add_executable (tab_separated_streams tab_separated_streams.cpp ${SRCS})

8
dbms/src/DataTypes/DataTypeNullable.cpp
View File

@ -242,8 +242,14 @@ void DataTypeNullable::serializeText(const IColumn & column, size_t row_num, Wri
{
const ColumnNullable & col = static_cast<const ColumnNullable &>(column);
/// In simple text format (like 'Pretty' format) (these formats are suitable only for output and cannot be parsed back),
/// data is printed without escaping.
/// It makes theoretically impossible to distinguish between NULL and some string value, regardless on how do we print NULL.
/// For this reason, we output NULL in a bit strange way.
/// This assumes UTF-8 and proper font support. This is Ok, because Pretty formats are "presentational", not for data exchange.
if (col.isNullAt(row_num))
writeCString("NULL", ostr);
writeCString("ᴺᵁᴸᴸ", ostr);
else
nested_data_type->serializeText(col.getNestedColumn(), row_num, ostr);
}

2
dbms/src/DataTypes/tests/CMakeLists.txt
View File

@ -1,5 +1,3 @@
include_directories (${CMAKE_CURRENT_BINARY_DIR})
set(SRCS )
add_executable (data_types_number_fixed data_types_number_fixed.cpp ${SRCS})

12
dbms/src/Functions/CMakeLists.txt
View File

@ -88,9 +88,9 @@ add_library(clickhouse_functions ${clickhouse_functions_sources})
target_link_libraries(clickhouse_functions PUBLIC dbms PRIVATE libconsistent-hashing ${FARMHASH_LIBRARIES} ${METROHASH_LIBRARIES})
target_include_directories (clickhouse_functions BEFORE PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/libfarmhash)
target_include_directories (clickhouse_functions BEFORE PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/libmetrohash/src)
target_include_directories (clickhouse_functions BEFORE PUBLIC ${DIVIDE_INCLUDE_DIR})
target_include_directories (clickhouse_functions SYSTEM BEFORE PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/libfarmhash)
target_include_directories (clickhouse_functions SYSTEM BEFORE PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/libmetrohash/src)
target_include_directories (clickhouse_functions SYSTEM BEFORE PUBLIC ${DIVIDE_INCLUDE_DIR})
if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" OR CMAKE_BUILD_TYPE_UC STREQUAL "MINSIZEREL")
# Won't generate debug info for files with heavy template instantiation to achieve faster linking and lower size.
@ -99,11 +99,11 @@ endif ()
if (USE_ICU)
#target_link_libraries (clickhouse_functions ${ICU_LIBS})
target_include_directories (clickhouse_functions PRIVATE ${ICU_INCLUDE_DIR})
target_include_directories (clickhouse_functions SYSTEM PRIVATE ${ICU_INCLUDE_DIR})
endif ()
if (USE_VECTORCLASS)
target_include_directories (clickhouse_functions BEFORE PUBLIC ${VECTORCLASS_INCLUDE_DIR})
target_include_directories (clickhouse_functions SYSTEM BEFORE PUBLIC ${VECTORCLASS_INCLUDE_DIR})
endif ()
if (ENABLE_TESTS)
@ -111,5 +111,5 @@ if (ENABLE_TESTS)
endif ()
if (USE_EMBEDDED_COMPILER)
target_include_directories (clickhouse_functions BEFORE PUBLIC ${LLVM_INCLUDE_DIRS})
target_include_directories (clickhouse_functions SYSTEM BEFORE PUBLIC ${LLVM_INCLUDE_DIRS})
endif ()

2
dbms/src/Functions/FunctionsMiscellaneous.cpp
View File

@ -1688,7 +1688,7 @@ void FunctionVisibleWidth::executeImpl(Block & block, const ColumnNumbers & argu
{
{
WriteBufferFromString out(tmp);
src.type->serializeTextEscaped(*src.column, i, out);
src.type->serializeText(*src.column, i, out);
}
res_data[i] = UTF8::countCodePoints(reinterpret_cast<const UInt8 *>(tmp.data()), tmp.size());

4
dbms/src/Functions/FunctionsRound.h
View File

@ -219,6 +219,8 @@ struct IntegerRoundingComputation
x = -x;
return x;
}
default:
__builtin_unreachable();
}
}
@ -232,6 +234,8 @@ struct IntegerRoundingComputation
return x;
case ScaleMode::Negative:
return computeImpl(x, scale);
default:
__builtin_unreachable();
}
}

124
dbms/src/Functions/FunctionsString.cpp
View File

@ -1084,6 +1084,125 @@ private:
};
struct NameStartsWith
{
static constexpr auto name = "startsWith";
};
struct NameEndsWith
{
static constexpr auto name = "endsWith";
};
template <typename Name>
class FunctionStartsEndsWith : public IFunction
{
public:
static constexpr auto name = Name::name;
static FunctionPtr create(const Context &)
{
return std::make_shared<FunctionStartsEndsWith>();
}
String getName() const override
{
return name;
}
size_t getNumberOfArguments() const override
{
return 2;
}
bool useDefaultImplementationForConstants() const override
{
return true;
}
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
if (!arguments[0]->isStringOrFixedString())
throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
if (!arguments[1]->isStringOrFixedString())
throw Exception("Illegal type " + arguments[1]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
return std::make_shared<DataTypeNumber<UInt8>>();
}
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override
{
const IColumn * haystack_column = block.getByPosition(arguments[0]).column.get();
const IColumn * needle_column = block.getByPosition(arguments[1]).column.get();
auto col_res = ColumnVector<UInt8>::create();
typename ColumnVector<UInt8>::Container & vec_res = col_res->getData();
vec_res.resize(input_rows_count);
if (const ColumnString * haystack = checkAndGetColumn<ColumnString>(haystack_column))
dispatch<StringSource>(StringSource(*haystack), needle_column, vec_res);
else if (const ColumnFixedString * haystack = checkAndGetColumn<ColumnFixedString>(haystack_column))
dispatch<FixedStringSource>(FixedStringSource(*haystack), needle_column, vec_res);
else if (const ColumnConst * haystack = checkAndGetColumnConst<ColumnString>(haystack_column))
dispatch<ConstSource<StringSource>>(ConstSource<StringSource>(*haystack), needle_column, vec_res);
else if (const ColumnConst * haystack = checkAndGetColumnConst<ColumnFixedString>(haystack_column))
dispatch<ConstSource<FixedStringSource>>(ConstSource<FixedStringSource>(*haystack), needle_column, vec_res);
else
throw Exception("Illegal combination of columns as arguments of function " + getName(), ErrorCodes::ILLEGAL_COLUMN);
block.getByPosition(result).column = std::move(col_res);
}
private:
template <typename HaystackSource>
void dispatch(HaystackSource haystack_source, const IColumn * needle_column, PaddedPODArray<UInt8> & res_data) const
{
if (const ColumnString * needle = checkAndGetColumn<ColumnString>(needle_column))
execute<HaystackSource, StringSource>(haystack_source, StringSource(*needle), res_data);
else if (const ColumnFixedString * needle = checkAndGetColumn<ColumnFixedString>(needle_column))
execute<HaystackSource, FixedStringSource>(haystack_source, FixedStringSource(*needle), res_data);
else if (const ColumnConst * needle = checkAndGetColumnConst<ColumnString>(needle_column))
execute<HaystackSource, ConstSource<StringSource>>(haystack_source, ConstSource<StringSource>(*needle), res_data);
else if (const ColumnConst * needle = checkAndGetColumnConst<ColumnFixedString>(needle_column))
execute<HaystackSource, ConstSource<FixedStringSource>>(haystack_source, ConstSource<FixedStringSource>(*needle), res_data);
else
throw Exception("Illegal combination of columns as arguments of function " + getName(), ErrorCodes::ILLEGAL_COLUMN);
}
template <typename HaystackSource, typename NeedleSource>
static void execute(HaystackSource haystack_source, NeedleSource needle_source, PaddedPODArray<UInt8> & res_data)
{
size_t row_num = 0;
while (!haystack_source.isEnd())
{
auto haystack = haystack_source.getWhole();
auto needle = needle_source.getWhole();
if (needle.size > haystack.size)
{
res_data[row_num] = false;
}
else
{
if constexpr (std::is_same_v<Name, NameStartsWith>)
{
res_data[row_num] = StringRef(haystack.data, needle.size) == StringRef(needle.data, needle.size);
}
else /// endsWith
{
res_data[row_num] = StringRef(haystack.data + haystack.size - needle.size, needle.size) == StringRef(needle.data, needle.size);
}
}
haystack_source.next();
needle_source.next();
++row_num;
}
}
};
struct NameEmpty
{
static constexpr auto name = "empty";
@ -1121,6 +1240,7 @@ struct NameConcatAssumeInjective
static constexpr auto name = "concatAssumeInjective";
};
using FunctionEmpty = FunctionStringOrArrayToT<EmptyImpl<false>, NameEmpty, UInt8>;
using FunctionNotEmpty = FunctionStringOrArrayToT<EmptyImpl<true>, NameNotEmpty, UInt8>;
using FunctionLength = FunctionStringOrArrayToT<LengthImpl, NameLength, UInt64>;
@ -1130,6 +1250,8 @@ using FunctionUpper = FunctionStringToString<LowerUpperImpl<'a', 'z'>, NameUpper
using FunctionReverseUTF8 = FunctionStringToString<ReverseUTF8Impl, NameReverseUTF8, true>;
using FunctionConcat = ConcatImpl<NameConcat, false>;
using FunctionConcatAssumeInjective = ConcatImpl<NameConcatAssumeInjective, true>;
using FunctionStartsWith = FunctionStartsEndsWith<NameStartsWith>;
using FunctionEndsWith = FunctionStartsEndsWith<NameEndsWith>;
void registerFunctionsString(FunctionFactory & factory)
@ -1149,5 +1271,7 @@ void registerFunctionsString(FunctionFactory & factory)
factory.registerFunction<FunctionSubstring>();
factory.registerFunction<FunctionSubstringUTF8>();
factory.registerFunction<FunctionAppendTrailingCharIfAbsent>();
factory.registerFunction<FunctionStartsWith>();
factory.registerFunction<FunctionEndsWith>();
}
}

2
dbms/src/Functions/tests/CMakeLists.txt
View File

@ -1,4 +1,2 @@
include_directories (${CMAKE_CURRENT_BINARY_DIR})
add_executable (number_traits number_traits.cpp)
target_link_libraries (number_traits dbms)

2
dbms/src/IO/tests/CMakeLists.txt
View File

@ -1,5 +1,3 @@
include_directories (${CMAKE_CURRENT_BINARY_DIR})
add_executable (read_buffer read_buffer.cpp)
target_link_libraries (read_buffer clickhouse_common_io)

2
dbms/src/Interpreters/Context.cpp
View File

@ -355,7 +355,7 @@ std::shared_ptr<Context> Context::acquireSession(const String & session_id, std:
if (session_check)
throw Exception("Session not found.", ErrorCodes::SESSION_NOT_FOUND);
auto new_session = std::make_shared<Context>(*global_context);
auto new_session = std::make_shared<Context>(*this);
new_session->scheduleCloseSession(key, timeout);

1
dbms/src/Interpreters/Context.h
View File

@ -6,6 +6,7 @@
#include <memory>
#include <mutex>
#include <thread>
#include <atomic>
#include <common/MultiVersion.h>
#include <Core/Types.h>

2
dbms/src/Interpreters/ExpressionAnalyzer.h
View File

@ -38,6 +38,8 @@ class ASTFunction;
class ASTExpressionList;
class ASTSelectQuery;
struct ProjectionManipulatorBase;
using ProjectionManipulatorPtr = std::shared_ptr<ProjectionManipulatorBase>;
/** Information on what to do when executing a subquery in the [GLOBAL] IN/JOIN section.
*/

5
dbms/src/Interpreters/InterpreterShowTablesQuery.cpp
View File

@ -1,17 +1,14 @@
#include <IO/ReadBufferFromString.h>
#include <Parsers/ASTShowTablesQuery.h>
#include <Parsers/ASTIdentifier.h>
#include <Interpreters/Context.h>
#include <Interpreters/executeQuery.h>
#include <Interpreters/InterpreterShowTablesQuery.h>
#include <Common/typeid_cast.h>
#include <iomanip>
#include <sstream>
namespace DB
{

10
dbms/src/Interpreters/tests/CMakeLists.txt
View File

@ -11,7 +11,7 @@ add_executable (aggregate aggregate.cpp)
target_link_libraries (aggregate dbms)
add_executable (hash_map hash_map.cpp)
target_include_directories (hash_map BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
target_include_directories (hash_map SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
target_link_libraries (hash_map dbms)
add_executable (hash_map3 hash_map3.cpp)
@ -26,15 +26,15 @@ target_link_libraries (hash_map_string_2 dbms)
add_executable (hash_map_string_3 hash_map_string_3.cpp)
target_link_libraries (hash_map_string_3 dbms ${FARMHASH_LIBRARIES} ${METROHASH_LIBRARIES})
target_include_directories (hash_map_string_3 BEFORE PRIVATE ${ClickHouse_SOURCE_DIR}/contrib/libfarmhash)
target_include_directories (hash_map_string_3 BEFORE PRIVATE ${ClickHouse_SOURCE_DIR}/contrib/libmetrohash/src)
target_include_directories (hash_map_string_3 SYSTEM BEFORE PRIVATE ${ClickHouse_SOURCE_DIR}/contrib/libfarmhash)
target_include_directories (hash_map_string_3 SYSTEM BEFORE PRIVATE ${ClickHouse_SOURCE_DIR}/contrib/libmetrohash/src)
add_executable (hash_map_string_small hash_map_string_small.cpp)
target_include_directories (hash_map_string_small BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
target_include_directories (hash_map_string_small SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
target_link_libraries (hash_map_string_small dbms)
add_executable (two_level_hash_map two_level_hash_map.cpp)
target_include_directories (two_level_hash_map BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
target_include_directories (two_level_hash_map SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
target_link_libraries (two_level_hash_map dbms)
add_executable (compiler_test compiler_test.cpp)

2
dbms/src/Parsers/tests/CMakeLists.txt
View File

@ -1,5 +1,3 @@
include_directories (${CMAKE_CURRENT_BINARY_DIR})
set(SRCS )
add_executable (lexer lexer.cpp ${SRCS})

26
dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp
View File

@ -22,11 +22,11 @@ MergeTreeBaseBlockInputStream::MergeTreeBaseBlockInputStream(
MergeTreeData & storage,
const ExpressionActionsPtr & prewhere_actions,
const String & prewhere_column_name,
size_t max_block_size_rows,
size_t preferred_block_size_bytes,
size_t preferred_max_column_in_block_size_bytes,
size_t min_bytes_to_use_direct_io,
size_t max_read_buffer_size,
UInt64 max_block_size_rows,
UInt64 preferred_block_size_bytes,
UInt64 preferred_max_column_in_block_size_bytes,
UInt64 min_bytes_to_use_direct_io,
UInt64 max_read_buffer_size,
bool use_uncompressed_cache,
bool save_marks_in_cache,
const Names & virt_column_names)
@ -89,7 +89,7 @@ Block MergeTreeBaseBlockInputStream::readFromPart()
/// Calculates number of rows will be read using preferred_block_size_bytes.
/// Can't be less than index_granularity.
size_t rows_to_read = task.size_predictor->estimateNumRows(preferred_block_size_bytes);
UInt64 rows_to_read = task.size_predictor->estimateNumRows(preferred_block_size_bytes);
if (!rows_to_read)
return rows_to_read;
rows_to_read = std::max(index_granularity, rows_to_read);
@ -97,21 +97,21 @@ Block MergeTreeBaseBlockInputStream::readFromPart()
if (preferred_max_column_in_block_size_bytes)
{
/// Calculates number of rows will be read using preferred_max_column_in_block_size_bytes.
size_t rows_to_read_for_max_size_column
UInt64 rows_to_read_for_max_size_column
= task.size_predictor->estimateNumRowsForMaxSizeColumn(preferred_max_column_in_block_size_bytes);
double filtration_ratio = std::max(min_filtration_ratio, 1.0 - task.size_predictor->filtered_rows_ratio);
auto rows_to_read_for_max_size_column_with_filtration
= static_cast<size_t>(rows_to_read_for_max_size_column / filtration_ratio);
= static_cast<UInt64>(rows_to_read_for_max_size_column / filtration_ratio);
/// If preferred_max_column_in_block_size_bytes is used, number of rows to read can be less than index_granularity.
rows_to_read = std::min(rows_to_read, rows_to_read_for_max_size_column_with_filtration);
}
size_t unread_rows_in_current_granule = reader.numPendingRowsInCurrentGranule();
UInt64 unread_rows_in_current_granule = reader.numPendingRowsInCurrentGranule();
if (unread_rows_in_current_granule >= rows_to_read)
return rows_to_read;
size_t granule_to_read = (rows_to_read + reader.numReadRowsInCurrentGranule() + index_granularity / 2) / index_granularity;
UInt64 granule_to_read = (rows_to_read + reader.numReadRowsInCurrentGranule() + index_granularity / 2) / index_granularity;
return index_granularity * granule_to_read - reader.numReadRowsInCurrentGranule();
};
@ -146,8 +146,8 @@ Block MergeTreeBaseBlockInputStream::readFromPart()
}
}
size_t recommended_rows = estimateNumRows(*task, task->range_reader);
size_t rows_to_read = std::max(static_cast<decltype(max_block_size_rows)>(1), std::min(max_block_size_rows, recommended_rows));
UInt64 recommended_rows = estimateNumRows(*task, task->range_reader);
UInt64 rows_to_read = std::max(UInt64(1), std::min(max_block_size_rows, recommended_rows));
auto read_result = task->range_reader.read(rows_to_read, task->mark_ranges);
@ -155,7 +155,7 @@ Block MergeTreeBaseBlockInputStream::readFromPart()
if (read_result.block.rows() == 0)
read_result.block.clear();
size_t num_filtered_rows = read_result.numReadRows() - read_result.block.rows();
UInt64 num_filtered_rows = read_result.numReadRows() - read_result.block.rows();
progressImpl({ read_result.numReadRows(), read_result.numBytesRead() });

22
dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.h
View File

@ -20,11 +20,11 @@ public:
MergeTreeData & storage,
const ExpressionActionsPtr & prewhere_actions,
const String & prewhere_column,
size_t max_block_size_rows,
size_t preferred_block_size_bytes,
size_t preferred_max_column_in_block_size_bytes,
size_t min_bytes_to_use_direct_io,
size_t max_read_buffer_size,
UInt64 max_block_size_rows,
UInt64 preferred_block_size_bytes,
UInt64 preferred_max_column_in_block_size_bytes,
UInt64 min_bytes_to_use_direct_io,
UInt64 max_read_buffer_size,
bool use_uncompressed_cache,
bool save_marks_in_cache = true,
const Names & virt_column_names = {});
@ -50,12 +50,12 @@ protected:
ExpressionActionsPtr prewhere_actions;
String prewhere_column_name;
size_t max_block_size_rows;
size_t preferred_block_size_bytes;
size_t preferred_max_column_in_block_size_bytes;
UInt64 max_block_size_rows;
UInt64 preferred_block_size_bytes;
UInt64 preferred_max_column_in_block_size_bytes;
size_t min_bytes_to_use_direct_io;
size_t max_read_buffer_size;
UInt64 min_bytes_to_use_direct_io;
UInt64 max_read_buffer_size;
bool use_uncompressed_cache;
bool save_marks_in_cache;
@ -71,7 +71,7 @@ protected:
MergeTreeReaderPtr reader;
MergeTreeReaderPtr pre_reader;
size_t max_block_size_marks;
UInt64 max_block_size_marks;
};
}

10
dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp
View File

@ -977,11 +977,11 @@ ReplicatedMergeTreeQueue::Status ReplicatedMergeTreeQueue::getStatus() const
res.inserts_in_queue = 0;
res.merges_in_queue = 0;
res.mutations_in_queue = 0;
res.part_mutations_in_queue = 0;
res.queue_oldest_time = 0;
res.inserts_oldest_time = 0;
res.merges_oldest_time = 0;
res.mutations_oldest_time = 0;
res.part_mutations_oldest_time = 0;
for (const LogEntryPtr & entry : queue)
{
@ -1012,11 +1012,11 @@ ReplicatedMergeTreeQueue::Status ReplicatedMergeTreeQueue::getStatus() const
if (entry->type == LogEntry::MUTATE_PART)
{
++res.mutations_in_queue;
++res.part_mutations_in_queue;
if (entry->create_time && (!res.mutations_oldest_time || entry->create_time < res.mutations_oldest_time))
if (entry->create_time && (!res.part_mutations_oldest_time || entry->create_time < res.part_mutations_oldest_time))
{
res.mutations_oldest_time = entry->create_time;
res.part_mutations_oldest_time = entry->create_time;
res.oldest_part_to_mutate_to = entry->new_part_name;
}
}

4
dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h
View File

@ -284,11 +284,11 @@ public:
UInt32 queue_size;
UInt32 inserts_in_queue;
UInt32 merges_in_queue;
UInt32 mutations_in_queue;
UInt32 part_mutations_in_queue;
UInt32 queue_oldest_time;
UInt32 inserts_oldest_time;
UInt32 merges_oldest_time;
UInt32 mutations_oldest_time;
UInt32 part_mutations_oldest_time;
String oldest_part_to_get;
String oldest_part_to_merge_to;
String oldest_part_to_mutate_to;

4
dbms/src/Storages/MutationCommands.cpp
View File

@ -36,7 +36,7 @@ void MutationCommand::writeText(WriteBuffer & out) const
{
std::stringstream ss;
formatAST(*predicate, ss, /* hilite = */ false, /* one_line = */ true);
out << "predicate: " << ss.str() << "\n";
out << "predicate: " << escape << ss.str() << "\n";
break;
}
default:
@ -54,7 +54,7 @@ void MutationCommand::readText(ReadBuffer & in)
type = DELETE;
String predicate_str;
in >> "predicate: " >> predicate_str >> "\n";
in >> "predicate: " >> escape >> predicate_str >> "\n";
ParserExpressionWithOptionalAlias p_expr(false);
predicate = parseQuery(
p_expr, predicate_str.data(), predicate_str.data() + predicate_str.length(), "mutation predicate", 0);

6
dbms/src/Storages/System/StorageSystemReplicas.cpp
View File

@ -33,11 +33,14 @@ StorageSystemReplicas::StorageSystemReplicas(const std::string & name_)
{ "queue_size", std::make_shared<DataTypeUInt32>() },
{ "inserts_in_queue", std::make_shared<DataTypeUInt32>() },
{ "merges_in_queue", std::make_shared<DataTypeUInt32>() },
{ "part_mutations_in_queue", std::make_shared<DataTypeUInt32>() },
{ "queue_oldest_time", std::make_shared<DataTypeDateTime>() },
{ "inserts_oldest_time", std::make_shared<DataTypeDateTime>() },
{ "merges_oldest_time", std::make_shared<DataTypeDateTime>() },
{ "part_mutations_oldest_time", std::make_shared<DataTypeDateTime>() },
{ "oldest_part_to_get", std::make_shared<DataTypeString>() },
{ "oldest_part_to_merge_to", std::make_shared<DataTypeString>() },
{ "oldest_part_to_mutate_to", std::make_shared<DataTypeString>() },
{ "log_max_index", std::make_shared<DataTypeUInt64>() },
{ "log_pointer", std::make_shared<DataTypeUInt64>() },
{ "last_queue_update", std::make_shared<DataTypeDateTime>() },
@ -140,11 +143,14 @@ BlockInputStreams StorageSystemReplicas::read(
res_columns[col_num++]->insert(UInt64(status.queue.queue_size));
res_columns[col_num++]->insert(UInt64(status.queue.inserts_in_queue));
res_columns[col_num++]->insert(UInt64(status.queue.merges_in_queue));
res_columns[col_num++]->insert(UInt64(status.queue.part_mutations_in_queue));
res_columns[col_num++]->insert(UInt64(status.queue.queue_oldest_time));
res_columns[col_num++]->insert(UInt64(status.queue.inserts_oldest_time));
res_columns[col_num++]->insert(UInt64(status.queue.merges_oldest_time));
res_columns[col_num++]->insert(UInt64(status.queue.part_mutations_oldest_time));
res_columns[col_num++]->insert(status.queue.oldest_part_to_get);
res_columns[col_num++]->insert(status.queue.oldest_part_to_merge_to);
res_columns[col_num++]->insert(status.queue.oldest_part_to_mutate_to);
res_columns[col_num++]->insert(status.log_max_index);
res_columns[col_num++]->insert(status.log_pointer);
res_columns[col_num++]->insert(UInt64(status.queue.last_queue_update));

2
dbms/src/Storages/tests/CMakeLists.txt
View File

@ -1,5 +1,3 @@
include_directories (${CMAKE_CURRENT_BINARY_DIR})
add_executable (system_numbers system_numbers.cpp)
target_link_libraries (system_numbers dbms clickhouse_storages_system)

4
dbms/src/TableFunctions/CMakeLists.txt
View File

@ -9,10 +9,10 @@ target_link_libraries(clickhouse_table_functions clickhouse_storages_system dbms
if (USE_POCO_SQLODBC)
target_link_libraries (clickhouse_table_functions ${Poco_SQLODBC_LIBRARY})
target_include_directories (clickhouse_table_functions PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIRS})
target_include_directories (clickhouse_table_functions SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIRS})
endif ()
if (USE_POCO_DATAODBC)
target_link_libraries (clickhouse_table_functions ${Poco_DataODBC_LIBRARY})
target_include_directories (clickhouse_table_functions PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIRS})
target_include_directories (clickhouse_table_functions SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIRS})
endif ()

7
dbms/tests/clickhouse-test-server
View File

@ -42,9 +42,8 @@ export CLICKHOUSE_PORT_INTERSERVER=${CLICKHOUSE_PORT_INTERSERVER:=`$CLICKHOUSE_E
export CLICKHOUSE_PORT_TCP_SECURE=${CLICKHOUSE_PORT_TCP_SECURE:=`$CLICKHOUSE_EXTRACT_CONFIG --key=tcp_port_secure`}
export CLICKHOUSE_PORT_HTTPS=${CLICKHOUSE_PORT_HTTPS:=`$CLICKHOUSE_EXTRACT_CONFIG --key=https_port`}
rm -rf $DATA_DIR
mkdir -p $LOG_DIR
rm -rf $DATA_DIR || true
mkdir -p $LOG_DIR || true
DHPARAM=`$CLICKHOUSE_EXTRACT_CONFIG --key=openSSL.server.dhParamsFile`
PRIVATEKEY=`${BIN_DIR}clickhouse-extract-from-config --config=$CLICKHOUSE_CONFIG --key=openSSL.server.privateKeyFile`
@ -78,8 +77,6 @@ $GDB ${BIN_DIR}clickhouse-server --config-file=$CLICKHOUSE_CONFIG -- \
CH_PID=$!
sleep 3
if [ "$GDB" ]; then
# Long symbols read
sleep 40

3
dbms/tests/queries/0_stateless/00231_format_vertical_raw.reference
View File

@ -1,7 +1,4 @@
Row 1:
──────
x: a\tb\nc\td
Row 1:
──────
x: a b
c d

1
dbms/tests/queries/0_stateless/00231_format_vertical_raw.sql
View File

@ -1,2 +1 @@
SELECT 'a\tb\nc\td' AS x FORMAT Vertical;
SELECT 'a\tb\nc\td' AS x FORMAT VerticalRaw;

10
dbms/tests/queries/0_stateless/00298_enum_width_and_cast.reference
View File

@ -3,15 +3,15 @@
└───────┴───┘
┌─x─────┬─y─┐
│ Hello │ 0 │
│ \\ │ 0 │
│ \ │ 0 │
└───────┴───┘
┌─x─────┬─y─┐
│ Hello │ 0 │
│ \\ │ 0 │
│ \t\\t │ 0 │
│ \ │ 0 │
│ \t │ 0 │
└───────┴───┘
┌─x─────┬─y─┬─toInt8(x)─┬─s─────┬─casted─┐
│ Hello │ 0 │ -100 │ Hello │ Hello │
│ \\ │ 0 │ 0 │ \\ │ \\ │
│ \t\\t │ 0 │ 111 │ \t\\t │ \t\\t │
│ \ │ 0 │ 0 │ \ │ \ │
│ \t │ 0 │ 111 │ \t │ \t │
└───────┴───┴───────────┴───────┴────────┘

84
dbms/tests/queries/0_stateless/00405_pretty_formats.reference
View File

@ -1,13 +1,13 @@
┏━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
┃ hello ┃ world ┃ tuple  ┃ sometimes_nulls ┃
┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
│ 0 │ 0 │ (0,'0') │ \N │
│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │
├───────┼───────┼─────────┼─────────────────┤
│ 1 │ 1 │ (1,'1') │ 1 │
├───────┼───────┼─────────┼─────────────────┤
│ 2 │ 2 │ (2,'2') │ 2 │
├───────┼───────┼─────────┼─────────────────┤
│ 3 │ 3 │ (3,'3') │ \N │
│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │
├───────┼───────┼─────────┼─────────────────┤
│ 4 │ 4 │ (4,'4') │ 1 │
└───────┴───────┴─────────┴─────────────────┘
@ -16,64 +16,64 @@
┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
│ 5 │ 5 │ (5,'5') │ 2 │
├───────┼───────┼─────────┼─────────────────┤
│ 6 │ 6 │ (6,'6') │ \N │
│ 6 │ 6 │ (6,'6') │ ᴺᵁᴸᴸ │
├───────┼───────┼─────────┼─────────────────┤
│ 7 │ 7 │ (7,'7') │ 1 │
├───────┼───────┼─────────┼─────────────────┤
│ 8 │ 8 │ (8,'8') │ 2 │
├───────┼───────┼─────────┼─────────────────┤
│ 9 │ 9 │ (9,'9') │ \N │
│ 9 │ 9 │ (9,'9') │ ᴺᵁᴸᴸ │
└───────┴───────┴─────────┴─────────────────┘
┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐
│ 0 │ 0 │ (0,'0') │ \N │
│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │
│ 1 │ 1 │ (1,'1') │ 1 │
│ 2 │ 2 │ (2,'2') │ 2 │
│ 3 │ 3 │ (3,'3') │ \N │
│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │
│ 4 │ 4 │ (4,'4') │ 1 │
└───────┴───────┴─────────┴─────────────────┘
┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐
│ 5 │ 5 │ (5,'5') │ 2 │
│ 6 │ 6 │ (6,'6') │ \N │
│ 6 │ 6 │ (6,'6') │ ᴺᵁᴸᴸ │
│ 7 │ 7 │ (7,'7') │ 1 │
│ 8 │ 8 │ (8,'8') │ 2 │
│ 9 │ 9 │ (9,'9') │ \N │
│ 9 │ 9 │ (9,'9') │ ᴺᵁᴸᴸ │
└───────┴───────┴─────────┴─────────────────┘
hello world tuple sometimes_nulls
0 0 (0,'0') \N
0 0 (0,'0') ᴺᵁᴸᴸ
1 1 (1,'1') 1
2 2 (2,'2') 2
3 3 (3,'3') \N
3 3 (3,'3') ᴺᵁᴸᴸ
4 4 (4,'4') 1
hello world tuple sometimes_nulls
5 5 (5,'5') 2
6 6 (6,'6') \N
6 6 (6,'6') ᴺᵁᴸᴸ
7 7 (7,'7') 1
8 8 (8,'8') 2
9 9 (9,'9') \N
9 9 (9,'9') ᴺᵁᴸᴸ
┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐
│ 0 │ 0 │ (0,'0') │ \N │
│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │
│ 1 │ 1 │ (1,'1') │ 1 │
│ 2 │ 2 │ (2,'2') │ 2 │
│ 3 │ 3 │ (3,'3') │ \N │
│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │
│ 4 │ 4 │ (4,'4') │ 1 │
│ 5 │ 5 │ (5,'5') │ 2 │
│ 6 │ 6 │ (6,'6') │ \N │
│ 6 │ 6 │ (6,'6') │ ᴺᵁᴸᴸ │
│ 7 │ 7 │ (7,'7') │ 1 │
│ 8 │ 8 │ (8,'8') │ 2 │
│ 9 │ 9 │ (9,'9') │ \N │
│ 9 │ 9 │ (9,'9') │ ᴺᵁᴸᴸ │
└───────┴───────┴─────────┴─────────────────┘
┏━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
┃ hello ┃ world ┃ tuple ┃ sometimes_nulls ┃
┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
│ 0 │ 0 │ (0,'0') │ \N │
│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │
├───────┼───────┼─────────┼─────────────────┤
│ 1 │ 1 │ (1,'1') │ 1 │
├───────┼───────┼─────────┼─────────────────┤
│ 2 │ 2 │ (2,'2') │ 2 │
├───────┼───────┼─────────┼─────────────────┤
│ 3 │ 3 │ (3,'3') │ \N │
│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │
├───────┼───────┼─────────┼─────────────────┤
│ 4 │ 4 │ (4,'4') │ 1 │
└───────┴───────┴─────────┴─────────────────┘
@ -82,52 +82,52 @@
┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
│ 5 │ 5 │ (5,'5') │ 2 │
├───────┼───────┼─────────┼─────────────────┤
│ 6 │ 6 │ (6,'6') │ \N │
│ 6 │ 6 │ (6,'6') │ ᴺᵁᴸᴸ │
├───────┼───────┼─────────┼─────────────────┤
│ 7 │ 7 │ (7,'7') │ 1 │
├───────┼───────┼─────────┼─────────────────┤
│ 8 │ 8 │ (8,'8') │ 2 │
├───────┼───────┼─────────┼─────────────────┤
│ 9 │ 9 │ (9,'9') │ \N │
│ 9 │ 9 │ (9,'9') │ ᴺᵁᴸᴸ │
└───────┴───────┴─────────┴─────────────────┘
┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐
│ 0 │ 0 │ (0,'0') │ \N │
│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │
│ 1 │ 1 │ (1,'1') │ 1 │
│ 2 │ 2 │ (2,'2') │ 2 │
│ 3 │ 3 │ (3,'3') │ \N │
│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │
│ 4 │ 4 │ (4,'4') │ 1 │
└───────┴───────┴─────────┴─────────────────┘
┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐
│ 5 │ 5 │ (5,'5') │ 2 │
│ 6 │ 6 │ (6,'6') │ \N │
│ 6 │ 6 │ (6,'6') │ ᴺᵁᴸᴸ │
│ 7 │ 7 │ (7,'7') │ 1 │
│ 8 │ 8 │ (8,'8') │ 2 │
│ 9 │ 9 │ (9,'9') │ \N │
│ 9 │ 9 │ (9,'9') │ ᴺᵁᴸᴸ │
└───────┴───────┴─────────┴─────────────────┘
hello world tuple sometimes_nulls
0 0 (0,'0') \N
0 0 (0,'0') ᴺᵁᴸᴸ
1 1 (1,'1') 1
2 2 (2,'2') 2
3 3 (3,'3') \N
3 3 (3,'3') ᴺᵁᴸᴸ
4 4 (4,'4') 1
hello world tuple sometimes_nulls
5 5 (5,'5') 2
6 6 (6,'6') \N
6 6 (6,'6') ᴺᵁᴸᴸ
7 7 (7,'7') 1
8 8 (8,'8') 2
9 9 (9,'9') \N
9 9 (9,'9') ᴺᵁᴸᴸ
┏━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
┃ hello ┃ world ┃ tuple  ┃ sometimes_nulls ┃
┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
│ 0 │ 0 │ (0,'0') │ \N │
│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │
├───────┼───────┼─────────┼─────────────────┤
│ 1 │ 1 │ (1,'1') │ 1 │
├───────┼───────┼─────────┼─────────────────┤
│ 2 │ 2 │ (2,'2') │ 2 │
├───────┼───────┼─────────┼─────────────────┤
│ 3 │ 3 │ (3,'3') │ \N │
│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │
├───────┼───────┼─────────┼─────────────────┤
│ 4 │ 4 │ (4,'4') │ 1 │
└───────┴───────┴─────────┴─────────────────┘
@ -138,10 +138,10 @@ hello world tuple sometimes_nulls
└───────┴───────┴─────────┴─────────────────┘
Showed first 6.
┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐
│ 0 │ 0 │ (0,'0') │ \N │
│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │
│ 1 │ 1 │ (1,'1') │ 1 │
│ 2 │ 2 │ (2,'2') │ 2 │
│ 3 │ 3 │ (3,'3') │ \N │
│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │
│ 4 │ 4 │ (4,'4') │ 1 │
└───────┴───────┴─────────┴─────────────────┘
┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐
@ -150,10 +150,10 @@ hello world tuple sometimes_nulls
Showed first 6.
hello world tuple sometimes_nulls
0 0 (0,'0') \N
0 0 (0,'0') ᴺᵁᴸᴸ
1 1 (1,'1') 1
2 2 (2,'2') 2
3 3 (3,'3') \N
3 3 (3,'3') ᴺᵁᴸᴸ
4 4 (4,'4') 1
hello world tuple sometimes_nulls
@ -161,10 +161,10 @@ hello world tuple sometimes_nulls
Showed first 6.
┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐
│ 0 │ 0 │ (0,'0') │ \N │
│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │
│ 1 │ 1 │ (1,'1') │ 1 │
│ 2 │ 2 │ (2,'2') │ 2 │
│ 3 │ 3 │ (3,'3') │ \N │
│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │
│ 4 │ 4 │ (4,'4') │ 1 │
│ 5 │ 5 │ (5,'5') │ 2 │
└───────┴───────┴─────────┴─────────────────┘
@ -172,13 +172,13 @@ Showed first 6.
┏━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
┃ hello ┃ world ┃ tuple ┃ sometimes_nulls ┃
┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
│ 0 │ 0 │ (0,'0') │ \N │
│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │
├───────┼───────┼─────────┼─────────────────┤
│ 1 │ 1 │ (1,'1') │ 1 │
├───────┼───────┼─────────┼─────────────────┤
│ 2 │ 2 │ (2,'2') │ 2 │
├───────┼───────┼─────────┼─────────────────┤
│ 3 │ 3 │ (3,'3') │ \N │
│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │
├───────┼───────┼─────────┼─────────────────┤
│ 4 │ 4 │ (4,'4') │ 1 │
└───────┴───────┴─────────┴─────────────────┘
@ -189,10 +189,10 @@ Showed first 6.
└───────┴───────┴─────────┴─────────────────┘
Showed first 6.
┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐
│ 0 │ 0 │ (0,'0') │ \N │
│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │
│ 1 │ 1 │ (1,'1') │ 1 │
│ 2 │ 2 │ (2,'2') │ 2 │
│ 3 │ 3 │ (3,'3') │ \N │
│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │
│ 4 │ 4 │ (4,'4') │ 1 │
└───────┴───────┴─────────┴─────────────────┘
┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐
@ -201,10 +201,10 @@ Showed first 6.
Showed first 6.
hello world tuple sometimes_nulls
0 0 (0,'0') \N
0 0 (0,'0') ᴺᵁᴸᴸ
1 1 (1,'1') 1
2 2 (2,'2') 2
3 3 (3,'3') \N
3 3 (3,'3') ᴺᵁᴸᴸ
4 4 (4,'4') 1
hello world tuple sometimes_nulls

6
dbms/tests/queries/0_stateless/00406_tuples_with_nulls.reference
View File

@ -10,6 +10,6 @@
│ (8,2,NULL) │
│ (9,NULL,'1') │
└───────────────┘
┌─x──┬─y──────┐
│ \N │ (NULL) │
└────┴────────┘
┌─x────┬─y──────┐
│ ᴺᵁᴸᴸ │ (NULL) │
└──────┴────────┘

151
dbms/tests/queries/0_stateless/00460_vertical_and_totals_extremes.reference
View File

@ -131,157 +131,6 @@ count(): 20
Showed first 4.
Totals:
───────
k: 0
count(): 100
Min:
────
k: 0
count(): 20
Max:
────
k: 4
count(): 20
Row 1:
──────
k: 0
count(): 20
Row 2:
──────
k: 1
count(): 20
Row 3:
──────
k: 2
count(): 20
Row 4:
──────
k: 3
count(): 20
Showed first 4.
Totals:
───────
k: 0
count(): 100
Min:
────
k: 0
count(): 20
Max:
────
k: 4
count(): 20
Row 1:
──────
k: 0
count(): 20
Row 2:
──────
k: 1
count(): 20
Row 3:
──────
k: 2
count(): 20
Row 4:
──────
k: 3
count(): 20
Showed first 4.
Totals:
───────
k: 0
count(): 100
Min:
────
k: 0
count(): 20
Max:
────
k: 4
count(): 20
Row 1:
──────
k: 0
count(): 20
Row 2:
──────
k: 1
count(): 20
Row 3:
──────
k: 2
count(): 20
Row 4:
──────
k: 3
count(): 20
Row 5:
──────
k: 4
count(): 20
Totals:
───────
k: 0
count(): 100
Min:
────
k: 0
count(): 20
Max:
────
k: 4
count(): 20
Row 1:
──────
k: 0
count(): 20
Row 2:
──────
k: 1
count(): 20
Row 3:
──────
k: 2
count(): 20
Row 4:
──────
k: 3
count(): 20
Showed first 4.
Totals:
───────
k: 0

12
dbms/tests/queries/0_stateless/00460_vertical_and_totals_extremes.sql
View File

@ -8,15 +8,3 @@ SELECT k, count() FROM (SELECT number % 5 AS k FROM system.numbers LIMIT 100) GR
SET output_format_pretty_max_rows = 4;
SELECT k, count() FROM (SELECT number % 5 AS k FROM system.numbers LIMIT 100) GROUP BY k WITH TOTALS ORDER BY k FORMAT Vertical;
SELECT k, count() FROM (SELECT number % 5 AS k FROM system.numbers LIMIT 100) GROUP BY k WITH TOTALS ORDER BY k FORMAT VerticalRaw;
SET extremes = 1;
SELECT k, count() FROM (SELECT number % 5 AS k FROM system.numbers LIMIT 100) GROUP BY k WITH TOTALS ORDER BY k FORMAT VerticalRaw;
SET output_format_pretty_max_rows = 5;
SELECT k, count() FROM (SELECT number % 5 AS k FROM system.numbers LIMIT 100) GROUP BY k WITH TOTALS ORDER BY k FORMAT VerticalRaw;
SET output_format_pretty_max_rows = 4;
SELECT k, count() FROM (SELECT number % 5 AS k FROM system.numbers LIMIT 100) GROUP BY k WITH TOTALS ORDER BY k FORMAT VerticalRaw;

14
dbms/tests/queries/0_stateless/00476_pretty_formats_and_widths.reference
View File

@ -57,12 +57,12 @@
│ 100000000 │ 100000000 │
│ 1000000000 │ 1000000000 │
└────────────┴────────────┘
┏━━━━━━━━━━━━━━━━━━┓
┃ \'\\\\\\\'\\\'\' ┃
┡━━━━━━━━━━━━━━━━━━┩
│ \\\'\' │
└──────────────────┘
┏━━━━━━━━━━┓
┃ '\\\'\'' ┃
┡━━━━━━━━━━┩
│ \'' │
└──────────┘
Row 1:
──────
\'\\\\\\\'\\\'\': \\\'\'
1: 1
'\\\'\'': \''
1: 1

12
dbms/tests/queries/0_stateless/00569_parse_date_time_best_effort.reference
View File

@ -19,7 +19,7 @@ s a b
19700102010203Z 1970-01-02 01:02:03 1970-01-02 01:02:03
1970/01/02 010203Z 1970-01-02 01:02:03 1970-01-02 01:02:03
20 2000-01-20 00:00:00 2000-01-20 00:00:00
201 \N 0000-00-00 00:00:00
201 ᴺᵁᴸᴸ 0000-00-00 00:00:00
20160101 2016-01-01 00:00:00 2016-01-01 00:00:00
2016-01-01 2016-01-01 00:00:00 2016-01-01 00:00:00
201601-01 2016-01-01 01:00:00 2016-01-01 01:00:00
@ -35,7 +35,7 @@ s a b
2017/01/01 2017-01-01 00:00:00 2017-01-01 00:00:00
201701 02 010203 UTC+0300 2017-01-01 22:02:03 2017-01-01 22:02:03
2017-01-02 03:04:05 2017-01-02 03:04:05 2017-01-02 03:04:05
2017-01-0203:04:05 \N 0000-00-00 00:00:00
2017-01-0203:04:05 ᴺᵁᴸᴸ 0000-00-00 00:00:00
2017-01-02 03:04:05+0 2017-01-02 03:04:05 2017-01-02 03:04:05
2017-01-02 03:04:05+00 2017-01-02 03:04:05 2017-01-02 03:04:05
2017-01-02 03:04:05+0000 2017-01-02 03:04:05 2017-01-02 03:04:05
@ -65,7 +65,7 @@ s a b
2017 25 1:2:3 0000-00-00 00:00:00 0000-00-00 00:00:00
2017 25 Apr 1:2:3 2017-04-01 01:02:03 2017-04-01 01:02:03
2017 Apr 01 11:22:33 2017-04-01 11:22:33 2017-04-01 11:22:33
2017 Apr 02 01/02/03 UTC+0300 \N 0000-00-00 00:00:00
2017 Apr 02 01/02/03 UTC+0300 ᴺᵁᴸᴸ 0000-00-00 00:00:00
2017 Apr 02 010203 UTC+0300 2017-04-01 22:02:03 2017-04-01 22:02:03
2017 Apr 02 01:2:3 UTC+0300 2017-04-01 22:02:03 2017-04-01 22:02:03
2017 Apr 02 1:02:3 2017-04-02 01:02:03 2017-04-02 01:02:03
@ -91,14 +91,14 @@ s a b
25 Jan 2017 1:2:3 Z 2017-01-25 01:02:03 2017-01-25 01:02:03
25 Jan 2017 1:2:3 Z +0300 2017-01-24 22:02:03 2017-01-24 22:02:03
25 Jan 2017 1:2:3 Z+03:00 2017-01-24 22:02:03 2017-01-24 22:02:03
25 Jan 2017 1:2:3 Z +0300 OM \N 0000-00-00 00:00:00
25 Jan 2017 1:2:3 Z +0300 OM ᴺᵁᴸᴸ 0000-00-00 00:00:00
25 Jan 2017 1:2:3 Z +03:00 PM 2017-01-25 10:02:03 2017-01-25 10:02:03
25 Jan 2017 1:2:3 Z +0300 PM 2017-01-25 10:02:03 2017-01-25 10:02:03
25 Jan 2017 1:2:3 Z+03:00 PM 2017-01-25 10:02:03 2017-01-25 10:02:03
25 Jan 2017 1:2:3 Z +03:30 PM 2017-01-25 09:32:03 2017-01-25 09:32:03
25 Jan 2017 1:2:3Z Mo \N 0000-00-00 00:00:00
25 Jan 2017 1:2:3Z Mo ᴺᵁᴸᴸ 0000-00-00 00:00:00
25 Jan 2017 1:2:3Z Mon 2017-01-25 01:02:03 2017-01-25 01:02:03
25 Jan 2017 1:2:3Z Moo \N 0000-00-00 00:00:00
25 Jan 2017 1:2:3Z Moo ᴺᵁᴸᴸ 0000-00-00 00:00:00
25 Jan 2017 1:2:3 Z PM 2017-01-25 13:02:03 2017-01-25 13:02:03
25 Jan 2017 1:2:3Z PM 2017-01-25 13:02:03 2017-01-25 13:02:03
25 Jan 2017 1:2:3 Z PM +03:00 2017-01-25 10:02:03 2017-01-25 10:02:03

1
dbms/tests/queries/0_stateless/00637_sessions_in_http_interface_and_settings.reference Normal file
View File

@ -0,0 +1 @@
readonly 1 1

12
dbms/tests/queries/0_stateless/00637_sessions_in_http_interface_and_settings.sh Executable file
View File

@ -0,0 +1,12 @@
#!/usr/bin/env bash
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. $CURDIR/../shell_config.sh
user=readonly
address=${CLICKHOUSE_HOST}
port=${CLICKHOUSE_PORT_HTTP}
url="${CLICKHOUSE_PORT_HTTP_PROTO}://readonly@$address:$port/?session_id=test"
select="SELECT name, value, changed FROM system.settings WHERE name = 'readonly'"
${CLICKHOUSE_CURL} -sS $url --data-binary "$select"

1
dbms/tests/queries/0_stateless/00638_remote_ssrf.reference Normal file
View File

@ -0,0 +1 @@
must not contain ASCII control characters

24
dbms/tests/queries/0_stateless/00638_remote_ssrf.sh Executable file
View File

@ -0,0 +1,24 @@
#!/usr/bin/env bash
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. $CURDIR/../shell_config.sh
user=readonly
address=${CLICKHOUSE_HOST}
port=${CLICKHOUSE_PORT_HTTP}
url="${CLICKHOUSE_PORT_HTTP_PROTO}://$address:$port/"
# Port is arbitary
nc -l -p 61845 -q 0 > /dev/null &
${CLICKHOUSE_CURL} -sS $url --data-binary "SELECT * FROM remote('localhost:61845', system.one, 'user', 'password')" > /dev/null 2>&1
wait
nc -l -p 61846 -q 0 > /dev/null &
${CLICKHOUSE_CURL} -sS $url --data-binary "SELECT * FROM remote('localhost:61846', system.one, 'user', 'passw
ord')" 2>&1 | grep -o 'must not contain ASCII control characters'
wait

16
dbms/tests/queries/0_stateless/00639_startsWith.reference Normal file
View File

@ -0,0 +1,16 @@
0
0
1
1
0
0
1
1
1
1
1
0
1
3
2
0

17
dbms/tests/queries/0_stateless/00639_startsWith.sql Normal file
View File

@ -0,0 +1,17 @@
USE test;
SELECT startsWith(s, 'He') FROM (SELECT arrayJoin(['', 'H', 'He', 'Hellow', '3434', 'fffffffffdHe']) AS s);
SELECT startsWith(s, '') FROM (SELECT arrayJoin(['', 'h', 'hi']) AS s);
SELECT startsWith('123', '123');
SELECT startsWith('123', '12');
SELECT startsWith('123', '1234');
SELECT startsWith('123', '');
DROP TABLE IF EXISTS startsWith_test;
CREATE TABLE startsWith_test(S1 String, S2 String, S3 FixedString(2)) ENGINE=Memory;
INSERT INTO startsWith_test values ('11', '22', '33'), ('a', 'a', 'bb'), ('abc', 'ab', '23');
SELECT COUNT() FROM startsWith_test WHERE startsWith(S1, S1);
SELECT COUNT() FROM startsWith_test WHERE startsWith(S1, S2);
SELECT COUNT() FROM startsWith_test WHERE startsWith(S2, S3);
DROP TABLE startsWith_test;

16
dbms/tests/queries/0_stateless/00640_endsWith.reference Normal file
View File

@ -0,0 +1,16 @@
0
0
1
1
0
0
1
1
1
1
1
0
1
3
2
0

17
dbms/tests/queries/0_stateless/00640_endsWith.sql Normal file
View File

@ -0,0 +1,17 @@
USE test;
SELECT endsWith(s, 'ow') FROM (SELECT arrayJoin(['', 'o', 'ow', 'Hellow', '3434', 'owfffffffdHe']) AS s);
SELECT endsWith(s, '') FROM (SELECT arrayJoin(['', 'h', 'hi']) AS s);
SELECT endsWith('123', '3');
SELECT endsWith('123', '23');
SELECT endsWith('123', '32');
SELECT endsWith('123', '');
DROP TABLE IF EXISTS endsWith_test;
CREATE TABLE endsWith_test(S1 String, S2 String, S3 FixedString(2)) ENGINE=Memory;
INSERT INTO endsWith_test values ('11', '22', '33'), ('a', 'a', 'bb'), ('abc', 'bc', '23');
SELECT COUNT() FROM endsWith_test WHERE endsWith(S1, S1);
SELECT COUNT() FROM endsWith_test WHERE endsWith(S1, S2);
SELECT COUNT() FROM endsWith_test WHERE endsWith(S2, S3);
DROP TABLE endsWith_test;

2
dbms/tests/users.xml
View File

@ -1 +1 @@
../src/Server/users.xml
../programs/server/users.xml

2
debian/control vendored
View File

@ -62,7 +62,7 @@ Description: debugging symbols for clickhouse-common-static
Package: clickhouse-test
Priority: optional
Architecture: all
Depends: ${shlibs:Depends}, ${misc:Depends}, clickhouse-client, bash, expect, python, python-lxml, python-termcolor, python-requests, curl, perl, sudo, openssl
Depends: ${shlibs:Depends}, ${misc:Depends}, clickhouse-client, bash, expect, python, python-lxml, python-termcolor, python-requests, curl, perl, sudo, openssl, netcat-openbsd
Description: Clickhouse tests

5
debian/rules vendored
View File

@ -99,11 +99,6 @@ override_dh_install:
mkdir -p $(DESTDIR)/etc/systemd/system/
cp debian/clickhouse-server.service $(DESTDIR)/etc/systemd/system/
#TODO: use from cmake:
# In case building clickhouse-server, adding to package binary of clang, ld and header files - for dynamic compilation.
#mkdir -p $(DESTDIR)/usr/share/clickhouse/headers
#CLANG=$(DESTDIR)/usr/bin/clickhouse-clang DESTDIR=$(DESTDIR) ./copy_headers.sh . $(DESTDIR)/usr/share/clickhouse/headers
# fake metrika files when private dir is empty
mkdir -p $(DESTDIR)/etc/clickhouse-server/metrika
touch $(DESTDIR)/etc/clickhouse-server/metrika/config.xml

2
docker/client/Dockerfile
View File

@ -9,7 +9,7 @@ RUN apt-get update && \
apt-key adv --keyserver keyserver.ubuntu.com --recv E0C56BD4 && \
echo $repository | tee /etc/apt/sources.list.d/clickhouse.list && \
apt-get update && \
env DEBIAN_FRONTEND=noninteractive apt-get install --allow-unauthenticated -y clickhouse-client=$version locales tzdata && \
env DEBIAN_FRONTEND=noninteractive apt-get install --allow-unauthenticated -y clickhouse-client=$version clickhouse-common-static=$version locales tzdata && \
rm -rf /var/lib/apt/lists/* /var/cache/debconf && \
apt-get clean

2
docker/server/Dockerfile
View File

@ -9,7 +9,7 @@ RUN apt-get update && \
apt-key adv --keyserver keyserver.ubuntu.com --recv E0C56BD4 && \
echo $repository | tee /etc/apt/sources.list.d/clickhouse.list && \
apt-get update && \
env DEBIAN_FRONTEND=noninteractive apt-get install --allow-unauthenticated -y "clickhouse-server=$version" libgcc-7-dev && \
env DEBIAN_FRONTEND=noninteractive apt-get install --allow-unauthenticated -y clickhouse-server=$version clickhouse-common-static=$version libgcc-7-dev && \
rm -rf /var/lib/apt/lists/* /var/cache/debconf && \
apt-get clean

4
libs/libmysqlxx/CMakeLists.txt
View File

@ -33,7 +33,7 @@ target_include_directories (mysqlxx PUBLIC include)
set(PLATFORM_LIBRARIES ${CMAKE_DL_LIBS})
if (USE_MYSQL)
target_include_directories (mysqlxx PRIVATE ${MYSQL_INCLUDE_DIR})
target_include_directories (mysqlxx SYSTEM PRIVATE ${MYSQL_INCLUDE_DIR})
endif ()
if (USE_STATIC_LIBRARIES AND STATIC_MYSQLCLIENT_LIB)
@ -49,7 +49,7 @@ if (USE_STATIC_LIBRARIES AND STATIC_MYSQLCLIENT_LIB)
endif ()
target_link_libraries (mysqlxx common ${MYSQLCLIENT_LIBRARIES} ${OPENSSL_LIBRARIES} ${ZLIB_LIBRARIES} ${PLATFORM_LIBRARIES})
target_include_directories (mysqlxx PRIVATE ${OPENSSL_INCLUDE_DIR})
target_include_directories (mysqlxx SYSTEM PRIVATE ${OPENSSL_INCLUDE_DIR})
if (ENABLE_TESTS)
add_subdirectory (src/tests)

4
libs/libpocoext/CMakeLists.txt
View File

@ -8,14 +8,14 @@ add_library (pocoext ${SPLIT_SHARED}
include/Poco/Ext/SessionPoolHelpers.h)
if (Poco_Data_FOUND)
target_include_directories (pocoext PRIVATE ${Poco_Data_INCLUDE_DIRS})
target_include_directories (pocoext SYSTEM PRIVATE ${Poco_Data_INCLUDE_DIRS})
target_link_libraries(pocoext ${Poco_Data_LIBRARY})
endif()
target_include_directories (pocoext PUBLIC include PRIVATE ${COMMON_INCLUDE_DIR})
if (NOT USE_INTERNAL_POCO_LIBRARY)
target_include_directories (pocoext BEFORE PUBLIC ${Poco_INCLUDE_DIRS})
target_include_directories (pocoext SYSTEM BEFORE PUBLIC ${Poco_INCLUDE_DIRS})
endif ()
target_link_libraries(pocoext ${Poco_Util_LIBRARY} ${Poco_Net_LIBRARY} ${Poco_XML_LIBRARY} ${Poco_Foundation_LIBRARY})

2
release
View File

@ -53,7 +53,7 @@ do
shift
elif [[ $1 == '--fast' ]]; then
# Wrong but fast pbuilder mode: create base package with all depends
EXTRAPACKAGES="$EXTRAPACKAGES debhelper cmake ninja-build gcc-7 g++-7 libc6-dev libmariadbclient-dev libicu-dev libltdl-dev libreadline-dev libssl-dev unixodbc-dev psmisc bash expect python python-lxml python-termcolor python-requests curl perl sudo openssl"
EXTRAPACKAGES="$EXTRAPACKAGES debhelper cmake ninja-build gcc-7 g++-7 libc6-dev libmariadbclient-dev libicu-dev libltdl-dev libreadline-dev libssl-dev unixodbc-dev psmisc bash expect python python-lxml python-termcolor python-requests curl perl sudo openssl netcat-openbsd"
shift
else
echo "Unknown option $1"

3
utils/check_include.sh
View File

@ -3,6 +3,7 @@
# Small .h isolated compile checker
# Finds missing #include <...>
# prints compile time, number of includes, use with sort: ./check_include.sh 2>&1 | sort -rk3
# use with clang: CXX=`which clang++-7 clang++-7.0 clang++-6.0 clang++-5.0 | head -n1` ./check_include.sh
pwd=`pwd`
BUILD_DIR=${BUILD_DIR:=./build}
inc="-I. \
@ -28,6 +29,8 @@ inc="-I. \
-I./contrib/poco/Foundation/include \
-I./contrib/boost/libs/*/include \
-I./contrib/boost \
-I./contrib/llvm/llvm/include \
-I${BUILD_DIR}/contrib/llvm/llvm/include \
-I./contrib/libbtrie/include \
-I./contrib/libpcg-random/include \
-I./libs/libmysqlxx/include \