diff --git a/CHANGELOG.md b/CHANGELOG.md index bfe4ed6e768..c29255e6026 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -293,7 +293,7 @@ ### New features: -* `DEFAULT` expressions are evaluated for missing fields when loading data in semi-structured input formats (`JSONEachRow`, `TSKV`). [#3555](https://github.com/yandex/ClickHouse/pull/3555) +* `DEFAULT` expressions are evaluated for missing fields when loading data in semi-structured input formats (`JSONEachRow`, `TSKV`). The feature is enabled with the `insert_sample_with_metadata` setting. [#3555](https://github.com/yandex/ClickHouse/pull/3555) * The `ALTER TABLE` query now has the `MODIFY ORDER BY` action for changing the sorting key when adding or removing a table column. This is useful for tables in the `MergeTree` family that perform additional tasks when merging based on this sorting key, such as `SummingMergeTree`, `AggregatingMergeTree`, and so on. [#3581](https://github.com/yandex/ClickHouse/pull/3581) [#3755](https://github.com/yandex/ClickHouse/pull/3755) * For tables in the `MergeTree` family, now you can specify a different sorting key (`ORDER BY`) and index (`PRIMARY KEY`). The sorting key can be longer than the index. [#3581](https://github.com/yandex/ClickHouse/pull/3581) * Added the `hdfs` table function and the `HDFS` table engine for importing and exporting data to HDFS. [chenxing-xc](https://github.com/yandex/ClickHouse/pull/3617) diff --git a/CHANGELOG_RU.md b/CHANGELOG_RU.md index d62086e78d7..3c3b425bb7f 100644 --- a/CHANGELOG_RU.md +++ b/CHANGELOG_RU.md @@ -302,7 +302,7 @@ ### Новые возможности: -* Вычисление `DEFAULT` выражений для отсутствующих полей при загрузке данных в полуструктурированных форматах (`JSONEachRow`, `TSKV`). [#3555](https://github.com/yandex/ClickHouse/pull/3555) +* Вычисление `DEFAULT` выражений для отсутствующих полей при загрузке данных в полуструктурированных форматах (`JSONEachRow`, `TSKV`) (требуется включить настройку запроса `insert_sample_with_metadata`). [#3555](https://github.com/yandex/ClickHouse/pull/3555) * Для запроса `ALTER TABLE` добавлено действие `MODIFY ORDER BY` для изменения ключа сортировки при одновременном добавлении или удалении столбца таблицы. Это полезно для таблиц семейства `MergeTree`, выполняющих дополнительную работу при слияниях, согласно этому ключу сортировки, как например, `SummingMergeTree`, `AggregatingMergeTree` и т. п. [#3581](https://github.com/yandex/ClickHouse/pull/3581) [#3755](https://github.com/yandex/ClickHouse/pull/3755) * Для таблиц семейства `MergeTree` появилась возможность указать различный ключ сортировки (`ORDER BY`) и индекс (`PRIMARY KEY`). Ключ сортировки может быть длиннее, чем индекс. [#3581](https://github.com/yandex/ClickHouse/pull/3581) * Добавлена табличная функция `hdfs` и движок таблиц `HDFS` для импорта и экспорта данных в HDFS. [chenxing-xc](https://github.com/yandex/ClickHouse/pull/3617) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8bbc7ca40ea..8c0627ce569 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -50,6 +50,7 @@ string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC) message (STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}") set (CMAKE_CONFIGURATION_TYPES "RelWithDebInfo;Debug;Release;MinSizeRel" CACHE STRING "" FORCE) +set (CMAKE_DEBUG_POSTFIX "d" CACHE STRING "Generate debug library name with a postfix.") # To be consistent with CMakeLists from contrib libs. option (USE_STATIC_LIBRARIES "Set to FALSE to use shared libraries" ON) option (MAKE_STATIC_LIBRARIES "Set to FALSE to make shared libraries" ${USE_STATIC_LIBRARIES}) @@ -98,10 +99,6 @@ if (CMAKE_SYSTEM_PROCESSOR MATCHES "amd64|x86_64") if (OS_LINUX AND NOT UNBUNDLED AND MAKE_STATIC_LIBRARIES AND CMAKE_VERSION VERSION_GREATER "3.9.0") option (GLIBC_COMPATIBILITY "Set to TRUE to enable compatibility with older glibc libraries. Only for x86_64, Linux. Implies USE_INTERNAL_MEMCPY." ON) - if (GLIBC_COMPATIBILITY) - message (STATUS "Some symbols from glibc will be replaced for compatibility") - link_libraries(glibc-compatibility) - endif () endif () endif () @@ -177,6 +174,60 @@ set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -g3 -ggdb3 include (cmake/use_libcxx.cmake) + +# Set standard, system and compiler libraries explicitly. +# This is intended for more control of what we are linking. + +set (DEFAULT_LIBS "") +if (OS_LINUX AND NOT UNBUNDLED) + # Note: this probably has no effict, but I'm not an expert in CMake. + set (CMAKE_C_IMPLICIT_LINK_LIBRARIES "") + set (CMAKE_CXX_IMPLICIT_LINK_LIBRARIES "") + + # Disable default linked libraries. + set (DEFAULT_LIBS "-nodefaultlibs") + + # Add C++ libraries. + # + # This consist of: + # - C++ standard library (like implementation of std::string); + # - C++ ABI implementation (functions for exceptions like __cxa_throw, RTTI, etc); + # - functions for internal implementation of exception handling (stack unwinding based on DWARF info; TODO replace with bundled libunwind); + # - compiler builtins (example: functions for implementation of __int128 operations); + # + # There are two variants of C++ library: libc++ (from LLVM compiler infrastructure) and libstdc++ (from GCC). + if (USE_LIBCXX) + set (BUILTINS_LIB_PATH "") + if (COMPILER_CLANG) + execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-${CMAKE_SYSTEM_PROCESSOR}.a OUTPUT_VARIABLE BUILTINS_LIB_PATH OUTPUT_STRIP_TRAILING_WHITESPACE) + endif () + + set (DEFAULT_LIBS "${DEFAULT_LIBS} -Wl,-Bstatic -lc++ -lc++abi -lgcc_eh ${BUILTINS_LIB_PATH} -Wl,-Bdynamic") + else () + set (DEFAULT_LIBS "${DEFAULT_LIBS} -Wl,-Bstatic -lstdc++ -lgcc_eh -lgcc -Wl,-Bdynamic") + endif () + + # Linking with GLIBC prevents portability of binaries to older systems. + # We overcome this behaviour by statically linking with our own implementation of all new symbols (that don't exist in older Libc or have infamous "symbol versioning"). + # The order of linking is important: 'glibc-compatibility' must be before libc but after all other libraries. + if (GLIBC_COMPATIBILITY) + message (STATUS "Some symbols from glibc will be replaced for compatibility") + + string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC) + set (CMAKE_POSTFIX_VARIABLE "CMAKE_${CMAKE_BUILD_TYPE_UC}_POSTFIX") + + set (DEFAULT_LIBS "${DEFAULT_LIBS} libs/libglibc-compatibility/libglibc-compatibility${${CMAKE_POSTFIX_VARIABLE}}.a") + endif () + + # Add Libc. GLIBC is actually a collection of interdependent libraries. + set (DEFAULT_LIBS "${DEFAULT_LIBS} -lrt -ldl -lpthread -lm -lc") + + # Note: we'd rather use Musl libc library, but it's little bit more difficult to use. + + message(STATUS "Default libraries: ${DEFAULT_LIBS}") +endif () + + if (NOT MAKE_STATIC_LIBRARIES) set(CMAKE_POSITION_INDEPENDENT_CODE ON) endif () @@ -284,3 +335,36 @@ add_subdirectory (utils) add_subdirectory (dbms) include (cmake/print_include_directories.cmake) + + +if (DEFAULT_LIBS) + # Add default libs to all targets as the last dependency. + # I have found no better way to specify default libs in CMake that will appear single time in specific order at the end of linker arguments. + + function(add_default_libs target_name) + if (TARGET ${target_name}) + # message(STATUS "Has target ${target_name}") + set_property(TARGET ${target_name} APPEND PROPERTY LINK_LIBRARIES "${DEFAULT_LIBS}") + set_property(TARGET ${target_name} APPEND PROPERTY INTERFACE_LINK_LIBRARIES "${DEFAULT_LIBS}") + if (GLIBC_COMPATIBILITY) + add_dependencies(${target_name} glibc-compatibility) + endif () + endif () + endfunction () + + add_default_libs(ltdl) + add_default_libs(zlibstatic) + add_default_libs(jemalloc) + add_default_libs(unwind) + add_default_libs(memcpy) + add_default_libs(Foundation) + add_default_libs(common) + add_default_libs(gtest) + add_default_libs(lz4) + add_default_libs(zstd) + add_default_libs(snappy) + add_default_libs(arrow) + add_default_libs(protoc) + add_default_libs(thrift_static) + add_default_libs(boost_regex_internal) +endif () diff --git a/README.md b/README.md index 61392a4136b..3e840d2cf10 100644 --- a/README.md +++ b/README.md @@ -13,5 +13,4 @@ ClickHouse is an open-source column-oriented database management system that all ## Upcoming Events -* [ClickHouse Community Meetup](https://www.eventbrite.com/e/meetup-clickhouse-in-the-wild-deployment-success-stories-registration-55305051899) in San Francisco on February 19. * [ClickHouse Community Meetup](https://www.eventbrite.com/e/clickhouse-meetup-in-madrid-registration-55376746339) in Madrid on April 2. diff --git a/cmake/find_brotli.cmake b/cmake/find_brotli.cmake index 6e93e88df9c..55350e21eed 100644 --- a/cmake/find_brotli.cmake +++ b/cmake/find_brotli.cmake @@ -1,3 +1,7 @@ +option (ENABLE_BROTLI "Enable brotli" ON) + +if (ENABLE_BROTLI) + option (USE_INTERNAL_BROTLI_LIBRARY "Set to FALSE to use system libbrotli library instead of bundled" ${NOT_UNBUNDLED}) if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/brotli/c/include/brotli/decode.h") @@ -27,4 +31,6 @@ elseif (NOT MISSING_INTERNAL_BROTLI_LIBRARY) set (USE_BROTLI 1) endif () +endif() + message (STATUS "Using brotli=${USE_BROTLI}: ${BROTLI_INCLUDE_DIR} : ${BROTLI_LIBRARY}") diff --git a/cmake/find_gtest.cmake b/cmake/find_gtest.cmake index c918f81f1cd..82ae0f8e229 100644 --- a/cmake/find_gtest.cmake +++ b/cmake/find_gtest.cmake @@ -20,11 +20,12 @@ if (NOT GTEST_SRC_DIR AND NOT GTEST_INCLUDE_DIRS AND NOT MISSING_INTERNAL_GTEST_ set (USE_INTERNAL_GTEST_LIBRARY 1) set (GTEST_MAIN_LIBRARIES gtest_main) set (GTEST_LIBRARIES gtest) + set (GTEST_BOTH_LIBRARIES ${GTEST_MAIN_LIBRARIES} ${GTEST_LIBRARIES}) set (GTEST_INCLUDE_DIRS ${ClickHouse_SOURCE_DIR}/contrib/googletest/googletest) endif () -if((GTEST_INCLUDE_DIRS AND GTEST_MAIN_LIBRARIES) OR GTEST_SRC_DIR) +if((GTEST_INCLUDE_DIRS AND GTEST_BOTH_LIBRARIES) OR GTEST_SRC_DIR) set(USE_GTEST 1) endif() -message (STATUS "Using gtest=${USE_GTEST}: ${GTEST_INCLUDE_DIRS} : ${GTEST_LIBRARIES}, ${GTEST_MAIN_LIBRARIES} : ${GTEST_SRC_DIR}") +message (STATUS "Using gtest=${USE_GTEST}: ${GTEST_INCLUDE_DIRS} : ${GTEST_BOTH_LIBRARIES} : ${GTEST_SRC_DIR}") diff --git a/cmake/find_parquet.cmake b/cmake/find_parquet.cmake index bfaf51abb46..39af93955f4 100644 --- a/cmake/find_parquet.cmake +++ b/cmake/find_parquet.cmake @@ -1,3 +1,7 @@ +option (ENABLE_PARQUET "Enable parquet" ON) + +if (ENABLE_PARQUET) + if (NOT OS_FREEBSD) # Freebsd: ../contrib/arrow/cpp/src/arrow/util/bit-util.h:27:10: fatal error: endian.h: No such file or directory option(USE_INTERNAL_PARQUET_LIBRARY "Set to FALSE to use system parquet library instead of bundled" ${NOT_UNBUNDLED}) endif() @@ -61,6 +65,8 @@ elseif(NOT MISSING_INTERNAL_PARQUET_LIBRARY AND NOT OS_FREEBSD) endif() endif() +endif() + if(USE_PARQUET) message(STATUS "Using Parquet: ${ARROW_LIBRARY}:${ARROW_INCLUDE_DIR} ; ${PARQUET_LIBRARY}:${PARQUET_INCLUDE_DIR} ; ${THRIFT_LIBRARY}") else() diff --git a/cmake/find_protobuf.cmake b/cmake/find_protobuf.cmake index e2fe9ca2fcd..57d546392c1 100644 --- a/cmake/find_protobuf.cmake +++ b/cmake/find_protobuf.cmake @@ -1,10 +1,8 @@ -option(USE_INTERNAL_PROTOBUF_LIBRARY "Set to FALSE to use system protobuf instead of bundled" ${NOT_UNBUNDLED}) +option (ENABLE_PROTOBUF "Enable protobuf" ON) -if(OS_FREEBSD AND SANITIZE STREQUAL "address") - # ../contrib/protobuf/src/google/protobuf/arena_impl.h:45:10: fatal error: 'sanitizer/asan_interface.h' file not found - set(MISSING_INTERNAL_PROTOBUF_LIBRARY 1) - set(USE_INTERNAL_PROTOBUF_LIBRARY 0) -endif() +if (ENABLE_PROTOBUF) + +option(USE_INTERNAL_PROTOBUF_LIBRARY "Set to FALSE to use system protobuf instead of bundled" ${NOT_UNBUNDLED}) if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/protobuf/cmake/CMakeLists.txt") if(USE_INTERNAL_PROTOBUF_LIBRARY) @@ -94,4 +92,16 @@ elseif(NOT MISSING_INTERNAL_PROTOBUF_LIBRARY) endfunction() endif() +if(OS_FREEBSD AND SANITIZE STREQUAL "address") + # ../contrib/protobuf/src/google/protobuf/arena_impl.h:45:10: fatal error: 'sanitizer/asan_interface.h' file not found + # #include + if(LLVM_INCLUDE_DIRS) + set(Protobuf_INCLUDE_DIR ${Protobuf_INCLUDE_DIR} ${LLVM_INCLUDE_DIRS}) + else() + set(USE_PROTOBUF 0) + endif() +endif() + +endif() + message(STATUS "Using protobuf=${USE_PROTOBUF}: ${Protobuf_INCLUDE_DIR} : ${Protobuf_LIBRARY}") diff --git a/cmake/use_libcxx.cmake b/cmake/use_libcxx.cmake index 618e71b11ce..29ac9406fe0 100644 --- a/cmake/use_libcxx.cmake +++ b/cmake/use_libcxx.cmake @@ -11,38 +11,13 @@ if (OS_LINUX AND COMPILER_CLANG) set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}") option (USE_LIBCXX "Use libc++ and libc++abi instead of libstdc++ (only make sense on Linux with Clang)" ${HAVE_LIBCXX}) - set (LIBCXX_PATH "" CACHE STRING "Use custom path for libc++. It should be used for MSan.") if (USE_LIBCXX) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") # Ok for clang6, for older can cause 'not used option' warning set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_LIBCPP_DEBUG=0") # More checks in debug build. - if (MAKE_STATIC_LIBRARIES) - execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-${CMAKE_SYSTEM_PROCESSOR}.a OUTPUT_VARIABLE BUILTINS_LIB_PATH OUTPUT_STRIP_TRAILING_WHITESPACE) - link_libraries (-nodefaultlibs -Wl,-Bstatic -stdlib=libc++ c++ c++abi gcc_eh ${BUILTINS_LIB_PATH} rt -Wl,-Bdynamic dl pthread m c) - else () - link_libraries (-stdlib=libc++ c++ c++abi) - endif () - - if (LIBCXX_PATH) - include_directories (SYSTEM BEFORE "${LIBCXX_PATH}/include" "${LIBCXX_PATH}/include/c++/v1") - link_directories ("${LIBCXX_PATH}/lib") - endif () endif () endif () -if (USE_LIBCXX) - set (STATIC_STDLIB_FLAGS "") -else () - set (STATIC_STDLIB_FLAGS "-static-libgcc -static-libstdc++") -endif () - -if (MAKE_STATIC_LIBRARIES AND NOT APPLE AND NOT (COMPILER_CLANG AND OS_FREEBSD)) - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${STATIC_STDLIB_FLAGS}") - - # Along with executables, we also build example of shared library for "library dictionary source"; and it also should be self-contained. - set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${STATIC_STDLIB_FLAGS}") -endif () - if (USE_STATIC_LIBRARIES AND HAVE_NO_PIE) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FLAG_NO_PIE}") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FLAG_NO_PIE}") diff --git a/contrib/cppkafka b/contrib/cppkafka index 860c90e92ee..9b184d881c1 160000 --- a/contrib/cppkafka +++ b/contrib/cppkafka @@ -1 +1 @@ -Subproject commit 860c90e92eee6690aa74a2ca7b7c5c6930dffecd +Subproject commit 9b184d881c15cc50784b28688c7c99d3d764db24 diff --git a/contrib/libhdfs3 b/contrib/libhdfs3 index bd6505cbb0c..e2131aa752d 160000 --- a/contrib/libhdfs3 +++ b/contrib/libhdfs3 @@ -1 +1 @@ -Subproject commit bd6505cbb0c130b0db695305b9a38546fa880e5a +Subproject commit e2131aa752d7e95441e08f9a18304c1445f2576a diff --git a/contrib/librdkafka b/contrib/librdkafka index 363dcad5a23..73295a702cd 160000 --- a/contrib/librdkafka +++ b/contrib/librdkafka @@ -1 +1 @@ -Subproject commit 363dcad5a23dc29381cc626620e68ae418b3af19 +Subproject commit 73295a702cd1c85c11749ade500d713db7099cca diff --git a/contrib/librdkafka-cmake/config.h b/contrib/librdkafka-cmake/config.h index 266baae8dae..ae4e370a628 100644 --- a/contrib/librdkafka-cmake/config.h +++ b/contrib/librdkafka-cmake/config.h @@ -2,6 +2,8 @@ #ifndef _CONFIG_H_ #define _CONFIG_H_ #define ARCH "x86_64" +#define BUILT_WITH "GCC GXX PKGCONFIG OSXLD LIBDL PLUGINS ZLIB SSL SASL_CYRUS ZSTD HDRHISTOGRAM LZ4_EXT SNAPPY SOCKEM SASL_SCRAM CRC32C_HW" + #define CPU "generic" #define WITHOUT_OPTIMIZATION 0 #define ENABLE_DEVEL 0 diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index 900b1e0a650..a07d658c4e6 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -184,7 +184,9 @@ target_link_libraries (clickhouse_common_io string_utils widechar_width ${LINK_LIBRARIES_ONLY_ON_X86_64} + PUBLIC ${DOUBLE_CONVERSION_LIBRARIES} + PRIVATE pocoext PUBLIC ${Poco_Net_LIBRARY} @@ -351,6 +353,6 @@ if (ENABLE_TESTS AND USE_GTEST) # attach all dbms gtest sources grep_gtest_sources(${ClickHouse_SOURCE_DIR}/dbms dbms_gtest_sources) add_executable(unit_tests_dbms ${dbms_gtest_sources}) - target_link_libraries(unit_tests_dbms PRIVATE gtest_main dbms clickhouse_common_zookeeper) + target_link_libraries(unit_tests_dbms PRIVATE ${GTEST_BOTH_LIBRARIES} dbms clickhouse_common_zookeeper) add_check(unit_tests_dbms) endif () diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake index 8cdbdc05344..7df40c7c0d4 100644 --- a/dbms/cmake/version.cmake +++ b/dbms/cmake/version.cmake @@ -1,11 +1,11 @@ # This strings autochanged from release_lib.sh: -set(VERSION_REVISION 54415) +set(VERSION_REVISION 54417) set(VERSION_MAJOR 19) -set(VERSION_MINOR 3) -set(VERSION_PATCH 4) -set(VERSION_GITHASH 263e69e861b769eae7e2bcc79d87673e3a08d376) -set(VERSION_DESCRIBE v19.3.4-testing) -set(VERSION_STRING 19.3.4) +set(VERSION_MINOR 5) +set(VERSION_PATCH 1) +set(VERSION_GITHASH 628ed349c335b79a441a1bd6e4bc791d61dfe62c) +set(VERSION_DESCRIBE v19.5.1.1-testing) +set(VERSION_STRING 19.5.1.1) # end of autochange set(VERSION_EXTRA "" CACHE STRING "") diff --git a/dbms/programs/CMakeLists.txt b/dbms/programs/CMakeLists.txt index 57067074527..be878cce1ab 100644 --- a/dbms/programs/CMakeLists.txt +++ b/dbms/programs/CMakeLists.txt @@ -22,7 +22,6 @@ endif() configure_file (config_tools.h.in ${CMAKE_CURRENT_BINARY_DIR}/config_tools.h) - macro(clickhouse_target_link_split_lib target name) if(NOT CLICKHOUSE_ONE_SHARED) target_link_libraries(${target} PRIVATE clickhouse-${name}-lib) @@ -91,9 +90,9 @@ endif () if (CLICKHOUSE_ONE_SHARED) add_library(clickhouse-lib SHARED ${CLICKHOUSE_SERVER_SOURCES} ${CLICKHOUSE_CLIENT_SOURCES} ${CLICKHOUSE_LOCAL_SOURCES} ${CLICKHOUSE_BENCHMARK_SOURCES} ${CLICKHOUSE_PERFORMANCE_TEST_SOURCES} ${CLICKHOUSE_COPIER_SOURCES} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES} ${CLICKHOUSE_COMPRESSOR_SOURCES} ${CLICKHOUSE_FORMAT_SOURCES} ${CLICKHOUSE_OBFUSCATOR_SOURCES} ${CLICKHOUSE_COMPILER_SOURCES} ${CLICKHOUSE_ODBC_BRIDGE_SOURCES}) - target_link_libraries(clickhouse-lib PUBLIC ${CLICKHOUSE_SERVER_LINK} ${CLICKHOUSE_CLIENT_LINK} ${CLICKHOUSE_LOCAL_LINK} ${CLICKHOUSE_BENCHMARK_LINK} ${CLICKHOUSE_PERFORMANCE_TEST_LINK} ${CLICKHOUSE_COPIER_LINK} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK} ${CLICKHOUSE_COMPRESSOR_LINK} ${CLICKHOUSE_FORMAT_LINK} ${CLICKHOUSE_OBFUSCATOR_LINK} ${CLICKHOUSE_COMPILER_LINK} ${CLICKHOUSE_ODBC_BRIDGE_LINK}) - set_target_properties(clickhouse-lib PROPERTIES SOVERSION ${VERSION_MAJOR}.${VERSION_MINOR} VERSION ${VERSION_SO} OUTPUT_NAME clickhouse) + target_link_libraries(clickhouse-lib ${CLICKHOUSE_SERVER_LINK} ${CLICKHOUSE_CLIENT_LINK} ${CLICKHOUSE_LOCAL_LINK} ${CLICKHOUSE_BENCHMARK_LINK} ${CLICKHOUSE_PERFORMANCE_TEST_LINK} ${CLICKHOUSE_COPIER_LINK} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK} ${CLICKHOUSE_COMPRESSOR_LINK} ${CLICKHOUSE_FORMAT_LINK} ${CLICKHOUSE_OBFUSCATOR_LINK} ${CLICKHOUSE_COMPILER_LINK} ${CLICKHOUSE_ODBC_BRIDGE_LINK}) target_include_directories(clickhouse-lib ${CLICKHOUSE_SERVER_INCLUDE} ${CLICKHOUSE_CLIENT_INCLUDE} ${CLICKHOUSE_LOCAL_INCLUDE} ${CLICKHOUSE_BENCHMARK_INCLUDE} ${CLICKHOUSE_PERFORMANCE_TEST_INCLUDE} ${CLICKHOUSE_COPIER_INCLUDE} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_INCLUDE} ${CLICKHOUSE_COMPRESSOR_INCLUDE} ${CLICKHOUSE_FORMAT_INCLUDE} ${CLICKHOUSE_OBFUSCATOR_INCLUDE} ${CLICKHOUSE_COMPILER_INCLUDE} ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE}) + set_target_properties(clickhouse-lib PROPERTIES SOVERSION ${VERSION_MAJOR}.${VERSION_MINOR} VERSION ${VERSION_SO} OUTPUT_NAME clickhouse DEBUG_POSTFIX "") endif() if (CLICKHOUSE_SPLIT_BINARY) @@ -112,6 +111,8 @@ if (CLICKHOUSE_SPLIT_BINARY) add_custom_target (clickhouse-bundle ALL DEPENDS ${CLICKHOUSE_ALL_TARGETS}) add_custom_target (clickhouse ALL DEPENDS clickhouse-bundle) + + install(PROGRAMS clickhouse-split-helper DESTINATION ${CMAKE_INSTALL_BINDIR} RENAME clickhouse COMPONENT clickhouse) else () if (USE_EMBEDDED_COMPILER) # before add_executable ! @@ -123,37 +124,37 @@ else () target_include_directories (clickhouse PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) if (ENABLE_CLICKHOUSE_SERVER) - target_link_libraries (clickhouse PRIVATE clickhouse-server-lib) + clickhouse_target_link_split_lib(clickhouse server) endif () if (ENABLE_CLICKHOUSE_CLIENT) - target_link_libraries (clickhouse PRIVATE clickhouse-client-lib) + clickhouse_target_link_split_lib(clickhouse client) endif () if (ENABLE_CLICKHOUSE_LOCAL) - target_link_libraries (clickhouse PRIVATE clickhouse-local-lib) + clickhouse_target_link_split_lib(clickhouse local) endif () if (ENABLE_CLICKHOUSE_BENCHMARK) - target_link_libraries (clickhouse PRIVATE clickhouse-benchmark-lib) + clickhouse_target_link_split_lib(clickhouse benchmark) endif () if (ENABLE_CLICKHOUSE_PERFORMANCE_TEST) - target_link_libraries (clickhouse PRIVATE clickhouse-performance-test-lib) + clickhouse_target_link_split_lib(clickhouse performance-test) endif () if (ENABLE_CLICKHOUSE_COPIER) - target_link_libraries (clickhouse PRIVATE clickhouse-copier-lib) + clickhouse_target_link_split_lib(clickhouse copier) endif () if (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG) - target_link_libraries (clickhouse PRIVATE clickhouse-extract-from-config-lib) + clickhouse_target_link_split_lib(clickhouse extract-from-config) endif () if (ENABLE_CLICKHOUSE_COMPRESSOR) - target_link_libraries (clickhouse PRIVATE clickhouse-compressor-lib) + clickhouse_target_link_split_lib(clickhouse compressor) endif () if (ENABLE_CLICKHOUSE_FORMAT) - target_link_libraries (clickhouse PRIVATE clickhouse-format-lib) + clickhouse_target_link_split_lib(clickhouse format) endif () if (ENABLE_CLICKHOUSE_OBFUSCATOR) - target_link_libraries (clickhouse PRIVATE clickhouse-obfuscator-lib) + clickhouse_target_link_split_lib(clickhouse obfuscator) endif () if (USE_EMBEDDED_COMPILER) - target_link_libraries (clickhouse PRIVATE clickhouse-compiler-lib) + clickhouse_target_link_split_lib(clickhouse compiler) endif () set (CLICKHOUSE_BUNDLE) diff --git a/dbms/programs/clickhouse-split-helper b/dbms/programs/clickhouse-split-helper new file mode 100755 index 00000000000..2495160e02a --- /dev/null +++ b/dbms/programs/clickhouse-split-helper @@ -0,0 +1,6 @@ +#!/bin/sh + +set -e +CMD=$1 +shift +clickhouse-$CMD $* diff --git a/dbms/programs/copier/CMakeLists.txt b/dbms/programs/copier/CMakeLists.txt index 1be75dff22d..55b2fc7e1cb 100644 --- a/dbms/programs/copier/CMakeLists.txt +++ b/dbms/programs/copier/CMakeLists.txt @@ -1,5 +1,5 @@ set(CLICKHOUSE_COPIER_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/ClusterCopier.cpp) -set(CLICKHOUSE_COPIER_LINK PRIVATE clickhouse_functions clickhouse_table_functions clickhouse_aggregate_functions daemon) -#set(CLICKHOUSE_COPIER_INCLUDE SYSTEM PRIVATE ...) +set(CLICKHOUSE_COPIER_LINK PRIVATE clickhouse_functions clickhouse_table_functions clickhouse_aggregate_functions PUBLIC daemon) +set(CLICKHOUSE_COPIER_INCLUDE SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR}) clickhouse_program_add(copier) diff --git a/dbms/programs/main.cpp b/dbms/programs/main.cpp index 9ee2df0fab6..15f1673985b 100644 --- a/dbms/programs/main.cpp +++ b/dbms/programs/main.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -17,12 +18,6 @@ #include // Y_IGNORE #endif -#if ENABLE_CLICKHOUSE_SERVER -#include "server/Server.h" -#endif -#if ENABLE_CLICKHOUSE_LOCAL -#include "local/LocalServer.h" -#endif #include /// Universal executable for various clickhouse applications @@ -145,6 +140,10 @@ bool isClickhouseApp(const std::string & app_suffix, std::vector & argv) int main(int argc_, char ** argv_) { + /// Reset new handler to default (that throws std::bad_alloc) + /// It is needed because LLVM library clobbers it. + std::set_new_handler(nullptr); + #if USE_EMBEDDED_COMPILER if (argc_ >= 2 && 0 == strcmp(argv_[1], "-cc1")) return mainEntryClickHouseClang(argc_, argv_); diff --git a/dbms/programs/odbc-bridge/CMakeLists.txt b/dbms/programs/odbc-bridge/CMakeLists.txt index b32fe363b73..143d7edf738 100644 --- a/dbms/programs/odbc-bridge/CMakeLists.txt +++ b/dbms/programs/odbc-bridge/CMakeLists.txt @@ -11,7 +11,7 @@ set(CLICKHOUSE_ODBC_BRIDGE_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/validateODBCConnectionString.cpp ) -set(CLICKHOUSE_ODBC_BRIDGE_LINK PRIVATE daemon dbms clickhouse_common_io) +set(CLICKHOUSE_ODBC_BRIDGE_LINK PRIVATE dbms clickhouse_common_io PUBLIC daemon) set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include) if (USE_POCO_SQLODBC) diff --git a/dbms/programs/server/CMakeLists.txt b/dbms/programs/server/CMakeLists.txt index 217447413d5..5cb08018065 100644 --- a/dbms/programs/server/CMakeLists.txt +++ b/dbms/programs/server/CMakeLists.txt @@ -10,7 +10,7 @@ set(CLICKHOUSE_SERVER_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/TCPHandler.cpp ) -set(CLICKHOUSE_SERVER_LINK PRIVATE clickhouse_dictionaries clickhouse_common_io daemon clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Poco_Net_LIBRARY}) +set(CLICKHOUSE_SERVER_LINK PRIVATE clickhouse_dictionaries clickhouse_common_io PUBLIC daemon PRIVATE clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Poco_Net_LIBRARY}) if (USE_POCO_NETSSL) set(CLICKHOUSE_SERVER_LINK ${CLICKHOUSE_SERVER_LINK} PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY}) endif () diff --git a/dbms/programs/server/Server.cpp b/dbms/programs/server/Server.cpp index 213a6e126df..2b10c9e3c98 100644 --- a/dbms/programs/server/Server.cpp +++ b/dbms/programs/server/Server.cpp @@ -260,6 +260,15 @@ int Server::main(const std::vector & /*args*/) StatusFile status{path + "status"}; SCOPE_EXIT({ + /** Ask to cancel background jobs all table engines, + * and also query_log. + * It is important to do early, not in destructor of Context, because + * table engines could use Context on destroy. + */ + LOG_INFO(log, "Shutting down storages."); + global_context->shutdown(); + LOG_DEBUG(log, "Shutted down storages."); + /** Explicitly destroy Context. It is more convenient than in destructor of Server, because logger is still available. * At this moment, no one could own shared part of Context. */ @@ -498,17 +507,6 @@ int Server::main(const std::vector & /*args*/) global_context->setCurrentDatabase(default_database); - SCOPE_EXIT({ - /** Ask to cancel background jobs all table engines, - * and also query_log. - * It is important to do early, not in destructor of Context, because - * table engines could use Context on destroy. - */ - LOG_INFO(log, "Shutting down storages."); - global_context->shutdown(); - LOG_DEBUG(log, "Shutted down storages."); - }); - if (has_zookeeper && config().has("distributed_ddl")) { /// DDL worker should be started after all tables were loaded diff --git a/dbms/src/AggregateFunctions/AggregateFunctionGroupArray.cpp b/dbms/src/AggregateFunctions/AggregateFunctionGroupArray.cpp index 51bc04ea39c..1efb1a82475 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionGroupArray.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionGroupArray.cpp @@ -12,6 +12,7 @@ namespace DB namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int BAD_ARGUMENTS; } namespace diff --git a/dbms/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h b/dbms/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h index ea7907b4d60..7f41e3f4fcf 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB { diff --git a/dbms/src/AggregateFunctions/AggregateFunctionQuantile.h b/dbms/src/AggregateFunctions/AggregateFunctionQuantile.h index ecfc0ba4df7..8968d990af5 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionQuantile.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionQuantile.h @@ -163,7 +163,7 @@ public: size_t old_size = data_to.size(); data_to.resize(data_to.size() + size); - data.getManyFloat(levels.levels.data(), levels.permutation.data(), size, &data_to[old_size]); + data.getManyFloat(levels.levels.data(), levels.permutation.data(), size, data_to.data() + old_size); } else { @@ -171,7 +171,7 @@ public: size_t old_size = data_to.size(); data_to.resize(data_to.size() + size); - data.getMany(levels.levels.data(), levels.permutation.data(), size, &data_to[old_size]); + data.getMany(levels.levels.data(), levels.permutation.data(), size, data_to.data() + old_size); } } else diff --git a/dbms/src/AggregateFunctions/AggregateFunctionTopK.cpp b/dbms/src/AggregateFunctions/AggregateFunctionTopK.cpp index 04e74c17434..242c1e4e4c0 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionTopK.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionTopK.cpp @@ -39,19 +39,19 @@ class AggregateFunctionTopKDateTime : public AggregateFunctionTopK -static IAggregateFunction * createWithExtraTypes(const DataTypePtr & argument_type, UInt64 threshold, const Array & params) +static IAggregateFunction * createWithExtraTypes(const DataTypePtr & argument_type, UInt64 threshold, UInt64 load_factor, const Array & params) { WhichDataType which(argument_type); if (which.idx == TypeIndex::Date) - return new AggregateFunctionTopKDate(threshold, {argument_type}, params); + return new AggregateFunctionTopKDate(threshold, load_factor, {argument_type}, params); if (which.idx == TypeIndex::DateTime) - return new AggregateFunctionTopKDateTime(threshold, {argument_type}, params); + return new AggregateFunctionTopKDateTime(threshold, load_factor, {argument_type}, params); /// Check that we can use plain version of AggregateFunctionTopKGeneric if (argument_type->isValueUnambiguouslyRepresentedInContiguousMemoryRegion()) - return new AggregateFunctionTopKGeneric(threshold, argument_type, params); + return new AggregateFunctionTopKGeneric(threshold, load_factor, argument_type, params); else - return new AggregateFunctionTopKGeneric(threshold, argument_type, params); + return new AggregateFunctionTopKGeneric(threshold, load_factor, argument_type, params); } @@ -65,19 +65,28 @@ AggregateFunctionPtr createAggregateFunctionTopK(const std::string & name, const else { assertBinary(name, argument_types); - if (!isNumber(argument_types[1])) - throw Exception("The second argument for aggregate function 'topKWeighted' must have numeric type", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (!isInteger(argument_types[1])) + throw Exception("The second argument for aggregate function 'topKWeighted' must have integer type", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } - UInt64 threshold = 10; /// default value + UInt64 threshold = 10; /// default values + UInt64 load_factor = 3; if (!params.empty()) { - if (params.size() != 1) - throw Exception("Aggregate function " + name + " requires one parameter or less.", + if (params.size() > 2) + throw Exception("Aggregate function " + name + " requires two parameters or less.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); UInt64 k = applyVisitor(FieldVisitorConvertToNumber(), params[0]); + if (params.size() == 2) + { + load_factor = applyVisitor(FieldVisitorConvertToNumber(), params[1]); + + if (load_factor < 1) + throw Exception("Too small parameter for aggregate function " + name + ". Minimum: 1", + ErrorCodes::ARGUMENT_OUT_OF_BOUND); + } if (k > TOP_K_MAX_SIZE) throw Exception("Too large parameter for aggregate function " + name + ". Maximum: " + toString(TOP_K_MAX_SIZE), @@ -90,10 +99,10 @@ AggregateFunctionPtr createAggregateFunctionTopK(const std::string & name, const threshold = k; } - AggregateFunctionPtr res(createWithNumericType(*argument_types[0], threshold, argument_types, params)); + AggregateFunctionPtr res(createWithNumericType(*argument_types[0], threshold, load_factor, argument_types, params)); if (!res) - res = AggregateFunctionPtr(createWithExtraTypes(argument_types[0], threshold, params)); + res = AggregateFunctionPtr(createWithExtraTypes(argument_types[0], threshold, load_factor, params)); if (!res) throw Exception("Illegal type " + argument_types[0]->getName() + diff --git a/dbms/src/AggregateFunctions/AggregateFunctionTopK.h b/dbms/src/AggregateFunctions/AggregateFunctionTopK.h index 76bca7dae86..72b724843a1 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionTopK.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionTopK.h @@ -20,10 +20,6 @@ namespace DB { -// Allow NxK more space before calculating top K to increase accuracy -#define TOP_K_LOAD_FACTOR 3 - - template struct AggregateFunctionTopKData { @@ -48,9 +44,9 @@ protected: UInt64 reserved; public: - AggregateFunctionTopK(UInt64 threshold, const DataTypes & argument_types_, const Array & params) + AggregateFunctionTopK(UInt64 threshold, UInt64 load_factor, const DataTypes & argument_types_, const Array & params) : IAggregateFunctionDataHelper, AggregateFunctionTopK>(argument_types_, params) - , threshold(threshold), reserved(TOP_K_LOAD_FACTOR * threshold) {} + , threshold(threshold), reserved(load_factor * threshold) {} String getName() const override { return is_weighted ? "topKWeighted" : "topK"; } @@ -143,9 +139,9 @@ private: public: AggregateFunctionTopKGeneric( - UInt64 threshold, const DataTypePtr & input_data_type, const Array & params) + UInt64 threshold, UInt64 load_factor, const DataTypePtr & input_data_type, const Array & params) : IAggregateFunctionDataHelper>({input_data_type}, params) - , threshold(threshold), reserved(TOP_K_LOAD_FACTOR * threshold), input_data_type(this->argument_types[0]) {} + , threshold(threshold), reserved(load_factor * threshold), input_data_type(this->argument_types[0]) {} String getName() const override { return is_weighted ? "topKWeighted" : "topK"; } @@ -238,6 +234,4 @@ public: const char * getHeaderFilePath() const override { return __FILE__; } }; -#undef TOP_K_LOAD_FACTOR - } diff --git a/dbms/src/AggregateFunctions/QuantileTDigest.h b/dbms/src/AggregateFunctions/QuantileTDigest.h index ca7d4f2fb1a..c4ee76b6eed 100644 --- a/dbms/src/AggregateFunctions/QuantileTDigest.h +++ b/dbms/src/AggregateFunctions/QuantileTDigest.h @@ -85,7 +85,7 @@ class QuantileTDigest Params params; /// The memory will be allocated to several elements at once, so that the state occupies 64 bytes. - static constexpr size_t bytes_in_arena = 64 - sizeof(PODArray) - sizeof(Count) - sizeof(UInt32); + static constexpr size_t bytes_in_arena = 128 - sizeof(PODArray) - sizeof(Count) - sizeof(UInt32); using Summary = PODArray, bytes_in_arena>>; diff --git a/dbms/src/Columns/ColumnAggregateFunction.cpp b/dbms/src/Columns/ColumnAggregateFunction.cpp index e491ff942f4..4652e4a08c8 100644 --- a/dbms/src/Columns/ColumnAggregateFunction.cpp +++ b/dbms/src/Columns/ColumnAggregateFunction.cpp @@ -152,7 +152,7 @@ void ColumnAggregateFunction::insertRangeFrom(const IColumn & from, size_t start size_t old_size = data.size(); data.resize(old_size + length); - memcpy(&data[old_size], &from_concrete.data[start], length * sizeof(data[0])); + memcpy(data.data() + old_size, &from_concrete.data[start], length * sizeof(data[0])); } } @@ -255,6 +255,11 @@ size_t ColumnAggregateFunction::allocatedBytes() const return res; } +void ColumnAggregateFunction::protect() +{ + data.protect(); +} + MutableColumnPtr ColumnAggregateFunction::cloneEmpty() const { return create(func, Arenas(1, std::make_shared())); diff --git a/dbms/src/Columns/ColumnAggregateFunction.h b/dbms/src/Columns/ColumnAggregateFunction.h index 3fc76b4c047..a028a95d68c 100644 --- a/dbms/src/Columns/ColumnAggregateFunction.h +++ b/dbms/src/Columns/ColumnAggregateFunction.h @@ -157,6 +157,8 @@ public: size_t allocatedBytes() const override; + void protect() override; + void insertRangeFrom(const IColumn & from, size_t start, size_t length) override; void popBack(size_t n) override; diff --git a/dbms/src/Columns/ColumnArray.cpp b/dbms/src/Columns/ColumnArray.cpp index 4ceda666db7..eeb06b64f49 100644 --- a/dbms/src/Columns/ColumnArray.cpp +++ b/dbms/src/Columns/ColumnArray.cpp @@ -311,6 +311,13 @@ size_t ColumnArray::allocatedBytes() const } +void ColumnArray::protect() +{ + getData().protect(); + getOffsets().protect(); +} + + bool ColumnArray::hasEqualOffsets(const ColumnArray & other) const { if (offsets == other.offsets) diff --git a/dbms/src/Columns/ColumnArray.h b/dbms/src/Columns/ColumnArray.h index 3e1b586e755..41e38953cf1 100644 --- a/dbms/src/Columns/ColumnArray.h +++ b/dbms/src/Columns/ColumnArray.h @@ -3,17 +3,12 @@ #include #include #include +#include + namespace DB { -namespace ErrorCodes -{ - extern const int ILLEGAL_COLUMN; - extern const int NOT_IMPLEMENTED; - extern const int BAD_ARGUMENTS; -} - /** A column of array values. * In memory, it is represented as one column of a nested type, whose size is equal to the sum of the sizes of all arrays, * and as an array of offsets in it, which allows you to get each element. @@ -78,6 +73,7 @@ public: void reserve(size_t n) override; size_t byteSize() const override; size_t allocatedBytes() const override; + void protect() override; ColumnPtr replicate(const Offsets & replicate_offsets) const override; ColumnPtr convertToFullColumnIfConst() const override; void getExtremes(Field & min, Field & max) const override; @@ -120,6 +116,13 @@ public: callback(data); } + bool structureEquals(const IColumn & rhs) const override + { + if (auto rhs_concrete = typeid_cast(&rhs)) + return data->structureEquals(*rhs_concrete->data); + return false; + } + private: ColumnPtr data; ColumnPtr offsets; diff --git a/dbms/src/Columns/ColumnConst.h b/dbms/src/Columns/ColumnConst.h index 248bb04a186..53ac5b24220 100644 --- a/dbms/src/Columns/ColumnConst.h +++ b/dbms/src/Columns/ColumnConst.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB @@ -190,6 +191,13 @@ public: callback(data); } + bool structureEquals(const IColumn & rhs) const override + { + if (auto rhs_concrete = typeid_cast(&rhs)) + return data->structureEquals(*rhs_concrete->data); + return false; + } + bool onlyNull() const override { return data->isNullAt(0); } bool isColumnConst() const override { return true; } bool isNumeric() const override { return data->isNumeric(); } diff --git a/dbms/src/Columns/ColumnDecimal.cpp b/dbms/src/Columns/ColumnDecimal.cpp index 3663981b632..9377f924f81 100644 --- a/dbms/src/Columns/ColumnDecimal.cpp +++ b/dbms/src/Columns/ColumnDecimal.cpp @@ -140,7 +140,7 @@ void ColumnDecimal::insertRangeFrom(const IColumn & src, size_t start, size_t size_t old_size = data.size(); data.resize(old_size + length); - memcpy(&data[old_size], &src_vec.data[start], length * sizeof(data[0])); + memcpy(data.data() + old_size, &src_vec.data[start], length * sizeof(data[0])); } template diff --git a/dbms/src/Columns/ColumnDecimal.h b/dbms/src/Columns/ColumnDecimal.h index 50a6d9d67fb..4a3c6153947 100644 --- a/dbms/src/Columns/ColumnDecimal.h +++ b/dbms/src/Columns/ColumnDecimal.h @@ -2,6 +2,7 @@ #include +#include #include #include @@ -87,6 +88,7 @@ public: size_t size() const override { return data.size(); } size_t byteSize() const override { return data.size() * sizeof(data[0]); } size_t allocatedBytes() const override { return data.allocated_bytes(); } + void protect() override { data.protect(); } void reserve(size_t n) override { data.reserve(n); } void insertFrom(const IColumn & src, size_t n) override { data.push_back(static_cast(src).getData()[n]); } @@ -132,6 +134,13 @@ public: void gather(ColumnGathererStream & gatherer_stream) override; + bool structureEquals(const IColumn & rhs) const override + { + if (auto rhs_concrete = typeid_cast *>(&rhs)) + return scale == rhs_concrete->scale; + return false; + } + void insert(const T value) { data.push_back(value); } Container & getData() { return data; } diff --git a/dbms/src/Columns/ColumnFixedString.cpp b/dbms/src/Columns/ColumnFixedString.cpp index 728ae095901..686223f712f 100644 --- a/dbms/src/Columns/ColumnFixedString.cpp +++ b/dbms/src/Columns/ColumnFixedString.cpp @@ -55,7 +55,7 @@ void ColumnFixedString::insert(const Field & x) size_t old_size = chars.size(); chars.resize_fill(old_size + n); - memcpy(&chars[old_size], s.data(), s.size()); + memcpy(chars.data() + old_size, s.data(), s.size()); } void ColumnFixedString::insertFrom(const IColumn & src_, size_t index) @@ -67,7 +67,7 @@ void ColumnFixedString::insertFrom(const IColumn & src_, size_t index) size_t old_size = chars.size(); chars.resize(old_size + n); - memcpySmallAllowReadWriteOverflow15(&chars[old_size], &src.chars[n * index], n); + memcpySmallAllowReadWriteOverflow15(chars.data() + old_size, &src.chars[n * index], n); } void ColumnFixedString::insertData(const char * pos, size_t length) @@ -77,7 +77,7 @@ void ColumnFixedString::insertData(const char * pos, size_t length) size_t old_size = chars.size(); chars.resize_fill(old_size + n); - memcpy(&chars[old_size], pos, length); + memcpy(chars.data() + old_size, pos, length); } StringRef ColumnFixedString::serializeValueIntoArena(size_t index, Arena & arena, char const *& begin) const @@ -91,7 +91,7 @@ const char * ColumnFixedString::deserializeAndInsertFromArena(const char * pos) { size_t old_size = chars.size(); chars.resize(old_size + n); - memcpy(&chars[old_size], pos, n); + memcpy(chars.data() + old_size, pos, n); return pos + n; } @@ -151,7 +151,7 @@ void ColumnFixedString::insertRangeFrom(const IColumn & src, size_t start, size_ size_t old_size = chars.size(); chars.resize(old_size + length * n); - memcpy(&chars[old_size], &src_concrete.chars[start * n], length * n); + memcpy(chars.data() + old_size, &src_concrete.chars[start * n], length * n); } ColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t result_size_hint) const diff --git a/dbms/src/Columns/ColumnFixedString.h b/dbms/src/Columns/ColumnFixedString.h index 941314b8888..1f79594b459 100644 --- a/dbms/src/Columns/ColumnFixedString.h +++ b/dbms/src/Columns/ColumnFixedString.h @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -57,6 +58,11 @@ public: return chars.allocated_bytes() + sizeof(n); } + void protect() override + { + chars.protect(); + } + Field operator[](size_t index) const override { return String(reinterpret_cast(&chars[n * index]), n); @@ -129,6 +135,12 @@ public: void getExtremes(Field & min, Field & max) const override; + bool structureEquals(const IColumn & rhs) const override + { + if (auto rhs_concrete = typeid_cast(&rhs)) + return n == rhs_concrete->n; + return false; + } bool canBeInsideNullable() const override { return true; } diff --git a/dbms/src/Columns/ColumnLowCardinality.cpp b/dbms/src/Columns/ColumnLowCardinality.cpp index c919116112c..c9a475fd8a6 100644 --- a/dbms/src/Columns/ColumnLowCardinality.cpp +++ b/dbms/src/Columns/ColumnLowCardinality.cpp @@ -363,7 +363,6 @@ ColumnPtr ColumnLowCardinality::countKeys() const } - ColumnLowCardinality::Index::Index() : positions(ColumnUInt8::create()), size_of_type(sizeof(UInt8)) {} ColumnLowCardinality::Index::Index(MutableColumnPtr && positions) : positions(std::move(positions)) diff --git a/dbms/src/Columns/ColumnLowCardinality.h b/dbms/src/Columns/ColumnLowCardinality.h index 0e9cbbadebe..3b816b2449c 100644 --- a/dbms/src/Columns/ColumnLowCardinality.h +++ b/dbms/src/Columns/ColumnLowCardinality.h @@ -5,6 +5,7 @@ #include #include "ColumnsNumber.h" + namespace DB { @@ -132,6 +133,14 @@ public: callback(dictionary.getColumnUniquePtr()); } + bool structureEquals(const IColumn & rhs) const override + { + if (auto rhs_low_cardinality = typeid_cast(&rhs)) + return idx.getPositions()->structureEquals(*rhs_low_cardinality->idx.getPositions()) + && dictionary.getColumnUnique().structureEquals(rhs_low_cardinality->dictionary.getColumnUnique()); + return false; + } + bool valuesHaveFixedSize() const override { return getDictionary().valuesHaveFixedSize(); } bool isFixedAndContiguous() const override { return false; } size_t sizeOfValueIfFixed() const override { return getDictionary().sizeOfValueIfFixed(); } diff --git a/dbms/src/Columns/ColumnNothing.h b/dbms/src/Columns/ColumnNothing.h index c9cde4f26ec..0c9f843a454 100644 --- a/dbms/src/Columns/ColumnNothing.h +++ b/dbms/src/Columns/ColumnNothing.h @@ -23,6 +23,11 @@ public: MutableColumnPtr cloneDummy(size_t s_) const override { return ColumnNothing::create(s_); } bool canBeInsideNullable() const override { return true; } + + bool structureEquals(const IColumn & rhs) const override + { + return typeid(rhs) == typeid(ColumnNothing); + } }; } diff --git a/dbms/src/Columns/ColumnNullable.cpp b/dbms/src/Columns/ColumnNullable.cpp index b88cf60581b..d9a8ea4f825 100644 --- a/dbms/src/Columns/ColumnNullable.cpp +++ b/dbms/src/Columns/ColumnNullable.cpp @@ -291,6 +291,12 @@ size_t ColumnNullable::allocatedBytes() const return getNestedColumn().allocatedBytes() + getNullMapColumn().allocatedBytes(); } +void ColumnNullable::protect() +{ + getNestedColumn().protect(); + getNullMapColumn().protect(); +} + namespace { diff --git a/dbms/src/Columns/ColumnNullable.h b/dbms/src/Columns/ColumnNullable.h index c8453a29689..8a3651b378b 100644 --- a/dbms/src/Columns/ColumnNullable.h +++ b/dbms/src/Columns/ColumnNullable.h @@ -2,6 +2,8 @@ #include #include +#include + namespace DB { @@ -71,6 +73,7 @@ public: void reserve(size_t n) override; size_t byteSize() const override; size_t allocatedBytes() const override; + void protect() override; ColumnPtr replicate(const Offsets & replicate_offsets) const override; void updateHashWithValue(size_t n, SipHash & hash) const override; void getExtremes(Field & min, Field & max) const override; @@ -88,6 +91,13 @@ public: callback(null_map); } + bool structureEquals(const IColumn & rhs) const override + { + if (auto rhs_nullable = typeid_cast(&rhs)) + return nested_column->structureEquals(*rhs_nullable->nested_column); + return false; + } + bool isColumnNullable() const override { return true; } bool isFixedAndContiguous() const override { return false; } bool valuesHaveFixedSize() const override { return nested_column->valuesHaveFixedSize(); } diff --git a/dbms/src/Columns/ColumnString.cpp b/dbms/src/Columns/ColumnString.cpp index 863f39f9a52..1443283783a 100644 --- a/dbms/src/Columns/ColumnString.cpp +++ b/dbms/src/Columns/ColumnString.cpp @@ -185,7 +185,7 @@ const char * ColumnString::deserializeAndInsertFromArena(const char * pos) const size_t old_size = chars.size(); const size_t new_size = old_size + string_size; chars.resize(new_size); - memcpy(&chars[old_size], pos, string_size); + memcpy(chars.data() + old_size, pos, string_size); offsets.push_back(new_size); return pos + string_size; @@ -412,4 +412,11 @@ void ColumnString::getPermutationWithCollation(const Collator & collator, bool r } } + +void ColumnString::protect() +{ + getChars().protect(); + getOffsets().protect(); +} + } diff --git a/dbms/src/Columns/ColumnString.h b/dbms/src/Columns/ColumnString.h index 7117bab0d05..9ae32c41fd9 100644 --- a/dbms/src/Columns/ColumnString.h +++ b/dbms/src/Columns/ColumnString.h @@ -1,6 +1,7 @@ #pragma once -#include +#include +#include #include #include @@ -67,25 +68,31 @@ public: return chars.allocated_bytes() + offsets.allocated_bytes(); } + void protect() override; + MutableColumnPtr cloneResized(size_t to_size) const override; Field operator[](size_t n) const override { + assert(n < size()); return Field(&chars[offsetAt(n)], sizeAt(n) - 1); } void get(size_t n, Field & res) const override { + assert(n < size()); res.assignString(&chars[offsetAt(n)], sizeAt(n) - 1); } StringRef getDataAt(size_t n) const override { + assert(n < size()); return StringRef(&chars[offsetAt(n)], sizeAt(n) - 1); } StringRef getDataAtWithTerminatingZero(size_t n) const override { + assert(n < size()); return StringRef(&chars[offsetAt(n)], sizeAt(n)); } @@ -103,7 +110,7 @@ public: const size_t new_size = old_size + size_to_append; chars.resize(new_size); - memcpy(&chars[old_size], s.c_str(), size_to_append); + memcpy(chars.data() + old_size, s.c_str(), size_to_append); offsets.push_back(new_size); } @@ -114,36 +121,22 @@ public: void insertFrom(const IColumn & src_, size_t n) override { const ColumnString & src = static_cast(src_); + const size_t size_to_append = src.offsets[n] - src.offsets[n - 1]; /// -1th index is Ok, see PaddedPODArray. - if (n != 0) + if (size_to_append == 1) { - const size_t size_to_append = src.offsets[n] - src.offsets[n - 1]; - - if (size_to_append == 1) - { - /// shortcut for empty string - chars.push_back(0); - offsets.push_back(chars.size()); - } - else - { - const size_t old_size = chars.size(); - const size_t offset = src.offsets[n - 1]; - const size_t new_size = old_size + size_to_append; - - chars.resize(new_size); - memcpySmallAllowReadWriteOverflow15(&chars[old_size], &src.chars[offset], size_to_append); - offsets.push_back(new_size); - } + /// shortcut for empty string + chars.push_back(0); + offsets.push_back(chars.size()); } else { const size_t old_size = chars.size(); - const size_t size_to_append = src.offsets[0]; + const size_t offset = src.offsets[n - 1]; const size_t new_size = old_size + size_to_append; chars.resize(new_size); - memcpySmallAllowReadWriteOverflow15(&chars[old_size], &src.chars[0], size_to_append); + memcpySmallAllowReadWriteOverflow15(chars.data() + old_size, &src.chars[offset], size_to_append); offsets.push_back(new_size); } } @@ -155,7 +148,7 @@ public: chars.resize(new_size); if (length) - memcpy(&chars[old_size], pos, length); + memcpy(chars.data() + old_size, pos, length); chars[old_size + length] = 0; offsets.push_back(new_size); } @@ -167,7 +160,7 @@ public: const size_t new_size = old_size + length; chars.resize(new_size); - memcpy(&chars[old_size], pos, length); + memcpy(chars.data() + old_size, pos, length); offsets.push_back(new_size); } @@ -238,6 +231,11 @@ public: bool canBeInsideNullable() const override { return true; } + bool structureEquals(const IColumn & rhs) const override + { + return typeid(rhs) == typeid(ColumnString); + } + Chars & getChars() { return chars; } const Chars & getChars() const { return chars; } diff --git a/dbms/src/Columns/ColumnTuple.cpp b/dbms/src/Columns/ColumnTuple.cpp index c235cd07c31..caed6c5d6f1 100644 --- a/dbms/src/Columns/ColumnTuple.cpp +++ b/dbms/src/Columns/ColumnTuple.cpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB @@ -315,6 +316,12 @@ size_t ColumnTuple::allocatedBytes() const return res; } +void ColumnTuple::protect() +{ + for (auto & column : columns) + column->assumeMutableRef().protect(); +} + void ColumnTuple::getExtremes(Field & min, Field & max) const { const size_t tuple_size = columns.size(); @@ -335,6 +342,23 @@ void ColumnTuple::forEachSubcolumn(ColumnCallback callback) callback(column); } +bool ColumnTuple::structureEquals(const IColumn & rhs) const +{ + if (auto rhs_tuple = typeid_cast(&rhs)) + { + const size_t tuple_size = columns.size(); + if (tuple_size != rhs_tuple->columns.size()) + return false; + + for (const auto i : ext::range(0, tuple_size)) + if (!columns[i]->structureEquals(*rhs_tuple->columns[i])) + return false; + + return true; + } + else + return false; +} } diff --git a/dbms/src/Columns/ColumnTuple.h b/dbms/src/Columns/ColumnTuple.h index d146c8bff6c..d0a65a03d37 100644 --- a/dbms/src/Columns/ColumnTuple.h +++ b/dbms/src/Columns/ColumnTuple.h @@ -71,7 +71,9 @@ public: void reserve(size_t n) override; size_t byteSize() const override; size_t allocatedBytes() const override; + void protect() override; void forEachSubcolumn(ColumnCallback callback) override; + bool structureEquals(const IColumn & rhs) const override; size_t tupleSize() const { return columns.size(); } diff --git a/dbms/src/Columns/ColumnUnique.h b/dbms/src/Columns/ColumnUnique.h index 85a9c498a94..a06863858ae 100644 --- a/dbms/src/Columns/ColumnUnique.h +++ b/dbms/src/Columns/ColumnUnique.h @@ -80,6 +80,7 @@ public: bool isNumeric() const override { return column_holder->isNumeric(); } size_t byteSize() const override { return column_holder->byteSize(); } + void protect() override { column_holder->assumeMutableRef().protect(); } size_t allocatedBytes() const override { return column_holder->allocatedBytes() @@ -94,6 +95,13 @@ public: nested_column_nullable = ColumnNullable::create(column_holder, nested_null_mask); } + bool structureEquals(const IColumn & rhs) const override + { + if (auto rhs_concrete = typeid_cast(&rhs)) + return column_holder->structureEquals(*rhs_concrete->column_holder); + return false; + } + const UInt64 * tryGetSavedHash() const override { return index.tryGetSavedHash(); } UInt128 getHash() const override { return hash.getHash(*getRawColumnPtr()); } diff --git a/dbms/src/Columns/ColumnVector.cpp b/dbms/src/Columns/ColumnVector.cpp index 3201a463f64..8322e3f74eb 100644 --- a/dbms/src/Columns/ColumnVector.cpp +++ b/dbms/src/Columns/ColumnVector.cpp @@ -141,7 +141,7 @@ void ColumnVector::insertRangeFrom(const IColumn & src, size_t start, size_t size_t old_size = data.size(); data.resize(old_size + length); - memcpy(&data[old_size], &src_vec.data[start], length * sizeof(data[0])); + memcpy(data.data() + old_size, &src_vec.data[start], length * sizeof(data[0])); } template diff --git a/dbms/src/Columns/ColumnVector.h b/dbms/src/Columns/ColumnVector.h index 1c5a45ef6ad..43f6b0a3d52 100644 --- a/dbms/src/Columns/ColumnVector.h +++ b/dbms/src/Columns/ColumnVector.h @@ -163,6 +163,11 @@ public: return data.allocated_bytes(); } + void protect() override + { + data.protect(); + } + void insertValue(const T value) { data.push_back(value); @@ -246,6 +251,12 @@ public: size_t sizeOfValueIfFixed() const override { return sizeof(T); } StringRef getRawData() const override { return StringRef(reinterpret_cast(data.data()), data.size()); } + + bool structureEquals(const IColumn & rhs) const override + { + return typeid(rhs) == typeid(ColumnVector); + } + /** More efficient methods of manipulation - to manipulate with data directly. */ Container & getData() { diff --git a/dbms/src/Columns/ColumnVectorHelper.h b/dbms/src/Columns/ColumnVectorHelper.h index 8a25812ffe7..d805f44218c 100644 --- a/dbms/src/Columns/ColumnVectorHelper.h +++ b/dbms/src/Columns/ColumnVectorHelper.h @@ -24,9 +24,10 @@ namespace DB class ColumnVectorHelper : public IColumn { public: + template const char * getRawDataBegin() const { - return *reinterpret_cast(reinterpret_cast(this) + sizeof(*this)); + return reinterpret_cast, 15, 16> *>(reinterpret_cast(this) + sizeof(*this))->raw_data(); } template diff --git a/dbms/src/Columns/IColumn.h b/dbms/src/Columns/IColumn.h index 2560b9639ad..b7df53ed148 100644 --- a/dbms/src/Columns/IColumn.h +++ b/dbms/src/Columns/IColumn.h @@ -253,11 +253,22 @@ public: /// Zero, if could be determined. virtual size_t allocatedBytes() const = 0; + /// Make memory region readonly with mprotect if it is large enough. + /// The operation is slow and performed only for debug builds. + virtual void protect() {} + /// If the column contains subcolumns (such as Array, Nullable, etc), do callback on them. /// Shallow: doesn't do recursive calls; don't do call for itself. using ColumnCallback = std::function; virtual void forEachSubcolumn(ColumnCallback) {} + /// Columns have equal structure. + /// If true - you can use "compareAt", "insertFrom", etc. methods. + virtual bool structureEquals(const IColumn &) const + { + throw Exception("Method structureEquals is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED); + } + MutablePtr mutate() const && { diff --git a/dbms/src/Columns/tests/CMakeLists.txt b/dbms/src/Columns/tests/CMakeLists.txt index cc989e5f0bf..302c554a1fd 100644 --- a/dbms/src/Columns/tests/CMakeLists.txt +++ b/dbms/src/Columns/tests/CMakeLists.txt @@ -1,4 +1,4 @@ if(USE_GTEST) add_executable(column_unique column_unique.cpp) - target_link_libraries(column_unique PRIVATE dbms gtest_main) + target_link_libraries(column_unique PRIVATE dbms ${GTEST_BOTH_LIBRARIES}) endif() \ No newline at end of file diff --git a/dbms/src/Common/Allocator.cpp b/dbms/src/Common/Allocator.cpp index 5c653a9a1c9..92ff10eafb7 100644 --- a/dbms/src/Common/Allocator.cpp +++ b/dbms/src/Common/Allocator.cpp @@ -5,6 +5,7 @@ #endif #include +#include #include #include @@ -42,11 +43,30 @@ namespace ErrorCodes * * PS. This is also required, because tcmalloc can not allocate a chunk of memory greater than 16 GB. */ -static constexpr size_t MMAP_THRESHOLD = 64 * (1ULL << 20); +#ifdef NDEBUG + static constexpr size_t MMAP_THRESHOLD = 64 * (1ULL << 20); +#else + /// In debug build, use small mmap threshold to reproduce more memory stomping bugs. + /// Along with ASLR it will hopefully detect more issues than ASan. + /// The program may fail due to the limit on number of memory mappings. + static constexpr size_t MMAP_THRESHOLD = 4096; +#endif + static constexpr size_t MMAP_MIN_ALIGNMENT = 4096; static constexpr size_t MALLOC_MIN_ALIGNMENT = 8; +template +void * Allocator::mmap_hint() +{ +#if ALLOCATOR_ASLR + return reinterpret_cast(std::uniform_int_distribution(0x100000000000UL, 0x700000000000UL)(rng)); +#else + return nullptr; +#endif +} + + template void * Allocator::alloc(size_t size, size_t alignment) { @@ -60,7 +80,7 @@ void * Allocator::alloc(size_t size, size_t alignment) throw DB::Exception("Too large alignment " + formatReadableSizeWithBinarySuffix(alignment) + ": more than page size when allocating " + formatReadableSizeWithBinarySuffix(size) + ".", DB::ErrorCodes::BAD_ARGUMENTS); - buf = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + buf = mmap(mmap_hint(), size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (MAP_FAILED == buf) DB::throwFromErrno("Allocator: Cannot mmap " + formatReadableSizeWithBinarySuffix(size) + ".", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); @@ -118,9 +138,11 @@ void * Allocator::realloc(void * buf, size_t old_size, size_t new if (old_size == new_size) { /// nothing to do. + /// BTW, it's not possible to change alignment while doing realloc. } else if (old_size < MMAP_THRESHOLD && new_size < MMAP_THRESHOLD && alignment <= MALLOC_MIN_ALIGNMENT) { + /// Resize malloc'd memory region with no special alignment requirement. CurrentMemoryTracker::realloc(old_size, new_size); void * new_buf = ::realloc(buf, new_size); @@ -133,6 +155,7 @@ void * Allocator::realloc(void * buf, size_t old_size, size_t new } else if (old_size >= MMAP_THRESHOLD && new_size >= MMAP_THRESHOLD) { + /// Resize mmap'd memory region. CurrentMemoryTracker::realloc(old_size, new_size); // On apple and freebsd self-implemented mremap used (common/mremap.h) @@ -142,21 +165,12 @@ void * Allocator::realloc(void * buf, size_t old_size, size_t new /// No need for zero-fill, because mmap guarantees it. } - else if (old_size >= MMAP_THRESHOLD && new_size < MMAP_THRESHOLD) - { - void * new_buf = alloc(new_size, alignment); - memcpy(new_buf, buf, new_size); - if (0 != munmap(buf, old_size)) - { - ::free(new_buf); - DB::throwFromErrno("Allocator: Cannot munmap " + formatReadableSizeWithBinarySuffix(old_size) + ".", DB::ErrorCodes::CANNOT_MUNMAP); - } - buf = new_buf; - } else { + /// All other cases that requires a copy. MemoryTracker is called inside 'alloc', 'free' methods. + void * new_buf = alloc(new_size, alignment); - memcpy(new_buf, buf, old_size); + memcpy(new_buf, buf, std::min(old_size, new_size)); free(buf, old_size); buf = new_buf; } diff --git a/dbms/src/Common/Allocator.h b/dbms/src/Common/Allocator.h index 9a2ab0b975c..d2a81f77b62 100644 --- a/dbms/src/Common/Allocator.h +++ b/dbms/src/Common/Allocator.h @@ -2,6 +2,19 @@ #include +#ifdef NDEBUG + /// If set to 1 - randomize memory mappings manually (address space layout randomization) to reproduce more memory stomping bugs. + /// Note that Linux doesn't do it by default. This may lead to worse TLB performance. + #define ALLOCATOR_ASLR 0 +#else + #define ALLOCATOR_ASLR 1 +#endif + +#if ALLOCATOR_ASLR + #include + #include +#endif + /** Responsible for allocating / freeing memory. Used, for example, in PODArray, Arena. * Also used in hash tables. @@ -14,6 +27,12 @@ template class Allocator { +#if ALLOCATOR_ASLR +private: + pcg64 rng{randomSeed()}; +#endif + void * mmap_hint(); + protected: static constexpr bool clear_memory = clear_memory_; diff --git a/dbms/src/Common/CompactArray.h b/dbms/src/Common/CompactArray.h index 434b475e262..785cd04b4d0 100644 --- a/dbms/src/Common/CompactArray.h +++ b/dbms/src/Common/CompactArray.h @@ -55,6 +55,28 @@ public: return locus; } + /// Used only in arcadia/metrika + void readText(ReadBuffer & in) + { + for (size_t i = 0; i < BITSET_SIZE; ++i) + { + if (i != 0) + assertChar(',', in); + readIntText(bitset[i], in); + } + } + + /// Used only in arcadia/metrika + void writeText(WriteBuffer & out) const + { + for (size_t i = 0; i < BITSET_SIZE; ++i) + { + if (i != 0) + writeCString(",", out); + writeIntText(bitset[i], out); + } + } + private: /// number of bytes in bitset static constexpr size_t BITSET_SIZE = (static_cast(bucket_count) * content_width + 7) / 8; diff --git a/dbms/src/Common/CurrentThread.cpp b/dbms/src/Common/CurrentThread.cpp index b6d161af67e..3eaf8bfa81d 100644 --- a/dbms/src/Common/CurrentThread.cpp +++ b/dbms/src/Common/CurrentThread.cpp @@ -21,6 +21,8 @@ namespace ErrorCodes void CurrentThread::updatePerformanceCounters() { + if (unlikely(!current_thread)) + return; get().updatePerformanceCounters(); } @@ -37,30 +39,38 @@ ProfileEvents::Counters & CurrentThread::getProfileEvents() return current_thread ? get().performance_counters : ProfileEvents::global_counters; } -MemoryTracker & CurrentThread::getMemoryTracker() +MemoryTracker * CurrentThread::getMemoryTracker() { - return get().memory_tracker; + if (unlikely(!current_thread)) + return nullptr; + return &get().memory_tracker; } void CurrentThread::updateProgressIn(const Progress & value) { + if (unlikely(!current_thread)) + return; get().progress_in.incrementPiecewiseAtomically(value); } void CurrentThread::updateProgressOut(const Progress & value) { + if (unlikely(!current_thread)) + return; get().progress_out.incrementPiecewiseAtomically(value); } void CurrentThread::attachInternalTextLogsQueue(const std::shared_ptr & logs_queue) { + if (unlikely(!current_thread)) + return; get().attachInternalTextLogsQueue(logs_queue); } std::shared_ptr CurrentThread::getInternalTextLogsQueue() { /// NOTE: this method could be called at early server startup stage - if (!current_thread) + if (unlikely(!current_thread)) return nullptr; if (get().getCurrentState() == ThreadStatus::ThreadState::Died) @@ -71,7 +81,7 @@ std::shared_ptr CurrentThread::getInternalTextLogsQueue() ThreadGroupStatusPtr CurrentThread::getGroup() { - if (!current_thread) + if (unlikely(!current_thread)) return nullptr; return get().getThreadGroup(); diff --git a/dbms/src/Common/CurrentThread.h b/dbms/src/Common/CurrentThread.h index c30555b22e8..49b46721008 100644 --- a/dbms/src/Common/CurrentThread.h +++ b/dbms/src/Common/CurrentThread.h @@ -45,7 +45,7 @@ public: static void updatePerformanceCounters(); static ProfileEvents::Counters & getProfileEvents(); - static MemoryTracker & getMemoryTracker(); + static MemoryTracker * getMemoryTracker(); /// Update read and write rows (bytes) statistics (used in system.query_thread_log) static void updateProgressIn(const Progress & value); diff --git a/dbms/src/Common/ErrorCodes.cpp b/dbms/src/Common/ErrorCodes.cpp index d3401427037..f974b2bdaf6 100644 --- a/dbms/src/Common/ErrorCodes.cpp +++ b/dbms/src/Common/ErrorCodes.cpp @@ -419,6 +419,7 @@ namespace ErrorCodes extern const int BAD_DATABASE_FOR_TEMPORARY_TABLE = 442; extern const int NO_COMMON_COLUMNS_WITH_PROTOBUF_SCHEMA = 443; extern const int UNKNOWN_PROTOBUF_FORMAT = 444; + extern const int CANNOT_MPROTECT = 445; extern const int KEEPER_EXCEPTION = 999; extern const int POCO_EXCEPTION = 1000; diff --git a/dbms/src/Common/MemoryTracker.cpp b/dbms/src/Common/MemoryTracker.cpp index 6a997e3b19a..bc324be4904 100644 --- a/dbms/src/Common/MemoryTracker.cpp +++ b/dbms/src/Common/MemoryTracker.cpp @@ -190,24 +190,27 @@ namespace CurrentMemoryTracker { void alloc(Int64 size) { - if (DB::current_thread) - DB::CurrentThread::getMemoryTracker().alloc(size); + if (auto memory_tracker = DB::CurrentThread::getMemoryTracker()) + memory_tracker->alloc(size); } void realloc(Int64 old_size, Int64 new_size) { - if (DB::current_thread) - DB::CurrentThread::getMemoryTracker().alloc(new_size - old_size); + if (auto memory_tracker = DB::CurrentThread::getMemoryTracker()) + memory_tracker->alloc(new_size - old_size); } void free(Int64 size) { - if (DB::current_thread) - DB::CurrentThread::getMemoryTracker().free(size); + if (auto memory_tracker = DB::CurrentThread::getMemoryTracker()) + memory_tracker->free(size); } } DB::SimpleActionLock getCurrentMemoryTrackerActionLock() { - return DB::CurrentThread::getMemoryTracker().blocker.cancel(); + auto memory_tracker = DB::CurrentThread::getMemoryTracker(); + if (!memory_tracker) + return {}; + return memory_tracker->blocker.cancel(); } diff --git a/dbms/src/Common/PODArray.h b/dbms/src/Common/PODArray.h index d74e5b5d2c1..0e7d547a7d0 100644 --- a/dbms/src/Common/PODArray.h +++ b/dbms/src/Common/PODArray.h @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -16,10 +17,19 @@ #include #include +#ifndef NDEBUG + #include +#endif + namespace DB { +namespace ErrorCodes +{ + extern const int CANNOT_MPROTECT; +} + inline constexpr size_t integerRoundUp(size_t value, size_t dividend) { return ((value + dividend - 1) / dividend) * dividend; @@ -107,6 +117,8 @@ protected: if (c_start == null) return; + unprotect(); + TAllocator::free(c_start - pad_left, allocated_bytes()); } @@ -119,6 +131,8 @@ protected: return; } + unprotect(); + ptrdiff_t end_diff = c_end - c_start; c_start = reinterpret_cast( @@ -154,11 +168,34 @@ protected: realloc(allocated_bytes() * 2, std::forward(allocator_params)...); } +#ifndef NDEBUG + /// Make memory region readonly with mprotect if it is large enough. + /// The operation is slow and performed only for debug builds. + void protectImpl(int prot) + { + static constexpr size_t PROTECT_PAGE_SIZE = 4096; + + char * left_rounded_up = reinterpret_cast((reinterpret_cast(c_start) - pad_left + PROTECT_PAGE_SIZE - 1) / PROTECT_PAGE_SIZE * PROTECT_PAGE_SIZE); + char * right_rounded_down = reinterpret_cast((reinterpret_cast(c_end_of_storage) + pad_right) / PROTECT_PAGE_SIZE * PROTECT_PAGE_SIZE); + + if (right_rounded_down > left_rounded_up) + { + size_t length = right_rounded_down - left_rounded_up; + if (0 != mprotect(left_rounded_up, length, prot)) + throwFromErrno("Cannot mprotect memory region", ErrorCodes::CANNOT_MPROTECT); + } + } + + /// Restore memory protection in destructor or realloc for further reuse by allocator. + bool mprotected = false; +#endif + public: bool empty() const { return c_end == c_start; } size_t size() const { return (c_end - c_start) / ELEMENT_SIZE; } size_t capacity() const { return (c_end_of_storage - c_start) / ELEMENT_SIZE; } + /// This method is safe to use only for information about memory usage. size_t allocated_bytes() const { return c_end_of_storage - c_start + pad_right + pad_left; } void clear() { c_end = c_start; } @@ -197,6 +234,23 @@ public: c_end += byte_size(1); } + void protect() + { +#ifndef NDEBUG + protectImpl(PROT_READ); + mprotected = true; +#endif + } + + void unprotect() + { +#ifndef NDEBUG + if (mprotected) + protectImpl(PROT_WRITE); + mprotected = false; +#endif + } + ~PODArrayBase() { dealloc(); @@ -271,8 +325,18 @@ public: const T * data() const { return t_start(); } /// The index is signed to access -1th element without pointer overflow. - T & operator[] (ssize_t n) { return t_start()[n]; } - const T & operator[] (ssize_t n) const { return t_start()[n]; } + T & operator[] (ssize_t n) + { + /// <= size, because taking address of one element past memory range is Ok in C++ (expression like &arr[arr.size()] is perfectly valid). + assert((n >= (static_cast(pad_left_) ? -1 : 0)) && (n <= static_cast(this->size()))); + return t_start()[n]; + } + + const T & operator[] (ssize_t n) const + { + assert((n >= (static_cast(pad_left_) ? -1 : 0)) && (n <= static_cast(this->size()))); + return t_start()[n]; + } T & front() { return t_start()[0]; } T & back() { return t_end()[-1]; } @@ -390,6 +454,11 @@ public: void swap(PODArray & rhs) { +#ifndef NDEBUG + this->unprotect(); + rhs.unprotect(); +#endif + /// Swap two PODArray objects, arr1 and arr2, that satisfy the following conditions: /// - The elements of arr1 are stored on stack. /// - The elements of arr2 are stored on heap. @@ -438,7 +507,9 @@ public: }; if (!this->isInitialized() && !rhs.isInitialized()) + { return; + } else if (!this->isInitialized() && rhs.isInitialized()) { do_move(rhs, *this); @@ -482,9 +553,13 @@ public: rhs.c_end = rhs.c_start + this->byte_size(lhs_size); } else if (this->isAllocatedFromStack() && !rhs.isAllocatedFromStack()) + { swap_stack_heap(*this, rhs); + } else if (!this->isAllocatedFromStack() && rhs.isAllocatedFromStack()) + { swap_stack_heap(rhs, *this); + } else { std::swap(this->c_start, rhs.c_start); diff --git a/dbms/src/Compression/CompressionFactory.cpp b/dbms/src/Compression/CompressionFactory.cpp index 7f7dbac21df..b5b2bfe6b5e 100644 --- a/dbms/src/Compression/CompressionFactory.cpp +++ b/dbms/src/Compression/CompressionFactory.cpp @@ -19,7 +19,6 @@ namespace ErrorCodes { extern const int UNKNOWN_CODEC; extern const int UNEXPECTED_AST_STRUCTURE; - extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE; extern const int DATA_TYPE_CANNOT_HAVE_ARGUMENTS; } @@ -85,7 +84,7 @@ CompressionCodecPtr CompressionCodecFactory::get(const UInt8 byte_code) const const auto family_code_and_creator = family_code_with_codec.find(byte_code); if (family_code_and_creator == family_code_with_codec.end()) - throw Exception("Unknown codec family code : " + toString(byte_code), ErrorCodes::UNKNOWN_CODEC); + throw Exception("Unknown codec family code: " + toString(byte_code), ErrorCodes::UNKNOWN_CODEC); return family_code_and_creator->second({}, nullptr); } diff --git a/dbms/src/Core/AccurateComparison.h b/dbms/src/Core/AccurateComparison.h index 62402c339f7..4fa33c7d099 100644 --- a/dbms/src/Core/AccurateComparison.h +++ b/dbms/src/Core/AccurateComparison.h @@ -2,9 +2,9 @@ #include #include - +#include "Defines.h" +#include "Types.h" #include -#include #include /** Preceptually-correct number comparisons. diff --git a/dbms/src/Core/BackgroundSchedulePool.cpp b/dbms/src/Core/BackgroundSchedulePool.cpp index 0493e13b2b9..ce67c895234 100644 --- a/dbms/src/Core/BackgroundSchedulePool.cpp +++ b/dbms/src/Core/BackgroundSchedulePool.cpp @@ -250,7 +250,8 @@ void BackgroundSchedulePool::threadFunction() attachToThreadGroup(); SCOPE_EXIT({ CurrentThread::detachQueryIfNotDetached(); }); - CurrentThread::getMemoryTracker().setMetric(CurrentMetrics::MemoryTrackingInBackgroundSchedulePool); + if (auto memory_tracker = CurrentThread::getMemoryTracker()) + memory_tracker->setMetric(CurrentMetrics::MemoryTrackingInBackgroundSchedulePool); while (!shutdown) { diff --git a/dbms/src/DataStreams/GraphiteRollupSortedBlockInputStream.h b/dbms/src/DataStreams/GraphiteRollupSortedBlockInputStream.h index bb2f81fc81f..00bd2f4b67e 100644 --- a/dbms/src/DataStreams/GraphiteRollupSortedBlockInputStream.h +++ b/dbms/src/DataStreams/GraphiteRollupSortedBlockInputStream.h @@ -113,6 +113,7 @@ namespace Graphite struct Pattern { std::shared_ptr regexp; + std::string regexp_str; AggregateFunctionPtr function; Retentions retentions; /// Must be ordered by 'age' descending. enum { TypeUndef, TypeRetention, TypeAggregation, TypeAll } type = TypeAll; /// The type of defined pattern, filled automatically @@ -124,6 +125,7 @@ namespace Graphite struct Params { + String config_name; String path_column_name; String time_column_name; String value_column_name; @@ -215,6 +217,7 @@ private: const Graphite::Pattern undef_pattern = { /// temporary empty pattern for selectPatternForPath nullptr, + "", nullptr, DB::Graphite::Retentions(), undef_pattern.TypeUndef, diff --git a/dbms/src/DataStreams/IBlockInputStream.h b/dbms/src/DataStreams/IBlockInputStream.h index 8a2b1e9b148..dfcd1c38802 100644 --- a/dbms/src/DataStreams/IBlockInputStream.h +++ b/dbms/src/DataStreams/IBlockInputStream.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -24,12 +25,9 @@ class IBlockInputStream; class ProcessListElement; class QuotaForIntervals; class QueryStatus; -class TableStructureReadLock; using BlockInputStreamPtr = std::shared_ptr; using BlockInputStreams = std::vector; -using TableStructureReadLockPtr = std::shared_ptr; -using TableStructureReadLocks = std::vector; /** Callback to track the progress of the query. * Used in IBlockInputStream and Context. @@ -117,7 +115,7 @@ public: size_t checkDepth(size_t max_depth) const { return checkDepthImpl(max_depth, max_depth); } /// Do not allow to change the table while the blocks stream and its children are alive. - void addTableLock(const TableStructureReadLockPtr & lock) { table_locks.push_back(lock); } + void addTableLock(const TableStructureReadLockHolder & lock) { table_locks.push_back(lock); } /// Get information about execution speed. const BlockStreamProfileInfo & getProfileInfo() const { return info; } @@ -244,7 +242,7 @@ public: protected: /// Order is important: `table_locks` must be destroyed after `children` so that tables from /// which child streams read are protected by the locks during the lifetime of the child streams. - TableStructureReadLocks table_locks; + std::vector table_locks; BlockInputStreams children; std::shared_mutex children_mutex; diff --git a/dbms/src/DataStreams/IBlockOutputStream.h b/dbms/src/DataStreams/IBlockOutputStream.h index 33494422479..f4269a66422 100644 --- a/dbms/src/DataStreams/IBlockOutputStream.h +++ b/dbms/src/DataStreams/IBlockOutputStream.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB @@ -12,13 +13,6 @@ namespace DB struct Progress; -class TableStructureReadLock; -using TableStructureReadLockPtr = std::shared_ptr; -using TableStructureReadLocks = std::vector; - -struct Progress; - - /** Interface of stream for writing data (into table, filesystem, network, terminal, etc.) */ class IBlockOutputStream : private boost::noncopyable @@ -64,10 +58,10 @@ public: /** Don't let to alter table while instance of stream is alive. */ - void addTableLock(const TableStructureReadLockPtr & lock) { table_locks.push_back(lock); } + void addTableLock(const TableStructureReadLockHolder & lock) { table_locks.push_back(lock); } private: - TableStructureReadLocks table_locks; + std::vector table_locks; }; using BlockOutputStreamPtr = std::shared_ptr; diff --git a/dbms/src/DataStreams/LimitBlockInputStream.cpp b/dbms/src/DataStreams/LimitBlockInputStream.cpp index c0ac474fb60..b1b8f5c95d1 100644 --- a/dbms/src/DataStreams/LimitBlockInputStream.cpp +++ b/dbms/src/DataStreams/LimitBlockInputStream.cpp @@ -6,9 +6,14 @@ namespace DB { -LimitBlockInputStream::LimitBlockInputStream(const BlockInputStreamPtr & input, UInt64 limit_, UInt64 offset_, bool always_read_till_end_) +LimitBlockInputStream::LimitBlockInputStream(const BlockInputStreamPtr & input, UInt64 limit_, UInt64 offset_, bool always_read_till_end_, bool use_limit_as_total_rows_approx) : limit(limit_), offset(offset_), always_read_till_end(always_read_till_end_) { + if (use_limit_as_total_rows_approx) + { + addTotalRowsApprox(static_cast(limit)); + } + children.push_back(input); } diff --git a/dbms/src/DataStreams/LimitBlockInputStream.h b/dbms/src/DataStreams/LimitBlockInputStream.h index 6e77c5a2f6a..ed6dac8c5ac 100644 --- a/dbms/src/DataStreams/LimitBlockInputStream.h +++ b/dbms/src/DataStreams/LimitBlockInputStream.h @@ -16,8 +16,9 @@ public: * returns an empty block, and this causes the query to be canceled. * If always_read_till_end = true - reads all the data to the end, but ignores them. This is necessary in rare cases: * when otherwise, due to the cancellation of the request, we would not have received the data for GROUP BY WITH TOTALS from the remote server. + * If use_limit_as_total_rows_approx = true, then addTotalRowsApprox is called to use the limit in progress & stats */ - LimitBlockInputStream(const BlockInputStreamPtr & input, UInt64 limit_, UInt64 offset_, bool always_read_till_end_ = false); + LimitBlockInputStream(const BlockInputStreamPtr & input, UInt64 limit_, UInt64 offset_, bool always_read_till_end_ = false, bool use_limit_as_total_rows_approx = false); String getName() const override { return "Limit"; } diff --git a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp index 5cb0a1e57e4..195c5edcb07 100644 --- a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp @@ -20,7 +20,7 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream( * Although now any insertion into the table is done via PushingToViewsBlockOutputStream, * but it's clear that here is not the best place for this functionality. */ - addTableLock(storage->lockStructure(true, context.getCurrentQueryId())); + addTableLock(storage->lockStructureForShare(true, context.getCurrentQueryId())); /// If the "root" table deduplactes blocks, there are no need to make deduplication for children /// Moreover, deduplication for AggregatingMergeTree children could produce false positives due to low size of inserting blocks @@ -45,7 +45,7 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream( auto & materialized_view = dynamic_cast(*dependent_table); if (StoragePtr inner_table = materialized_view.tryGetTargetTable()) - addTableLock(inner_table->lockStructure(true, context.getCurrentQueryId())); + addTableLock(inner_table->lockStructureForShare(true, context.getCurrentQueryId())); auto query = materialized_view.getInnerQuery(); BlockOutputStreamPtr out = std::make_shared( diff --git a/dbms/src/DataTypes/DataTypeFixedString.cpp b/dbms/src/DataTypes/DataTypeFixedString.cpp index 356c69351ef..64c94602f8c 100644 --- a/dbms/src/DataTypes/DataTypeFixedString.cpp +++ b/dbms/src/DataTypes/DataTypeFixedString.cpp @@ -69,7 +69,7 @@ void DataTypeFixedString::deserializeBinary(IColumn & column, ReadBuffer & istr) data.resize(old_size + n); try { - istr.readStrict(reinterpret_cast(&data[old_size]), n); + istr.readStrict(reinterpret_cast(data.data() + old_size), n); } catch (...) { diff --git a/dbms/src/DataTypes/DataTypeString.cpp b/dbms/src/DataTypes/DataTypeString.cpp index a97a80ffc5e..d3334ef93bf 100644 --- a/dbms/src/DataTypes/DataTypeString.cpp +++ b/dbms/src/DataTypes/DataTypeString.cpp @@ -130,9 +130,9 @@ static NO_INLINE void deserializeBinarySSE2(ColumnString::Chars & data, ColumnSt if (size) { -#ifdef __x86_64__ +#ifdef __SSE2__ /// An optimistic branch in which more efficient copying is possible. - if (offset + 16 * UNROLL_TIMES <= data.allocated_bytes() && istr.position() + size + 16 * UNROLL_TIMES <= istr.buffer().end()) + if (offset + 16 * UNROLL_TIMES <= data.capacity() && istr.position() + size + 16 * UNROLL_TIMES <= istr.buffer().end()) { const __m128i * sse_src_pos = reinterpret_cast(istr.position()); const __m128i * sse_src_end = sse_src_pos + (size + (16 * UNROLL_TIMES - 1)) / 16 / UNROLL_TIMES * UNROLL_TIMES; @@ -140,22 +140,11 @@ static NO_INLINE void deserializeBinarySSE2(ColumnString::Chars & data, ColumnSt while (sse_src_pos < sse_src_end) { - /// NOTE gcc 4.9.2 unrolls the loop, but for some reason uses only one xmm register. - /// for (size_t j = 0; j < UNROLL_TIMES; ++j) - /// _mm_storeu_si128(sse_dst_pos + j, _mm_loadu_si128(sse_src_pos + j)); + for (size_t j = 0; j < UNROLL_TIMES; ++j) + _mm_storeu_si128(sse_dst_pos + j, _mm_loadu_si128(sse_src_pos + j)); sse_src_pos += UNROLL_TIMES; sse_dst_pos += UNROLL_TIMES; - - if (UNROLL_TIMES >= 4) __asm__("movdqu %0, %%xmm0" :: "m"(sse_src_pos[-4])); - if (UNROLL_TIMES >= 3) __asm__("movdqu %0, %%xmm1" :: "m"(sse_src_pos[-3])); - if (UNROLL_TIMES >= 2) __asm__("movdqu %0, %%xmm2" :: "m"(sse_src_pos[-2])); - if (UNROLL_TIMES >= 1) __asm__("movdqu %0, %%xmm3" :: "m"(sse_src_pos[-1])); - - if (UNROLL_TIMES >= 4) __asm__("movdqu %%xmm0, %0" : "=m"(sse_dst_pos[-4])); - if (UNROLL_TIMES >= 3) __asm__("movdqu %%xmm1, %0" : "=m"(sse_dst_pos[-3])); - if (UNROLL_TIMES >= 2) __asm__("movdqu %%xmm2, %0" : "=m"(sse_dst_pos[-2])); - if (UNROLL_TIMES >= 1) __asm__("movdqu %%xmm3, %0" : "=m"(sse_dst_pos[-1])); } istr.position() += size; diff --git a/dbms/src/DataTypes/tests/CMakeLists.txt b/dbms/src/DataTypes/tests/CMakeLists.txt index 5f699b29c59..aa4cb34620b 100644 --- a/dbms/src/DataTypes/tests/CMakeLists.txt +++ b/dbms/src/DataTypes/tests/CMakeLists.txt @@ -7,5 +7,5 @@ target_link_libraries (data_type_string PRIVATE dbms) if(USE_GTEST) add_executable(data_type_get_common_type data_type_get_common_type.cpp) - target_link_libraries(data_type_get_common_type PRIVATE dbms gtest_main) + target_link_libraries(data_type_get_common_type PRIVATE dbms ${GTEST_BOTH_LIBRARIES}) endif() diff --git a/dbms/src/Databases/DatabasesCommon.cpp b/dbms/src/Databases/DatabasesCommon.cpp index 780140969ad..006d65ede7b 100644 --- a/dbms/src/Databases/DatabasesCommon.cpp +++ b/dbms/src/Databases/DatabasesCommon.cpp @@ -35,6 +35,7 @@ String getTableDefinitionFromCreateQuery(const ASTPtr & query) create.as_table.clear(); create.if_not_exists = false; create.is_populate = false; + create.replace_view = false; /// For views it is necessary to save the SELECT query itself, for the rest - on the contrary if (!create.is_view && !create.is_materialized_view) diff --git a/dbms/src/Dictionaries/DictionaryFactory.h b/dbms/src/Dictionaries/DictionaryFactory.h index 2c101425f41..e0a2dd642cb 100644 --- a/dbms/src/Dictionaries/DictionaryFactory.h +++ b/dbms/src/Dictionaries/DictionaryFactory.h @@ -3,26 +3,29 @@ #include #include "IDictionary.h" + namespace Poco { + namespace Util { class AbstractConfiguration; } class Logger; + } + namespace DB { + class Context; class DictionaryFactory : public ext::singleton { public: - DictionaryPtr - create(const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Context & context) - const; + DictionaryPtr create(const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Context & context) const; using Creator = std::functiongetSourceName() + " is wrong."); new_chars.resize(old_size + name_entry.name.length() + 1); - memcpy(&new_chars[old_size], name_entry.name.c_str(), name_entry.name.length() + 1); + memcpy(new_chars.data() + old_size, name_entry.name.c_str(), name_entry.name.length() + 1); if (name_entry.id > max_region_id) { @@ -92,7 +92,7 @@ void RegionsNames::reload() while (name_entry.id >= new_names_refs.size()) new_names_refs.resize(new_names_refs.size() * 2, StringRef("", 0)); - new_names_refs[name_entry.id] = StringRef(&new_chars[old_size], name_entry.name.length()); + new_names_refs[name_entry.id] = StringRef(new_chars.data() + old_size, name_entry.name.length()); } chars[language_id].swap(new_chars); diff --git a/dbms/src/Formats/ProtobufWriter.cpp b/dbms/src/Formats/ProtobufWriter.cpp index c6bfcbe01c4..07cdb413c86 100644 --- a/dbms/src/Formats/ProtobufWriter.cpp +++ b/dbms/src/Formats/ProtobufWriter.cpp @@ -59,7 +59,7 @@ namespace { size_t old_size = buf.size(); buf.reserve(old_size + MAX_VARINT_SIZE); - UInt8 * ptr = &buf[old_size]; + UInt8 * ptr = buf.data() + old_size; ptr = writeVarint(value, ptr); buf.resize_assume_reserved(ptr - buf.data()); } @@ -200,7 +200,7 @@ void ProtobufWriter::SimpleWriter::writeUInt(UInt32 field_number, UInt64 value) { size_t old_size = buffer.size(); buffer.reserve(old_size + 2 * MAX_VARINT_SIZE); - UInt8 * ptr = &buffer[old_size]; + UInt8 * ptr = buffer.data() + old_size; ptr = writeFieldNumber(field_number, VARINT, ptr); ptr = writeVarint(value, ptr); buffer.resize_assume_reserved(ptr - buffer.data()); @@ -223,7 +223,7 @@ void ProtobufWriter::SimpleWriter::writeFixed(UInt32 field_number, T value) constexpr WireType wire_type = (sizeof(T) == 4) ? BITS32 : BITS64; size_t old_size = buffer.size(); buffer.reserve(old_size + MAX_VARINT_SIZE + sizeof(T)); - UInt8 * ptr = &buffer[old_size]; + UInt8 * ptr = buffer.data() + old_size; ptr = writeFieldNumber(field_number, wire_type, ptr); memcpy(ptr, &value, sizeof(T)); ptr += sizeof(T); @@ -234,7 +234,7 @@ void ProtobufWriter::SimpleWriter::writeString(UInt32 field_number, const String { size_t old_size = buffer.size(); buffer.reserve(old_size + 2 * MAX_VARINT_SIZE + str.size); - UInt8 * ptr = &buffer[old_size]; + UInt8 * ptr = buffer.data() + old_size; ptr = writeFieldNumber(field_number, LENGTH_DELIMITED, ptr); ptr = writeVarint(str.size, ptr); memcpy(ptr, str.data, str.size); @@ -294,7 +294,7 @@ void ProtobufWriter::SimpleWriter::addFixedToRepeatedPack(T value) static_assert((sizeof(T) == 4) || (sizeof(T) == 8)); size_t old_size = buffer.size(); buffer.resize(old_size + sizeof(T)); - memcpy(&buffer[old_size], &value, sizeof(T)); + memcpy(buffer.data() + old_size, &value, sizeof(T)); } diff --git a/dbms/src/Functions/FunctionJoinGet.cpp b/dbms/src/Functions/FunctionJoinGet.cpp index 3ee3145dcfc..5201a0ba5c2 100644 --- a/dbms/src/Functions/FunctionJoinGet.cpp +++ b/dbms/src/Functions/FunctionJoinGet.cpp @@ -65,7 +65,7 @@ FunctionBasePtr FunctionBuilderJoinGet::buildImpl(const ColumnsWithTypeAndName & auto join = storage_join->getJoin(); DataTypes data_types(arguments.size()); - auto table_lock = storage_join->lockStructure(false, context.getCurrentQueryId()); + auto table_lock = storage_join->lockStructureForShare(false, context.getCurrentQueryId()); for (size_t i = 0; i < arguments.size(); ++i) data_types[i] = arguments[i].type; diff --git a/dbms/src/Functions/FunctionJoinGet.h b/dbms/src/Functions/FunctionJoinGet.h index edf45adab6a..556574b1346 100644 --- a/dbms/src/Functions/FunctionJoinGet.h +++ b/dbms/src/Functions/FunctionJoinGet.h @@ -1,4 +1,5 @@ #include +#include namespace DB { @@ -7,8 +8,6 @@ class IStorage; using StoragePtr = std::shared_ptr; class Join; using JoinPtr = std::shared_ptr; -class TableStructureReadLock; -using TableStructureReadLockPtr = std::shared_ptr; class FunctionJoinGet final : public IFunction, public std::enable_shared_from_this { @@ -16,7 +15,7 @@ public: static constexpr auto name = "joinGet"; FunctionJoinGet( - TableStructureReadLockPtr table_lock, StoragePtr storage_join, JoinPtr join, const String & attr_name, DataTypePtr return_type) + TableStructureReadLockHolder table_lock, StoragePtr storage_join, JoinPtr join, const String & attr_name, DataTypePtr return_type) : table_lock(std::move(table_lock)) , storage_join(std::move(storage_join)) , join(std::move(join)) @@ -36,7 +35,7 @@ private: size_t getNumberOfArguments() const override { return 0; } private: - TableStructureReadLockPtr table_lock; + TableStructureReadLockHolder table_lock; StoragePtr storage_join; JoinPtr join; const String attr_name; diff --git a/dbms/src/Functions/FunctionsCoding.h b/dbms/src/Functions/FunctionsCoding.h index 6192102b529..1f88482aa49 100644 --- a/dbms/src/Functions/FunctionsCoding.h +++ b/dbms/src/Functions/FunctionsCoding.h @@ -33,6 +33,7 @@ namespace ErrorCodes { extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; extern const int LOGICAL_ERROR; + extern const int ILLEGAL_COLUMN; } @@ -123,8 +124,8 @@ public: } else throw Exception("Illegal column " + block.getByPosition(arguments[0]).column->getName() - + " of argument of function " + getName(), - ErrorCodes::ILLEGAL_COLUMN); + + " of argument of function " + getName(), + ErrorCodes::ILLEGAL_COLUMN); } }; diff --git a/dbms/src/Functions/FunctionsEmbeddedDictionaries.h b/dbms/src/Functions/FunctionsEmbeddedDictionaries.h index 7e5093d1ce8..e51b847f47d 100644 --- a/dbms/src/Functions/FunctionsEmbeddedDictionaries.h +++ b/dbms/src/Functions/FunctionsEmbeddedDictionaries.h @@ -36,6 +36,7 @@ namespace ErrorCodes { extern const int DICTIONARIES_WAS_NOT_LOADED; extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_COLUMN; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } diff --git a/dbms/src/Functions/FunctionsExternalDictionaries.h b/dbms/src/Functions/FunctionsExternalDictionaries.h index 85a5c9cc6a5..e9e0f94bb40 100644 --- a/dbms/src/Functions/FunctionsExternalDictionaries.h +++ b/dbms/src/Functions/FunctionsExternalDictionaries.h @@ -44,6 +44,8 @@ namespace ErrorCodes extern const int UNKNOWN_TYPE; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int TYPE_MISMATCH; + extern const int ILLEGAL_COLUMN; + extern const int BAD_ARGUMENTS; } /** Functions that use plug-ins (external) dictionaries. diff --git a/dbms/src/Functions/FunctionsFindCluster.h b/dbms/src/Functions/FunctionsFindCluster.h index 5b8db96d1ba..1b2bde45d29 100644 --- a/dbms/src/Functions/FunctionsFindCluster.h +++ b/dbms/src/Functions/FunctionsFindCluster.h @@ -20,6 +20,7 @@ namespace DB namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int ILLEGAL_COLUMN; } enum ClusterOperation diff --git a/dbms/src/Functions/FunctionsHashing.h b/dbms/src/Functions/FunctionsHashing.h index f14a91d422e..e756e8b79e7 100644 --- a/dbms/src/Functions/FunctionsHashing.h +++ b/dbms/src/Functions/FunctionsHashing.h @@ -48,6 +48,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int NOT_IMPLEMENTED; + extern const int ILLEGAL_COLUMN; } diff --git a/dbms/src/Functions/FunctionsRound.h b/dbms/src/Functions/FunctionsRound.h index 9e93e16e7bb..bff8248b3ea 100644 --- a/dbms/src/Functions/FunctionsRound.h +++ b/dbms/src/Functions/FunctionsRound.h @@ -8,7 +8,7 @@ #include #include #include - +#include "IFunction.h" #include #include #include diff --git a/dbms/src/Functions/FunctionsStringArray.h b/dbms/src/Functions/FunctionsStringArray.h index 641a23e225c..fb89dfceda2 100644 --- a/dbms/src/Functions/FunctionsStringArray.h +++ b/dbms/src/Functions/FunctionsStringArray.h @@ -21,6 +21,7 @@ namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_COLUMN; } diff --git a/dbms/src/Functions/FunctionsStringSimilarity.cpp b/dbms/src/Functions/FunctionsStringSimilarity.cpp index 7f0267d6d59..7c77857345a 100644 --- a/dbms/src/Functions/FunctionsStringSimilarity.cpp +++ b/dbms/src/Functions/FunctionsStringSimilarity.cpp @@ -8,164 +8,271 @@ #include +#include + #include +#include #include #include #include +#include #ifdef __SSE4_2__ -#include +# include #endif namespace DB { /** Distance function implementation. - * We calculate all the trigrams from left string and count by the index of + * We calculate all the n-grams from left string and count by the index of * 16 bits hash of them in the map. - * Then calculate all the trigrams from the right string and calculate - * the trigram distance on the flight by adding and subtracting from the hashmap. + * Then calculate all the n-grams from the right string and calculate + * the n-gram distance on the flight by adding and subtracting from the hashmap. * Then return the map into the condition of which it was after the left string * calculation. If the right string size is big (more than 2**15 bytes), * the strings are not similar at all and we return 1. */ -struct TrigramDistanceImpl +template +struct NgramDistanceImpl { using ResultType = Float32; - using CodePoint = UInt32; - /// map_size for trigram difference + /// map_size for ngram difference. static constexpr size_t map_size = 1u << 16; - /// If the haystack size is bigger than this, behaviour is unspecified for this function + /// If the haystack size is bigger than this, behaviour is unspecified for this function. static constexpr size_t max_string_size = 1u << 15; + /// Default padding to read safely. + static constexpr size_t default_padding = 16; + + /// Max codepoints to store at once. 16 is for batching usage and PODArray has this padding. + static constexpr size_t simultaneously_codepoints_num = default_padding + N - 1; + /** This fits mostly in L2 cache all the time. * Actually use UInt16 as addings and subtractions do not UB overflow. But think of it as a signed * integer array. */ - using TrigramStats = UInt16[map_size]; + using NgramStats = UInt16[map_size]; - static ALWAYS_INLINE UInt16 trigramHash(CodePoint one, CodePoint two, CodePoint three) + static ALWAYS_INLINE UInt16 ASCIIHash(const CodePoint * code_points) { - UInt64 combined = (static_cast(one) << 32) | two; + return intHashCRC32(unalignedLoad(code_points)) & 0xFFFFu; + } + + static ALWAYS_INLINE UInt16 UTF8Hash(const CodePoint * code_points) + { + UInt64 combined = (static_cast(code_points[0]) << 32) | code_points[1]; #ifdef __SSE4_2__ - return _mm_crc32_u64(three, combined) & 0xFFFFu; + return _mm_crc32_u64(code_points[2], combined) & 0xFFFFu; #else - return (intHashCRC32(combined) ^ intHashCRC32(three)) & 0xFFFFu; + return (intHashCRC32(combined) ^ intHashCRC32(code_points[2])) & 0xFFFFu; #endif } - static ALWAYS_INLINE CodePoint readCodePoint(const char *& pos, const char * end) noexcept + template + static ALWAYS_INLINE inline void unrollLowering(Container & cont, const std::index_sequence &) { - size_t length = UTF8::seqLength(*pos); - - if (pos + length > end) - length = end - pos; - - CodePoint res; - /// This is faster than just memcpy because of compiler optimizations with moving bytes. - switch (length) - { - case 1: - res = 0; - memcpy(&res, pos, 1); - break; - case 2: - res = 0; - memcpy(&res, pos, 2); - break; - case 3: - res = 0; - memcpy(&res, pos, 3); - break; - default: - memcpy(&res, pos, 4); - } - - pos += length; - return res; + ((cont[Offset + I] = std::tolower(cont[Offset + I])), ...); } - static inline size_t calculateNeedleStats(const char * data, const size_t size, TrigramStats & trigram_stats) noexcept + static ALWAYS_INLINE size_t readASCIICodePoints(CodePoint * code_points, const char *& pos, const char * end) { - size_t len = 0; - const char * start = data; - const char * end = data + size; - CodePoint cp1 = 0; - CodePoint cp2 = 0; - CodePoint cp3 = 0; + /// Offset before which we copy some data. + constexpr size_t padding_offset = default_padding - N + 1; + /// We have an array like this for ASCII (N == 4, other cases are similar) + /// |a0|a1|a2|a3|a4|a5|a6|a7|a8|a9|a10|a11|a12|a13|a14|a15|a16|a17|a18| + /// And we copy ^^^^^^^^^^^^^^^ these bytes to the start + /// Actually it is enough to copy 3 bytes, but memcpy for 4 bytes translates into 1 instruction + memcpy(code_points, code_points + padding_offset, roundUpToPowerOfTwoOrZero(N - 1) * sizeof(CodePoint)); + /// Now we have an array + /// |a13|a14|a15|a16|a4|a5|a6|a7|a8|a9|a10|a11|a12|a13|a14|a15|a16|a17|a18| + /// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + /// Doing unaligned read of 16 bytes and copy them like above + /// 16 is also chosen to do two `movups`. + /// Such copying allow us to have 3 codepoints from the previous read to produce the 4-grams with them. + memcpy(code_points + (N - 1), pos, default_padding * sizeof(CodePoint)); - while (start != end) + if constexpr (CaseInsensitive) { - cp1 = cp2; - cp2 = cp3; - cp3 = readCodePoint(start, end); - ++len; - if (len < 3) - continue; - ++trigram_stats[trigramHash(cp1, cp2, cp3)]; + /// We really need template lambdas with C++20 to do it inline + unrollLowering(code_points, std::make_index_sequence()); } - return std::max(static_cast(0), static_cast(len) - 2); + pos += padding_offset; + if (pos > end) + return default_padding - (pos - end); + return default_padding; } - static inline UInt64 calculateHaystackStatsAndMetric(const char * data, const size_t size, TrigramStats & trigram_stats, size_t & distance) + static ALWAYS_INLINE size_t readUTF8CodePoints(CodePoint * code_points, const char *& pos, const char * end) { - size_t len = 0; - size_t trigram_cnt = 0; + /// The same copying as described in the function above. + memcpy(code_points, code_points + default_padding - N + 1, roundUpToPowerOfTwoOrZero(N - 1) * sizeof(CodePoint)); + + size_t num = N - 1; + while (num < default_padding && pos < end) + { + size_t length = UTF8::seqLength(*pos); + + if (pos + length > end) + length = end - pos; + + CodePoint res; + /// This is faster than just memcpy because of compiler optimizations with moving bytes. + switch (length) + { + case 1: + res = 0; + memcpy(&res, pos, 1); + break; + case 2: + res = 0; + memcpy(&res, pos, 2); + break; + case 3: + res = 0; + memcpy(&res, pos, 3); + break; + default: + memcpy(&res, pos, 4); + } + + /// This is not a really true case insensitive utf8. We zero the 5-th bit of every byte. + /// For ASCII it works https://catonmat.net/ascii-case-conversion-trick. For most cyrrilic letters also does. + /// For others, we don't care now. Lowering UTF is not a cheap operation. + if constexpr (CaseInsensitive) + { + switch (length) + { + case 4: + res &= ~(1u << (5 + 3 * CHAR_BIT)); + [[fallthrough]]; + case 3: + res &= ~(1u << (5 + 2 * CHAR_BIT)); + [[fallthrough]]; + case 2: + res &= ~(1u << (5 + CHAR_BIT)); + [[fallthrough]]; + default: + res &= ~(1u << 5); + } + } + + pos += length; + code_points[num++] = res; + } + return num; + } + + static ALWAYS_INLINE inline size_t calculateNeedleStats( + const char * data, + const size_t size, + NgramStats & ngram_stats, + size_t (*read_code_points)(CodePoint *, const char *&, const char *), + UInt16 (*hash_functor)(const CodePoint *)) + { + // To prevent size_t overflow below. + if (size < N) + return 0; + const char * start = data; const char * end = data + size; - CodePoint cp1 = 0; - CodePoint cp2 = 0; - CodePoint cp3 = 0; + CodePoint cp[simultaneously_codepoints_num] = {}; + + /// read_code_points returns the position of cp where it stopped reading codepoints. + size_t found = read_code_points(cp, start, end); + /// We need to start for the first time here, because first N - 1 codepoints mean nothing. + size_t i = N - 1; + /// Initialize with this value because for the first time `found` does not initialize first N - 1 codepoints. + size_t len = -N + 1; + do + { + len += found - N + 1; + for (; i + N <= found; ++i) + ++ngram_stats[hash_functor(cp + i)]; + i = 0; + } while (start < end && (found = read_code_points(cp, start, end))); + + return len; + } + + static ALWAYS_INLINE inline UInt64 calculateHaystackStatsAndMetric( + const char * data, + const size_t size, + NgramStats & ngram_stats, + size_t & distance, + size_t (*read_code_points)(CodePoint *, const char *&, const char *), + UInt16 (*hash_functor)(const CodePoint *)) + { + size_t ngram_cnt = 0; + const char * start = data; + const char * end = data + size; + CodePoint cp[simultaneously_codepoints_num] = {}; /// allocation tricks, most strings are relatively small static constexpr size_t small_buffer_size = 256; std::unique_ptr big_buffer; UInt16 small_buffer[small_buffer_size]; - UInt16 * trigram_storage = small_buffer; + UInt16 * ngram_storage = small_buffer; if (size > small_buffer_size) { - trigram_storage = new UInt16[size]; - big_buffer.reset(trigram_storage); + ngram_storage = new UInt16[size]; + big_buffer.reset(ngram_storage); } - while (start != end) + /// read_code_points returns the position of cp where it stopped reading codepoints. + size_t found = read_code_points(cp, start, end); + /// We need to start for the first time here, because first N - 1 codepoints mean nothing. + size_t iter = N - 1; + + do { - cp1 = cp2; - cp2 = cp3; - cp3 = readCodePoint(start, end); - ++len; - if (len < 3) - continue; + for (; iter + N <= found; ++iter) + { + UInt16 hash = hash_functor(cp + iter); + if (static_cast(ngram_stats[hash]) > 0) + --distance; + else + ++distance; - UInt16 hash = trigramHash(cp1, cp2, cp3); - - if (static_cast(trigram_stats[hash]) > 0) - --distance; - else - ++distance; - - trigram_storage[trigram_cnt++] = hash; - --trigram_stats[hash]; - } + ngram_storage[ngram_cnt++] = hash; + --ngram_stats[hash]; + } + iter = 0; + } while (start < end && (found = read_code_points(cp, start, end))); /// Return the state of hash map to its initial. - for (size_t i = 0; i < trigram_cnt; ++i) - ++trigram_stats[trigram_storage[i]]; - - return trigram_cnt; + for (size_t i = 0; i < ngram_cnt; ++i) + ++ngram_stats[ngram_storage[i]]; + return ngram_cnt; } - static void constant_constant(const std::string & data, const std::string & needle, Float32 & res) + template + static inline size_t dispatchSearcher(Callback callback, Args &&... args) { - TrigramStats common_stats; + if constexpr (!UTF8) + return callback(std::forward(args)..., readASCIICodePoints, ASCIIHash); + else + return callback(std::forward(args)..., readUTF8CodePoints, UTF8Hash); + } + + static void constant_constant(std::string data, std::string needle, Float32 & res) + { + NgramStats common_stats; memset(common_stats, 0, sizeof(common_stats)); - size_t second_size = calculateNeedleStats(needle.data(), needle.size(), common_stats); + + /// We use unsafe versions of getting ngrams, so I decided to use padded strings. + const size_t needle_size = needle.size(); + const size_t data_size = data.size(); + needle.resize(needle_size + default_padding); + data.resize(data_size + default_padding); + + size_t second_size = dispatchSearcher(calculateNeedleStats, needle.data(), needle_size, common_stats); size_t distance = second_size; - if (data.size() <= max_string_size) + if (data_size <= max_string_size) { - size_t first_size = calculateHaystackStatsAndMetric(data.data(), data.size(), common_stats, distance); + size_t first_size = dispatchSearcher(calculateHaystackStatsAndMetric, data.data(), data_size, common_stats, distance); res = distance * 1.f / std::max(first_size + second_size, size_t(1)); } else @@ -175,11 +282,18 @@ struct TrigramDistanceImpl } static void vector_constant( - const ColumnString::Chars & data, const ColumnString::Offsets & offsets, const std::string & needle, PaddedPODArray & res) + const ColumnString::Chars & data, const ColumnString::Offsets & offsets, std::string needle, PaddedPODArray & res) { - TrigramStats common_stats; + /// zeroing our map + NgramStats common_stats; memset(common_stats, 0, sizeof(common_stats)); - const size_t needle_stats_size = calculateNeedleStats(needle.data(), needle.size(), common_stats); + + /// We use unsafe versions of getting ngrams, so I decided to use padded_data even in needle case. + const size_t needle_size = needle.size(); + needle.resize(needle_size + default_padding); + + const size_t needle_stats_size = dispatchSearcher(calculateNeedleStats, needle.data(), needle_size, common_stats); + size_t distance = needle_stats_size; size_t prev_offset = 0; for (size_t i = 0; i < offsets.size(); ++i) @@ -188,12 +302,13 @@ struct TrigramDistanceImpl const size_t haystack_size = offsets[i] - prev_offset - 1; if (haystack_size <= max_string_size) { - size_t haystack_stats_size - = calculateHaystackStatsAndMetric(reinterpret_cast(haystack), haystack_size, common_stats, distance); + size_t haystack_stats_size = dispatchSearcher( + calculateHaystackStatsAndMetric, reinterpret_cast(haystack), haystack_size, common_stats, distance); res[i] = distance * 1.f / std::max(haystack_stats_size + needle_stats_size, size_t(1)); } else { + /// if the strings are too big, we say they are completely not the same res[i] = 1.f; } distance = needle_stats_size; @@ -203,16 +318,39 @@ struct TrigramDistanceImpl }; -struct TrigramDistanceName +struct NgramDistanceName { - static constexpr auto name = "trigramDistance"; + static constexpr auto name = "ngramDistance"; }; -using FunctionTrigramsDistance = FunctionsStringSimilarity; +struct NgramDistanceCaseInsensitiveName +{ + static constexpr auto name = "ngramDistanceCaseInsensitive"; +}; + +struct NgramDistanceUTF8Name +{ + static constexpr auto name = "ngramDistanceUTF8"; +}; + +struct NgramDistanceUTF8CaseInsensitiveName +{ + static constexpr auto name = "ngramDistanceCaseInsensitiveUTF8"; +}; + +using FunctionNgramDistance = FunctionsStringSimilarity, NgramDistanceName>; +using FunctionNgramDistanceCaseInsensitive + = FunctionsStringSimilarity, NgramDistanceCaseInsensitiveName>; +using FunctionNgramDistanceUTF8 = FunctionsStringSimilarity, NgramDistanceUTF8Name>; +using FunctionNgramDistanceCaseInsensitiveUTF8 + = FunctionsStringSimilarity, NgramDistanceUTF8CaseInsensitiveName>; void registerFunctionsStringSimilarity(FunctionFactory & factory) { - factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); } } diff --git a/dbms/src/Functions/FunctionsStringSimilarity.h b/dbms/src/Functions/FunctionsStringSimilarity.h index 00c90e20569..c23d9be999a 100644 --- a/dbms/src/Functions/FunctionsStringSimilarity.h +++ b/dbms/src/Functions/FunctionsStringSimilarity.h @@ -12,8 +12,9 @@ namespace DB /** Calculate similarity metrics: * - * trigramDistance(haystack, needle) --- calculate so called 3-gram distance between haystack and needle. + * ngramDistance(haystack, needle) --- calculate n-gram distance between haystack and needle. * Returns float number from 0 to 1 - the closer to zero, the more strings are similar to each other. + * Also support CaseInsensitive and UTF8 formats. */ namespace ErrorCodes diff --git a/dbms/src/Functions/FunctionsVisitParam.h b/dbms/src/Functions/FunctionsVisitParam.h index 5b20a53135d..09cc3106719 100644 --- a/dbms/src/Functions/FunctionsVisitParam.h +++ b/dbms/src/Functions/FunctionsVisitParam.h @@ -38,6 +38,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; +} + struct HasParam { using ResultType = UInt8; diff --git a/dbms/src/Functions/GatherUtils/Algorithms.h b/dbms/src/Functions/GatherUtils/Algorithms.h index aaf1828f47d..9cc2d8ba21a 100644 --- a/dbms/src/Functions/GatherUtils/Algorithms.h +++ b/dbms/src/Functions/GatherUtils/Algorithms.h @@ -53,7 +53,7 @@ inline ALWAYS_INLINE void writeSlice(const StringSource::Slice & slice, FixedStr /// Assuming same types of underlying columns for slice and sink if (ArraySlice, ArraySink) is (GenericArraySlice, GenericArraySink). inline ALWAYS_INLINE void writeSlice(const GenericArraySlice & slice, GenericArraySink & sink) { - if (typeid(slice.elements) == typeid(static_cast(&sink.elements))) + if (slice.elements->structureEquals(sink.elements)) { sink.elements.insertRangeFrom(*slice.elements, slice.begin, slice.size); sink.current_offset += slice.size; @@ -125,7 +125,7 @@ void writeSlice(const NumericValueSlice & slice, NumericArraySink & sink) /// Assuming same types of underlying columns for slice and sink if (ArraySlice, ArraySink) is (GenericValueSlice, GenericArraySink). inline ALWAYS_INLINE void writeSlice(const GenericValueSlice & slice, GenericArraySink & sink) { - if (typeid(slice.elements) == typeid(static_cast(&sink.elements))) + if (slice.elements->structureEquals(sink.elements)) { sink.elements.insertFrom(*slice.elements, slice.position); ++sink.current_offset; @@ -457,7 +457,7 @@ template bool sliceHas(const GenericArraySlice & first, const GenericArraySlice & second) { /// Generic arrays should have the same type in order to use column.compareAt(...) - if (typeid(*first.elements) != typeid(*second.elements)) + if (!first.elements->structureEquals(*second.elements)) return false; auto impl = sliceHasImpl; diff --git a/dbms/src/Functions/GatherUtils/Sources.h b/dbms/src/Functions/GatherUtils/Sources.h index 40115a5a240..a70e06c98e2 100644 --- a/dbms/src/Functions/GatherUtils/Sources.h +++ b/dbms/src/Functions/GatherUtils/Sources.h @@ -14,7 +14,15 @@ #include #include -namespace DB::GatherUtils +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; +} + +namespace GatherUtils { template @@ -660,3 +668,5 @@ struct NullableValueSource : public ValueSource }; } + +} diff --git a/dbms/src/Functions/arrayEnumerateRanked.cpp b/dbms/src/Functions/arrayEnumerateRanked.cpp index 705d92ecd19..a559d64c9a0 100644 --- a/dbms/src/Functions/arrayEnumerateRanked.cpp +++ b/dbms/src/Functions/arrayEnumerateRanked.cpp @@ -6,6 +6,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + ArraysDepths getArraysDepths(const ColumnsWithTypeAndName & arguments) { const size_t num_arguments = arguments.size(); diff --git a/dbms/src/Functions/arrayReduce.cpp b/dbms/src/Functions/arrayReduce.cpp index 1bee774412d..ffab005e949 100644 --- a/dbms/src/Functions/arrayReduce.cpp +++ b/dbms/src/Functions/arrayReduce.cpp @@ -22,6 +22,7 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int ILLEGAL_COLUMN; extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int BAD_ARGUMENTS; } diff --git a/dbms/src/Functions/flatten.cpp b/dbms/src/Functions/flatten.cpp index eb1ff7212e6..8fe743db8ea 100644 --- a/dbms/src/Functions/flatten.cpp +++ b/dbms/src/Functions/flatten.cpp @@ -7,6 +7,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; +} + /// flatten([[1, 2, 3], [4, 5]]) = [1, 2, 3, 4, 5] - flatten array. class FunctionFlatten : public IFunction { diff --git a/dbms/src/Functions/hasAllAny.h b/dbms/src/Functions/hasAllAny.h index 45780d0aa3c..b688406fd91 100644 --- a/dbms/src/Functions/hasAllAny.h +++ b/dbms/src/Functions/hasAllAny.h @@ -53,10 +53,8 @@ public: size_t rows = input_rows_count; size_t num_args = arguments.size(); - auto result_column = ColumnUInt8::create(rows); - DataTypePtr common_type = nullptr; - auto commonType = [& common_type, & block, & arguments]() + auto commonType = [&common_type, &block, &arguments]() { if (common_type == nullptr) { @@ -106,6 +104,7 @@ public: throw Exception{"Arguments for function " + getName() + " must be arrays.", ErrorCodes::LOGICAL_ERROR}; } + auto result_column = ColumnUInt8::create(rows); auto result_column_ptr = typeid_cast(result_column.get()); GatherUtils::sliceHas(*sources[0], *sources[1], all, *result_column_ptr); diff --git a/dbms/src/Functions/replicate.cpp b/dbms/src/Functions/replicate.cpp index 756b745ccde..10b82953a57 100644 --- a/dbms/src/Functions/replicate.cpp +++ b/dbms/src/Functions/replicate.cpp @@ -8,6 +8,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; +} + /** Creates an array, multiplying the column (the first argument) by the number of elements in the array (the second argument). */ class FunctionReplicate : public IFunction @@ -54,7 +59,7 @@ public: array_column = checkAndGetColumn(temp_column.get()); } block.getByPosition(result).column - = ColumnArray::create(first_column->replicate(array_column->getOffsets()), array_column->getOffsetsPtr()); + = ColumnArray::create(first_column->replicate(array_column->getOffsets())->convertToFullColumnIfConst(), array_column->getOffsetsPtr()); } }; diff --git a/dbms/src/Functions/timeSlots.cpp b/dbms/src/Functions/timeSlots.cpp index 7ee0d1a1b2f..ce10d96043e 100644 --- a/dbms/src/Functions/timeSlots.cpp +++ b/dbms/src/Functions/timeSlots.cpp @@ -17,6 +17,7 @@ namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int ILLEGAL_COLUMN; } /** timeSlots(StartTime, Duration) diff --git a/dbms/src/IO/Progress.h b/dbms/src/IO/Progress.h index 54c1d230455..d7366d187aa 100644 --- a/dbms/src/IO/Progress.h +++ b/dbms/src/IO/Progress.h @@ -55,14 +55,11 @@ struct Progress /// Each value separately is changed atomically (but not whole object). bool incrementPiecewiseAtomically(const Progress & rhs) { - if (!rhs.rows) - return false; - rows += rhs.rows; bytes += rhs.bytes; total_rows += rhs.total_rows; - return true; + return rhs.rows ? true : false; } void reset() diff --git a/dbms/src/IO/WriteBufferFromVector.h b/dbms/src/IO/WriteBufferFromVector.h index 7cc7eaf25cb..e52ff7d8411 100644 --- a/dbms/src/IO/WriteBufferFromVector.h +++ b/dbms/src/IO/WriteBufferFromVector.h @@ -34,7 +34,7 @@ private: size_t old_size = vector.size(); vector.resize(old_size * 2); - internal_buffer = Buffer(reinterpret_cast(&vector[old_size]), reinterpret_cast(vector.data() + vector.size())); + internal_buffer = Buffer(reinterpret_cast(vector.data() + old_size), reinterpret_cast(vector.data() + vector.size())); working_buffer = internal_buffer; } diff --git a/dbms/src/IO/WriteIntText.h b/dbms/src/IO/WriteIntText.h index 15276bba63f..4e18826600e 100644 --- a/dbms/src/IO/WriteIntText.h +++ b/dbms/src/IO/WriteIntText.h @@ -3,6 +3,7 @@ #include #include #include +#include /// 40 digits or 39 digits and a sign #define WRITE_HELPERS_MAX_INT_WIDTH 40U diff --git a/dbms/src/Interpreters/AggregationCommon.h b/dbms/src/Interpreters/AggregationCommon.h index 12c2d53819b..74836d4463d 100644 --- a/dbms/src/Interpreters/AggregationCommon.h +++ b/dbms/src/Interpreters/AggregationCommon.h @@ -102,23 +102,23 @@ static inline T ALWAYS_INLINE packFixed( switch (key_sizes[j]) { case 1: - memcpy(bytes + offset, static_cast(column)->getRawDataBegin() + index, 1); + memcpy(bytes + offset, static_cast(column)->getRawDataBegin<1>() + index, 1); offset += 1; break; case 2: - memcpy(bytes + offset, static_cast(column)->getRawDataBegin() + index * 2, 2); + memcpy(bytes + offset, static_cast(column)->getRawDataBegin<2>() + index * 2, 2); offset += 2; break; case 4: - memcpy(bytes + offset, static_cast(column)->getRawDataBegin() + index * 4, 4); + memcpy(bytes + offset, static_cast(column)->getRawDataBegin<4>() + index * 4, 4); offset += 4; break; case 8: - memcpy(bytes + offset, static_cast(column)->getRawDataBegin() + index * 8, 8); + memcpy(bytes + offset, static_cast(column)->getRawDataBegin<8>() + index * 8, 8); offset += 8; break; default: - memcpy(bytes + offset, static_cast(column)->getRawDataBegin() + index * key_sizes[j], key_sizes[j]); + memcpy(bytes + offset, static_cast(column)->getRawDataBegin<1>() + index * key_sizes[j], key_sizes[j]); offset += key_sizes[j]; } } @@ -168,23 +168,23 @@ static inline T ALWAYS_INLINE packFixed( switch (key_sizes[j]) { case 1: - memcpy(bytes + offset, static_cast(key_columns[j])->getRawDataBegin() + i, 1); + memcpy(bytes + offset, static_cast(key_columns[j])->getRawDataBegin<1>() + i, 1); offset += 1; break; case 2: - memcpy(bytes + offset, static_cast(key_columns[j])->getRawDataBegin() + i * 2, 2); + memcpy(bytes + offset, static_cast(key_columns[j])->getRawDataBegin<2>() + i * 2, 2); offset += 2; break; case 4: - memcpy(bytes + offset, static_cast(key_columns[j])->getRawDataBegin() + i * 4, 4); + memcpy(bytes + offset, static_cast(key_columns[j])->getRawDataBegin<4>() + i * 4, 4); offset += 4; break; case 8: - memcpy(bytes + offset, static_cast(key_columns[j])->getRawDataBegin() + i * 8, 8); + memcpy(bytes + offset, static_cast(key_columns[j])->getRawDataBegin<8>() + i * 8, 8); offset += 8; break; default: - memcpy(bytes + offset, static_cast(key_columns[j])->getRawDataBegin() + i * key_sizes[j], key_sizes[j]); + memcpy(bytes + offset, static_cast(key_columns[j])->getRawDataBegin<1>() + i * key_sizes[j], key_sizes[j]); offset += key_sizes[j]; } } diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 40d06c3228b..0c6477942dd 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -145,8 +145,9 @@ Aggregator::Aggregator(const Params & params_) isCancelled([]() { return false; }) { /// Use query-level memory tracker - if (auto memory_tracker = CurrentThread::getMemoryTracker().getParent()) - memory_usage_before_aggregation = memory_tracker->get(); + if (auto memory_tracker_child = CurrentThread::getMemoryTracker()) + if (auto memory_tracker = memory_tracker_child->getParent()) + memory_usage_before_aggregation = memory_tracker->get(); aggregate_functions.resize(params.aggregates_size); for (size_t i = 0; i < params.aggregates_size; ++i) @@ -835,8 +836,9 @@ bool Aggregator::executeOnBlock(const Block & block, AggregatedDataVariants & re size_t result_size = result.sizeWithoutOverflowRow(); Int64 current_memory_usage = 0; - if (auto memory_tracker = CurrentThread::getMemoryTracker().getParent()) - current_memory_usage = memory_tracker->get(); + if (auto memory_tracker_child = CurrentThread::getMemoryTracker()) + if (auto memory_tracker = memory_tracker_child->getParent()) + current_memory_usage = memory_tracker->get(); auto result_size_bytes = current_memory_usage - memory_usage_before_aggregation; /// Here all the results in the sum are taken into account, from different threads. diff --git a/dbms/src/Interpreters/Aliases.h b/dbms/src/Interpreters/Aliases.h index 80976c7551f..b51dc80ae5f 100644 --- a/dbms/src/Interpreters/Aliases.h +++ b/dbms/src/Interpreters/Aliases.h @@ -1,7 +1,7 @@ #pragma once #include - +#include #include namespace DB diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index b7613116f2d..2c1a9a58567 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -1203,6 +1203,8 @@ EmbeddedDictionaries & Context::getEmbeddedDictionariesImpl(const bool throw_on_ ExternalDictionaries & Context::getExternalDictionariesImpl(const bool throw_on_error) const { + const auto & config = getConfigRef(); + std::lock_guard lock(shared->external_dictionaries_mutex); if (!shared->external_dictionaries) @@ -1214,6 +1216,7 @@ ExternalDictionaries & Context::getExternalDictionariesImpl(const bool throw_on_ shared->external_dictionaries.emplace( std::move(config_repository), + config, *this->global_context, throw_on_error); } diff --git a/dbms/src/Interpreters/CrossToInnerJoinVisitor.cpp b/dbms/src/Interpreters/CrossToInnerJoinVisitor.cpp index 8c74ddf699a..ede49b90538 100644 --- a/dbms/src/Interpreters/CrossToInnerJoinVisitor.cpp +++ b/dbms/src/Interpreters/CrossToInnerJoinVisitor.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include #include #include @@ -18,6 +20,66 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; + extern const int NOT_IMPLEMENTED; +} + +namespace +{ + +struct JoinedTable +{ + DatabaseAndTableWithAlias table; + ASTTablesInSelectQueryElement * element = nullptr; + ASTTableJoin * join = nullptr; + ASTPtr array_join = nullptr; + bool has_using = false; + + JoinedTable(ASTPtr table_element) + { + element = typeid_cast(table_element.get()); + if (!element) + throw Exception("Logical error: TablesInSelectQueryElement expected", ErrorCodes::LOGICAL_ERROR); + + if (element->table_join) + { + join = typeid_cast(element->table_join.get()); + if (join->kind == ASTTableJoin::Kind::Cross || + join->kind == ASTTableJoin::Kind::Comma) + { + if (!join->children.empty()) + throw Exception("Logical error: CROSS JOIN has expressions", ErrorCodes::LOGICAL_ERROR); + } + + if (join->using_expression_list) + has_using = true; + } + + if (element->table_expression) + { + auto & expr = typeid_cast(*element->table_expression); + table = DatabaseAndTableWithAlias(expr); + } + + array_join = element->array_join; + } + + void rewriteCommaToCross() + { + if (join) + join->kind = ASTTableJoin::Kind::Cross; + } + + bool canAttachOnExpression() const { return join && !join->on_expression; } +}; + +bool isComparison(const String & name) +{ + return name == NameEquals::name || + name == NameNotEquals::name || + name == NameLess::name || + name == NameGreater::name || + name == NameLessOrEquals::name || + name == NameGreaterOrEquals::name; } /// It checks if where expression could be moved to JOIN ON expression partially or entirely. @@ -26,15 +88,17 @@ class CheckExpressionVisitorData public: using TypeToVisit = const ASTFunction; - CheckExpressionVisitorData(const std::vector & tables_) + CheckExpressionVisitorData(const std::vector & tables_) : tables(tables_) - , save_where(false) - , flat_ands(true) + , ands_only(true) {} void visit(const ASTFunction & node, ASTPtr & ast) { - if (node.name == "and") + if (!ands_only) + return; + + if (node.name == NameAnd::name) { if (!node.arguments || node.arguments->children.empty()) throw Exception("Logical error: function requires argiment", ErrorCodes::LOGICAL_ERROR); @@ -42,50 +106,54 @@ public: for (auto & child : node.arguments->children) { if (auto func = typeid_cast(child.get())) - { - if (func->name == "and") - flat_ands = false; visit(*func, child); - } else - save_where = true; + ands_only = false; } } - else if (node.name == "equals") + else if (node.name == NameEquals::name) { - if (checkEquals(node)) - asts_to_join_on.push_back(ast); - else - save_where = true; + if (size_t min_table = canMoveEqualsToJoinOn(node)) + asts_to_join_on[min_table].push_back(ast); + } + else if (isComparison(node.name)) + { + /// leave other comparisons as is } else - save_where = true; + { + ands_only = false; + asts_to_join_on.clear(); + } } - bool matchAny() const { return !asts_to_join_on.empty(); } - bool matchAll() const { return matchAny() && !save_where; } - bool canReuseWhere() const { return matchAll() && flat_ands; } + bool complex() const { return !ands_only; } + bool matchAny(size_t t) const { return asts_to_join_on.count(t); } - ASTPtr makeOnExpression() + ASTPtr makeOnExpression(size_t table_pos) { - if (asts_to_join_on.size() == 1) - return asts_to_join_on[0]->clone(); + if (!asts_to_join_on.count(table_pos)) + return {}; + + std::vector & expressions = asts_to_join_on[table_pos]; + + if (expressions.size() == 1) + return expressions[0]->clone(); std::vector arguments; - arguments.reserve(asts_to_join_on.size()); - for (auto & ast : asts_to_join_on) + arguments.reserve(expressions.size()); + for (auto & ast : expressions) arguments.emplace_back(ast->clone()); - return makeASTFunction("and", std::move(arguments)); + return makeASTFunction(NameAnd::name, std::move(arguments)); } private: - const std::vector & tables; - std::vector asts_to_join_on; - bool save_where; - bool flat_ands; + const std::vector & tables; + std::map> asts_to_join_on; + bool ands_only; - bool checkEquals(const ASTFunction & node) + size_t canMoveEqualsToJoinOn(const ASTFunction & node) { if (!node.arguments) throw Exception("Logical error: function requires argiment", ErrorCodes::LOGICAL_ERROR); @@ -102,7 +170,8 @@ private: /// Check if the identifiers are from different joined tables. If it's a self joint, tables should have aliases. /// select * from t1 a cross join t2 b where a.x = b.x - bool checkIdentifiers(const ASTIdentifier & left, const ASTIdentifier & right) + /// @return table position to attach expression to or 0. + size_t checkIdentifiers(const ASTIdentifier & left, const ASTIdentifier & right) { /// {best_match, berst_table_pos} std::pair left_best{0, 0}; @@ -110,14 +179,14 @@ private: for (size_t i = 0; i < tables.size(); ++i) { - size_t match = IdentifierSemantic::canReferColumnToTable(left, tables[i]); + size_t match = IdentifierSemantic::canReferColumnToTable(left, tables[i].table); if (match > left_best.first) { left_best.first = match; left_best.second = i; } - match = IdentifierSemantic::canReferColumnToTable(right, tables[i]); + match = IdentifierSemantic::canReferColumnToTable(right, tables[i].table); if (match > right_best.first) { right_best.first = match; @@ -125,59 +194,51 @@ private: } } - return left_best.first && right_best.first && (left_best.second != right_best.second); + if (left_best.first && right_best.first && (left_best.second != right_best.second)) + { + size_t table_pos = std::max(left_best.second, right_best.second); + if (tables[table_pos].canAttachOnExpression()) + return table_pos; + } + return 0; } }; - -static bool extractTableName(const ASTTableExpression & expr, std::vector & names) -{ - /// Subselects are not supported. - if (!expr.database_and_table_name) - return false; - - names.emplace_back(DatabaseAndTableWithAlias(expr)); - return true; -} +using CheckExpressionMatcher = OneTypeMatcher; +using CheckExpressionVisitor = InDepthNodeVisitor; -static ASTPtr getCrossJoin(ASTSelectQuery & select, std::vector & table_names) +bool getTables(ASTSelectQuery & select, std::vector & joined_tables, size_t & num_comma) { if (!select.tables) - return {}; + return false; auto tables = typeid_cast(select.tables.get()); if (!tables) - return {}; + return false; size_t num_tables = tables->children.size(); - if (num_tables != 2) - return {}; + if (num_tables < 2) + return false; - auto left = typeid_cast(tables->children[0].get()); - auto right = typeid_cast(tables->children[1].get()); - if (!left || !right || !right->table_join) - return {}; - - if (auto join = typeid_cast(right->table_join.get())) + joined_tables.reserve(num_tables); + for (auto & child : tables->children) { - if (join->kind == ASTTableJoin::Kind::Cross || - join->kind == ASTTableJoin::Kind::Comma) - { - if (!join->children.empty()) - throw Exception("Logical error: CROSS JOIN has expressions", ErrorCodes::LOGICAL_ERROR); + joined_tables.emplace_back(JoinedTable(child)); + JoinedTable & t = joined_tables.back(); + if (t.array_join) + return false; - auto & left_expr = typeid_cast(*left->table_expression); - auto & right_expr = typeid_cast(*right->table_expression); + if (num_tables > 2 && t.has_using) + throw Exception("Multiple CROSS/COMMA JOIN do not support USING", ErrorCodes::NOT_IMPLEMENTED); - table_names.reserve(2); - if (extractTableName(left_expr, table_names) && - extractTableName(right_expr, table_names)) - return right->table_join; - } + if (ASTTableJoin * join = t.join) + if (join->kind == ASTTableJoin::Kind::Comma) + ++num_comma; } + return true; +} - return {}; } @@ -187,40 +248,47 @@ void CrossToInnerJoinMatcher::visit(ASTPtr & ast, Data & data) visit(*t, ast, data); } -void CrossToInnerJoinMatcher::visit(ASTSelectQuery & select, ASTPtr & ast, Data & data) +void CrossToInnerJoinMatcher::visit(ASTSelectQuery & select, ASTPtr &, Data & data) { - using CheckExpressionMatcher = OneTypeMatcher; - using CheckExpressionVisitor = InDepthNodeVisitor; + size_t num_comma = 0; + std::vector joined_tables; + if (!getTables(select, joined_tables, num_comma)) + return; + + /// COMMA to CROSS + + if (num_comma) + { + if (num_comma != (joined_tables.size() - 1)) + throw Exception("Mix of COMMA and other JOINS is not supported", ErrorCodes::NOT_IMPLEMENTED); + + for (auto & table : joined_tables) + table.rewriteCommaToCross(); + } + + /// CROSS to INNER if (!select.where_expression) return; - std::vector table_names; - ASTPtr ast_join = getCrossJoin(select, table_names); - if (!ast_join) - return; - - CheckExpressionVisitor::Data visitor_data{table_names}; + CheckExpressionVisitor::Data visitor_data{joined_tables}; CheckExpressionVisitor(visitor_data).visit(select.where_expression); - if (visitor_data.matchAny()) + if (visitor_data.complex()) + return; + + for (size_t i = 1; i < joined_tables.size(); ++i) { - auto & join = typeid_cast(*ast_join); - join.kind = ASTTableJoin::Kind::Inner; - join.strictness = ASTTableJoin::Strictness::All; + if (visitor_data.matchAny(i)) + { + ASTTableJoin & join = *joined_tables[i].join; + join.kind = ASTTableJoin::Kind::Inner; + join.strictness = ASTTableJoin::Strictness::All; - if (visitor_data.canReuseWhere()) - join.on_expression.swap(select.where_expression); - else - join.on_expression = visitor_data.makeOnExpression(); - - if (visitor_data.matchAll()) - select.where_expression.reset(); - - join.children.push_back(join.on_expression); - - ast = ast->clone(); /// rewrite AST in right manner - data.done = true; + join.on_expression = visitor_data.makeOnExpression(i); + join.children.push_back(join.on_expression); + data.done = true; + } } } diff --git a/dbms/src/Interpreters/DDLWorker.cpp b/dbms/src/Interpreters/DDLWorker.cpp index a0777422e87..35639bf213b 100644 --- a/dbms/src/Interpreters/DDLWorker.cpp +++ b/dbms/src/Interpreters/DDLWorker.cpp @@ -224,7 +224,7 @@ DDLWorker::DDLWorker(const std::string & zk_root_dir, Context & context_, const { task_max_lifetime = config->getUInt64(prefix + ".task_max_lifetime", static_cast(task_max_lifetime)); cleanup_delay_period = config->getUInt64(prefix + ".cleanup_delay_period", static_cast(cleanup_delay_period)); - max_tasks_in_queue = std::max(static_cast(1), config->getUInt64(prefix + ".max_tasks_in_queue", max_tasks_in_queue)); + max_tasks_in_queue = std::max(1, config->getUInt64(prefix + ".max_tasks_in_queue", max_tasks_in_queue)); if (config->has(prefix + ".profile")) context.setSetting("profile", config->getString(prefix + ".profile")); diff --git a/dbms/src/Interpreters/ExpressionActions.cpp b/dbms/src/Interpreters/ExpressionActions.cpp index 08bb663a75b..21477badac2 100644 --- a/dbms/src/Interpreters/ExpressionActions.cpp +++ b/dbms/src/Interpreters/ExpressionActions.cpp @@ -1112,7 +1112,7 @@ void ExpressionActions::optimizeArrayJoin() BlockInputStreamPtr ExpressionActions::createStreamWithNonJoinedDataIfFullOrRightJoin(const Block & source_header, UInt64 max_block_size) const { for (const auto & action : actions) - if (action.join && (action.join->getKind() == ASTTableJoin::Kind::Full || action.join->getKind() == ASTTableJoin::Kind::Right)) + if (action.join && isRightOrFull(action.join->getKind())) return action.join->createStreamWithNonJoinedRows( source_header, action.join_key_names_left, action.columns_added_by_join, max_block_size); diff --git a/dbms/src/Interpreters/ExternalDictionaries.cpp b/dbms/src/Interpreters/ExternalDictionaries.cpp index ba0a3a9e5fd..719b13670fa 100644 --- a/dbms/src/Interpreters/ExternalDictionaries.cpp +++ b/dbms/src/Interpreters/ExternalDictionaries.cpp @@ -26,17 +26,19 @@ namespace } +/// Must not acquire Context lock in constructor to avoid possibility of deadlocks. ExternalDictionaries::ExternalDictionaries( std::unique_ptr config_repository, + const Poco::Util::AbstractConfiguration & config, Context & context, bool throw_on_error) - : ExternalLoader(context.getConfigRef(), + : ExternalLoader(config, externalDictionariesUpdateSettings, getExternalDictionariesConfigSettings(), std::move(config_repository), &Logger::get("ExternalDictionaries"), "external dictionary"), - context(context) + context(context) { init(throw_on_error); } diff --git a/dbms/src/Interpreters/ExternalDictionaries.h b/dbms/src/Interpreters/ExternalDictionaries.h index 0b00ab9f632..a5581dd2237 100644 --- a/dbms/src/Interpreters/ExternalDictionaries.h +++ b/dbms/src/Interpreters/ExternalDictionaries.h @@ -20,6 +20,7 @@ public: /// Dictionaries will be loaded immediately and then will be updated in separate thread, each 'reload_period' seconds. ExternalDictionaries( std::unique_ptr config_repository, + const Poco::Util::AbstractConfiguration & config, Context & context, bool throw_on_error); diff --git a/dbms/src/Interpreters/ExternalLoader.cpp b/dbms/src/Interpreters/ExternalLoader.cpp index 4e0f12ea254..089bda1752c 100644 --- a/dbms/src/Interpreters/ExternalLoader.cpp +++ b/dbms/src/Interpreters/ExternalLoader.cpp @@ -155,14 +155,6 @@ void ExternalLoader::reloadAndUpdate(bool throw_on_error) /// periodic update std::vector> objects_to_update; - auto getNextUpdateTime = [this](const LoadablePtr & current) - { - /// calculate next update time - const auto & lifetime = current->getLifetime(); - std::uniform_int_distribution distribution{lifetime.min_sec, lifetime.max_sec}; - return std::chrono::system_clock::now() + std::chrono::seconds{distribution(rnd_engine)}; - }; - /// Collect objects that needs to be updated under lock. Then create new versions without lock, and assign under lock. { std::lock_guard lock{map_mutex}; @@ -185,18 +177,12 @@ void ExternalLoader::reloadAndUpdate(bool throw_on_error) if (!current->supportUpdates()) continue; - auto & update_time = update_times[current->getName()]; + auto update_time = update_times[current->getName()]; /// check that timeout has passed if (std::chrono::system_clock::now() < update_time) continue; - if (!current->isModified()) - { - update_time = getNextUpdateTime(current); - continue; - } - objects_to_update.emplace_back(loadable_object.first, current); } catch (...) @@ -209,6 +195,14 @@ void ExternalLoader::reloadAndUpdate(bool throw_on_error) } } + auto getNextUpdateTime = [this](const LoadablePtr & current) + { + /// Calculate next update time. + const auto & lifetime = current->getLifetime(); + std::uniform_int_distribution distribution{lifetime.min_sec, lifetime.max_sec}; + return std::chrono::system_clock::now() + std::chrono::seconds{distribution(rnd_engine)}; + }; + for (auto & [name, current] : objects_to_update) { LoadablePtr new_version; @@ -216,9 +210,12 @@ void ExternalLoader::reloadAndUpdate(bool throw_on_error) try { - /// create new version of loadable object - new_version = current->clone(); - exception = new_version->getCreationException(); + if (current->isModified()) + { + /// Create new version of loadable object. + new_version = current->clone(); + exception = new_version->getCreationException(); + } } catch (...) { @@ -235,8 +232,12 @@ void ExternalLoader::reloadAndUpdate(bool throw_on_error) it->second.exception = exception; if (!exception) { - it->second.loadable.reset(); - it->second.loadable = std::move(new_version); + /// If the dictionary is not modified - leave old version. + if (new_version) + { + it->second.loadable.reset(); + it->second.loadable = std::move(new_version); + } } else { diff --git a/dbms/src/Interpreters/ExternalLoader.h b/dbms/src/Interpreters/ExternalLoader.h index c2ce161f0e1..765ee1bbfdb 100644 --- a/dbms/src/Interpreters/ExternalLoader.h +++ b/dbms/src/Interpreters/ExternalLoader.h @@ -19,8 +19,6 @@ namespace DB { -class Context; - struct ExternalLoaderUpdateSettings { UInt64 check_period_sec = 5; diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.cpp b/dbms/src/Interpreters/InterpreterAlterQuery.cpp index c28c8aef32f..c80001dc2fc 100644 --- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp +++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp @@ -74,8 +74,9 @@ BlockIO InterpreterAlterQuery::execute() if (!alter_commands.empty()) { + auto table_lock_holder = table->lockAlterIntention(context.getCurrentQueryId()); alter_commands.validate(*table, context); - table->alter(alter_commands, database_name, table_name, context); + table->alter(alter_commands, database_name, table_name, context, table_lock_holder); } return {}; diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index 708bd616828..e4eb78c2b01 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -45,6 +45,7 @@ #include #include +#include namespace DB @@ -587,11 +588,11 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) String as_table_name = create.as_table; StoragePtr as_storage; - TableStructureReadLockPtr as_storage_lock; + TableStructureReadLockHolder as_storage_lock; if (!as_table_name.empty()) { as_storage = context.getTable(as_database_name, as_table_name); - as_storage_lock = as_storage->lockStructure(false, context.getCurrentQueryId()); + as_storage_lock = as_storage->lockStructureForShare(false, context.getCurrentQueryId()); } /// Set and retrieve list of columns. @@ -623,6 +624,17 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) { if (create.if_not_exists) return {}; + else if (create.replace_view) + { + /// when executing CREATE OR REPLACE VIEW, drop current existing view + auto drop_ast = std::make_shared(); + drop_ast->database = database_name; + drop_ast->table = table_name; + drop_ast->no_ddl_lock = true; + + InterpreterDropQuery interpreter(drop_ast, context); + interpreter.execute(); + } else throw Exception("Table " + database_name + "." + table_name + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS); } diff --git a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp index 19cb167168e..dfca483f7ad 100644 --- a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp @@ -93,7 +93,7 @@ BlockInputStreamPtr InterpreterDescribeQuery::executeImpl() table = context.getTable(database_name, table_name); } - auto table_lock = table->lockStructure(false, context.getCurrentQueryId()); + auto table_lock = table->lockStructureForShare(false, context.getCurrentQueryId()); columns = table->getColumns().getAll(); column_defaults = table->getColumns().defaults; column_comments = table->getColumns().comments; diff --git a/dbms/src/Interpreters/InterpreterDropQuery.cpp b/dbms/src/Interpreters/InterpreterDropQuery.cpp index c90017721c1..cc422c445fc 100644 --- a/dbms/src/Interpreters/InterpreterDropQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDropQuery.cpp @@ -39,7 +39,7 @@ BlockIO InterpreterDropQuery::execute() return executeDDLQueryOnCluster(query_ptr, context, {drop.database}); if (!drop.table.empty()) - return executeToTable(drop.database, drop.table, drop.kind, drop.if_exists, drop.temporary); + return executeToTable(drop.database, drop.table, drop.kind, drop.if_exists, drop.temporary, drop.no_ddl_lock); else if (!drop.database.empty()) return executeToDatabase(drop.database, drop.kind, drop.if_exists); else @@ -47,7 +47,7 @@ BlockIO InterpreterDropQuery::execute() } -BlockIO InterpreterDropQuery::executeToTable(String & database_name_, String & table_name, ASTDropQuery::Kind kind, bool if_exists, bool if_temporary) +BlockIO InterpreterDropQuery::executeToTable(String & database_name_, String & table_name, ASTDropQuery::Kind kind, bool if_exists, bool if_temporary, bool no_ddl_lock) { if (if_temporary || database_name_.empty()) { @@ -59,7 +59,7 @@ BlockIO InterpreterDropQuery::executeToTable(String & database_name_, String & t String database_name = database_name_.empty() ? context.getCurrentDatabase() : database_name_; - auto ddl_guard = context.getDDLGuard(database_name, table_name); + auto ddl_guard = (!no_ddl_lock ? context.getDDLGuard(database_name, table_name) : nullptr); DatabaseAndTable database_and_table = tryGetDatabaseAndTable(database_name, table_name, if_exists); @@ -69,7 +69,7 @@ BlockIO InterpreterDropQuery::executeToTable(String & database_name_, String & t { database_and_table.second->shutdown(); /// If table was already dropped by anyone, an exception will be thrown - auto table_lock = database_and_table.second->lockForAlter(context.getCurrentQueryId()); + auto table_lock = database_and_table.second->lockExclusively(context.getCurrentQueryId()); /// Drop table from memory, don't touch data and metadata database_and_table.first->detachTable(database_and_table.second->getTableName()); } @@ -78,7 +78,7 @@ BlockIO InterpreterDropQuery::executeToTable(String & database_name_, String & t database_and_table.second->checkTableCanBeDropped(); /// If table was already dropped by anyone, an exception will be thrown - auto table_lock = database_and_table.second->lockForAlter(context.getCurrentQueryId()); + auto table_lock = database_and_table.second->lockExclusively(context.getCurrentQueryId()); /// Drop table data, don't touch metadata database_and_table.second->truncate(query_ptr, context); } @@ -89,7 +89,7 @@ BlockIO InterpreterDropQuery::executeToTable(String & database_name_, String & t database_and_table.second->shutdown(); /// If table was already dropped by anyone, an exception will be thrown - auto table_lock = database_and_table.second->lockForAlter(context.getCurrentQueryId()); + auto table_lock = database_and_table.second->lockExclusively(context.getCurrentQueryId()); /// Delete table metadata and table itself from memory database_and_table.first->removeTable(context, database_and_table.second->getTableName()); @@ -126,7 +126,7 @@ BlockIO InterpreterDropQuery::executeToTemporaryTable(String & table_name, ASTDr if (kind == ASTDropQuery::Kind::Truncate) { /// If table was already dropped by anyone, an exception will be thrown - auto table_lock = table->lockForAlter(context.getCurrentQueryId()); + auto table_lock = table->lockExclusively(context.getCurrentQueryId()); /// Drop table data, don't touch metadata table->truncate(query_ptr, context); } @@ -135,7 +135,7 @@ BlockIO InterpreterDropQuery::executeToTemporaryTable(String & table_name, ASTDr context_handle.tryRemoveExternalTable(table_name); table->shutdown(); /// If table was already dropped by anyone, an exception will be thrown - auto table_lock = table->lockForAlter(context.getCurrentQueryId()); + auto table_lock = table->lockExclusively(context.getCurrentQueryId()); /// Delete table data table->drop(); table->is_dropped = true; @@ -166,7 +166,7 @@ BlockIO InterpreterDropQuery::executeToDatabase(String & database_name, ASTDropQ for (auto iterator = database->getIterator(context); iterator->isValid(); iterator->next()) { String current_table_name = iterator->table()->getTableName(); - executeToTable(database_name, current_table_name, kind, false, false); + executeToTable(database_name, current_table_name, kind, false, false, false); } auto context_lock = context.getLock(); diff --git a/dbms/src/Interpreters/InterpreterDropQuery.h b/dbms/src/Interpreters/InterpreterDropQuery.h index 2bea9f85c99..986c73f8465 100644 --- a/dbms/src/Interpreters/InterpreterDropQuery.h +++ b/dbms/src/Interpreters/InterpreterDropQuery.h @@ -32,7 +32,7 @@ private: BlockIO executeToDatabase(String & database_name, ASTDropQuery::Kind kind, bool if_exists); - BlockIO executeToTable(String & database_name, String & table_name, ASTDropQuery::Kind kind, bool if_exists, bool if_temporary); + BlockIO executeToTable(String & database_name, String & table_name, ASTDropQuery::Kind kind, bool if_exists, bool if_temporary, bool no_ddl_lock); DatabasePtr tryGetDatabase(String & database_name, bool exists); diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index 00cf8e925cd..d5c2600eda4 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -96,7 +96,7 @@ BlockIO InterpreterInsertQuery::execute() checkAccess(query); StoragePtr table = getTable(query); - auto table_lock = table->lockStructure(true, context.getCurrentQueryId()); + auto table_lock = table->lockStructureForShare(true, context.getCurrentQueryId()); /// We create a pipeline of several streams, into which we will write data. BlockOutputStreamPtr out; diff --git a/dbms/src/Interpreters/InterpreterOptimizeQuery.cpp b/dbms/src/Interpreters/InterpreterOptimizeQuery.cpp index 78978794386..47e77172eae 100644 --- a/dbms/src/Interpreters/InterpreterOptimizeQuery.cpp +++ b/dbms/src/Interpreters/InterpreterOptimizeQuery.cpp @@ -23,7 +23,7 @@ BlockIO InterpreterOptimizeQuery::execute() return executeDDLQueryOnCluster(query_ptr, context, {ast.database}); StoragePtr table = context.getTable(ast.database, ast.table); - auto table_lock = table->lockStructure(true, context.getCurrentQueryId()); + auto table_lock = table->lockStructureForShare(true, context.getCurrentQueryId()); table->optimize(query_ptr, ast.partition, ast.final, ast.deduplicate, context); return {}; } diff --git a/dbms/src/Interpreters/InterpreterRenameQuery.cpp b/dbms/src/Interpreters/InterpreterRenameQuery.cpp index e3fbe1f1de8..77a0c862905 100644 --- a/dbms/src/Interpreters/InterpreterRenameQuery.cpp +++ b/dbms/src/Interpreters/InterpreterRenameQuery.cpp @@ -96,12 +96,12 @@ BlockIO InterpreterRenameQuery::execute() table_guards.emplace(to, context.getDDLGuard(to.database_name, to.table_name)); } - std::vector locks; + std::vector locks; locks.reserve(unique_tables_from.size()); for (const auto & names : unique_tables_from) if (auto table = context.tryGetTable(names.database_name, names.table_name)) - locks.emplace_back(table->lockForAlter(context.getCurrentQueryId())); + locks.emplace_back(table->lockExclusively(context.getCurrentQueryId())); /** All tables are locked. If there are more than one rename in chain, * we need to hold global lock while doing all renames. Order matters to avoid deadlocks. diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index 2b07ee9c881..83d53ca37b8 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -38,6 +38,8 @@ #include #include #include +#include +#include #include #include @@ -139,7 +141,6 @@ InterpreterSelectQuery::InterpreterSelectQuery( bool modify_inplace) /// NOTE: the query almost always should be cloned because it will be modified during analysis. : query_ptr(modify_inplace ? query_ptr_ : query_ptr_->clone()) - , query(typeid_cast(*query_ptr)) , context(context_) , to_stage(to_stage_) , subquery_depth(subquery_depth_) @@ -155,7 +156,20 @@ InterpreterSelectQuery::InterpreterSelectQuery( throw Exception("Too deep subqueries. Maximum: " + settings.max_subquery_depth.toString(), ErrorCodes::TOO_DEEP_SUBQUERIES); + if (settings.allow_experimental_cross_to_join_conversion) + { + CrossToInnerJoinVisitor::Data cross_to_inner; + CrossToInnerJoinVisitor(cross_to_inner).visit(query_ptr); + } + + if (settings.allow_experimental_multiple_joins_emulation) + { + JoinToSubqueryTransformVisitor::Data join_to_subs_data; + JoinToSubqueryTransformVisitor(join_to_subs_data).visit(query_ptr); + } + max_streams = settings.max_threads; + ASTSelectQuery & query = selectQuery(); ASTPtr table_expression = extractTableExpression(query, 0); @@ -200,7 +214,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( } if (storage) - table_lock = storage->lockStructure(false, context.getCurrentQueryId()); + table_lock = storage->lockStructureForShare(false, context.getCurrentQueryId()); syntax_analyzer_result = SyntaxAnalyzer(context, subquery_depth).analyze( query_ptr, source_header.getNamesAndTypesList(), required_result_column_names, storage); @@ -263,9 +277,15 @@ InterpreterSelectQuery::InterpreterSelectQuery( } +ASTSelectQuery & InterpreterSelectQuery::selectQuery() +{ + return typeid_cast(*query_ptr); +} + + void InterpreterSelectQuery::getDatabaseAndTableNames(String & database_name, String & table_name) { - if (auto db_and_table = getDatabaseAndTable(query, 0)) + if (auto db_and_table = getDatabaseAndTable(selectQuery(), 0)) { table_name = db_and_table->table; database_name = db_and_table->database; @@ -364,6 +384,7 @@ InterpreterSelectQuery::AnalysisResult InterpreterSelectQuery::analyzeExpression { ExpressionActionsChain chain(context); + ASTSelectQuery & query = selectQuery(); Names additional_required_columns_after_prewhere; @@ -487,6 +508,7 @@ void InterpreterSelectQuery::executeImpl(Pipeline & pipeline, const BlockInputSt * then perform the remaining operations with one resulting stream. */ + ASTSelectQuery & query = selectQuery(); const Settings & settings = context.getSettingsRef(); QueryProcessingStage::Enum from_stage = QueryProcessingStage::FetchColumns; @@ -569,7 +591,7 @@ void InterpreterSelectQuery::executeImpl(Pipeline & pipeline, const BlockInputSt if (expressions.hasJoin()) { const ASTTableJoin & join = static_cast(*query.join()->table_join); - if (join.kind == ASTTableJoin::Kind::Full || join.kind == ASTTableJoin::Kind::Right) + if (isRightOrFull(join.kind)) pipeline.stream_with_non_joined_data = expressions.before_join->createStreamWithNonJoinedDataIfFullOrRightJoin( pipeline.firstStream()->getHeader(), settings.max_block_size); @@ -780,6 +802,7 @@ void InterpreterSelectQuery::executeFetchColumns( QueryProcessingStage::Enum processing_stage, Pipeline & pipeline, const PrewhereInfoPtr & prewhere_info, const Names & columns_to_remove_after_prewhere) { + ASTSelectQuery & query = selectQuery(); const Settings & settings = context.getSettingsRef(); /// Actions to calculate ALIAS if required. @@ -1074,7 +1097,7 @@ void InterpreterSelectQuery::executeWhere(Pipeline & pipeline, const ExpressionA { pipeline.transform([&](auto & stream) { - stream = std::make_shared(stream, expression, query.where_expression->getColumnName(), remove_fiter); + stream = std::make_shared(stream, expression, selectQuery().where_expression->getColumnName(), remove_fiter); }); } @@ -1202,7 +1225,7 @@ void InterpreterSelectQuery::executeHaving(Pipeline & pipeline, const Expression { pipeline.transform([&](auto & stream) { - stream = std::make_shared(stream, expression, query.having_expression->getColumnName()); + stream = std::make_shared(stream, expression, selectQuery().having_expression->getColumnName()); }); } @@ -1215,7 +1238,7 @@ void InterpreterSelectQuery::executeTotalsAndHaving(Pipeline & pipeline, bool ha pipeline.firstStream() = std::make_shared( pipeline.firstStream(), overflow_row, expression, - has_having ? query.having_expression->getColumnName() : "", settings.totals_mode, settings.totals_auto_threshold, final); + has_having ? selectQuery().having_expression->getColumnName() : "", settings.totals_mode, settings.totals_auto_threshold, final); } void InterpreterSelectQuery::executeRollupOrCube(Pipeline & pipeline, Modificator modificator) @@ -1280,6 +1303,7 @@ static SortDescription getSortDescription(ASTSelectQuery & query) void InterpreterSelectQuery::executeOrder(Pipeline & pipeline) { + ASTSelectQuery & query = selectQuery(); SortDescription order_descr = getSortDescription(query); UInt64 limit = getLimitForSorting(query, context); @@ -1311,6 +1335,7 @@ void InterpreterSelectQuery::executeOrder(Pipeline & pipeline) void InterpreterSelectQuery::executeMergeSorted(Pipeline & pipeline) { + ASTSelectQuery & query = selectQuery(); SortDescription order_descr = getSortDescription(query); UInt64 limit = getLimitForSorting(query, context); @@ -1347,6 +1372,7 @@ void InterpreterSelectQuery::executeProjection(Pipeline & pipeline, const Expres void InterpreterSelectQuery::executeDistinct(Pipeline & pipeline, bool before_order, Names columns) { + ASTSelectQuery & query = selectQuery(); if (query.distinct) { const Settings & settings = context.getSettingsRef(); @@ -1389,6 +1415,7 @@ void InterpreterSelectQuery::executeUnion(Pipeline & pipeline) /// Preliminary LIMIT - is used in every source, if there are several sources, before they are combined. void InterpreterSelectQuery::executePreLimit(Pipeline & pipeline) { + ASTSelectQuery & query = selectQuery(); /// If there is LIMIT if (query.limit_length) { @@ -1403,6 +1430,7 @@ void InterpreterSelectQuery::executePreLimit(Pipeline & pipeline) void InterpreterSelectQuery::executeLimitBy(Pipeline & pipeline) { + ASTSelectQuery & query = selectQuery(); if (!query.limit_by_value || !query.limit_by_expression_list) return; @@ -1444,6 +1472,7 @@ bool hasWithTotalsInAnySubqueryInFromClause(const ASTSelectQuery & query) void InterpreterSelectQuery::executeLimit(Pipeline & pipeline) { + ASTSelectQuery & query = selectQuery(); /// If there is LIMIT if (query.limit_length) { @@ -1515,12 +1544,13 @@ void InterpreterSelectQuery::unifyStreams(Pipeline & pipeline) void InterpreterSelectQuery::ignoreWithTotals() { - query.group_by_with_totals = false; + selectQuery().group_by_with_totals = false; } void InterpreterSelectQuery::initSettings() { + ASTSelectQuery & query = selectQuery(); if (query.settings) InterpreterSetQuery(query.settings, context).executeForCurrentContext(); } diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.h b/dbms/src/Interpreters/InterpreterSelectQuery.h index de5a11e727b..89fdc35eb7b 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.h +++ b/dbms/src/Interpreters/InterpreterSelectQuery.h @@ -133,6 +133,7 @@ private: } }; + ASTSelectQuery & selectQuery(); void executeImpl(Pipeline & pipeline, const BlockInputStreamPtr & prepared_input, bool dry_run); @@ -222,7 +223,6 @@ private: void initSettings(); ASTPtr query_ptr; - ASTSelectQuery & query; Context context; QueryProcessingStage::Enum to_stage; size_t subquery_depth = 0; @@ -248,7 +248,7 @@ private: /// Table from where to read data, if not subquery. StoragePtr storage; - TableStructureReadLockPtr table_lock; + TableStructureReadLockHolder table_lock; /// Used when we read from prepared input, not table or subquery. BlockInputStreamPtr input; diff --git a/dbms/src/Interpreters/InterpreterSystemQuery.cpp b/dbms/src/Interpreters/InterpreterSystemQuery.cpp index e1233a04180..20bd860fb26 100644 --- a/dbms/src/Interpreters/InterpreterSystemQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSystemQuery.cpp @@ -239,7 +239,7 @@ StoragePtr InterpreterSystemQuery::tryRestartReplica(const String & database_nam table->shutdown(); /// If table was already dropped by anyone, an exception will be thrown - auto table_lock = table->lockForAlter(context.getCurrentQueryId()); + auto table_lock = table->lockExclusively(context.getCurrentQueryId()); create_ast = system_context.getCreateTableQuery(database_name, table_name); database->detachTable(table_name); diff --git a/dbms/src/Interpreters/Join.cpp b/dbms/src/Interpreters/Join.cpp index 7b4b1783fa6..7ce39e12a00 100644 --- a/dbms/src/Interpreters/Join.cpp +++ b/dbms/src/Interpreters/Join.cpp @@ -220,13 +220,6 @@ struct KeyGetterForType }; -/// Do I need to use the hash table maps_*_full, in which we remember whether the row was joined. -static bool getFullness(ASTTableJoin::Kind kind) -{ - return kind == ASTTableJoin::Kind::Right || kind == ASTTableJoin::Kind::Full; -} - - void Join::init(Type type_) { type = type_; @@ -340,7 +333,7 @@ void Join::setSampleBlock(const Block & block) } /// In case of LEFT and FULL joins, if use_nulls, convert joined columns to Nullable. - if (use_nulls && (kind == ASTTableJoin::Kind::Left || kind == ASTTableJoin::Kind::Full)) + if (use_nulls && isLeftOrFull(kind)) for (size_t i = 0; i < num_columns_to_add; ++i) convertColumnToNullable(sample_block_with_columns_to_add.getByPosition(i)); } @@ -476,7 +469,7 @@ bool Join::insertFromBlock(const Block & block) blocks.push_back(block); Block * stored_block = &blocks.back(); - if (getFullness(kind)) + if (isRightOrFull(kind)) { /** Move the key columns to the beginning of the block. * This is where NonJoinedBlockInputStream will expect. @@ -511,9 +504,9 @@ bool Join::insertFromBlock(const Block & block) stored_block->safeGetByPosition(i).column = stored_block->safeGetByPosition(i).column->convertToFullColumnIfConst(); /// In case of LEFT and FULL joins, if use_nulls, convert joined columns to Nullable. - if (use_nulls && (kind == ASTTableJoin::Kind::Left || kind == ASTTableJoin::Kind::Full)) + if (use_nulls && isLeftOrFull(kind)) { - for (size_t i = getFullness(kind) ? keys_size : 0; i < size; ++i) + for (size_t i = isFull(kind) ? keys_size : 0; i < size; ++i) { convertColumnToNullable(stored_block->getByPosition(i)); } @@ -721,7 +714,7 @@ void Join::joinBlockImpl( * Because if they are constants, then in the "not joined" rows, they may have different values * - default values, which can differ from the values of these constants. */ - if (getFullness(kind)) + if (isRightOrFull(kind)) { for (size_t i = 0; i < existing_columns; ++i) { @@ -741,7 +734,7 @@ void Join::joinBlockImpl( * but they will not be used at this stage of joining (and will be in `AdderNonJoined`), and they need to be skipped. */ size_t num_columns_to_skip = 0; - if (getFullness(kind)) + if (isRightOrFull(kind)) num_columns_to_skip = keys_size; /// Add new columns to the block. @@ -798,7 +791,7 @@ void Join::joinBlockImpl( if (strictness == ASTTableJoin::Strictness::Any) { - if (kind == ASTTableJoin::Kind::Inner || kind == ASTTableJoin::Kind::Right) + if (isInnerOrRight(kind)) { /// If ANY INNER | RIGHT JOIN - filter all the columns except the new ones. for (size_t i = 0; i < existing_columns; ++i) diff --git a/dbms/src/Interpreters/JoinToSubqueryTransformVisitor.cpp b/dbms/src/Interpreters/JoinToSubqueryTransformVisitor.cpp index 5a1f7260a4f..1a110a6c8e0 100644 --- a/dbms/src/Interpreters/JoinToSubqueryTransformVisitor.cpp +++ b/dbms/src/Interpreters/JoinToSubqueryTransformVisitor.cpp @@ -210,13 +210,13 @@ bool needRewrite(ASTSelectQuery & select) if (!table || !table->table_join) throw Exception("Multiple JOIN expects joined tables", ErrorCodes::LOGICAL_ERROR); - auto join = typeid_cast(table->table_join.get()); - if (join->kind == ASTTableJoin::Kind::Comma) - throw Exception("Multiple COMMA JOIN is not supported", ErrorCodes::NOT_IMPLEMENTED); + auto join = typeid_cast(*table->table_join); + if (isComma(join.kind)) + throw Exception("COMMA to CROSS JOIN rewriter is not enabled or cannot rewrite query", ErrorCodes::NOT_IMPLEMENTED); /// it's not trivial to support mix of JOIN ON & JOIN USING cause of short names - if (!join || !join->on_expression) - throw Exception("Multiple JOIN expects JOIN with ON section", ErrorCodes::NOT_IMPLEMENTED); + if (join.using_expression_list) + throw Exception("Multiple JOIN does not support USING", ErrorCodes::NOT_IMPLEMENTED); } return true; @@ -264,8 +264,9 @@ void JoinToSubqueryTransformMatcher::visit(ASTSelectQuery & select, ASTPtr &, Da auto table = typeid_cast(child.get()); if (table->table_join) { - auto * join = typeid_cast(table->table_join.get()); - ColumnAliasesVisitor(aliases_data).visit(join->on_expression); + auto & join = typeid_cast(*table->table_join); + if (join.on_expression) + ColumnAliasesVisitor(aliases_data).visit(join.on_expression); } } diff --git a/dbms/src/Interpreters/PredicateExpressionsOptimizer.h b/dbms/src/Interpreters/PredicateExpressionsOptimizer.h index 9281247dd4e..0492600ca85 100644 --- a/dbms/src/Interpreters/PredicateExpressionsOptimizer.h +++ b/dbms/src/Interpreters/PredicateExpressionsOptimizer.h @@ -1,7 +1,7 @@ #pragma once -#include - +#include "DatabaseAndTableWithAlias.h" +#include "ExpressionAnalyzer.h" #include namespace DB diff --git a/dbms/src/Interpreters/Set.h b/dbms/src/Interpreters/Set.h index 00e5113c574..ab6299a5891 100644 --- a/dbms/src/Interpreters/Set.h +++ b/dbms/src/Interpreters/Set.h @@ -192,4 +192,4 @@ private: std::vector indexes_mapping; }; - } +} diff --git a/dbms/src/Interpreters/SyntaxAnalyzer.cpp b/dbms/src/Interpreters/SyntaxAnalyzer.cpp index 6d274e326b4..bc808c3d37f 100644 --- a/dbms/src/Interpreters/SyntaxAnalyzer.cpp +++ b/dbms/src/Interpreters/SyntaxAnalyzer.cpp @@ -573,7 +573,7 @@ void collectJoinedColumns(AnalyzedJoin & analyzed_join, const ASTSelectQuery & s else if (table_join.on_expression) collectJoinedColumnsFromJoinOnExpr(analyzed_join, table_join); - bool make_nullable = join_use_nulls && (table_join.kind == ASTTableJoin::Kind::Left || table_join.kind == ASTTableJoin::Kind::Full); + bool make_nullable = join_use_nulls && isLeftOrFull(table_join.kind); analyzed_join.calculateAvailableJoinedColumns(make_nullable); } diff --git a/dbms/src/Interpreters/ThreadStatusExt.cpp b/dbms/src/Interpreters/ThreadStatusExt.cpp index 4917f90851e..96ab18fde04 100644 --- a/dbms/src/Interpreters/ThreadStatusExt.cpp +++ b/dbms/src/Interpreters/ThreadStatusExt.cpp @@ -37,6 +37,8 @@ const std::string & ThreadStatus::getQueryId() const void CurrentThread::defaultThreadDeleter() { + if (unlikely(!current_thread)) + return; ThreadStatus & thread = CurrentThread::get(); thread.detachQuery(true, true); } @@ -191,44 +193,63 @@ void ThreadStatus::logToQueryThreadLog(QueryThreadLog & thread_log) void CurrentThread::initializeQuery() { + if (unlikely(!current_thread)) + return; get().initializeQuery(); get().deleter = CurrentThread::defaultThreadDeleter; } void CurrentThread::attachTo(const ThreadGroupStatusPtr & thread_group) { + if (unlikely(!current_thread)) + return; get().attachQuery(thread_group, true); get().deleter = CurrentThread::defaultThreadDeleter; } void CurrentThread::attachToIfDetached(const ThreadGroupStatusPtr & thread_group) { + if (unlikely(!current_thread)) + return; get().attachQuery(thread_group, false); get().deleter = CurrentThread::defaultThreadDeleter; } const std::string & CurrentThread::getQueryId() { + if (unlikely(!current_thread)) + { + const static std::string empty; + return empty; + } return get().getQueryId(); } void CurrentThread::attachQueryContext(Context & query_context) { + if (unlikely(!current_thread)) + return; return get().attachQueryContext(query_context); } void CurrentThread::finalizePerformanceCounters() { + if (unlikely(!current_thread)) + return; get().finalizePerformanceCounters(); } void CurrentThread::detachQuery() { + if (unlikely(!current_thread)) + return; get().detachQuery(false); } void CurrentThread::detachQueryIfNotDetached() { + if (unlikely(!current_thread)) + return; get().detachQuery(true); } @@ -241,8 +262,12 @@ CurrentThread::QueryScope::QueryScope(Context & query_context) void CurrentThread::QueryScope::logPeakMemoryUsage() { + auto group = CurrentThread::getGroup(); + if (!group) + return; + log_peak_memory_usage_in_destructor = false; - CurrentThread::getGroup()->memory_tracker.logPeakMemoryUsage(); + group->memory_tracker.logPeakMemoryUsage(); } CurrentThread::QueryScope::~QueryScope() diff --git a/dbms/src/Interpreters/executeQuery.cpp b/dbms/src/Interpreters/executeQuery.cpp index bba4202e7c0..069c5c67abc 100644 --- a/dbms/src/Interpreters/executeQuery.cpp +++ b/dbms/src/Interpreters/executeQuery.cpp @@ -21,8 +21,6 @@ #include #include -#include -#include #include #include #include @@ -200,22 +198,6 @@ static std::tuple executeQueryImpl( { logQuery(query.substr(0, settings.log_queries_cut_to_length), context, internal); - if (!internal && settings.allow_experimental_multiple_joins_emulation) - { - JoinToSubqueryTransformVisitor::Data join_to_subs_data; - JoinToSubqueryTransformVisitor(join_to_subs_data).visit(ast); - if (join_to_subs_data.done) - logQuery(queryToString(*ast), context, internal); - } - - if (!internal && settings.allow_experimental_cross_to_join_conversion) - { - CrossToInnerJoinVisitor::Data cross_to_inner; - CrossToInnerJoinVisitor(cross_to_inner).visit(ast); - if (cross_to_inner.done) - logQuery(queryToString(*ast), context, internal); - } - /// Check the limits. checkASTSizeLimits(*ast, settings); diff --git a/dbms/src/Parsers/ASTCreateQuery.h b/dbms/src/Parsers/ASTCreateQuery.h index 6be7451529d..5b6d7398452 100644 --- a/dbms/src/Parsers/ASTCreateQuery.h +++ b/dbms/src/Parsers/ASTCreateQuery.h @@ -180,6 +180,7 @@ public: bool is_view{false}; bool is_materialized_view{false}; bool is_populate{false}; + bool replace_view{false}; /// CREATE OR REPLACE VIEW ASTColumns * columns_list = nullptr; String to_database; /// For CREATE MATERIALIZED VIEW mv TO table. String to_table; @@ -244,6 +245,7 @@ protected: << (settings.hilite ? hilite_keyword : "") << (attach ? "ATTACH " : "CREATE ") << (temporary ? "TEMPORARY " : "") + << (replace_view ? "OR REPLACE " : "") << what << " " << (if_not_exists ? "IF NOT EXISTS " : "") << (settings.hilite ? hilite_none : "") diff --git a/dbms/src/Parsers/ASTDropQuery.h b/dbms/src/Parsers/ASTDropQuery.h index 1c230e30aea..fce4cee8131 100644 --- a/dbms/src/Parsers/ASTDropQuery.h +++ b/dbms/src/Parsers/ASTDropQuery.h @@ -22,6 +22,9 @@ public: Kind kind; bool if_exists{false}; + /// Useful if we already have a DDL lock + bool no_ddl_lock{false}; + /** Get the text that identifies this element. */ String getID(char) const override; ASTPtr clone() const override; diff --git a/dbms/src/Parsers/ASTTablesInSelectQuery.h b/dbms/src/Parsers/ASTTablesInSelectQuery.h index 4d4d0471ca1..5565d0ba3f8 100644 --- a/dbms/src/Parsers/ASTTablesInSelectQuery.h +++ b/dbms/src/Parsers/ASTTablesInSelectQuery.h @@ -106,6 +106,13 @@ struct ASTTableJoin : public IAST void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; }; +inline bool isFull(ASTTableJoin::Kind kind) { return kind == ASTTableJoin::Kind::Full; } +inline bool isCross(ASTTableJoin::Kind kind) { return kind == ASTTableJoin::Kind::Cross; } +inline bool isComma(ASTTableJoin::Kind kind) { return kind == ASTTableJoin::Kind::Comma; } +inline bool isRightOrFull(ASTTableJoin::Kind kind) { return kind == ASTTableJoin::Kind::Right || kind == ASTTableJoin::Kind::Full; } +inline bool isLeftOrFull(ASTTableJoin::Kind kind) { return kind == ASTTableJoin::Kind::Left || kind == ASTTableJoin::Kind::Full; } +inline bool isInnerOrRight(ASTTableJoin::Kind kind) { return kind == ASTTableJoin::Kind::Inner || kind == ASTTableJoin::Kind::Right; } + /// Specification of ARRAY JOIN. struct ASTArrayJoin : public IAST diff --git a/dbms/src/Parsers/ParserCreateQuery.cpp b/dbms/src/Parsers/ParserCreateQuery.cpp index fe343b31618..5a8ad919b58 100644 --- a/dbms/src/Parsers/ParserCreateQuery.cpp +++ b/dbms/src/Parsers/ParserCreateQuery.cpp @@ -298,6 +298,7 @@ bool ParserCreateQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserKeyword s_view("VIEW"); ParserKeyword s_materialized("MATERIALIZED"); ParserKeyword s_populate("POPULATE"); + ParserKeyword s_or_replace("OR REPLACE"); ParserToken s_dot(TokenType::Dot); ParserToken s_lparen(TokenType::OpeningRoundBracket); ParserToken s_rparen(TokenType::ClosingRoundBracket); @@ -322,6 +323,7 @@ bool ParserCreateQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) bool is_materialized_view = false; bool is_populate = false; bool is_temporary = false; + bool replace_view = false; if (!s_create.ignore(pos, expected)) { @@ -432,7 +434,12 @@ bool ParserCreateQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) else { /// VIEW or MATERIALIZED VIEW - if (s_materialized.ignore(pos, expected)) + if (s_or_replace.ignore(pos, expected)) + { + replace_view = true; + } + + if (!replace_view && s_materialized.ignore(pos, expected)) { is_materialized_view = true; } @@ -442,7 +449,7 @@ bool ParserCreateQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) if (!s_view.ignore(pos, expected)) return false; - if (s_if_not_exists.ignore(pos, expected)) + if (!replace_view && s_if_not_exists.ignore(pos, expected)) if_not_exists = true; if (!name_p.parse(pos, table, expected)) @@ -512,6 +519,7 @@ bool ParserCreateQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) query->is_materialized_view = is_materialized_view; query->is_populate = is_populate; query->temporary = is_temporary; + query->replace_view = replace_view; getIdentifierName(database, query->database); getIdentifierName(table, query->table); diff --git a/dbms/src/Parsers/ParserCreateQuery.h b/dbms/src/Parsers/ParserCreateQuery.h index 46dee1c8fa3..140a153e990 100644 --- a/dbms/src/Parsers/ParserCreateQuery.h +++ b/dbms/src/Parsers/ParserCreateQuery.h @@ -285,7 +285,7 @@ protected: * CREATE|ATTACH DATABASE db [ENGINE = engine] * * Or: - * CREATE|ATTACH [MATERIALIZED] VIEW [IF NOT EXISTS] [db.]name [TO [db.]name] [ENGINE = engine] [POPULATE] AS SELECT ... + * CREATE [OR REPLACE]|ATTACH [MATERIALIZED] VIEW [IF NOT EXISTS] [db.]name [TO [db.]name] [ENGINE = engine] [POPULATE] AS SELECT ... */ class ParserCreateQuery : public IParserBase { diff --git a/dbms/src/Storages/IStorage.cpp b/dbms/src/Storages/IStorage.cpp index e3d143adbb7..697e919987c 100644 --- a/dbms/src/Storages/IStorage.cpp +++ b/dbms/src/Storages/IStorage.cpp @@ -5,16 +5,7 @@ namespace DB { -TableStructureReadLock::TableStructureReadLock(StoragePtr storage_, bool lock_structure, bool lock_data, const String & query_id) - : storage(storage_) -{ - if (lock_data) - data_lock = storage->data_lock->getLock(RWLockImpl::Read, query_id); - if (lock_structure) - structure_lock = storage->structure_lock->getLock(RWLockImpl::Read, query_id); -} - -void IStorage::alter(const AlterCommands & params, const String & database_name, const String & table_name, const Context & context) +void IStorage::alter(const AlterCommands & params, const String & database_name, const String & table_name, const Context & context, TableStructureWriteLockHolder & table_lock_holder) { for (const auto & param : params) { @@ -22,7 +13,7 @@ void IStorage::alter(const AlterCommands & params, const String & database_name, throw Exception("Method alter supports only change comment of column for storage " + getName(), ErrorCodes::NOT_IMPLEMENTED); } - auto lock = lockStructureForAlter(context.getCurrentQueryId()); + lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); auto new_columns = getColumns(); auto new_indices = getIndicesDescription(); params.apply(new_columns); diff --git a/dbms/src/Storages/IStorage.h b/dbms/src/Storages/IStorage.h index 5841126e844..6d52afe5314 100644 --- a/dbms/src/Storages/IStorage.h +++ b/dbms/src/Storages/IStorage.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -47,35 +48,6 @@ class MutationCommands; class PartitionCommands; -/** Does not allow changing the table description (including rename and delete the table). - * If during any operation the table structure should remain unchanged, you need to hold such a lock for all of its time. - * For example, you need to hold such a lock for the duration of the entire SELECT or INSERT query and for the whole time the merge of the set of parts - * (but between the selection of parts for the merge and their merging, the table structure can change). - * NOTE: This is a lock to "read" the table's description. To change the table description, you need to take the TableStructureWriteLock. - */ -class TableStructureReadLock -{ -private: - friend class IStorage; - - StoragePtr storage; - /// Order is important. - RWLockImpl::LockHolder data_lock; - RWLockImpl::LockHolder structure_lock; - -public: - TableStructureReadLock(StoragePtr storage_, bool lock_structure, bool lock_data, const String & query_id); -}; - - -using TableStructureReadLockPtr = std::shared_ptr; -using TableStructureReadLocks = std::vector; - -using TableStructureWriteLock = RWLockImpl::LockHolder; -using TableDataWriteLock = RWLockImpl::LockHolder; -using TableFullWriteLock = std::pair; - - /** Storage. Responsible for * - storage of the table data; * - the definition in which files (or not in files) the data is stored; @@ -111,50 +83,72 @@ public: /** Returns true if the storage supports deduplication of inserted data blocks . */ virtual bool supportsDeduplication() const { return false; } - /** Does not allow you to change the structure or name of the table. - * If you change the data in the table, you will need to specify will_modify_data = true. - * This will take an extra lock that does not allow starting ALTER MODIFY. - * - * WARNING: You need to call methods from ITableDeclaration under such a lock. Without it, they are not thread safe. - * WARNING: To avoid deadlocks, this method must not be called under lock of Context. - */ - TableStructureReadLockPtr lockStructure(bool will_modify_data, const String & query_id) + + /// Acquire this lock if you need the table structure to remain constant during the execution of + /// the query. If will_add_new_data is true, this means that the query will add new data to the table + /// (INSERT or a parts merge). + TableStructureReadLockHolder lockStructureForShare(bool will_add_new_data, const String & query_id) { - TableStructureReadLockPtr res = std::make_shared(shared_from_this(), true, will_modify_data, query_id); + TableStructureReadLockHolder result; + if (will_add_new_data) + result.new_data_structure_lock = new_data_structure_lock->getLock(RWLockImpl::Read, query_id); + result.structure_lock = structure_lock->getLock(RWLockImpl::Read, query_id); + if (is_dropped) throw Exception("Table is dropped", ErrorCodes::TABLE_IS_DROPPED); - return res; + return result; } - /** Does not allow reading the table structure. It is taken for ALTER, RENAME and DROP, TRUNCATE. - */ - TableFullWriteLock lockForAlter(const String & query_id) + /// Acquire this lock at the start of ALTER to lock out other ALTERs and make sure that only you + /// can modify the table structure. It can later be upgraded to the exclusive lock. + TableStructureWriteLockHolder lockAlterIntention(const String & query_id) { - /// The calculation order is important. - auto res_data_lock = lockDataForAlter(query_id); - auto res_structure_lock = lockStructureForAlter(query_id); + TableStructureWriteLockHolder result; + result.alter_intention_lock = alter_intention_lock->getLock(RWLockImpl::Write, query_id); - return {std::move(res_data_lock), std::move(res_structure_lock)}; - } - - /** Does not allow changing the data in the table. (Moreover, does not give a look at the structure of the table with the intention to change the data). - * It is taken during write temporary data in ALTER MODIFY. - * Under this lock, you can take lockStructureForAlter() to change the structure of the table. - */ - TableDataWriteLock lockDataForAlter(const String & query_id) - { - auto res = data_lock->getLock(RWLockImpl::Write, query_id); if (is_dropped) throw Exception("Table is dropped", ErrorCodes::TABLE_IS_DROPPED); - return res; + return result; } - TableStructureWriteLock lockStructureForAlter(const String & query_id) + /// Upgrade alter intention lock and make sure that no new data is inserted into the table. + /// This is used by the ALTER MODIFY of the MergeTree storage to consistently determine + /// the set of parts that needs to be altered. + void lockNewDataStructureExclusively(TableStructureWriteLockHolder & lock_holder, const String & query_id) { - auto res = structure_lock->getLock(RWLockImpl::Write, query_id); + if (!lock_holder.alter_intention_lock) + throw Exception("Alter intention lock for table " + getTableName() + " was not taken. This is a bug.", + ErrorCodes::LOGICAL_ERROR); + + lock_holder.new_data_structure_lock = new_data_structure_lock->getLock(RWLockImpl::Write, query_id); + } + + /// Upgrade alter intention lock to the full exclusive structure lock. This is done by ALTER queries + /// to ensure that no other query uses the table structure and it can be safely changed. + void lockStructureExclusively(TableStructureWriteLockHolder & lock_holder, const String & query_id) + { + if (!lock_holder.alter_intention_lock) + throw Exception("Alter intention lock for table " + getTableName() + " was not taken. This is a bug.", + ErrorCodes::LOGICAL_ERROR); + + if (!lock_holder.new_data_structure_lock) + lock_holder.new_data_structure_lock = new_data_structure_lock->getLock(RWLockImpl::Write, query_id); + lock_holder.structure_lock = structure_lock->getLock(RWLockImpl::Write, query_id); + } + + /// Acquire the full exclusive lock immediately. No other queries can run concurrently. + TableStructureWriteLockHolder lockExclusively(const String & query_id) + { + TableStructureWriteLockHolder result; + result.alter_intention_lock = alter_intention_lock->getLock(RWLockImpl::Write, query_id); + if (is_dropped) throw Exception("Table is dropped", ErrorCodes::TABLE_IS_DROPPED); - return res; + + result.new_data_structure_lock = new_data_structure_lock->getLock(RWLockImpl::Write, query_id); + result.structure_lock = structure_lock->getLock(RWLockImpl::Write, query_id); + + return result; } /** Returns stage to which query is going to be processed in read() function. @@ -233,7 +227,7 @@ public: * This method must fully execute the ALTER query, taking care of the locks itself. * To update the table metadata on disk, this method should call InterpreterAlterQuery::updateMetadata. */ - virtual void alter(const AlterCommands & params, const String & database_name, const String & table_name, const Context & context); + virtual void alter(const AlterCommands & params, const String & database_name, const String & table_name, const Context & context, TableStructureWriteLockHolder & table_lock_holder); /** ALTER tables with regard to its partitions. * Should handle locks for each command on its own. @@ -345,28 +339,21 @@ public: using std::enable_shared_from_this::shared_from_this; private: - friend class TableStructureReadLock; + /// You always need to take the next three locks in this order. - /// You always need to take the next two locks in this order. + /// If you hold this lock exclusively, you can be sure that no other structure modifying queries + /// (e.g. ALTER, DROP) are concurrently executing. But queries that only read table structure + /// (e.g. SELECT, INSERT) can continue to execute. + mutable RWLock alter_intention_lock = RWLockImpl::create(); - /** It is taken for read for the entire INSERT query and the entire merge of the parts (for MergeTree). - * It is taken for write for the entire time ALTER MODIFY. - * - * Formally: - * Taking a write lock ensures that: - * 1) the data in the table will not change while the lock is alive, - * 2) all changes to the data after releasing the lock will be based on the structure of the table at the time after the lock was released. - * You need to take for read for the entire time of the operation that changes the data. - */ - mutable RWLock data_lock = RWLockImpl::create(); + /// It is taken for share for the entire INSERT query and the entire merge of the parts (for MergeTree). + /// ALTER COLUMN queries acquire an exclusive lock to ensure that no new parts with the old structure + /// are added to the table and thus the set of parts to modify doesn't change. + mutable RWLock new_data_structure_lock = RWLockImpl::create(); - /** Lock for multiple columns and path to table. It is taken for write at RENAME, ALTER (for ALTER MODIFY for a while) and DROP. - * It is taken for read for the whole time of SELECT, INSERT and merge parts (for MergeTree). - * - * Taking this lock for writing is a strictly "stronger" operation than taking parts_writing_lock for write record. - * That is, if this lock is taken for write, you should not worry about `parts_writing_lock`. - * parts_writing_lock is only needed for cases when you do not want to take `table_structure_lock` for long operations (ALTER MODIFY). - */ + /// Lock for the table column structure (names, types, etc.) and data path. + /// It is taken in exclusive mode by queries that modify them (e.g. RENAME, ALTER and DROP) + /// and in share mode by other queries. mutable RWLock structure_lock = RWLockImpl::create(); }; diff --git a/dbms/src/Storages/Kafka/KafkaBlockInputStream.cpp b/dbms/src/Storages/Kafka/KafkaBlockInputStream.cpp index 4b4d66849aa..c511a1053b3 100644 --- a/dbms/src/Storages/Kafka/KafkaBlockInputStream.cpp +++ b/dbms/src/Storages/Kafka/KafkaBlockInputStream.cpp @@ -7,7 +7,7 @@ namespace DB { KafkaBlockInputStream::KafkaBlockInputStream( - StorageKafka & storage_, const Context & context_, const String & schema, UInt64 max_block_size_) + StorageKafka & storage_, const Context & context_, const String & schema, size_t max_block_size_) : storage(storage_), context(context_), max_block_size(max_block_size_) { context.setSetting("input_format_skip_unknown_fields", 1u); // Always skip unknown fields regardless of the context (JSON or TSKV) diff --git a/dbms/src/Storages/MergeTree/BackgroundProcessingPool.cpp b/dbms/src/Storages/MergeTree/BackgroundProcessingPool.cpp index b60d860ec6c..b63ffeefd5e 100644 --- a/dbms/src/Storages/MergeTree/BackgroundProcessingPool.cpp +++ b/dbms/src/Storages/MergeTree/BackgroundProcessingPool.cpp @@ -140,7 +140,8 @@ void BackgroundProcessingPool::threadFunction() } SCOPE_EXIT({ CurrentThread::detachQueryIfNotDetached(); }); - CurrentThread::getMemoryTracker().setMetric(CurrentMetrics::MemoryTrackingInBackgroundProcessingPool); + if (auto memory_tracker = CurrentThread::getMemoryTracker()) + memory_tracker->setMetric(CurrentMetrics::MemoryTrackingInBackgroundProcessingPool); pcg64 rng(randomSeed()); std::this_thread::sleep_for(std::chrono::duration(std::uniform_real_distribution(0, thread_sleep_seconds_random_part)(rng))); diff --git a/dbms/src/Storages/MergeTree/DataPartsExchange.cpp b/dbms/src/Storages/MergeTree/DataPartsExchange.cpp index 9a8346d36a7..42c668c9fcb 100644 --- a/dbms/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/dbms/src/Storages/MergeTree/DataPartsExchange.cpp @@ -79,7 +79,7 @@ void Service::processQuery(const Poco::Net::HTMLForm & params, ReadBuffer & /*bo try { - auto storage_lock = owned_storage->lockStructure(false, RWLockImpl::NO_QUERY); + auto storage_lock = owned_storage->lockStructureForShare(false, RWLockImpl::NO_QUERY); MergeTreeData::DataPartPtr part = findPart(part_name); diff --git a/dbms/src/Storages/MergeTree/MergeList.cpp b/dbms/src/Storages/MergeTree/MergeList.cpp index b65d53d032d..9490a4f1068 100644 --- a/dbms/src/Storages/MergeTree/MergeList.cpp +++ b/dbms/src/Storages/MergeTree/MergeList.cpp @@ -38,7 +38,7 @@ MergeListElement::MergeListElement(const std::string & database, const std::stri } /// Each merge is executed into separate background processing pool thread - background_thread_memory_tracker = &CurrentThread::getMemoryTracker(); + background_thread_memory_tracker = CurrentThread::getMemoryTracker(); if (background_thread_memory_tracker) { memory_tracker.setMetric(CurrentMetrics::MemoryTrackingForMerges); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp index bf9c5b3409d..01ff4c4cdac 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp @@ -513,13 +513,16 @@ void MergeTreeDataPart::loadIndex() for (size_t i = 0; i < marks_count; ++i) //-V756 for (size_t j = 0; j < key_size; ++j) - storage.primary_key_data_types[j]->deserializeBinary(*loaded_index[j].get(), index_file); + storage.primary_key_data_types[j]->deserializeBinary(*loaded_index[j], index_file); for (size_t i = 0; i < key_size; ++i) + { + loaded_index[i]->protect(); if (loaded_index[i]->size() != marks_count) throw Exception("Cannot read all data from index file " + index_path + "(expected size: " + toString(marks_count) + ", read: " + toString(loaded_index[i]->size()) + ")", ErrorCodes::CANNOT_READ_ALL_DATA); + } if (!index_file.eof()) throw Exception("Index file " + index_path + " is unexpectedly long", ErrorCodes::EXPECTED_END_OF_FILE); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 8886b0d157b..759980a4fab 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -368,14 +368,11 @@ BlockInputStreams MergeTreeDataSelectExecutor::readFromParts( * It is also important that the entire universe can be covered using SAMPLE 0.1 OFFSET 0, ... OFFSET 0.9 and similar decimals. */ - bool use_sampling = relative_sample_size > 0 || settings.parallel_replicas_count > 1; + bool use_sampling = relative_sample_size > 0 || (settings.parallel_replicas_count > 1 && data.supportsSampling()); bool no_data = false; /// There is nothing left after sampling. if (use_sampling) { - if (!data.supportsSampling()) - throw Exception("Illegal SAMPLE: table doesn't support sampling", ErrorCodes::SAMPLING_NOT_SUPPORTED); - if (sample_factor_column_queried && relative_sample_size != RelativeSize(0)) used_sample_factor = 1.0 / boost::rational_cast(relative_sample_size); diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndices.h b/dbms/src/Storages/MergeTree/MergeTreeIndices.h index 1d62e9e9e9c..fac3bc4c858 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndices.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIndices.h @@ -34,6 +34,19 @@ struct IMergeTreeIndexGranule virtual void deserializeBinary(ReadBuffer & istr) = 0; virtual bool empty() const = 0; +}; + +using MergeTreeIndexGranulePtr = std::shared_ptr; +using MergeTreeIndexGranules = std::vector; + + +/// Aggregates info about a single block of data. +struct IMergeTreeIndexAggregator +{ + virtual ~IMergeTreeIndexAggregator() = default; + + virtual bool empty() const = 0; + virtual MergeTreeIndexGranulePtr getGranuleAndReset() = 0; /// Updates the stored info using rows of the specified block. /// Reads no more than `limit` rows. @@ -41,8 +54,8 @@ struct IMergeTreeIndexGranule virtual void update(const Block & block, size_t * pos, size_t limit) = 0; }; -using MergeTreeIndexGranulePtr = std::shared_ptr; -using MergeTreeIndexGranules = std::vector; +using MergeTreeIndexAggregatorPtr = std::shared_ptr; +using MergeTreeIndexAggregators = std::vector; /// Condition on the index. @@ -83,6 +96,7 @@ public: String getFileName() const { return INDEX_FILE_PREFIX + name; } virtual MergeTreeIndexGranulePtr createIndexGranule() const = 0; + virtual MergeTreeIndexAggregatorPtr createIndexAggregator() const = 0; virtual IndexConditionPtr createIndexCondition( const SelectQueryInfo & query_info, const Context & context) const = 0; diff --git a/dbms/src/Storages/MergeTree/MergeTreeMinMaxIndex.cpp b/dbms/src/Storages/MergeTree/MergeTreeMinMaxIndex.cpp index d9816c3e119..7f27785f911 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeMinMaxIndex.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeMinMaxIndex.cpp @@ -17,9 +17,11 @@ namespace ErrorCodes MergeTreeMinMaxGranule::MergeTreeMinMaxGranule(const MergeTreeMinMaxIndex & index) - : IMergeTreeIndexGranule(), index(index), parallelogram() -{ -} + : IMergeTreeIndexGranule(), index(index), parallelogram() {} + +MergeTreeMinMaxGranule::MergeTreeMinMaxGranule( + const MergeTreeMinMaxIndex & index, std::vector && parallelogram) + : IMergeTreeIndexGranule(), index(index), parallelogram(std::move(parallelogram)) {} void MergeTreeMinMaxGranule::serializeBinary(WriteBuffer & ostr) const { @@ -51,7 +53,16 @@ void MergeTreeMinMaxGranule::deserializeBinary(ReadBuffer & istr) } } -void MergeTreeMinMaxGranule::update(const Block & block, size_t * pos, size_t limit) + +MergeTreeMinMaxAggregator::MergeTreeMinMaxAggregator(const MergeTreeMinMaxIndex & index) + : index(index) {} + +MergeTreeIndexGranulePtr MergeTreeMinMaxAggregator::getGranuleAndReset() +{ + return std::make_shared(index, std::move(parallelogram)); +} + +void MergeTreeMinMaxAggregator::update(const Block & block, size_t * pos, size_t limit) { if (*pos >= block.rows()) throw Exception( @@ -109,6 +120,13 @@ MergeTreeIndexGranulePtr MergeTreeMinMaxIndex::createIndexGranule() const return std::make_shared(*this); } + +MergeTreeIndexAggregatorPtr MergeTreeMinMaxIndex::createIndexAggregator() const +{ + return std::make_shared(*this); +} + + IndexConditionPtr MergeTreeMinMaxIndex::createIndexCondition( const SelectQueryInfo & query, const Context & context) const { diff --git a/dbms/src/Storages/MergeTree/MergeTreeMinMaxIndex.h b/dbms/src/Storages/MergeTree/MergeTreeMinMaxIndex.h index 7a6206dccc7..24beada455b 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeMinMaxIndex.h +++ b/dbms/src/Storages/MergeTree/MergeTreeMinMaxIndex.h @@ -16,14 +16,27 @@ class MergeTreeMinMaxIndex; struct MergeTreeMinMaxGranule : public IMergeTreeIndexGranule { explicit MergeTreeMinMaxGranule(const MergeTreeMinMaxIndex & index); + MergeTreeMinMaxGranule(const MergeTreeMinMaxIndex & index, std::vector && parallelogram); + ~MergeTreeMinMaxGranule() override = default; void serializeBinary(WriteBuffer & ostr) const override; void deserializeBinary(ReadBuffer & istr) override; bool empty() const override { return parallelogram.empty(); } - void update(const Block & block, size_t * pos, size_t limit) override; - ~MergeTreeMinMaxGranule() override = default; + const MergeTreeMinMaxIndex & index; + std::vector parallelogram; +}; + + +struct MergeTreeMinMaxAggregator : IMergeTreeIndexAggregator +{ + explicit MergeTreeMinMaxAggregator(const MergeTreeMinMaxIndex & index); + ~MergeTreeMinMaxAggregator() override = default; + + bool empty() const override { return parallelogram.empty(); } + MergeTreeIndexGranulePtr getGranuleAndReset() override; + void update(const Block & block, size_t * pos, size_t limit) override; const MergeTreeMinMaxIndex & index; std::vector parallelogram; @@ -64,6 +77,7 @@ public: ~MergeTreeMinMaxIndex() override = default; MergeTreeIndexGranulePtr createIndexGranule() const override; + MergeTreeIndexAggregatorPtr createIndexAggregator() const override; IndexConditionPtr createIndexCondition( const SelectQueryInfo & query, const Context & context) const override; diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp index 9091228d80a..89f5aaeafd5 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp @@ -132,6 +132,7 @@ void MergeTreeReaderStream::loadMarks() if (buffer.eof() || buffer.buffer().size() != file_size) throw Exception("Cannot read all marks from file " + mrk_path, ErrorCodes::CANNOT_READ_ALL_DATA); + res->protect(); return res; }; diff --git a/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp index 2cfbe5f5539..0b4d56be8ea 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp @@ -26,7 +26,7 @@ MergeTreeSequentialBlockInputStream::MergeTreeSequentialBlockInputStream( { std::stringstream message; message << "Reading " << data_part->marks_count << " marks from part " << data_part->name - << ", totaly " << data_part->rows_count + << ", total " << data_part->rows_count << " rows starting from the beginning of the part, columns: "; for (size_t i = 0, size = columns_to_read.size(); i < size; ++i) message << (i == 0 ? "" : ", ") << columns_to_read[i]; diff --git a/dbms/src/Storages/MergeTree/MergeTreeSetSkippingIndex.cpp b/dbms/src/Storages/MergeTree/MergeTreeSetSkippingIndex.cpp index 227c6bfd7f7..742d3971930 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSetSkippingIndex.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSetSkippingIndex.cpp @@ -22,10 +22,15 @@ const Field UNKNOWN_FIELD(3u); MergeTreeSetIndexGranule::MergeTreeSetIndexGranule(const MergeTreeSetSkippingIndex & index) - : IMergeTreeIndexGranule(), index(index), set(new Set(SizeLimits{}, true)) -{ - set->setHeader(index.header); -} + : IMergeTreeIndexGranule() + , index(index) + , block(index.header.cloneEmpty()) {} + +MergeTreeSetIndexGranule::MergeTreeSetIndexGranule( + const MergeTreeSetSkippingIndex & index, MutableColumns && mutable_columns) + : IMergeTreeIndexGranule() + , index(index) + , block(index.header.cloneWithColumns(std::move(mutable_columns))) {} void MergeTreeSetIndexGranule::serializeBinary(WriteBuffer & ostr) const { @@ -33,10 +38,9 @@ void MergeTreeSetIndexGranule::serializeBinary(WriteBuffer & ostr) const throw Exception( "Attempt to write empty set index `" + index.name + "`", ErrorCodes::LOGICAL_ERROR); - const auto & columns = set->getSetElements(); const auto & size_type = DataTypePtr(std::make_shared()); - if (size() > index.max_rows) + if (index.max_rows && size() > index.max_rows) { size_type->serializeBinary(0, ostr); return; @@ -55,21 +59,15 @@ void MergeTreeSetIndexGranule::serializeBinary(WriteBuffer & ostr) const IDataType::SerializeBinaryBulkStatePtr state; type->serializeBinaryBulkStatePrefix(settings, state); - type->serializeBinaryBulkWithMultipleStreams(*columns[i], 0, size(), settings, state); + type->serializeBinaryBulkWithMultipleStreams(*block.getByPosition(i).column, 0, size(), settings, state); type->serializeBinaryBulkStateSuffix(settings, state); } } void MergeTreeSetIndexGranule::deserializeBinary(ReadBuffer & istr) { - if (!set->empty()) - { - auto new_set = std::make_unique(SizeLimits{}, true); - new_set->setHeader(index.header); - set.swap(new_set); - } + block.clear(); - Block block; Field field_rows; const auto & size_type = DataTypePtr(std::make_shared()); size_type->deserializeBinary(field_rows, istr); @@ -93,47 +91,124 @@ void MergeTreeSetIndexGranule::deserializeBinary(ReadBuffer & istr) block.insert(ColumnWithTypeAndName(new_column->getPtr(), type, index.columns[i])); } - - set->insertFromBlock(block); } -void MergeTreeSetIndexGranule::update(const Block & new_block, size_t * pos, size_t limit) + +MergeTreeSetIndexAggregator::MergeTreeSetIndexAggregator(const MergeTreeSetSkippingIndex & index) + : index(index), columns(index.header.cloneEmptyColumns()) { - if (*pos >= new_block.rows()) + ColumnRawPtrs column_ptrs; + column_ptrs.reserve(index.columns.size()); + Columns materialized_columns; + for (const auto & column : index.header.getColumns()) + { + materialized_columns.emplace_back(column->convertToFullColumnIfConst()->convertToFullColumnIfLowCardinality()); + column_ptrs.emplace_back(materialized_columns.back().get()); + } + + data.init(ClearableSetVariants::chooseMethod(column_ptrs, key_sizes)); + + columns = index.header.cloneEmptyColumns(); +} + +void MergeTreeSetIndexAggregator::update(const Block & block, size_t * pos, size_t limit) +{ + if (*pos >= block.rows()) throw Exception( "The provided position is not less than the number of block rows. Position: " - + toString(*pos) + ", Block rows: " + toString(new_block.rows()) + ".", ErrorCodes::LOGICAL_ERROR); + + toString(*pos) + ", Block rows: " + toString(block.rows()) + ".", ErrorCodes::LOGICAL_ERROR); - size_t rows_read = std::min(limit, new_block.rows() - *pos); + size_t rows_read = std::min(limit, block.rows() - *pos); - if (size() > index.max_rows) + if (index.max_rows && size() > index.max_rows) { *pos += rows_read; return; } - Block key_block; - for (size_t i = 0; i < index.columns.size(); ++i) + ColumnRawPtrs index_column_ptrs; + index_column_ptrs.reserve(index.columns.size()); + Columns materialized_columns; + for (const auto & column_name : index.columns) { - const auto & name = index.columns[i]; - const auto & type = index.data_types[i]; - key_block.insert( - ColumnWithTypeAndName( - new_block.getByName(name).column->cut(*pos, rows_read), - type, - name)); + materialized_columns.emplace_back( + block.getByName(column_name).column->convertToFullColumnIfConst()->convertToFullColumnIfLowCardinality()); + index_column_ptrs.emplace_back(materialized_columns.back().get()); } - set->insertFromBlock(key_block); + IColumn::Filter filter(block.rows(), 0); + + bool has_new_data = false; + switch (data.type) + { + case ClearableSetVariants::Type::EMPTY: + break; +#define M(NAME) \ + case ClearableSetVariants::Type::NAME: \ + has_new_data = buildFilter(*data.NAME, index_column_ptrs, filter, *pos, rows_read, data); \ + break; + APPLY_FOR_SET_VARIANTS(M) +#undef M + } + + if (has_new_data) + { + for (size_t i = 0; i < columns.size(); ++i) + { + auto filtered_column = block.getByName(index.columns[i]).column->filter(filter, block.rows()); + columns[i]->insertRangeFrom(*filtered_column, 0, filtered_column->size()); + } + } *pos += rows_read; } -Block MergeTreeSetIndexGranule::getElementsBlock() const +template +bool MergeTreeSetIndexAggregator::buildFilter( + Method & method, + const ColumnRawPtrs & column_ptrs, + IColumn::Filter & filter, + size_t pos, + size_t limit, + ClearableSetVariants & variants) const { - if (size() > index.max_rows) - return index.header; - return index.header.cloneWithColumns(set->getSetElements()); + /// Like DistinctSortedBlockInputStream. + typename Method::State state(column_ptrs, key_sizes, nullptr); + + bool has_new_data = false; + for (size_t i = 0; i < limit; ++i) + { + auto emplace_result = state.emplaceKey(method.data, pos + i, variants.string_pool); + + if (emplace_result.isInserted()) + has_new_data = true; + + /// Emit the record if there is no such key in the current set yet. + /// Skip it otherwise. + filter[pos + i] = emplace_result.isInserted(); + } + return has_new_data; +} + +MergeTreeIndexGranulePtr MergeTreeSetIndexAggregator::getGranuleAndReset() +{ + auto granule = std::make_shared(index, std::move(columns)); + + switch (data.type) + { + case ClearableSetVariants::Type::EMPTY: + break; +#define M(NAME) \ + case ClearableSetVariants::Type::NAME: \ + data.NAME->data.clear(); \ + break; + APPLY_FOR_SET_VARIANTS(M) +#undef M + } + + columns = index.header.cloneEmptyColumns(); + + return granule; } @@ -190,10 +265,10 @@ bool SetIndexCondition::mayBeTrueOnGranule(MergeTreeIndexGranulePtr idx_granule) throw Exception( "Set index condition got a granule with the wrong type.", ErrorCodes::LOGICAL_ERROR); - if (useless || !granule->size() || granule->size() > index.max_rows) + if (useless || !granule->size() || (index.max_rows && granule->size() > index.max_rows)) return true; - Block result = granule->getElementsBlock(); + Block result = granule->block; actions->execute(result); auto column = result.getByName(expression_ast->getColumnName()).column->convertToFullColumnIfLowCardinality(); @@ -377,8 +452,13 @@ MergeTreeIndexGranulePtr MergeTreeSetSkippingIndex::createIndexGranule() const return std::make_shared(*this); } +MergeTreeIndexAggregatorPtr MergeTreeSetSkippingIndex::createIndexAggregator() const +{ + return std::make_shared(*this); +} + IndexConditionPtr MergeTreeSetSkippingIndex::createIndexCondition( - const SelectQueryInfo & query, const Context & context) const + const SelectQueryInfo & query, const Context & context) const { return std::make_shared(query, context, *this); }; diff --git a/dbms/src/Storages/MergeTree/MergeTreeSetSkippingIndex.h b/dbms/src/Storages/MergeTree/MergeTreeSetSkippingIndex.h index 28be8788f98..9cdf9a1e3d4 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSetSkippingIndex.h +++ b/dbms/src/Storages/MergeTree/MergeTreeSetSkippingIndex.h @@ -3,7 +3,7 @@ #include #include -#include +#include #include #include @@ -17,20 +17,48 @@ class MergeTreeSetSkippingIndex; struct MergeTreeSetIndexGranule : public IMergeTreeIndexGranule { explicit MergeTreeSetIndexGranule(const MergeTreeSetSkippingIndex & index); + MergeTreeSetIndexGranule(const MergeTreeSetSkippingIndex & index, MutableColumns && columns); void serializeBinary(WriteBuffer & ostr) const override; void deserializeBinary(ReadBuffer & istr) override; - size_t size() const { return set->getTotalRowCount(); } + size_t size() const { return block.rows(); } bool empty() const override { return !size(); } - void update(const Block & block, size_t * pos, size_t limit) override; - Block getElementsBlock() const; - ~MergeTreeSetIndexGranule() override = default; const MergeTreeSetSkippingIndex & index; - std::unique_ptr set; + Block block; +}; + + +struct MergeTreeSetIndexAggregator : IMergeTreeIndexAggregator +{ + explicit MergeTreeSetIndexAggregator(const MergeTreeSetSkippingIndex & index); + ~MergeTreeSetIndexAggregator() override = default; + + size_t size() const { return data.getTotalRowCount(); } + bool empty() const override { return !size(); } + + MergeTreeIndexGranulePtr getGranuleAndReset() override; + + void update(const Block & block, size_t * pos, size_t limit) override; + +private: + /// return true if has new data + template + bool buildFilter( + Method & method, + const ColumnRawPtrs & column_ptrs, + IColumn::Filter & filter, + size_t pos, + size_t limit, + ClearableSetVariants & variants) const; + + const MergeTreeSetSkippingIndex & index; + ClearableSetVariants data; + Sizes key_sizes; + MutableColumns columns; }; @@ -79,6 +107,7 @@ public: ~MergeTreeSetSkippingIndex() override = default; MergeTreeIndexGranulePtr createIndexGranule() const override; + MergeTreeIndexAggregatorPtr createIndexAggregator() const override; IndexConditionPtr createIndexCondition( const SelectQueryInfo & query, const Context & context) const override; diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 22a7610e83b..49fa5da2afa 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -11,6 +11,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + namespace { @@ -330,11 +335,8 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( for (size_t i = 0; i < storage.skip_indices.size(); ++i) { auto & stream = *skip_indices_streams[i]; - if (skip_indices_granules[i] && !skip_indices_granules[i]->empty()) - { - skip_indices_granules[i]->serializeBinary(stream.compressed); - skip_indices_granules[i].reset(); - } + if (!skip_indices_aggregators[i]->empty()) + skip_indices_aggregators[i]->getGranuleAndReset()->serializeBinary(stream.compressed); } @@ -362,7 +364,7 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( } skip_indices_streams.clear(); - skip_indices_granules.clear(); + skip_indices_aggregators.clear(); skip_index_filling.clear(); for (ColumnStreams::iterator it = column_streams.begin(); it != column_streams.end(); ++it) @@ -432,8 +434,7 @@ void MergedBlockOutputStream::init() part_path + stream_name, MARKS_FILE_EXTENSION, codec, max_compress_block_size, 0, aio_threshold)); - - skip_indices_granules.emplace_back(nullptr); + skip_indices_aggregators.push_back(index->createIndexAggregator()); skip_index_filling.push_back(0); } } @@ -563,9 +564,9 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm else { limit = storage.index_granularity; - if (!skip_indices_granules[i]) + if (skip_indices_aggregators[i]->empty()) { - skip_indices_granules[i] = index->createIndexGranule(); + skip_indices_aggregators[i] = index->createIndexAggregator(); skip_index_filling[i] = 0; if (stream.compressed.offset() >= min_compress_block_size) @@ -577,7 +578,7 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm } size_t pos = prev_pos; - skip_indices_granules[i]->update(indices_update_block, &pos, limit); + skip_indices_aggregators[i]->update(indices_update_block, &pos, limit); if (pos == prev_pos + limit) { @@ -586,8 +587,7 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm /// write index if it is filled if (skip_index_filling[i] == index->granularity) { - skip_indices_granules[i]->serializeBinary(stream.compressed); - skip_indices_granules[i].reset(); + skip_indices_aggregators[i]->getGranuleAndReset()->serializeBinary(stream.compressed); skip_index_filling[i] = 0; } } diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.h b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.h index 6bc6e90e887..06acba26804 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.h +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.h @@ -151,7 +151,7 @@ private: MutableColumns index_columns; std::vector> skip_indices_streams; - MergeTreeIndexGranules skip_indices_granules; + MergeTreeIndexAggregators skip_indices_aggregators; std::vector skip_index_filling; }; diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp index 03b397e0390..ba3bc77af6b 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp @@ -108,7 +108,7 @@ void ReplicatedMergeTreeAlterThread::run() LOG_INFO(log, "Version of metadata nodes in ZooKeeper changed. Waiting for structure write lock."); - auto table_lock = storage.lockStructureForAlter(RWLockImpl::NO_QUERY); + auto table_lock = storage.lockExclusively(RWLockImpl::NO_QUERY); if (columns_in_zk == storage.getColumns() && metadata_diff.empty()) { @@ -134,7 +134,7 @@ void ReplicatedMergeTreeAlterThread::run() /// Update parts. if (changed_columns_version || force_recheck_parts) { - auto table_lock = storage.lockStructure(false, RWLockImpl::NO_QUERY); + auto table_lock = storage.lockStructureForShare(false, RWLockImpl::NO_QUERY); if (changed_columns_version) LOG_INFO(log, "ALTER-ing parts"); diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index f0dad30025c..a745af1e414 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -202,7 +202,7 @@ void ReplicatedMergeTreePartCheckThread::checkPart(const String & part_name) else if (part->name == part_name) { auto zookeeper = storage.getZooKeeper(); - auto table_lock = storage.lockStructure(false, RWLockImpl::NO_QUERY); + auto table_lock = storage.lockStructureForShare(false, RWLockImpl::NO_QUERY); auto local_part_header = ReplicatedMergeTreePartHeader::fromColumnsAndChecksums( part->columns, part->checksums); diff --git a/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp b/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp index 103be508564..a64f376e3de 100644 --- a/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -101,7 +101,8 @@ static void appendGraphitePattern( { if (key == "regexp") { - pattern.regexp = std::make_shared(config.getString(config_element + ".regexp")); + pattern.regexp_str = config.getString(config_element + ".regexp"); + pattern.regexp = std::make_shared(pattern.regexp_str); } else if (key == "function") { @@ -165,6 +166,7 @@ static void setGraphitePatternsFromConfig(const Context & context, throw Exception("No '" + config_element + "' element in configuration file", ErrorCodes::NO_ELEMENTS_IN_CONFIG); + params.config_name = config_element; params.path_column_name = config.getString(config_element + ".path_column_name", "Path"); params.time_column_name = config.getString(config_element + ".time_column_name", "Time"); params.value_column_name = config.getString(config_element + ".value_column_name", "Value"); diff --git a/dbms/src/Storages/StorageBuffer.cpp b/dbms/src/Storages/StorageBuffer.cpp index b3cb485a6b4..5487db29703 100644 --- a/dbms/src/Storages/StorageBuffer.cpp +++ b/dbms/src/Storages/StorageBuffer.cpp @@ -150,7 +150,7 @@ BlockInputStreams StorageBuffer::read( if (destination.get() == this) throw Exception("Destination table is myself. Read will cause infinite loop.", ErrorCodes::INFINITE_LOOP); - auto destination_lock = destination->lockStructure(false, context.getCurrentQueryId()); + auto destination_lock = destination->lockStructureForShare(false, context.getCurrentQueryId()); const bool dst_has_same_structure = std::all_of(column_names.begin(), column_names.end(), [this, destination](const String& column_name) { @@ -677,9 +677,9 @@ void StorageBuffer::flushThread() } -void StorageBuffer::alter(const AlterCommands & params, const String & database_name, const String & table_name, const Context & context) +void StorageBuffer::alter(const AlterCommands & params, const String & database_name, const String & table_name, const Context & context, TableStructureWriteLockHolder & table_lock_holder) { - auto lock = lockStructureForAlter(context.getCurrentQueryId()); + lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); /// So that no blocks of the old structure remain. optimize({} /*query*/, {} /*partition_id*/, false /*final*/, false /*deduplicate*/, context); diff --git a/dbms/src/Storages/StorageBuffer.h b/dbms/src/Storages/StorageBuffer.h index 854d4efd05d..f32a4c72c43 100644 --- a/dbms/src/Storages/StorageBuffer.h +++ b/dbms/src/Storages/StorageBuffer.h @@ -81,7 +81,9 @@ public: bool mayBenefitFromIndexForIn(const ASTPtr & left_in_operand, const Context & query_context) const override; /// The structure of the subordinate table is not checked and does not change. - void alter(const AlterCommands & params, const String & database_name, const String & table_name, const Context & context) override; + void alter( + const AlterCommands & params, const String & database_name, const String & table_name, + const Context & context, TableStructureWriteLockHolder & table_lock_holder) override; private: String name; diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index 644576272d0..582dd976c0d 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -336,9 +336,11 @@ BlockOutputStreamPtr StorageDistributed::write(const ASTPtr &, const Context & c } -void StorageDistributed::alter(const AlterCommands & params, const String & database_name, const String & current_table_name, const Context & context) +void StorageDistributed::alter( + const AlterCommands & params, const String & database_name, const String & current_table_name, + const Context & context, TableStructureWriteLockHolder & table_lock_holder) { - auto lock = lockStructureForAlter(context.getCurrentQueryId()); + lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); auto new_columns = getColumns(); auto new_indices = getIndicesDescription(); diff --git a/dbms/src/Storages/StorageDistributed.h b/dbms/src/Storages/StorageDistributed.h index eecab76ea8e..a628f2c542a 100644 --- a/dbms/src/Storages/StorageDistributed.h +++ b/dbms/src/Storages/StorageDistributed.h @@ -81,7 +81,9 @@ public: void rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & new_table_name) override { table_name = new_table_name; } /// in the sub-tables, you need to manually add and delete columns /// the structure of the sub-table is not checked - void alter(const AlterCommands & params, const String & database_name, const String & table_name, const Context & context) override; + void alter( + const AlterCommands & params, const String & database_name, const String & table_name, + const Context & context, TableStructureWriteLockHolder & table_lock_holder) override; void startup() override; void shutdown() override; diff --git a/dbms/src/Storages/StorageMaterializedView.cpp b/dbms/src/Storages/StorageMaterializedView.cpp index 93663badf56..b19dd53ff49 100644 --- a/dbms/src/Storages/StorageMaterializedView.cpp +++ b/dbms/src/Storages/StorageMaterializedView.cpp @@ -190,7 +190,7 @@ BlockInputStreams StorageMaterializedView::read( const unsigned num_streams) { auto storage = getTargetTable(); - auto lock = storage->lockStructure(false, context.getCurrentQueryId()); + auto lock = storage->lockStructureForShare(false, context.getCurrentQueryId()); auto streams = storage->read(column_names, query_info, context, processed_stage, max_block_size, num_streams); for (auto & stream : streams) stream->addTableLock(lock); @@ -200,7 +200,7 @@ BlockInputStreams StorageMaterializedView::read( BlockOutputStreamPtr StorageMaterializedView::write(const ASTPtr & query, const Context & context) { auto storage = getTargetTable(); - auto lock = storage->lockStructure(true, context.getCurrentQueryId()); + auto lock = storage->lockStructureForShare(true, context.getCurrentQueryId()); auto stream = storage->write(query, context); stream->addTableLock(lock); return stream; diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp index d8e3dfce649..4521083bc03 100644 --- a/dbms/src/Storages/StorageMerge.cpp +++ b/dbms/src/Storages/StorageMerge.cpp @@ -224,7 +224,7 @@ BlockInputStreams StorageMerge::read( current_streams = std::max(size_t(1), current_streams); StoragePtr storage = it->first; - TableStructureReadLockPtr struct_lock = it->second; + TableStructureReadLockHolder struct_lock = it->second; BlockInputStreams source_streams; @@ -262,7 +262,7 @@ BlockInputStreams StorageMerge::read( BlockInputStreams StorageMerge::createSourceStreams(const SelectQueryInfo & query_info, const QueryProcessingStage::Enum & processed_stage, const UInt64 max_block_size, const Block & header, const StoragePtr & storage, - const TableStructureReadLockPtr & struct_lock, Names & real_column_names, + const TableStructureReadLockHolder & struct_lock, Names & real_column_names, Context & modified_context, size_t streams_num, bool has_table_virtual_column, bool concat_streams) { @@ -345,7 +345,7 @@ StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables(const String { auto & table = iterator->table(); if (table.get() != this) - selected_tables.emplace_back(table, table->lockStructure(false, query_id)); + selected_tables.emplace_back(table, table->lockStructureForShare(false, query_id)); } iterator->next(); @@ -375,7 +375,7 @@ StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables(const ASTPtr if (storage.get() != this) { virtual_column->insert(storage->getTableName()); - selected_tables.emplace_back(storage, get_lock ? storage->lockStructure(false, query_id) : TableStructureReadLockPtr{}); + selected_tables.emplace_back(storage, get_lock ? storage->lockStructureForShare(false, query_id) : TableStructureReadLockHolder{}); } } @@ -395,9 +395,11 @@ StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables(const ASTPtr return selected_tables; } -void StorageMerge::alter(const AlterCommands & params, const String & database_name, const String & table_name, const Context & context) +void StorageMerge::alter( + const AlterCommands & params, const String & database_name, const String & table_name, + const Context & context, TableStructureWriteLockHolder & table_lock_holder) { - auto lock = lockStructureForAlter(context.getCurrentQueryId()); + lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); auto new_columns = getColumns(); auto new_indices = getIndicesDescription(); diff --git a/dbms/src/Storages/StorageMerge.h b/dbms/src/Storages/StorageMerge.h index 000d8daed1a..752872de885 100644 --- a/dbms/src/Storages/StorageMerge.h +++ b/dbms/src/Storages/StorageMerge.h @@ -44,7 +44,9 @@ public: /// you need to add and remove columns in the sub-tables manually /// the structure of sub-tables is not checked - void alter(const AlterCommands & params, const String & database_name, const String & table_name, const Context & context) override; + void alter( + const AlterCommands & params, const String & database_name, const String & table_name, + const Context & context, TableStructureWriteLockHolder & table_lock_holder) override; bool mayBenefitFromIndexForIn(const ASTPtr & left_in_operand, const Context & query_context) const override; @@ -54,7 +56,7 @@ private: OptimizedRegularExpression table_name_regexp; Context global_context; - using StorageListWithLocks = std::list>; + using StorageListWithLocks = std::list>; StorageListWithLocks getSelectedTables(const String & query_id) const; @@ -76,7 +78,7 @@ protected: BlockInputStreams createSourceStreams(const SelectQueryInfo & query_info, const QueryProcessingStage::Enum & processed_stage, const UInt64 max_block_size, const Block & header, const StoragePtr & storage, - const TableStructureReadLockPtr & struct_lock, Names & real_column_names, + const TableStructureReadLockHolder & struct_lock, Names & real_column_names, Context & modified_context, size_t streams_num, bool has_table_virtual_column, bool concat_streams = false); diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index ba3fe04dd89..856976ca35d 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -195,11 +195,12 @@ void StorageMergeTree::alter( const AlterCommands & params, const String & current_database_name, const String & current_table_name, - const Context & context) + const Context & context, + TableStructureWriteLockHolder & table_lock_holder) { if (!params.is_mutable()) { - auto table_soft_lock = lockStructureForAlter(context.getCurrentQueryId()); + lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); auto new_columns = getColumns(); auto new_indices = getIndicesDescription(); params.apply(new_columns); @@ -211,7 +212,7 @@ void StorageMergeTree::alter( /// NOTE: Here, as in ReplicatedMergeTree, you can do ALTER which does not block the writing of data for a long time. auto merge_blocker = merger_mutator.actions_blocker.cancel(); - auto table_soft_lock = lockDataForAlter(context.getCurrentQueryId()); + lockNewDataStructureExclusively(table_lock_holder, context.getCurrentQueryId()); data.checkAlter(params, context); @@ -230,7 +231,7 @@ void StorageMergeTree::alter( transactions.push_back(std::move(transaction)); } - auto table_hard_lock = lockStructureForAlter(context.getCurrentQueryId()); + lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); IDatabase::ASTModifier storage_modifier = [&] (IAST & ast) { @@ -452,7 +453,7 @@ bool StorageMergeTree::merge( bool deduplicate, String * out_disable_reason) { - auto structure_lock = lockStructure(true, RWLockImpl::NO_QUERY); + auto table_lock_holder = lockStructureForShare(true, RWLockImpl::NO_QUERY); FutureMergedMutatedPart future_part; @@ -562,7 +563,7 @@ bool StorageMergeTree::merge( bool StorageMergeTree::tryMutatePart() { - auto structure_lock = lockStructure(true, RWLockImpl::NO_QUERY); + auto table_lock_holder = lockStructureForShare(true, RWLockImpl::NO_QUERY); FutureMergedMutatedPart future_part; MutationCommands commands; @@ -774,7 +775,7 @@ void StorageMergeTree::clearColumnInPartition(const ASTPtr & partition, const Fi auto merge_blocker = merger_mutator.actions_blocker.cancel(); /// We don't change table structure, only data in some parts, parts are locked inside alterDataPart() function - auto lock_read_structure = lockStructure(false, context.getCurrentQueryId()); + auto lock_read_structure = lockStructureForShare(false, context.getCurrentQueryId()); String partition_id = data.getPartitionIDFromQuery(partition, context); auto parts = data.getDataPartsVectorInPartition(MergeTreeDataPartState::Committed, partition_id); @@ -879,7 +880,7 @@ void StorageMergeTree::alterPartition(const ASTPtr & query, const PartitionComma case PartitionCommand::FREEZE_PARTITION: { - auto lock = lockStructure(false, context.getCurrentQueryId()); + auto lock = lockStructureForShare(false, context.getCurrentQueryId()); data.freezePartition(command.partition, command.with_name, context); } break; @@ -890,7 +891,7 @@ void StorageMergeTree::alterPartition(const ASTPtr & query, const PartitionComma case PartitionCommand::FREEZE_ALL_PARTITIONS: { - auto lock = lockStructure(false, context.getCurrentQueryId()); + auto lock = lockStructureForShare(false, context.getCurrentQueryId()); data.freezeAll(command.with_name, context); } break; @@ -908,7 +909,7 @@ void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, cons /// This protects against "revival" of data for a removed partition after completion of merge. auto merge_blocker = merger_mutator.actions_blocker.cancel(); /// Waits for completion of merge and does not start new ones. - auto lock = lockForAlter(context.getCurrentQueryId()); + auto lock = lockExclusively(context.getCurrentQueryId()); String partition_id = data.getPartitionIDFromQuery(partition, context); @@ -991,8 +992,8 @@ void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_par void StorageMergeTree::replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, const Context & context) { - auto lock1 = lockStructure(false, context.getCurrentQueryId()); - auto lock2 = source_table->lockStructure(false, context.getCurrentQueryId()); + auto lock1 = lockStructureForShare(false, context.getCurrentQueryId()); + auto lock2 = source_table->lockStructureForShare(false, context.getCurrentQueryId()); Stopwatch watch; MergeTreeData * src_data = data.checkStructureAndGetMergeTreeData(source_table); diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index d17b496bd97..f7e4e10fc4e 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -77,7 +77,9 @@ public: void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override; - void alter(const AlterCommands & params, const String & database_name, const String & table_name, const Context & context) override; + void alter( + const AlterCommands & params, const String & database_name, const String & table_name, + const Context & context, TableStructureWriteLockHolder & table_lock_holder) override; void checkTableCanBeDropped() const override; diff --git a/dbms/src/Storages/StorageNull.cpp b/dbms/src/Storages/StorageNull.cpp index a920fe3c395..176f70d3e9f 100644 --- a/dbms/src/Storages/StorageNull.cpp +++ b/dbms/src/Storages/StorageNull.cpp @@ -30,9 +30,11 @@ void registerStorageNull(StorageFactory & factory) }); } -void StorageNull::alter(const AlterCommands & params, const String & current_database_name, const String & current_table_name, const Context & context) +void StorageNull::alter( + const AlterCommands & params, const String & current_database_name, const String & current_table_name, + const Context & context, TableStructureWriteLockHolder & table_lock_holder) { - auto lock = lockStructureForAlter(context.getCurrentQueryId()); + lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); ColumnsDescription new_columns = getColumns(); IndicesDescription new_indices = getIndicesDescription(); diff --git a/dbms/src/Storages/StorageNull.h b/dbms/src/Storages/StorageNull.h index bf070480316..14ce1140680 100644 --- a/dbms/src/Storages/StorageNull.h +++ b/dbms/src/Storages/StorageNull.h @@ -41,7 +41,9 @@ public: table_name = new_table_name; } - void alter(const AlterCommands & params, const String & database_name, const String & table_name, const Context & context) override; + void alter( + const AlterCommands & params, const String & database_name, const String & table_name, + const Context & context, TableStructureWriteLockHolder & table_lock_holder) override; private: String table_name; diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index b9bbf762c66..235e9ee1cb0 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -277,6 +277,13 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( return; } + if (attach && !current_zookeeper->exists(zookeeper_path + "/metadata")) + { + LOG_WARNING(log, "No metadata in ZooKeeper: table will be in readonly mode."); + is_readonly = true; + return; + } + if (!attach) { if (!data.getDataParts().empty()) @@ -540,45 +547,25 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) /// There are no PreCommitted parts at startup. auto parts = data.getDataParts({MergeTreeDataPartState::Committed, MergeTreeDataPartState::Outdated}); - /// Local parts that are not in ZK. + /** Local parts that are not in ZK. + * In very rare cases they may cover missing parts + * and someone may think that pushing them to zookeeper is good idea. + * But actually we can't precisely determine that ALL missing parts + * covered by this unexpected part. So missing parts will be downloaded. + */ MergeTreeData::DataParts unexpected_parts; + /// Collect unexpected parts for (const auto & part : parts) - { - if (expected_parts.count(part->name)) - expected_parts.erase(part->name); - else - unexpected_parts.insert(part); - } - - /// Which local parts to added into ZK. - MergeTreeData::DataPartsVector parts_to_add; - UInt64 parts_to_add_rows = 0; + if (!expected_parts.count(part->name)) + unexpected_parts.insert(part); /// this parts we will place to detached with ignored_ prefix /// Which parts should be taken from other replicas. Strings parts_to_fetch; for (const String & missing_name : expected_parts) - { - /// If locally some part is missing, but there is a part covering it, you can replace it in ZK with the covering one. - auto containing = data.getActiveContainingPart(missing_name); - if (containing) - { - LOG_ERROR(log, "Ignoring missing local part " << missing_name << " because part " << containing->name << " exists"); - if (unexpected_parts.count(containing)) - { - parts_to_add.push_back(containing); - unexpected_parts.erase(containing); - parts_to_add_rows += containing->rows_count; - } - } - else + if (!data.getActiveContainingPart(missing_name)) parts_to_fetch.push_back(missing_name); - } - - for (const String & name : parts_to_fetch) - expected_parts.erase(name); - /** To check the adequacy, for the parts that are in the FS, but not in ZK, we will only consider not the most recent parts. * Because unexpected new parts usually arise only because they did not have time to enroll in ZK with a rough restart of the server. @@ -613,16 +600,10 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) for (const String & name : parts_to_fetch) parts_to_fetch_blocks += get_blocks_count_in_data_part(name); - UInt64 expected_parts_blocks = 0; - for (const String & name : expected_parts) - expected_parts_blocks += get_blocks_count_in_data_part(name); - std::stringstream sanity_report; sanity_report << "There are " << unexpected_parts.size() << " unexpected parts with " << unexpected_parts_rows << " rows (" << unexpected_parts_nonnew << " of them is not just-written with " << unexpected_parts_rows << " rows), " - << parts_to_add.size() << " unexpectedly merged parts with " << parts_to_add_rows << " rows, " - << expected_parts.size() << " missing obsolete parts (with " << expected_parts_blocks << " blocks), " << parts_to_fetch.size() << " missing parts (with " << parts_to_fetch_blocks << " blocks)."; /** We can automatically synchronize data, @@ -638,45 +619,23 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) for (const auto & part : parts) total_rows_on_filesystem += part->rows_count; - UInt64 total_suspicious_rows = parts_to_add_rows + unexpected_parts_rows; - UInt64 total_suspicious_rows_no_new = parts_to_add_rows + unexpected_parts_nonnew_rows; - - bool insane = total_suspicious_rows > total_rows_on_filesystem * data.settings.replicated_max_ratio_of_wrong_parts; + bool insane = unexpected_parts_rows > total_rows_on_filesystem * data.settings.replicated_max_ratio_of_wrong_parts; if (insane && !skip_sanity_checks) { std::stringstream why; why << "The local set of parts of table " << database_name << "." << table_name << " doesn't look like the set of parts " << "in ZooKeeper: " - << formatReadableQuantity(total_suspicious_rows) << " rows of " << formatReadableQuantity(total_rows_on_filesystem) + << formatReadableQuantity(unexpected_parts_rows) << " rows of " << formatReadableQuantity(total_rows_on_filesystem) << " total rows in filesystem are suspicious."; throw Exception(why.str() + " " + sanity_report.str(), ErrorCodes::TOO_MANY_UNEXPECTED_DATA_PARTS); } - if (total_suspicious_rows_no_new > 0) + if (unexpected_parts_nonnew_rows > 0) LOG_WARNING(log, sanity_report.str()); - /// Add information to the ZK about the parts that cover the missing parts. - for (const MergeTreeData::DataPartPtr & part : parts_to_add) - { - LOG_ERROR(log, "Adding unexpected local part to ZooKeeper: " << part->name); - - Coordination::Requests ops; - checkPartChecksumsAndAddCommitOps(zookeeper, part, ops); - zookeeper->multi(ops); - } - - /// Remove from ZK information about the parts covered by the newly added ones. - { - for (const String & name : expected_parts) - LOG_ERROR(log, "Removing unexpectedly merged local part from ZooKeeper: " << name); - - removePartsFromZooKeeper(zookeeper, Strings(expected_parts.begin(), expected_parts.end())); - } - /// Add to the queue jobs to pick up the missing parts from other replicas and remove from ZK the information that we have them. - std::vector> exists_futures; exists_futures.reserve(parts_to_fetch.size()); for (const String & part_name : parts_to_fetch) @@ -1089,7 +1048,7 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) /// Can throw an exception. DiskSpaceMonitor::ReservationPtr reserved_space = DiskSpaceMonitor::reserve(full_path, estimated_space_for_merge); - auto table_lock = lockStructure(false, RWLockImpl::NO_QUERY); + auto table_lock = lockStructureForShare(false, RWLockImpl::NO_QUERY); FutureMergedMutatedPart future_merged_part(parts); if (future_merged_part.name != entry.new_part_name) @@ -1219,7 +1178,7 @@ bool StorageReplicatedMergeTree::tryExecutePartMutation(const StorageReplicatedM /// Can throw an exception. DiskSpaceMonitor::ReservationPtr reserved_space = DiskSpaceMonitor::reserve(full_path, estimated_space_for_result); - auto table_lock = lockStructure(false, RWLockImpl::NO_QUERY); + auto table_lock = lockStructureForShare(false, RWLockImpl::NO_QUERY); MergeTreeData::MutableDataPartPtr new_part; MergeTreeData::Transaction transaction(data); @@ -1528,7 +1487,7 @@ void StorageReplicatedMergeTree::executeClearColumnInPartition(const LogEntry & /// We don't change table structure, only data in some parts /// To disable reading from these parts, we will sequentially acquire write lock for each part inside alterDataPart() /// If we will lock the whole table here, a deadlock can occur. For example, if use use Buffer table (CLICKHOUSE-3238) - auto lock_read_structure = lockStructure(false, RWLockImpl::NO_QUERY); + auto lock_read_structure = lockStructureForShare(false, RWLockImpl::NO_QUERY); auto zookeeper = getZooKeeper(); @@ -1624,7 +1583,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) PartDescriptions parts_to_add; MergeTreeData::DataPartsVector parts_to_remove; - auto structure_lock_dst_table = lockStructure(false, RWLockImpl::NO_QUERY); + auto table_lock_holder_dst_table = lockStructureForShare(false, RWLockImpl::NO_QUERY); for (size_t i = 0; i < entry_replace.new_part_names.size(); ++i) { @@ -1662,7 +1621,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) } StoragePtr source_table; - TableStructureReadLockPtr structure_lock_src_table; + TableStructureReadLockHolder table_lock_holder_src_table; String source_table_name = entry_replace.from_database + "." + entry_replace.from_table; auto clone_data_parts_from_source_table = [&] () -> size_t @@ -1686,7 +1645,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) return 0; } - structure_lock_src_table = source_table->lockStructure(false, RWLockImpl::NO_QUERY); + table_lock_holder_src_table = source_table->lockStructureForShare(false, RWLockImpl::NO_QUERY); MergeTreeData::DataPartStates valid_states{MergeTreeDataPartState::PreCommitted, MergeTreeDataPartState::Committed, MergeTreeDataPartState::Outdated}; @@ -2719,9 +2678,9 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Strin LOG_DEBUG(log, "Fetching part " << part_name << " from " << source_replica_path); - TableStructureReadLockPtr table_lock; + TableStructureReadLockHolder table_lock_holder; if (!to_detached) - table_lock = lockStructure(true, RWLockImpl::NO_QUERY); + table_lock_holder = lockStructureForShare(true, RWLockImpl::NO_QUERY); /// Logging Stopwatch stopwatch; @@ -3014,7 +2973,7 @@ bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const ASTPtr & p return true; } - ReplicatedMergeTreeLogEntryData merge_entry; + std::vector merge_entries; { /// We must select parts for merge under merge_selecting_mutex because other threads /// (merge_selecting_thread or OPTIMIZE queries) could assign new merges. @@ -3045,9 +3004,12 @@ bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const ASTPtr & p FutureMergedMutatedPart future_merged_part; bool selected = merger_mutator.selectAllPartsToMergeWithinPartition( future_merged_part, disk_space, can_merge, partition_id, true, nullptr); + ReplicatedMergeTreeLogEntryData merge_entry; if (selected && !createLogEntryToMergeParts(zookeeper, future_merged_part.parts, future_merged_part.name, deduplicate, &merge_entry)) return handle_noop("Can't create merge queue node in ZooKeeper"); + if (merge_entry.type != ReplicatedMergeTreeLogEntryData::Type::EMPTY) + merge_entries.push_back(std::move(merge_entry)); } } else @@ -3074,21 +3036,28 @@ bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const ASTPtr & p return handle_noop(disable_reason); } + ReplicatedMergeTreeLogEntryData merge_entry; if (!createLogEntryToMergeParts(zookeeper, future_merged_part.parts, future_merged_part.name, deduplicate, &merge_entry)) return handle_noop("Can't create merge queue node in ZooKeeper"); + if (merge_entry.type != ReplicatedMergeTreeLogEntryData::Type::EMPTY) + merge_entries.push_back(std::move(merge_entry)); } } /// TODO: Bad setting name for such purpose if (query_context.getSettingsRef().replication_alter_partitions_sync != 0) - waitForAllReplicasToProcessLogEntry(merge_entry); + { + for (auto & merge_entry : merge_entries) + waitForAllReplicasToProcessLogEntry(merge_entry); + } return true; } -void StorageReplicatedMergeTree::alter(const AlterCommands & params, - const String & /*database_name*/, const String & /*table_name*/, const Context & query_context) +void StorageReplicatedMergeTree::alter( + const AlterCommands & params, const String & /*database_name*/, const String & /*table_name*/, + const Context & query_context, TableStructureWriteLockHolder & table_lock_holder) { assertNotReadonly(); @@ -3124,7 +3093,7 @@ void StorageReplicatedMergeTree::alter(const AlterCommands & params, { /// Just to read current structure. Alter will be done in separate thread. - auto table_lock = lockStructure(false, query_context.getCurrentQueryId()); + auto table_lock = lockStructureForShare(false, query_context.getCurrentQueryId()); if (is_readonly) throw Exception("Can't ALTER readonly table", ErrorCodes::TABLE_IS_READ_ONLY); @@ -3166,6 +3135,8 @@ void StorageReplicatedMergeTree::alter(const AlterCommands & params, LOG_DEBUG(log, "Updated shared metadata nodes in ZooKeeper. Waiting for replicas to apply changes."); + table_lock_holder.release(); + /// Wait until all replicas will apply ALTER. for (const auto & node : changed_nodes) @@ -3382,7 +3353,7 @@ void StorageReplicatedMergeTree::alterPartition(const ASTPtr & query, const Part case PartitionCommand::FREEZE_PARTITION: { - auto lock = lockStructure(false, query_context.getCurrentQueryId()); + auto lock = lockStructureForShare(false, query_context.getCurrentQueryId()); data.freezePartition(command.partition, command.with_name, query_context); } break; @@ -3393,7 +3364,7 @@ void StorageReplicatedMergeTree::alterPartition(const ASTPtr & query, const Part case PartitionCommand::FREEZE_ALL_PARTITIONS: { - auto lock = lockStructure(false, query_context.getCurrentQueryId()); + auto lock = lockStructureForShare(false, query_context.getCurrentQueryId()); data.freezeAll(command.with_name, query_context); } break; @@ -4448,7 +4419,7 @@ void StorageReplicatedMergeTree::clearOldPartsAndRemoveFromZK() { /// Critical section is not required (since grabOldParts() returns unique part set on each call) - auto table_lock = lockStructure(false, RWLockImpl::NO_QUERY); + auto table_lock = lockStructureForShare(false, RWLockImpl::NO_QUERY); auto zookeeper = getZooKeeper(); MergeTreeData::DataPartsVector parts = data.grabOldParts(); @@ -4740,8 +4711,8 @@ void StorageReplicatedMergeTree::clearBlocksInPartition( void StorageReplicatedMergeTree::replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, const Context & context) { - auto lock1 = lockStructure(false, context.getCurrentQueryId()); - auto lock2 = source_table->lockStructure(false, context.getCurrentQueryId()); + auto lock1 = lockStructureForShare(false, context.getCurrentQueryId()); + auto lock2 = source_table->lockStructureForShare(false, context.getCurrentQueryId()); Stopwatch watch; MergeTreeData * src_data = data.checkStructureAndGetMergeTreeData(source_table); diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index e3a7d023066..caa75000d70 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -116,7 +116,9 @@ public: bool optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & query_context) override; - void alter(const AlterCommands & params, const String & database_name, const String & table_name, const Context & query_context) override; + void alter( + const AlterCommands & params, const String & database_name, const String & table_name, + const Context & query_context, TableStructureWriteLockHolder & table_lock_holder) override; void alterPartition(const ASTPtr & query, const PartitionCommands & commands, const Context & query_context) override; diff --git a/dbms/src/Storages/System/StorageSystemColumns.cpp b/dbms/src/Storages/System/StorageSystemColumns.cpp index fe0c3d90d69..d94b8c0ab10 100644 --- a/dbms/src/Storages/System/StorageSystemColumns.cpp +++ b/dbms/src/Storages/System/StorageSystemColumns.cpp @@ -100,11 +100,11 @@ protected: { StoragePtr storage = storages.at(std::make_pair(database_name, table_name)); - TableStructureReadLockPtr table_lock; + TableStructureReadLockHolder table_lock; try { - table_lock = storage->lockStructure(false, query_id); + table_lock = storage->lockStructureForShare(false, query_id); } catch (const Exception & e) { diff --git a/dbms/src/Storages/System/StorageSystemGraphite.cpp b/dbms/src/Storages/System/StorageSystemGraphite.cpp index d75eb71841e..e315bc04fd1 100644 --- a/dbms/src/Storages/System/StorageSystemGraphite.cpp +++ b/dbms/src/Storages/System/StorageSystemGraphite.cpp @@ -1,175 +1,139 @@ #include +#include +#include -#include -#include -#include -#include -#include -#include -#include #include -#include - namespace DB { -namespace ErrorCodes -{ - extern const int NO_ELEMENTS_IN_CONFIG; -} - -namespace -{ - -using namespace Poco::Util; - -struct Pattern -{ - struct Retention - { - UInt64 age; - UInt64 precision; - }; - - std::string regexp; - std::string function; - std::vector retentions; - UInt16 priority; - UInt8 is_default; -}; - -static Pattern readOnePattern( - const AbstractConfiguration & config, - const std::string & path) -{ - Pattern pattern; - AbstractConfiguration::Keys keys; - - config.keys(path, keys); - - if (keys.empty()) - throw Exception("Empty pattern in Graphite rollup configuration", ErrorCodes::NO_ELEMENTS_IN_CONFIG); - - for (const auto & key : keys) - { - const String key_path = path + "." + key; - - if (startsWith(key, "regexp")) - { - pattern.regexp = config.getString(key_path); - } - else if (startsWith(key, "function")) - { - pattern.function = config.getString(key_path); - } - else if (startsWith(key, "retention")) - { - pattern.retentions.push_back(Pattern::Retention{0, 0}); - pattern.retentions.back().age = config.getUInt64(key_path + ".age", 0); - pattern.retentions.back().precision = config.getUInt64(key_path + ".precision", 0); - } - } - - return pattern; -} - -static std::vector readPatterns( - const AbstractConfiguration & config, - const std::string & section) -{ - AbstractConfiguration::Keys keys; - std::vector result; - size_t count = 0; - - config.keys(section, keys); - - for (const auto & key : keys) - { - if (startsWith(key, "pattern")) - { - Pattern pattern(readOnePattern(config, section + "." + key)); - pattern.is_default = false; - pattern.priority = ++count; - result.push_back(pattern); - } - else if (startsWith(key, "default")) - { - Pattern pattern(readOnePattern(config, section + "." + key)); - pattern.is_default = true; - pattern.priority = std::numeric_limits::max(); - result.push_back(pattern); - } - } - - return result; -} - -static Strings getAllGraphiteSections(const AbstractConfiguration & config) -{ - Strings result; - - AbstractConfiguration::Keys keys; - config.keys(keys); - - for (const auto & key : keys) - { - if (startsWith(key, "graphite_")) - result.push_back(key); - } - - return result; -} - -} // namespace - NamesAndTypesList StorageSystemGraphite::getNamesAndTypes() { return { - {"config_name", std::make_shared()}, - {"regexp", std::make_shared()}, - {"function", std::make_shared()}, - {"age", std::make_shared()}, - {"precision", std::make_shared()}, - {"priority", std::make_shared()}, - {"is_default", std::make_shared()}, + {"config_name", std::make_shared()}, + {"regexp", std::make_shared()}, + {"function", std::make_shared()}, + {"age", std::make_shared()}, + {"precision", std::make_shared()}, + {"priority", std::make_shared()}, + {"is_default", std::make_shared()}, + {"Tables.database", std::make_shared(std::make_shared())}, + {"Tables.table", std::make_shared(std::make_shared())}, }; } +/* + * Looking for (Replicated)*GraphiteMergeTree and get all configuration parameters for them + */ +StorageSystemGraphite::Configs StorageSystemGraphite::getConfigs(const Context & context) const +{ + const Databases databases = context.getDatabases(); + Configs graphite_configs; + + for (const auto & db : databases) + { + for (auto iterator = db.second->getIterator(context); iterator->isValid(); iterator->next()) + { + auto & table = iterator->table(); + const MergeTreeData * table_data = nullptr; + + if (const StorageMergeTree * merge_tree = dynamic_cast(table.get())) + { + table_data = &merge_tree->getData(); + } + else if (const StorageReplicatedMergeTree * replicated_merge_tree = dynamic_cast(table.get())) + { + table_data = &replicated_merge_tree->getData(); + } + else + { + continue; + } + + if (table_data->merging_params.mode == MergeTreeData::MergingParams::Graphite) + { + const String & config_name = table_data->merging_params.graphite_params.config_name; + + if (!graphite_configs.count(config_name)) + { + Config new_config = + { + table_data->merging_params.graphite_params, + { table_data->getDatabaseName() }, + { table_data->getTableName() }, + }; + graphite_configs.emplace(config_name, new_config); + } + else + { + graphite_configs[config_name].databases.emplace_back(table_data->getDatabaseName()); + graphite_configs[config_name].tables.emplace_back(table_data->getTableName()); + } + } + } + } + + return graphite_configs; +} + void StorageSystemGraphite::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const { - const auto & config = context.getConfigRef(); + Configs graphite_configs = StorageSystemGraphite::getConfigs(context); - Strings sections = getAllGraphiteSections(config); - for (const auto & section : sections) + for (const auto & config : graphite_configs) { - const auto patterns = readPatterns(config, section); - for (const auto & pattern : patterns) + UInt16 priority = 0; + for (const auto & pattern : config.second.graphite_params.patterns) { + bool is_default = pattern.regexp == nullptr; + String regexp; + String function; + + if (is_default) + { + priority = std::numeric_limits::max(); + } + else + { + priority++; + regexp = pattern.regexp_str; + } + + if (pattern.function) + { + function = pattern.function->getName(); + } + if (!pattern.retentions.empty()) { - for (const auto & ret : pattern.retentions) + for (const auto & retention : pattern.retentions) { - res_columns[0]->insert(section); - res_columns[1]->insert(pattern.regexp); - res_columns[2]->insert(pattern.function); - res_columns[3]->insert(ret.age); - res_columns[4]->insert(ret.precision); - res_columns[5]->insert(pattern.priority); - res_columns[6]->insert(pattern.is_default); + size_t i = 0; + res_columns[i++]->insert(config.first); + res_columns[i++]->insert(regexp); + res_columns[i++]->insert(function); + res_columns[i++]->insert(retention.age); + res_columns[i++]->insert(retention.precision); + res_columns[i++]->insert(priority); + res_columns[i++]->insert(is_default); + res_columns[i++]->insert(config.second.databases); + res_columns[i++]->insert(config.second.tables); } } else { - res_columns[0]->insert(section); - res_columns[1]->insert(pattern.regexp); - res_columns[2]->insert(pattern.function); - res_columns[3]->insert(0); - res_columns[4]->insert(0); - res_columns[5]->insert(pattern.priority); - res_columns[6]->insert(pattern.is_default); + size_t i = 0; + res_columns[i++]->insert(config.first); + res_columns[i++]->insert(regexp); + res_columns[i++]->insert(function); + res_columns[i++]->insertDefault(); + res_columns[i++]->insertDefault(); + res_columns[i++]->insert(priority); + res_columns[i++]->insert(is_default); + res_columns[i++]->insert(config.second.databases); + res_columns[i++]->insert(config.second.tables); } } } diff --git a/dbms/src/Storages/System/StorageSystemGraphite.h b/dbms/src/Storages/System/StorageSystemGraphite.h index fa63c839857..b874e294782 100644 --- a/dbms/src/Storages/System/StorageSystemGraphite.h +++ b/dbms/src/Storages/System/StorageSystemGraphite.h @@ -1,7 +1,10 @@ #pragma once +#include #include +#include #include +#include #include namespace DB @@ -15,10 +18,21 @@ public: static NamesAndTypesList getNamesAndTypes(); + struct Config + { + Graphite::Params graphite_params; + Array databases; + Array tables; + }; + + using Configs = std::map; + + protected: using IStorageSystemOneBlock::IStorageSystemOneBlock; void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo & query_info) const override; + StorageSystemGraphite::Configs getConfigs(const Context & context) const; }; } diff --git a/dbms/src/Storages/System/StorageSystemNumbers.cpp b/dbms/src/Storages/System/StorageSystemNumbers.cpp index 15b31212e9d..c909a338453 100644 --- a/dbms/src/Storages/System/StorageSystemNumbers.cpp +++ b/dbms/src/Storages/System/StorageSystemNumbers.cpp @@ -49,7 +49,6 @@ StorageSystemNumbers::StorageSystemNumbers(const std::string & name_, bool multi setColumns(ColumnsDescription({{"number", std::make_shared()}})); } - BlockInputStreams StorageSystemNumbers::read( const Names & column_names, const SelectQueryInfo &, @@ -75,7 +74,7 @@ BlockInputStreams StorageSystemNumbers::read( res[i] = std::make_shared(max_block_size, offset + i * max_block_size, num_streams * max_block_size); if (limit) /// This formula is how to split 'limit' elements to 'num_streams' chunks almost uniformly. - res[i] = std::make_shared(res[i], *limit * (i + 1) / num_streams - *limit * i / num_streams, 0); + res[i] = std::make_shared(res[i], *limit * (i + 1) / num_streams - *limit * i / num_streams, 0, false, true); } return res; diff --git a/dbms/src/Storages/System/StorageSystemPartsBase.cpp b/dbms/src/Storages/System/StorageSystemPartsBase.cpp index f8449e7a937..e65d780f629 100644 --- a/dbms/src/Storages/System/StorageSystemPartsBase.cpp +++ b/dbms/src/Storages/System/StorageSystemPartsBase.cpp @@ -167,7 +167,7 @@ public: try { /// For table not to be dropped and set of columns to remain constant. - info.table_lock = info.storage->lockStructure(false, query_id); + info.table_lock = info.storage->lockStructureForShare(false, query_id); } catch (const Exception & e) { diff --git a/dbms/src/Storages/System/StorageSystemPartsBase.h b/dbms/src/Storages/System/StorageSystemPartsBase.h index 920ae184e94..6fa92b02854 100644 --- a/dbms/src/Storages/System/StorageSystemPartsBase.h +++ b/dbms/src/Storages/System/StorageSystemPartsBase.h @@ -34,7 +34,7 @@ public: struct StoragesInfo { StoragePtr storage; - TableStructureReadLockPtr table_lock; + TableStructureReadLockHolder table_lock; String database; String table; diff --git a/dbms/src/Storages/TableStructureLockHolder.h b/dbms/src/Storages/TableStructureLockHolder.h new file mode 100644 index 00000000000..1413ead80a8 --- /dev/null +++ b/dbms/src/Storages/TableStructureLockHolder.h @@ -0,0 +1,37 @@ +#pragma once + +#include + +namespace DB +{ + +/// Structs that hold table structure (columns, their types, default values etc.) locks when executing queries. +/// See IStorage::lock* methods for comments. + +struct TableStructureWriteLockHolder +{ + void release() + { + *this = TableStructureWriteLockHolder(); + } + +private: + friend class IStorage; + + /// Order is important. + RWLockImpl::LockHolder alter_intention_lock; + RWLockImpl::LockHolder new_data_structure_lock; + RWLockImpl::LockHolder structure_lock; +}; + +struct TableStructureReadLockHolder +{ +private: + friend class IStorage; + + /// Order is important. + RWLockImpl::LockHolder new_data_structure_lock; + RWLockImpl::LockHolder structure_lock; +}; + +} diff --git a/dbms/tests/clickhouse-test b/dbms/tests/clickhouse-test index 1cf4882ddc8..8956a687766 100755 --- a/dbms/tests/clickhouse-test +++ b/dbms/tests/clickhouse-test @@ -30,6 +30,10 @@ MSG_UNKNOWN = OP_SQUARE_BRACKET + colored(" UNKNOWN ", "yellow", attrs=['bold']) MSG_OK = OP_SQUARE_BRACKET + colored(" OK ", "green", attrs=['bold']) + CL_SQUARE_BRACKET MSG_SKIPPED = OP_SQUARE_BRACKET + colored(" SKIPPED ", "cyan", attrs=['bold']) + CL_SQUARE_BRACKET +MESSAGES_TO_RETRY = [ + "Coordination::Exception: Connection loss", +] + def remove_control_characters(s): """ @@ -44,6 +48,52 @@ def remove_control_characters(s): s = re.sub(ur"[\x00-\x08\x0b\x0e-\x1f\x7f]", "", s) return s +def run_single_test(args, ext, server_logs_level, case_file, stdout_file, stderr_file): + if ext == '.sql': + command = "{0} --send_logs_level={1} --testmode --multiquery < {2} > {3} 2> {4}".format(args.client, server_logs_level, case_file, stdout_file, stderr_file) + else: + command = "{} > {} 2> {}".format(case_file, stdout_file, stderr_file) + + proc = Popen(command, shell = True) + start_time = datetime.now() + while (datetime.now() - start_time).total_seconds() < args.timeout and proc.poll() is None: + sleep(0.01) + + stdout = open(stdout_file, 'r').read() if os.path.exists(stdout_file) else '' + stdout = unicode(stdout, errors='replace', encoding='utf-8') + stderr = open(stderr_file, 'r').read() if os.path.exists(stderr_file) else '' + stderr = unicode(stderr, errors='replace', encoding='utf-8') + + return proc, stdout, stderr + +def need_retry(stderr): + return any(msg in stderr for msg in MESSAGES_TO_RETRY) + +def get_processlist(client_cmd): + try: + return subprocess.check_output("{} --query 'SHOW PROCESSLIST FORMAT Vertical'".format(client_cmd), shell=True) + except: + return "" # server seems dead + +def get_stacktraces(server_pid): + cmd = "gdb -q -ex 'set pagination off' -ex 'backtrace' -ex 'thread apply all backtrace' -ex 'detach' -ex 'quit' --pid {} 2>/dev/null".format(server_pid) + try: + return subprocess.check_output(cmd, shell=True) + except Exception as ex: + return "Error occured while receiving stack traces {}".format(str(ex)) + +def get_server_pid(server_tcp_port): + cmd = "lsof -i tcp:{port} | grep '*:{port}'".format(port=server_tcp_port) + try: + output = subprocess.check_output(cmd, shell=True) + if output: + columns = output.strip().split(' ') + return int(columns[1]) + else: + return None # server dead + except Exception as ex: + return None + def main(args): SERVER_DIED = False @@ -110,7 +160,21 @@ def main(args): clickhouse_proc_create = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE) clickhouse_proc_create.communicate("CREATE DATABASE IF NOT EXISTS test") - for suite in sorted(os.listdir(base_dir)): + def sute_key_func(item): + if args.order == 'random': + return random() + + if -1 == item.find('_'): + return 99998 + + prefix, suffix = item.split('_', 1) + + try: + return int(prefix), suffix + except ValueError: + return 99997 + + for suite in sorted(os.listdir(base_dir), key=sute_key_func): if SERVER_DIED: break @@ -201,16 +265,7 @@ def main(args): stdout_file = os.path.join(suite_tmp_dir, name) + '.stdout' stderr_file = os.path.join(suite_tmp_dir, name) + '.stderr' - if ext == '.sql': - command = "{0} --send_logs_level={1} --testmode --multiquery < {2} > {3} 2> {4}".format(args.client, server_logs_level, case_file, stdout_file, stderr_file) - else: - command = "{} > {} 2> {}".format(case_file, stdout_file, stderr_file) - - proc = Popen(command, shell = True) - start_time = datetime.now() - while (datetime.now() - start_time).total_seconds() < args.timeout and proc.poll() is None: - sleep(0.01) - + proc, stdout, stderr = run_single_test(args, ext, server_logs_level, case_file, stdout_file, stderr_file) if proc.returncode is None: try: proc.kill() @@ -224,10 +279,13 @@ def main(args): failures += 1 print("{0} - Timeout!".format(MSG_FAIL)) else: - stdout = open(stdout_file, 'r').read() if os.path.exists(stdout_file) else '' - stdout = unicode(stdout, errors='replace', encoding='utf-8') - stderr = open(stderr_file, 'r').read() if os.path.exists(stderr_file) else '' - stderr = unicode(stderr, errors='replace', encoding='utf-8') + counter = 1 + while proc.returncode != 0 and need_retry(stderr): + proc, stdout, stderr = run_single_test(args, ext, server_logs_level, case_file, stdout_file, stderr_file) + sleep(2**counter) + counter += 1 + if counter > 6: + break if proc.returncode != 0: failure = et.Element("failure", attrib = {"message": "return code {}".format(proc.returncode)}) @@ -322,12 +380,28 @@ def main(args): failures_total = failures_total + failures + exit_code = 0 if failures_total > 0: print(colored("\nHaving {failures_total} errors! {passed_total} tests passed. {skipped_total} tests skipped.".format(passed_total = passed_total, skipped_total = skipped_total, failures_total = failures_total), "red", attrs=["bold"])) - sys.exit(1) + exit_code = 1 else: print(colored("\n{passed_total} tests passed. {skipped_total} tests skipped.".format(passed_total = passed_total, skipped_total = skipped_total), "green", attrs=["bold"])) - sys.exit(0) + + if args.hung_check: + processlist = get_processlist(args.client) + if processlist: + server_pid = get_server_pid(os.getenv("CLICKHOUSE_PORT_TCP", '9000')) + print(colored("\nFound hung queries in processlist:", "red", attrs=["bold"])) + print(processlist) + if server_pid: + print("\nStacktraces of all threads:") + print(get_stacktraces(server_pid)) + exit_code = 1 + else: + print(colored("\nNo queries hung.", "green", attrs=["bold"])) + + sys.exit(exit_code) + def find_binary(name): paths = os.environ.get("PATH").split(':') @@ -339,31 +413,32 @@ def find_binary(name): return os.access(os.path.join('/usr/bin', name), os.X_OK) if __name__ == '__main__': - parser = ArgumentParser(description = 'ClickHouse functional tests') - parser.add_argument('-q', '--queries', help = 'Path to queries dir') - parser.add_argument('--tmp', help = 'Path to tmp dir') - parser.add_argument('-b', '--binary', default = 'clickhouse', help = 'Main clickhouse binary') - parser.add_argument('-c', '--client', help = 'Client program') - parser.add_argument('--extract_from_config', help = 'extract-from-config program') - parser.add_argument('--configclient', help = 'Client config (if you use not default ports)') - parser.add_argument('--configserver', default= '/etc/clickhouse-server/config.xml', help = 'Preprocessed server config') - parser.add_argument('-o', '--output', help = 'Output xUnit compliant test report directory') - parser.add_argument('-t', '--timeout', type = int, default = 600, help = 'Timeout for each test case in seconds') - parser.add_argument('test', nargs = '?', help = 'Optional test case name regex') - parser.add_argument('-d', '--disabled', action = 'store_true', default = False, help = 'Also run disabled tests') - parser.add_argument('--stop', action = 'store_true', default = None, dest = 'stop', help = 'Stop on network errors') - parser.add_argument('--order', default = 'desc', help = 'Run order (asc, desc, random)') - parser.add_argument('--testname', action = 'store_true', default = None, dest = 'testname', help = 'Make query with test name before test run') + parser=ArgumentParser(description='ClickHouse functional tests') + parser.add_argument('-q', '--queries', help='Path to queries dir') + parser.add_argument('--tmp', help='Path to tmp dir') + parser.add_argument('-b', '--binary', default='clickhouse', help='Main clickhouse binary') + parser.add_argument('-c', '--client', help='Client program') + parser.add_argument('--extract_from_config', help='extract-from-config program') + parser.add_argument('--configclient', help='Client config (if you use not default ports)') + parser.add_argument('--configserver', default= '/etc/clickhouse-server/config.xml', help='Preprocessed server config') + parser.add_argument('-o', '--output', help='Output xUnit compliant test report directory') + parser.add_argument('-t', '--timeout', type=int, default=600, help='Timeout for each test case in seconds') + parser.add_argument('test', nargs='?', help='Optional test case name regex') + parser.add_argument('-d', '--disabled', action='store_true', default=False, help='Also run disabled tests') + parser.add_argument('--stop', action='store_true', default=None, dest='stop', help='Stop on network errors') + parser.add_argument('--order', default='desc', help='Run order (asc, desc, random)') + parser.add_argument('--testname', action='store_true', default=None, dest='testname', help='Make query with test name before test run') + parser.add_argument('--hung-check', action='store_true', default=False) - parser.add_argument('--no-stateless', action = 'store_true', help = 'Disable all stateless tests') - parser.add_argument('--skip', nargs='+', help = "Skip these tests") - parser.add_argument('--no-long', action = 'store_false', dest = 'no_long', help = 'Do not run long tests') - group = parser.add_mutually_exclusive_group(required = False) - group.add_argument('--zookeeper', action = 'store_true', default = None, dest = 'zookeeper', help = 'Run zookeeper related tests') - group.add_argument('--no-zookeeper', action = 'store_false', default = None, dest = 'zookeeper', help = 'Do not run zookeeper related tests') - group = parser.add_mutually_exclusive_group(required = False) - group.add_argument('--shard', action = 'store_true', default = None, dest = 'shard', help = 'Run sharding related tests (required to clickhouse-server listen 127.0.0.2 127.0.0.3)') - group.add_argument('--no-shard', action = 'store_false', default = None, dest = 'shard', help = 'Do not run shard related tests') + parser.add_argument('--no-stateless', action='store_true', help='Disable all stateless tests') + parser.add_argument('--skip', nargs='+', help="Skip these tests") + parser.add_argument('--no-long', action='store_false', dest='no_long', help='Do not run long tests') + group=parser.add_mutually_exclusive_group(required=False) + group.add_argument('--zookeeper', action='store_true', default=None, dest='zookeeper', help='Run zookeeper related tests') + group.add_argument('--no-zookeeper', action='store_false', default=None, dest='zookeeper', help='Do not run zookeeper related tests') + group=parser.add_mutually_exclusive_group(required=False) + group.add_argument('--shard', action='store_true', default=None, dest='shard', help='Run sharding related tests (required to clickhouse-server listen 127.0.0.2 127.0.0.3)') + group.add_argument('--no-shard', action='store_false', default=None, dest='shard', help='Do not run shard related tests') args = parser.parse_args() diff --git a/dbms/tests/integration/helpers/cluster.py b/dbms/tests/integration/helpers/cluster.py index 8bd930ac1b0..240cc2c8695 100644 --- a/dbms/tests/integration/helpers/cluster.py +++ b/dbms/tests/integration/helpers/cluster.py @@ -482,6 +482,13 @@ class ClickHouseInstance: def get_query_request(self, *args, **kwargs): return self.client.get_query_request(*args, **kwargs) + def restart_clickhouse(self, stop_start_wait_sec=5): + if not self.stay_alive: + raise Exception("clickhouse can be restarted only with stay_alive=True instance") + + self.exec_in_container(["bash", "-c", "pkill clickhouse"], user='root') + time.sleep(stop_start_wait_sec) + self.exec_in_container(["bash", "-c", "{} --daemon".format(CLICKHOUSE_START_COMMAND)], user='root') def exec_in_container(self, cmd, detach=False, **kwargs): container = self.get_docker_handle() diff --git a/dbms/tests/integration/test_graphite_merge_tree/test.py b/dbms/tests/integration/test_graphite_merge_tree/test.py index 8e98c97e077..509fbac97d0 100644 --- a/dbms/tests/integration/test_graphite_merge_tree/test.py +++ b/dbms/tests/integration/test_graphite_merge_tree/test.py @@ -231,6 +231,50 @@ SELECT * FROM test.graphite; assert TSV(result) == TSV(expected) +def test_system_graphite_retentions(graphite_table): + expected = ''' +graphite_rollup \\\\.count$ sum 0 0 1 0 ['test'] ['graphite'] +graphite_rollup \\\\.max$ max 0 0 2 0 ['test'] ['graphite'] +graphite_rollup ^five_min\\\\. 31536000 14400 3 0 ['test'] ['graphite'] +graphite_rollup ^five_min\\\\. 5184000 3600 3 0 ['test'] ['graphite'] +graphite_rollup ^five_min\\\\. 0 300 3 0 ['test'] ['graphite'] +graphite_rollup ^one_min avg 31536000 600 4 0 ['test'] ['graphite'] +graphite_rollup ^one_min avg 7776000 300 4 0 ['test'] ['graphite'] +graphite_rollup ^one_min avg 0 60 4 0 ['test'] ['graphite'] + ''' + result = q('SELECT * from system.graphite_retentions') + + assert TSV(result) == TSV(expected) + + q(''' +DROP TABLE IF EXISTS test.graphite2; +CREATE TABLE test.graphite2 + (metric String, value Float64, timestamp UInt32, date Date, updated UInt32) + ENGINE = GraphiteMergeTree('graphite_rollup') + PARTITION BY toYYYYMM(date) + ORDER BY (metric, timestamp) + SETTINGS index_granularity=8192; + ''') + expected = ''' +graphite_rollup ['test','test'] ['graphite','graphite2'] +graphite_rollup ['test','test'] ['graphite','graphite2'] +graphite_rollup ['test','test'] ['graphite','graphite2'] +graphite_rollup ['test','test'] ['graphite','graphite2'] +graphite_rollup ['test','test'] ['graphite','graphite2'] +graphite_rollup ['test','test'] ['graphite','graphite2'] +graphite_rollup ['test','test'] ['graphite','graphite2'] +graphite_rollup ['test','test'] ['graphite','graphite2'] + ''' + result = q(''' + SELECT + config_name, + Tables.database, + Tables.table + FROM system.graphite_retentions + ''') + assert TSV(result) == TSV(expected) + + def test_path_dangling_pointer(graphite_table): q(''' DROP TABLE IF EXISTS test.graphite2; diff --git a/dbms/tests/integration/test_replication_without_zookeeper/__init__.py b/dbms/tests/integration/test_replication_without_zookeeper/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_replication_without_zookeeper/configs/remote_servers.xml b/dbms/tests/integration/test_replication_without_zookeeper/configs/remote_servers.xml new file mode 100644 index 00000000000..538aa72d386 --- /dev/null +++ b/dbms/tests/integration/test_replication_without_zookeeper/configs/remote_servers.xml @@ -0,0 +1,14 @@ + + + + + true + + shard_0 + node1 + 9000 + + + + + diff --git a/dbms/tests/integration/test_replication_without_zookeeper/test.py b/dbms/tests/integration/test_replication_without_zookeeper/test.py new file mode 100644 index 00000000000..408a8f8844d --- /dev/null +++ b/dbms/tests/integration/test_replication_without_zookeeper/test.py @@ -0,0 +1,48 @@ +import time +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import assert_eq_with_retry + +cluster = ClickHouseCluster(__file__) +node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True, stay_alive=True) + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + + node1.query( + ''' + CREATE DATABASE test; + CREATE TABLE test_table(date Date, id UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/replicated', 'node1') ORDER BY id PARTITION BY toYYYYMM(date); + ''') + + yield cluster + + except Exception as ex: + print ex + + finally: + cluster.shutdown() + +def drop_zk(zk): + zk.delete(path="/clickhouse", recursive=True) + +def test_startup_without_zookeeper(start_cluster): + node1.query("INSERT INTO test_table VALUES ('2018-10-01', 1), ('2018-10-02', 2), ('2018-10-03', 3)") + node1.query("SELECT COUNT(*) from test_table") == "3\n" + node1.query("SELECT is_readonly from system.replicas where table='test_table'") == "0\n" + + cluster.run_kazoo_commands_with_retries(drop_zk) + + time.sleep(5) + node1.query("SELECT COUNT(*) from test_table") == "3\n" + node1.query("SELECT is_readonly from system.replicas where table='test_table'") == "1\n" + + node1.restart_clickhouse() + time.sleep(5) + + node1.query("SELECT COUNT(*) from test_table") == "3\n" + node1.query("SELECT is_readonly from system.replicas where table='test_table'") == "1\n" diff --git a/dbms/tests/performance/string_search/ngram_distance.xml b/dbms/tests/performance/string_search/ngram_distance.xml new file mode 100644 index 00000000000..16960811067 --- /dev/null +++ b/dbms/tests/performance/string_search/ngram_distance.xml @@ -0,0 +1,42 @@ + + Distance search performance test + + + search + + + + hits_100m_single + + + loop + + + + 5 + 10000 + + + 50 + 60000 + + + + SELECT DISTINCT Title, ngramDistance(Title, 'what is love') AS distance FROM hits_100m_single ORDER BY distance ASC LIMIT 50 + SELECT DISTINCT Title, ngramDistance(Title, 'baby dont hurt me') AS distance FROM hits_100m_single ORDER BY distance ASC LIMIT 50 + SELECT DISTINCT Title, ngramDistance(Title, 'no more') AS distance FROM hits_100m_single ORDER BY distance ASC LIMIT 50 + SELECT DISTINCT Title, ngramDistanceCaseInsensitive(Title, 'wHAt Is lovE') AS distance FROM hits_100m_single ORDER BY distance ASC LIMIT 50 + SELECT DISTINCT Title, ngramDistanceCaseInsensitive(Title, 'BABY DonT hUrT me') AS distance FROM hits_100m_single ORDER BY distance ASC LIMIT 50 + SELECT DISTINCT Title, ngramDistanceCaseInsensitive(Title, 'nO MOrE') AS distance FROM hits_100m_single ORDER BY distance ASC LIMIT 50 + SELECT DISTINCT Title, ngramDistanceUTF8(Title, 'метрика') AS distance FROM hits_100m_single ORDER BY distance ASC LIMIT 50 + SELECT DISTINCT URL, ngramDistanceUTF8(URL, 'как дела') AS distance FROM hits_100m_single ORDER BY distance ASC LIMIT 50 + SELECT DISTINCT URL, ngramDistanceUTF8(URL, 'чем занимаешься') AS distance FROM hits_100m_single ORDER BY distance ASC LIMIT 50 + + SELECT DISTINCT Title, ngramDistanceCaseInsensitiveUTF8(Title, 'Метрика') AS distance FROM hits_100m_single ORDER BY distance ASC LIMIT 50 + SELECT DISTINCT URL, ngramDistanceCaseInsensitiveUTF8(URL, 'как дЕлА') AS distance FROM hits_100m_single ORDER BY distance ASC LIMIT 50 + SELECT DISTINCT URL, ngramDistanceCaseInsensitiveUTF8(URL, 'Чем зАнимаешЬся') AS distance FROM hits_100m_single ORDER BY distance ASC LIMIT 50 + + + + + diff --git a/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.reference b/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.reference index afe27bfae32..42036e1a81f 100644 --- a/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.reference +++ b/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.reference @@ -1,3 +1,7 @@ +< X-ClickHouse-Progress: {"read_rows":"0","read_bytes":"0","total_rows":"10"} +< X-ClickHouse-Progress: {"read_rows":"5","read_bytes":"40","total_rows":"10"} +< X-ClickHouse-Progress: {"read_rows":"10","read_bytes":"80","total_rows":"10"} +9 < X-ClickHouse-Progress: {"read_rows":"1","read_bytes":"8","total_rows":"0"} < X-ClickHouse-Progress: {"read_rows":"2","read_bytes":"16","total_rows":"0"} < X-ClickHouse-Progress: {"read_rows":"3","read_bytes":"24","total_rows":"0"} diff --git a/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh b/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh index 8e762515da1..4b1ca06aacc 100755 --- a/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh +++ b/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh @@ -3,6 +3,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh +${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}?max_block_size=5&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0" -d 'SELECT max(number) FROM numbers(10)' 2>&1 | grep -E 'Content-Encoding|X-ClickHouse-Progress|^[0-9]' # This test will fail with external poco (progress not supported) ${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}?max_block_size=1&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0" -d 'SELECT number FROM system.numbers LIMIT 10' 2>&1 | grep -E 'Content-Encoding|X-ClickHouse-Progress|^[0-9]' diff --git a/dbms/tests/queries/0_stateless/00446_clear_column_in_partition_zookeeper.sql b/dbms/tests/queries/0_stateless/00446_clear_column_in_partition_zookeeper.sql index 41d83ebe263..72f97d50886 100644 --- a/dbms/tests/queries/0_stateless/00446_clear_column_in_partition_zookeeper.sql +++ b/dbms/tests/queries/0_stateless/00446_clear_column_in_partition_zookeeper.sql @@ -26,6 +26,7 @@ CREATE TABLE test.clear_column1 (d Date, i Int64) ENGINE = ReplicatedMergeTree(' CREATE TABLE test.clear_column2 (d Date, i Int64) ENGINE = ReplicatedMergeTree('/clickhouse/test/tables/clear_column', '2', d, d, 8192); INSERT INTO test.clear_column1 (d) VALUES ('2000-01-01'), ('2000-02-01'); +SYSTEM SYNC REPLICA test.clear_column2; SET replication_alter_partitions_sync=2; ALTER TABLE test.clear_column1 ADD COLUMN s String; @@ -33,9 +34,10 @@ ALTER TABLE test.clear_column1 CLEAR COLUMN s IN PARTITION '200001'; INSERT INTO test.clear_column1 VALUES ('2000-01-01', 1, 'a'), ('2000-02-01', 2, 'b'); INSERT INTO test.clear_column1 VALUES ('2000-01-01', 3, 'c'), ('2000-02-01', 4, 'd'); +SYSTEM SYNC REPLICA test.clear_column2; SELECT 'all'; -SELECT * FROM test.clear_column1 ORDER BY d, i, s; +SELECT * FROM test.clear_column2 ORDER BY d, i, s; SELECT 'w/o i 1'; ALTER TABLE test.clear_column1 CLEAR COLUMN i IN PARTITION '200001'; diff --git a/dbms/tests/queries/0_stateless/00816_concurrent_alter_column.reference b/dbms/tests/queries/0_stateless/00816_concurrent_alter_column.reference new file mode 100644 index 00000000000..85dfb778810 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00816_concurrent_alter_column.reference @@ -0,0 +1 @@ +did not crash diff --git a/dbms/tests/queries/0_stateless/00816_concurrent_alter_column.sh b/dbms/tests/queries/0_stateless/00816_concurrent_alter_column.sh new file mode 100755 index 00000000000..c1fc0ebce6a --- /dev/null +++ b/dbms/tests/queries/0_stateless/00816_concurrent_alter_column.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -e + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +echo "DROP TABLE IF EXISTS test.concurrent_alter_column" | ${CLICKHOUSE_CLIENT} +echo "CREATE TABLE test.concurrent_alter_column (ts DATETIME) ENGINE = MergeTree PARTITION BY toStartOfDay(ts) ORDER BY tuple()" | ${CLICKHOUSE_CLIENT} + +for i in {1..500}; do echo "ALTER TABLE test.concurrent_alter_column ADD COLUMN c$i DOUBLE;"; done | ${CLICKHOUSE_CLIENT} -n + +for i in {1..100}; do echo "ALTER TABLE test.concurrent_alter_column ADD COLUMN d DOUBLE" | ${CLICKHOUSE_CLIENT}; sleep `echo 0.0$RANDOM`; echo "ALTER TABLE test.concurrent_alter_column DROP COLUMN d" | ${CLICKHOUSE_CLIENT} -n; done & +for i in {1..100}; do echo "ALTER TABLE test.concurrent_alter_column ADD COLUMN e DOUBLE" | ${CLICKHOUSE_CLIENT}; sleep `echo 0.0$RANDOM`; echo "ALTER TABLE test.concurrent_alter_column DROP COLUMN e" | ${CLICKHOUSE_CLIENT} -n; done & +for i in {1..100}; do echo "ALTER TABLE test.concurrent_alter_column ADD COLUMN f DOUBLE" | ${CLICKHOUSE_CLIENT}; sleep `echo 0.0$RANDOM`; echo "ALTER TABLE test.concurrent_alter_column DROP COLUMN f" | ${CLICKHOUSE_CLIENT} -n; done & + +wait + +echo "DROP TABLE test.concurrent_alter_column" | ${CLICKHOUSE_CLIENT} + +echo 'did not crash' diff --git a/dbms/tests/queries/0_stateless/00819_ast_refactoring_bugs.sql b/dbms/tests/queries/0_stateless/00819_ast_refactoring_bugs.sql index 010d03920c7..780dcd7e602 100644 --- a/dbms/tests/queries/0_stateless/00819_ast_refactoring_bugs.sql +++ b/dbms/tests/queries/0_stateless/00819_ast_refactoring_bugs.sql @@ -1,5 +1,5 @@ -DROP TABLE IF EXISTS test.visits; -CREATE TABLE test.visits +DROP TABLE IF EXISTS test.visits1; +CREATE TABLE test.visits1 ( Sign Int8, Arr Array(Int8), @@ -8,10 +8,10 @@ CREATE TABLE test.visits CounterID UInt32 ) ENGINE = Memory; -SELECT arrayMap(x -> x * Sign, Arr) FROM test.visits; +SELECT arrayMap(x -> x * Sign, Arr) FROM test.visits1; SELECT PP.Key2 AS `ym:s:pl2` -FROM test.visits +FROM test.visits1 ARRAY JOIN `ParsedParams.Key2` AS `PP.Key2`, `ParsedParams.Key1` AS `PP.Key1`, @@ -19,7 +19,7 @@ ARRAY JOIN arrayEnumerateUniq(`ParsedParams.Key2`) AS _uniq_ParsedParams WHERE CounterID = 100500; -DROP TABLE test.visits; +DROP TABLE test.visits1; select u, cumSum from ( select u, min(d) mn, max(d) mx, groupArray(d) dg, groupArray(v) vg, diff --git a/dbms/tests/queries/0_stateless/00826_cross_to_inner_join.reference b/dbms/tests/queries/0_stateless/00826_cross_to_inner_join.reference index 93cdf438a0f..244ff8c9600 100644 --- a/dbms/tests/queries/0_stateless/00826_cross_to_inner_join.reference +++ b/dbms/tests/queries/0_stateless/00826_cross_to_inner_join.reference @@ -56,26 +56,26 @@ comma nullable 1 1 1 1 2 2 1 2 cross -Explain ParsedAST (children 1)\n SelectWithUnionQuery (children 1)\n ExpressionList (children 1)\n SelectQuery (children 3)\n ExpressionList (children 1)\n Asterisk\n TablesInSelectQuery (children 2)\n TablesInSelectQueryElement (children 1)\n TableExpression (children 1)\n Identifier t1\n TablesInSelectQueryElement (children 2)\n TableExpression (children 1)\n Identifier t2\n TableJoin\n Function equals (children 1)\n ExpressionList (children 2)\n Identifier t1.a\n Identifier t2.a\n -Explain ParsedAST (children 1)\n SelectWithUnionQuery (children 1)\n ExpressionList (children 1)\n SelectQuery (children 2)\n ExpressionList (children 1)\n Asterisk\n TablesInSelectQuery (children 2)\n TablesInSelectQueryElement (children 1)\n TableExpression (children 1)\n Identifier t1\n TablesInSelectQueryElement (children 2)\n TableJoin (children 1)\n Function equals (children 1)\n ExpressionList (children 2)\n Identifier t1.a\n Identifier t2.a\n TableExpression (children 1)\n Identifier t2\n +SELECT \n a, \n b, \n t2.a, \n t2.b\nFROM t1 \nCROSS JOIN t2 \nWHERE a = t2.a +SELECT \n a, \n b, \n t2.a, \n t2.b\nFROM t1 \nALL INNER JOIN t2 ON a = t2.a\nWHERE a = t2.a cross nullable -Explain ParsedAST (children 1)\n SelectWithUnionQuery (children 1)\n ExpressionList (children 1)\n SelectQuery (children 3)\n ExpressionList (children 1)\n Asterisk\n TablesInSelectQuery (children 2)\n TablesInSelectQueryElement (children 1)\n TableExpression (children 1)\n Identifier t1\n TablesInSelectQueryElement (children 2)\n TableExpression (children 1)\n Identifier t2\n TableJoin\n Function equals (children 1)\n ExpressionList (children 2)\n Identifier t1.a\n Identifier t2.a\n -Explain ParsedAST (children 1)\n SelectWithUnionQuery (children 1)\n ExpressionList (children 1)\n SelectQuery (children 2)\n ExpressionList (children 1)\n Asterisk\n TablesInSelectQuery (children 2)\n TablesInSelectQueryElement (children 1)\n TableExpression (children 1)\n Identifier t1\n TablesInSelectQueryElement (children 2)\n TableJoin (children 1)\n Function equals (children 1)\n ExpressionList (children 2)\n Identifier t1.a\n Identifier t2.a\n TableExpression (children 1)\n Identifier t2\n +SELECT \n a, \n b, \n t2.a, \n t2.b\nFROM t1 \n, t2 \nWHERE a = t2.a +SELECT \n a, \n b, \n t2.a, \n t2.b\nFROM t1 \nALL INNER JOIN t2 ON a = t2.a\nWHERE a = t2.a cross nullable vs not nullable -Explain ParsedAST (children 1)\n SelectWithUnionQuery (children 1)\n ExpressionList (children 1)\n SelectQuery (children 3)\n ExpressionList (children 1)\n Asterisk\n TablesInSelectQuery (children 2)\n TablesInSelectQueryElement (children 1)\n TableExpression (children 1)\n Identifier t1\n TablesInSelectQueryElement (children 2)\n TableExpression (children 1)\n Identifier t2\n TableJoin\n Function equals (children 1)\n ExpressionList (children 2)\n Identifier t1.a\n Identifier t2.b\n -Explain ParsedAST (children 1)\n SelectWithUnionQuery (children 1)\n ExpressionList (children 1)\n SelectQuery (children 2)\n ExpressionList (children 1)\n Asterisk\n TablesInSelectQuery (children 2)\n TablesInSelectQueryElement (children 1)\n TableExpression (children 1)\n Identifier t1\n TablesInSelectQueryElement (children 2)\n TableJoin (children 1)\n Function equals (children 1)\n ExpressionList (children 2)\n Identifier t1.a\n Identifier t2.b\n TableExpression (children 1)\n Identifier t2\n +SELECT \n a, \n b, \n t2.a, \n t2.b\nFROM t1 \nCROSS JOIN t2 \nWHERE a = t2.b +SELECT \n a, \n b, \n t2.a, \n t2.b\nFROM t1 \nALL INNER JOIN t2 ON a = t2.b\nWHERE a = t2.b cross self -Explain ParsedAST (children 1)\n SelectWithUnionQuery (children 1)\n ExpressionList (children 1)\n SelectQuery (children 3)\n ExpressionList (children 1)\n Asterisk\n TablesInSelectQuery (children 2)\n TablesInSelectQueryElement (children 1)\n TableExpression (children 1)\n Identifier t1 (alias x)\n TablesInSelectQueryElement (children 2)\n TableExpression (children 1)\n Identifier t1 (alias y)\n TableJoin\n Function and (children 1)\n ExpressionList (children 2)\n Function equals (children 1)\n ExpressionList (children 2)\n Identifier x.a\n Identifier y.a\n Function equals (children 1)\n ExpressionList (children 2)\n Identifier x.b\n Identifier y.b\n -Explain ParsedAST (children 1)\n SelectWithUnionQuery (children 1)\n ExpressionList (children 1)\n SelectQuery (children 2)\n ExpressionList (children 1)\n Asterisk\n TablesInSelectQuery (children 2)\n TablesInSelectQueryElement (children 1)\n TableExpression (children 1)\n Identifier t1 (alias x)\n TablesInSelectQueryElement (children 2)\n TableJoin (children 1)\n Function and (children 1)\n ExpressionList (children 2)\n Function equals (children 1)\n ExpressionList (children 2)\n Identifier x.a\n Identifier y.a\n Function equals (children 1)\n ExpressionList (children 2)\n Identifier x.b\n Identifier y.b\n TableExpression (children 1)\n Identifier t1 (alias y)\n +SELECT \n a, \n b, \n y.a, \n y.b\nFROM t1 AS x \nCROSS JOIN t1 AS y \nWHERE (a = y.a) AND (b = y.b) +SELECT \n a, \n b, \n y.a, \n y.b\nFROM t1 AS x \nALL INNER JOIN t1 AS y ON (a = y.a) AND (b = y.b)\nWHERE (a = y.a) AND (b = y.b) cross one table expr -Explain ParsedAST (children 1)\n SelectWithUnionQuery (children 1)\n ExpressionList (children 1)\n SelectQuery (children 3)\n ExpressionList (children 1)\n Asterisk\n TablesInSelectQuery (children 2)\n TablesInSelectQueryElement (children 1)\n TableExpression (children 1)\n Identifier t1\n TablesInSelectQueryElement (children 2)\n TableExpression (children 1)\n Identifier t2\n TableJoin\n Function equals (children 1)\n ExpressionList (children 2)\n Identifier t1.a\n Identifier t1.b\n -Explain ParsedAST (children 1)\n SelectWithUnionQuery (children 1)\n ExpressionList (children 1)\n SelectQuery (children 3)\n ExpressionList (children 1)\n Asterisk\n TablesInSelectQuery (children 2)\n TablesInSelectQueryElement (children 1)\n TableExpression (children 1)\n Identifier t1\n TablesInSelectQueryElement (children 2)\n TableExpression (children 1)\n Identifier t2\n TableJoin\n Function equals (children 1)\n ExpressionList (children 2)\n Identifier t1.a\n Identifier t1.b\n +SELECT \n a, \n b, \n t2.a, \n t2.b\nFROM t1 \nCROSS JOIN t2 \nWHERE a = b +SELECT \n a, \n b, \n t2.a, \n t2.b\nFROM t1 \nCROSS JOIN t2 \nWHERE a = b cross multiple ands -Explain ParsedAST (children 1)\n SelectWithUnionQuery (children 1)\n ExpressionList (children 1)\n SelectQuery (children 3)\n ExpressionList (children 1)\n Asterisk\n TablesInSelectQuery (children 2)\n TablesInSelectQueryElement (children 1)\n TableExpression (children 1)\n Identifier t1\n TablesInSelectQueryElement (children 2)\n TableExpression (children 1)\n Identifier t2\n TableJoin\n Function and (children 1)\n ExpressionList (children 2)\n Function equals (children 1)\n ExpressionList (children 2)\n Identifier t1.a\n Identifier t2.a\n Function equals (children 1)\n ExpressionList (children 2)\n Identifier t1.b\n Identifier t2.b\n -Explain ParsedAST (children 1)\n SelectWithUnionQuery (children 1)\n ExpressionList (children 1)\n SelectQuery (children 2)\n ExpressionList (children 1)\n Asterisk\n TablesInSelectQuery (children 2)\n TablesInSelectQueryElement (children 1)\n TableExpression (children 1)\n Identifier t1\n TablesInSelectQueryElement (children 2)\n TableJoin (children 1)\n Function and (children 1)\n ExpressionList (children 2)\n Function equals (children 1)\n ExpressionList (children 2)\n Identifier t1.a\n Identifier t2.a\n Function equals (children 1)\n ExpressionList (children 2)\n Identifier t1.b\n Identifier t2.b\n TableExpression (children 1)\n Identifier t2\n +SELECT \n a, \n b, \n t2.a, \n t2.b\nFROM t1 \nCROSS JOIN t2 \nWHERE (a = t2.a) AND (b = t2.b) +SELECT \n a, \n b, \n t2.a, \n t2.b\nFROM t1 \nALL INNER JOIN t2 ON (a = t2.a) AND (b = t2.b)\nWHERE (a = t2.a) AND (b = t2.b) cross and inside and -Explain ParsedAST (children 1)\n SelectWithUnionQuery (children 1)\n ExpressionList (children 1)\n SelectQuery (children 3)\n ExpressionList (children 1)\n Asterisk\n TablesInSelectQuery (children 2)\n TablesInSelectQueryElement (children 1)\n TableExpression (children 1)\n Identifier t1\n TablesInSelectQueryElement (children 2)\n TableExpression (children 1)\n Identifier t2\n TableJoin\n Function and (children 1)\n ExpressionList (children 2)\n Function equals (children 1)\n ExpressionList (children 2)\n Identifier t1.a\n Identifier t2.a\n Function and (children 1)\n ExpressionList (children 2)\n Function equals (children 1)\n ExpressionList (children 2)\n Identifier t1.a\n Identifier t2.a\n Function and (children 1)\n ExpressionList (children 2)\n Function equals (children 1)\n ExpressionList (children 2)\n Identifier t1.a\n Identifier t2.a\n Function equals (children 1)\n ExpressionList (children 2)\n Identifier t1.b\n Identifier t2.b\n -Explain ParsedAST (children 1)\n SelectWithUnionQuery (children 1)\n ExpressionList (children 1)\n SelectQuery (children 2)\n ExpressionList (children 1)\n Asterisk\n TablesInSelectQuery (children 2)\n TablesInSelectQueryElement (children 1)\n TableExpression (children 1)\n Identifier t1\n TablesInSelectQueryElement (children 2)\n TableJoin (children 1)\n Function and (children 1)\n ExpressionList (children 4)\n Function equals (children 1)\n ExpressionList (children 2)\n Identifier t1.a\n Identifier t2.a\n Function equals (children 1)\n ExpressionList (children 2)\n Identifier t1.a\n Identifier t2.a\n Function equals (children 1)\n ExpressionList (children 2)\n Identifier t1.a\n Identifier t2.a\n Function equals (children 1)\n ExpressionList (children 2)\n Identifier t1.b\n Identifier t2.b\n TableExpression (children 1)\n Identifier t2\n +SELECT \n a, \n b, \n t2.a, \n t2.b\nFROM t1 \nCROSS JOIN t2 \nWHERE (a = t2.a) AND ((a = t2.a) AND ((a = t2.a) AND (b = t2.b))) +SELECT \n a, \n b, \n t2.a, \n t2.b\nFROM t1 \nALL INNER JOIN t2 ON (a = t2.a) AND (a = t2.a) AND (a = t2.a) AND (b = t2.b)\nWHERE (a = t2.a) AND ((a = t2.a) AND ((a = t2.a) AND (b = t2.b))) cross split conjunction -Explain ParsedAST (children 1)\n SelectWithUnionQuery (children 1)\n ExpressionList (children 1)\n SelectQuery (children 3)\n ExpressionList (children 1)\n Asterisk\n TablesInSelectQuery (children 2)\n TablesInSelectQueryElement (children 1)\n TableExpression (children 1)\n Identifier t1\n TablesInSelectQueryElement (children 2)\n TableExpression (children 1)\n Identifier t2\n TableJoin\n Function and (children 1)\n ExpressionList (children 4)\n Function equals (children 1)\n ExpressionList (children 2)\n Identifier t1.a\n Identifier t2.a\n Function equals (children 1)\n ExpressionList (children 2)\n Identifier t1.b\n Identifier t2.b\n Function greaterOrEquals (children 1)\n ExpressionList (children 2)\n Identifier t1.a\n Literal UInt64_1\n Function greater (children 1)\n ExpressionList (children 2)\n Identifier t2.b\n Literal UInt64_0\n -Explain ParsedAST (children 1)\n SelectWithUnionQuery (children 1)\n ExpressionList (children 1)\n SelectQuery (children 3)\n ExpressionList (children 1)\n Asterisk\n TablesInSelectQuery (children 2)\n TablesInSelectQueryElement (children 1)\n TableExpression (children 1)\n Identifier t1\n TablesInSelectQueryElement (children 2)\n TableJoin (children 1)\n Function and (children 1)\n ExpressionList (children 2)\n Function equals (children 1)\n ExpressionList (children 2)\n Identifier t1.a\n Identifier t2.a\n Function equals (children 1)\n ExpressionList (children 2)\n Identifier t1.b\n Identifier t2.b\n TableExpression (children 1)\n Identifier t2\n Function and (children 1)\n ExpressionList (children 4)\n Function equals (children 1)\n ExpressionList (children 2)\n Identifier t1.a\n Identifier t2.a\n Function equals (children 1)\n ExpressionList (children 2)\n Identifier t1.b\n Identifier t2.b\n Function greaterOrEquals (children 1)\n ExpressionList (children 2)\n Identifier t1.a\n Literal UInt64_1\n Function greater (children 1)\n ExpressionList (children 2)\n Identifier t2.b\n Literal UInt64_0\n +SELECT \n a, \n b, \n t2.a, \n t2.b\nFROM t1 \nCROSS JOIN t2 \nWHERE (a = t2.a) AND (b = t2.b) AND (a >= 1) AND (t2.b > 0) +SELECT \n a, \n b, \n t2.a, \n t2.b\nFROM t1 \nALL INNER JOIN t2 ON (a = t2.a) AND (b = t2.b)\nWHERE (a = t2.a) AND (b = t2.b) AND (a >= 1) AND (t2.b > 0) diff --git a/dbms/tests/queries/0_stateless/00826_cross_to_inner_join.sql b/dbms/tests/queries/0_stateless/00826_cross_to_inner_join.sql index 218ea1f1e45..b84cb268107 100644 --- a/dbms/tests/queries/0_stateless/00826_cross_to_inner_join.sql +++ b/dbms/tests/queries/0_stateless/00826_cross_to_inner_join.sql @@ -69,30 +69,30 @@ SELECT * FROM t1, t2 where t1.b = t2.b; SELECT 'cross'; -SET allow_experimental_cross_to_join_conversion = 0; AST SELECT * FROM t1 cross join t2 where t1.a = t2.a; -SET allow_experimental_cross_to_join_conversion = 1; AST SELECT * FROM t1 cross join t2 where t1.a = t2.a; +SET allow_experimental_cross_to_join_conversion = 0; ANALYZE SELECT * FROM t1 cross join t2 where t1.a = t2.a; +SET allow_experimental_cross_to_join_conversion = 1; ANALYZE SELECT * FROM t1 cross join t2 where t1.a = t2.a; SELECT 'cross nullable'; -SET allow_experimental_cross_to_join_conversion = 0; AST SELECT * FROM t1, t2 where t1.a = t2.a; -SET allow_experimental_cross_to_join_conversion = 1; AST SELECT * FROM t1, t2 where t1.a = t2.a; +SET allow_experimental_cross_to_join_conversion = 0; ANALYZE SELECT * FROM t1, t2 where t1.a = t2.a; +SET allow_experimental_cross_to_join_conversion = 1; ANALYZE SELECT * FROM t1, t2 where t1.a = t2.a; SELECT 'cross nullable vs not nullable'; -SET allow_experimental_cross_to_join_conversion = 0; AST SELECT * FROM t1 cross join t2 where t1.a = t2.b; -SET allow_experimental_cross_to_join_conversion = 1; AST SELECT * FROM t1 cross join t2 where t1.a = t2.b; +SET allow_experimental_cross_to_join_conversion = 0; ANALYZE SELECT * FROM t1 cross join t2 where t1.a = t2.b; +SET allow_experimental_cross_to_join_conversion = 1; ANALYZE SELECT * FROM t1 cross join t2 where t1.a = t2.b; SELECT 'cross self'; -SET allow_experimental_cross_to_join_conversion = 0; AST SELECT * FROM t1 x cross join t1 y where x.a = y.a and x.b = y.b; -SET allow_experimental_cross_to_join_conversion = 1; AST SELECT * FROM t1 x cross join t1 y where x.a = y.a and x.b = y.b; +SET allow_experimental_cross_to_join_conversion = 0; ANALYZE SELECT * FROM t1 x cross join t1 y where x.a = y.a and x.b = y.b; +SET allow_experimental_cross_to_join_conversion = 1; ANALYZE SELECT * FROM t1 x cross join t1 y where x.a = y.a and x.b = y.b; SELECT 'cross one table expr'; -SET allow_experimental_cross_to_join_conversion = 0; AST SELECT * FROM t1 cross join t2 where t1.a = t1.b; -SET allow_experimental_cross_to_join_conversion = 1; AST SELECT * FROM t1 cross join t2 where t1.a = t1.b; +SET allow_experimental_cross_to_join_conversion = 0; ANALYZE SELECT * FROM t1 cross join t2 where t1.a = t1.b; +SET allow_experimental_cross_to_join_conversion = 1; ANALYZE SELECT * FROM t1 cross join t2 where t1.a = t1.b; SELECT 'cross multiple ands'; -SET allow_experimental_cross_to_join_conversion = 0; AST SELECT * FROM t1 cross join t2 where t1.a = t2.a and t1.b = t2.b; -SET allow_experimental_cross_to_join_conversion = 1; AST SELECT * FROM t1 cross join t2 where t1.a = t2.a and t1.b = t2.b; +SET allow_experimental_cross_to_join_conversion = 0; ANALYZE SELECT * FROM t1 cross join t2 where t1.a = t2.a and t1.b = t2.b; +SET allow_experimental_cross_to_join_conversion = 1; ANALYZE SELECT * FROM t1 cross join t2 where t1.a = t2.a and t1.b = t2.b; SELECT 'cross and inside and'; -SET allow_experimental_cross_to_join_conversion = 0; AST SELECT * FROM t1 cross join t2 where t1.a = t2.a and (t1.a = t2.a and (t1.a = t2.a and t1.b = t2.b)); -SET allow_experimental_cross_to_join_conversion = 1; AST SELECT * FROM t1 cross join t2 where t1.a = t2.a and (t1.a = t2.a and (t1.a = t2.a and t1.b = t2.b)); +SET allow_experimental_cross_to_join_conversion = 0; ANALYZE SELECT * FROM t1 cross join t2 where t1.a = t2.a and (t1.a = t2.a and (t1.a = t2.a and t1.b = t2.b)); +SET allow_experimental_cross_to_join_conversion = 1; ANALYZE SELECT * FROM t1 cross join t2 where t1.a = t2.a and (t1.a = t2.a and (t1.a = t2.a and t1.b = t2.b)); SELECT 'cross split conjunction'; -SET allow_experimental_cross_to_join_conversion = 0; AST SELECT * FROM t1 cross join t2 where t1.a = t2.a and t1.b = t2.b and t1.a >= 1 and t2.b > 0; -SET allow_experimental_cross_to_join_conversion = 1; AST SELECT * FROM t1 cross join t2 where t1.a = t2.a and t1.b = t2.b and t1.a >= 1 and t2.b > 0; +SET allow_experimental_cross_to_join_conversion = 0; ANALYZE SELECT * FROM t1 cross join t2 where t1.a = t2.a and t1.b = t2.b and t1.a >= 1 and t2.b > 0; +SET allow_experimental_cross_to_join_conversion = 1; ANALYZE SELECT * FROM t1 cross join t2 where t1.a = t2.a and t1.b = t2.b and t1.a >= 1 and t2.b > 0; DROP TABLE t1; DROP TABLE t2; diff --git a/dbms/tests/queries/0_stateless/00849_multiple_comma_join.reference b/dbms/tests/queries/0_stateless/00849_multiple_comma_join.reference new file mode 100644 index 00000000000..15333f14e53 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00849_multiple_comma_join.reference @@ -0,0 +1,55 @@ +SELECT a\nFROM t1 \nCROSS JOIN t2 +SELECT a\nFROM t1 \nALL INNER JOIN t2 ON a = t2.a\nWHERE a = t2.a +SELECT a\nFROM t1 \nALL INNER JOIN t2 ON b = t2.b\nWHERE b = t2.b +SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT \n a AS `--t1.a`, \n b, \n t2.a AS `--t2.a`, \n t2.b\n FROM t1 \n ALL INNER JOIN t2 ON `--t1.a` = `--t2.a`\n) \nALL INNER JOIN t3 ON `--t1.a` = a\nWHERE (`--t1.a` = `--t2.a`) AND (`--t1.a` = a) +SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT \n a AS `--t1.a`, \n b AS `--t1.b`, \n t2.a, \n t2.b AS `--t2.b`\n FROM t1 \n ALL INNER JOIN t2 ON `--t1.b` = `--t2.b`\n) \nALL INNER JOIN t3 ON `--t1.b` = b\nWHERE (`--t1.b` = `--t2.b`) AND (`--t1.b` = b) +SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT \n `--t1.a`, \n b, \n `--t2.a`, \n `t2.b`, \n a AS `--t3.a`, \n t3.b\n FROM \n (\n SELECT \n a AS `--t1.a`, \n b, \n t2.a AS `--t2.a`, \n t2.b\n FROM t1 \n ALL INNER JOIN t2 ON `--t1.a` = `--t2.a`\n ) \n ALL INNER JOIN t3 ON `--t1.a` = `--t3.a`\n) \nALL INNER JOIN t4 ON `--t1.a` = a\nWHERE (`--t1.a` = `--t2.a`) AND (`--t1.a` = `--t3.a`) AND (`--t1.a` = a) +SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT \n `--t1.a`, \n `--t1.b`, \n `t2.a`, \n `--t2.b`, \n a, \n b AS `--t3.b`\n FROM \n (\n SELECT \n a AS `--t1.a`, \n b AS `--t1.b`, \n t2.a, \n t2.b AS `--t2.b`\n FROM t1 \n ALL INNER JOIN t2 ON `--t1.b` = `--t2.b`\n ) \n ALL INNER JOIN t3 ON `--t1.b` = `--t3.b`\n) \nALL INNER JOIN t4 ON `--t1.b` = b\nWHERE (`--t1.b` = `--t2.b`) AND (`--t1.b` = `--t3.b`) AND (`--t1.b` = b) +SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT \n `--t1.a`, \n b, \n `--t2.a`, \n `t2.b`, \n a AS `--t3.a`, \n t3.b\n FROM \n (\n SELECT \n a AS `--t1.a`, \n b, \n t2.a AS `--t2.a`, \n t2.b\n FROM t1 \n ALL INNER JOIN t2 ON `--t2.a` = `--t1.a`\n ) \n ALL INNER JOIN t3 ON `--t2.a` = `--t3.a`\n) \nALL INNER JOIN t4 ON `--t2.a` = a\nWHERE (`--t2.a` = `--t1.a`) AND (`--t2.a` = `--t3.a`) AND (`--t2.a` = a) +SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT \n `--t1.a`, \n b, \n `--t2.a`, \n `t2.b`, \n a AS `--t3.a`, \n t3.b\n FROM \n (\n SELECT \n a AS `--t1.a`, \n b, \n t2.a AS `--t2.a`, \n t2.b\n FROM t1 \n CROSS JOIN t2 \n ) \n ALL INNER JOIN t3 ON (`--t3.a` = `--t1.a`) AND (`--t3.a` = `--t2.a`)\n) \nALL INNER JOIN t4 ON `--t3.a` = a\nWHERE (`--t3.a` = `--t1.a`) AND (`--t3.a` = `--t2.a`) AND (`--t3.a` = a) +SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT \n `--t1.a`, \n b, \n `--t2.a`, \n `t2.b`, \n a AS `--t3.a`, \n t3.b\n FROM \n (\n SELECT \n a AS `--t1.a`, \n b, \n t2.a AS `--t2.a`, \n t2.b\n FROM t1 \n CROSS JOIN t2 \n ) \n CROSS JOIN t3 \n) \nALL INNER JOIN t4 ON (a = `--t1.a`) AND (a = `--t2.a`) AND (a = `--t3.a`)\nWHERE (a = `--t1.a`) AND (a = `--t2.a`) AND (a = `--t3.a`) +SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT \n `--t1.a`, \n b, \n `--t2.a`, \n `t2.b`, \n a AS `--t3.a`, \n t3.b\n FROM \n (\n SELECT \n a AS `--t1.a`, \n b, \n t2.a AS `--t2.a`, \n t2.b\n FROM t1 \n ALL INNER JOIN t2 ON `--t1.a` = `--t2.a`\n ) \n ALL INNER JOIN t3 ON `--t2.a` = `--t3.a`\n) \nALL INNER JOIN t4 ON `--t3.a` = a\nWHERE (`--t1.a` = `--t2.a`) AND (`--t2.a` = `--t3.a`) AND (`--t3.a` = a) +SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT \n `--t1.a`, \n b, \n `t2.a`, \n `t2.b`, \n a, \n t3.b\n FROM \n (\n SELECT \n a AS `--t1.a`, \n b, \n t2.a, \n t2.b\n FROM t1 \n CROSS JOIN t2 \n ) \n CROSS JOIN t3 \n) \nCROSS JOIN t4 +SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT \n `--t1.a`, \n b, \n `t2.a`, \n `t2.b`, \n a, \n t3.b\n FROM \n (\n SELECT \n a AS `--t1.a`, \n b, \n t2.a, \n t2.b\n FROM t1 \n CROSS JOIN t2 \n ) \n CROSS JOIN t3 \n) \nCROSS JOIN t4 +SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT \n a AS `--t1.a`, \n b, \n t2.a AS `--t2.a`, \n t2.b\n FROM t1 \n ALL INNER JOIN t2 ON `--t1.a` = `--t2.a`\n) \nCROSS JOIN t3 +SELECT * FROM t1, t2 +1 1 1 1 +1 1 1 \N +2 2 1 1 +2 2 1 \N +3 3 1 1 +3 3 1 \N +4 4 1 1 +4 4 1 \N +SELECT * FROM t1, t2 WHERE t1.a = t2.a +1 1 1 1 +1 1 1 \N +SELECT t1.a, t2.a FROM t1, t2 WHERE t1.b = t2.b +1 1 +SELECT t1.a, t2.b, t3.b FROM t1, t2, t3 WHERE t1.a = t2.a AND t1.a = t3.a +1 1 1 +1 1 \N +1 \N 1 +1 \N \N +SELECT t1.a, t2.b, t3.b FROM t1, t2, t3 WHERE t1.b = t2.b AND t1.b = t3.b +1 1 1 +SELECT t1.a, t2.b, t3.b, t4.b FROM t1, t2, t3, t4 WHERE t1.a = t2.a AND t1.a = t3.a AND t1.a = t4.a +1 1 1 1 +1 1 1 \N +1 1 \N 1 +1 1 \N \N +1 \N 1 1 +1 \N 1 \N +1 \N \N 1 +1 \N \N \N +SELECT t1.a, t2.b, t3.b, t4.b FROM t1, t2, t3, t4 WHERE t1.b = t2.b AND t1.b = t3.b AND t1.b = t4.b +1 1 1 1 +SELECT t1.a, t2.b, t3.b, t4.b FROM t1, t2, t3, t4 WHERE t1.a = t2.a AND t2.a = t3.a AND t3.a = t4.a +1 1 1 1 +1 1 1 \N +1 1 \N 1 +1 1 \N \N +1 \N 1 1 +1 \N 1 \N +1 \N \N 1 +1 \N \N \N diff --git a/dbms/tests/queries/0_stateless/00849_multiple_comma_join.sql b/dbms/tests/queries/0_stateless/00849_multiple_comma_join.sql new file mode 100644 index 00000000000..0e5aa4884a5 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00849_multiple_comma_join.sql @@ -0,0 +1,59 @@ +SET enable_debug_queries = 1; +USE test; + +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +DROP TABLE IF EXISTS t3; +DROP TABLE IF EXISTS t4; + +CREATE TABLE t1 (a UInt32, b Nullable(Int32)) ENGINE = Memory; +CREATE TABLE t2 (a UInt32, b Nullable(Int32)) ENGINE = Memory; +CREATE TABLE t3 (a UInt32, b Nullable(Int32)) ENGINE = Memory; +CREATE TABLE t4 (a UInt32, b Nullable(Int32)) ENGINE = Memory; + +ANALYZE SELECT t1.a FROM t1, t2; +ANALYZE SELECT t1.a FROM t1, t2 WHERE t1.a = t2.a; +ANALYZE SELECT t1.a FROM t1, t2 WHERE t1.b = t2.b; +ANALYZE SELECT t1.a FROM t1, t2, t3 WHERE t1.a = t2.a AND t1.a = t3.a; +ANALYZE SELECT t1.a FROM t1, t2, t3 WHERE t1.b = t2.b AND t1.b = t3.b; +ANALYZE SELECT t1.a FROM t1, t2, t3, t4 WHERE t1.a = t2.a AND t1.a = t3.a AND t1.a = t4.a; +ANALYZE SELECT t1.a FROM t1, t2, t3, t4 WHERE t1.b = t2.b AND t1.b = t3.b AND t1.b = t4.b; + +ANALYZE SELECT t1.a FROM t1, t2, t3, t4 WHERE t2.a = t1.a AND t2.a = t3.a AND t2.a = t4.a; +ANALYZE SELECT t1.a FROM t1, t2, t3, t4 WHERE t3.a = t1.a AND t3.a = t2.a AND t3.a = t4.a; +ANALYZE SELECT t1.a FROM t1, t2, t3, t4 WHERE t4.a = t1.a AND t4.a = t2.a AND t4.a = t3.a; +ANALYZE SELECT t1.a FROM t1, t2, t3, t4 WHERE t1.a = t2.a AND t2.a = t3.a AND t3.a = t4.a; + +ANALYZE SELECT t1.a FROM t1, t2, t3, t4; +ANALYZE SELECT t1.a FROM t1 CROSS JOIN t2 CROSS JOIN t3 CROSS JOIN t4; + +ANALYZE SELECT t1.a FROM t1, t2 CROSS JOIN t3; -- { serverError 48 } +ANALYZE SELECT t1.a FROM t1 JOIN t2 USING a CROSS JOIN t3; -- { serverError 48 } +ANALYZE SELECT t1.a FROM t1 JOIN t2 ON t1.a = t2.a CROSS JOIN t3; + +INSERT INTO t1 values (1,1), (2,2), (3,3), (4,4); +INSERT INTO t2 values (1,1), (1, Null); +INSERT INTO t3 values (1,1), (1, Null); +INSERT INTO t4 values (1,1), (1, Null); + +SELECT 'SELECT * FROM t1, t2'; +SELECT * FROM t1, t2; +SELECT 'SELECT * FROM t1, t2 WHERE t1.a = t2.a'; +SELECT * FROM t1, t2 WHERE t1.a = t2.a; +SELECT 'SELECT t1.a, t2.a FROM t1, t2 WHERE t1.b = t2.b'; +SELECT t1.a, t2.b FROM t1, t2 WHERE t1.b = t2.b; +SELECT 'SELECT t1.a, t2.b, t3.b FROM t1, t2, t3 WHERE t1.a = t2.a AND t1.a = t3.a'; +SELECT t1.a, t2.b, t3.b FROM t1, t2, t3 WHERE t1.a = t2.a AND t1.a = t3.a; +SELECT 'SELECT t1.a, t2.b, t3.b FROM t1, t2, t3 WHERE t1.b = t2.b AND t1.b = t3.b'; +SELECT t1.a, t2.b, t3.b FROM t1, t2, t3 WHERE t1.b = t2.b AND t1.b = t3.b; +SELECT 'SELECT t1.a, t2.b, t3.b, t4.b FROM t1, t2, t3, t4 WHERE t1.a = t2.a AND t1.a = t3.a AND t1.a = t4.a'; +SELECT t1.a, t2.b, t3.b, t4.b FROM t1, t2, t3, t4 WHERE t1.a = t2.a AND t1.a = t3.a AND t1.a = t4.a; +SELECT 'SELECT t1.a, t2.b, t3.b, t4.b FROM t1, t2, t3, t4 WHERE t1.b = t2.b AND t1.b = t3.b AND t1.b = t4.b'; +SELECT t1.a, t2.b, t3.b, t4.b FROM t1, t2, t3, t4 WHERE t1.b = t2.b AND t1.b = t3.b AND t1.b = t4.b; +SELECT 'SELECT t1.a, t2.b, t3.b, t4.b FROM t1, t2, t3, t4 WHERE t1.a = t2.a AND t2.a = t3.a AND t3.a = t4.a'; +SELECT t1.a, t2.b, t3.b, t4.b FROM t1, t2, t3, t4 WHERE t1.a = t2.a AND t2.a = t3.a AND t3.a = t4.a; + +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +DROP TABLE t4; diff --git a/dbms/tests/queries/0_stateless/00850_global_join_dups.reference b/dbms/tests/queries/0_stateless/00850_global_join_dups.reference new file mode 100644 index 00000000000..b261da18d51 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00850_global_join_dups.reference @@ -0,0 +1,2 @@ +1 +0 diff --git a/dbms/tests/queries/0_stateless/00850_global_join_dups.sql b/dbms/tests/queries/0_stateless/00850_global_join_dups.sql new file mode 100644 index 00000000000..faf0397374a --- /dev/null +++ b/dbms/tests/queries/0_stateless/00850_global_join_dups.sql @@ -0,0 +1,83 @@ +USE test; +DROP TABLE IF EXISTS t_local; +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; + +CREATE TABLE t_local (dummy UInt8) ENGINE = Memory; +CREATE TABLE t1 (dummy UInt8) ENGINE = Distributed(test_shard_localhost, 'test', 't_local'); +CREATE TABLE t2 (dummy UInt8) ENGINE = Distributed(test_shard_localhost, 'test', 't_local'); + +INSERT INTO t_local VALUES (1); + +SET asterisk_left_columns_only = 1; + +SELECT * FROM t1 +GLOBAL INNER JOIN +( + SELECT * + FROM ( SELECT * FROM t2 ) + INNER JOIN ( SELECT * FROM t1 ) + USING dummy +) USING dummy; + +DROP TABLE t_local; +DROP TABLE t1; +DROP TABLE t2; + + +SELECT * FROM remote('127.0.0.2', system.one) +GLOBAL INNER JOIN +( + SELECT * + FROM ( SELECT dummy FROM remote('127.0.0.2', system.one) ) t1 + GLOBAL INNER JOIN ( SELECT dummy FROM remote('127.0.0.3', system.one) ) t2 + USING dummy +) USING dummy; + + +-- SET asterisk_left_columns_only = 0; +-- +-- SELECT * FROM remote('127.0.0.2', system.one) +-- GLOBAL INNER JOIN +-- ( +-- SELECT *, dummy +-- FROM ( SELECT dummy FROM remote('127.0.0.2', system.one) ) t1 +-- GLOBAL INNER JOIN ( SELECT dummy FROM remote('127.0.0.3', system.one) ) t2 +-- USING dummy +-- ) USING dummy; +-- +-- SELECT * FROM remote('127.0.0.2', system.one) +-- GLOBAL INNER JOIN +-- ( +-- SELECT *, t1.*, t2.* +-- FROM ( SELECT toUInt8(1) AS dummy ) t1 +-- INNER JOIN ( SELECT toUInt8(1) AS dummy ) t2 +-- USING dummy +-- ) USING dummy; +-- +-- SELECT * FROM remote('127.0.0.2', system.one) +-- GLOBAL INNER JOIN +-- ( +-- SELECT *, dummy +-- FROM ( SELECT toUInt8(1) AS dummy ) t1 +-- INNER JOIN ( SELECT toUInt8(1) AS dummy ) t2 +-- USING dummy +-- ) USING dummy; +-- +-- SELECT * FROM remote('127.0.0.2', system.one) +-- GLOBAL INNER JOIN +-- ( +-- SELECT * +-- FROM ( SELECT dummy FROM remote('127.0.0.3', system.one) ) t1 +-- GLOBAL INNER JOIN ( SELECT toUInt8(1) AS dummy ) t2 +-- USING dummy +-- ) USING dummy; +-- +-- SELECT * FROM remote('127.0.0.2', system.one) +-- GLOBAL INNER JOIN +-- ( +-- SELECT * +-- FROM ( SELECT toUInt8(1) AS dummy ) t1 +-- GLOBAL INNER JOIN ( SELECT dummy FROM remote('127.0.0.3', system.one) ) t2 +-- USING dummy +-- ) USING dummy; diff --git a/dbms/tests/queries/0_stateless/00909_kill_not_initialized_query.reference b/dbms/tests/queries/0_stateless/00909_kill_not_initialized_query.reference index aa47d0d46d4..b1616b08f0b 100644 --- a/dbms/tests/queries/0_stateless/00909_kill_not_initialized_query.reference +++ b/dbms/tests/queries/0_stateless/00909_kill_not_initialized_query.reference @@ -1,2 +1 @@ -0 -0 +killed diff --git a/dbms/tests/queries/0_stateless/00909_kill_not_initialized_query.sh b/dbms/tests/queries/0_stateless/00909_kill_not_initialized_query.sh index 76d6424e690..72c61b55cbf 100755 --- a/dbms/tests/queries/0_stateless/00909_kill_not_initialized_query.sh +++ b/dbms/tests/queries/0_stateless/00909_kill_not_initialized_query.sh @@ -36,11 +36,16 @@ sleep 1 # Kill $query_for_pending SYNC. This query is not blocker, so it should be killed fast. timeout 5 $CLICKHOUSE_CLIENT -q "KILL QUERY WHERE query='$query_for_pending' SYNC" &>/dev/null -# But let's sleep a little time, just to be sure -sleep 3 - # Both queries have to be killed, doesn't matter with SYNC or ASYNC kill -$CLICKHOUSE_CLIENT -q "SELECT count() FROM system.processes where query='$query_for_pending'" -$CLICKHOUSE_CLIENT -q "SELECT count() FROM system.processes where query='$query_to_kill'" +for run in {1..15} +do + sleep 1 + no_first_query=`$CLICKHOUSE_CLIENT -q "SELECT count() FROM system.processes where query='$query_for_pending'"` + no_second_query=`$CLICKHOUSE_CLIENT -q "SELECT count() FROM system.processes where query='$query_to_kill'"` + if [ $no_first_query == "0" ] && [ $no_second_query == "0" ]; then + echo "killed" + break + fi +done $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS test.cannot_kill_query" &>/dev/null diff --git a/dbms/tests/queries/0_stateless/00909_ngram_distance.reference b/dbms/tests/queries/0_stateless/00909_ngram_distance.reference new file mode 100644 index 00000000000..356cc5db466 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00909_ngram_distance.reference @@ -0,0 +1,509 @@ +0 +0 +0 +0 +0 +1000 +1000 +1000 +1000 +1000 +1000 +1000 +1000 +1000 +1000 +0 +0 +0 +0 +0 +77 +77 +77 +77 +77 +636 +636 +636 +636 +636 +1000 +1000 +1000 +1000 +1000 +0 +1000 +1000 +0 +77 +636 +1000 +привет как дела?... Херсон 297 +пап привет как дела - Яндекс.Видео 422 +привет как дела клип - Яндекс.Видео 435 +привет братан как дела - Яндекс.Видео 500 +привет 529 +http://metric.ru/ 1000 +http://autometric.ru/ 1000 +http://metrica.yandex.com/ 1000 +http://metris.ru/ 1000 +http://metrika.ru/ 1000 + 1000 +привет как дела?... Херсон 459 +пап привет как дела - Яндекс.Видео 511 +привет 529 +привет как дела клип - Яндекс.Видео 565 +привет братан как дела - Яндекс.Видео 583 +http://metric.ru/ 1000 +http://autometric.ru/ 1000 +http://metrica.yandex.com/ 1000 +http://metris.ru/ 1000 +http://metrika.ru/ 1000 + 1000 +http://metrika.ru/ 524 +http://metric.ru/ 700 +http://metris.ru/ 700 +http://autometric.ru/ 750 +http://metrica.yandex.com/ 793 +привет как дела?... Херсон 1000 +привет как дела клип - Яндекс.Видео 1000 +привет 1000 +пап привет как дела - Яндекс.Видео 1000 +привет братан как дела - Яндекс.Видео 1000 + 1000 +http://metric.ru/ 600 +http://metrica.yandex.com/ 655 +http://autometric.ru/ 667 +http://metris.ru/ 700 +http://metrika.ru/ 714 +привет как дела?... Херсон 1000 +привет как дела клип - Яндекс.Видео 1000 +привет 1000 +пап привет как дела - Яндекс.Видео 1000 +привет братан как дела - Яндекс.Видео 1000 + 1000 +http://metrika.ru/ 619 +http://metric.ru/ 700 +http://metris.ru/ 700 +http://autometric.ru/ 750 +http://metrica.yandex.com/ 793 +привет как дела?... Херсон 1000 +привет как дела клип - Яндекс.Видео 1000 +привет 1000 +пап привет как дела - Яндекс.Видео 1000 +привет братан как дела - Яндекс.Видео 1000 + 1000 +http://metric.ru/ 600 +http://autometric.ru/ 667 +http://metris.ru/ 700 +http://metrika.ru/ 714 +http://metrica.yandex.com/ 724 +привет как дела?... Херсон 1000 +привет как дела клип - Яндекс.Видео 1000 +привет 1000 +пап привет как дела - Яндекс.Видео 1000 +привет братан как дела - Яндекс.Видео 1000 + 1000 +http://metrica.yandex.com/ 714 +привет как дела?... Херсон 1000 +привет как дела клип - Яндекс.Видео 1000 +привет 1000 +пап привет как дела - Яндекс.Видео 1000 +привет братан как дела - Яндекс.Видео 1000 +http://metric.ru/ 1000 +http://autometric.ru/ 1000 +http://metris.ru/ 1000 +http://metrika.ru/ 1000 + 1000 +0 +0 +0 +0 +0 +1000 +1000 +1000 +1000 +1000 +1000 +1000 +1000 +1000 +1000 +0 +0 +0 +0 +0 +77 +77 +77 +77 +77 +636 +636 +636 +636 +636 +1000 +1000 +1000 +1000 +1000 +0 +1000 +1000 +429 +77 +636 +1000 +привет как дела?... Херсон 297 +пап привет как дела - Яндекс.Видео 422 +привет как дела клип - Яндекс.Видео 435 +привет братан как дела - Яндекс.Видео 500 +привет 529 +http://metric.ru/ 1000 +http://autometric.ru/ 1000 +http://metrica.yandex.com/ 1000 +http://metris.ru/ 1000 +http://metrika.ru/ 1000 + 1000 +привет как дела?... Херсон 676 +пап привет как дела - Яндекс.Видео 733 +привет как дела клип - Яндекс.Видео 739 +привет братан как дела - Яндекс.Видео 750 +привет 882 +http://metric.ru/ 1000 +http://autometric.ru/ 1000 +http://metrica.yandex.com/ 1000 +http://metris.ru/ 1000 +http://metrika.ru/ 1000 + 1000 +http://metrika.ru/ 524 +http://metric.ru/ 700 +http://metris.ru/ 700 +http://autometric.ru/ 750 +http://metrica.yandex.com/ 793 +привет как дела?... Херсон 1000 +привет как дела клип - Яндекс.Видео 1000 +привет 1000 +пап привет как дела - Яндекс.Видео 1000 +привет братан как дела - Яндекс.Видео 1000 + 1000 +http://metrika.ru/ 524 +http://metric.ru/ 700 +http://metris.ru/ 700 +http://autometric.ru/ 750 +http://metrica.yandex.com/ 793 +привет как дела?... Херсон 1000 +привет как дела клип - Яндекс.Видео 1000 +привет 1000 +пап привет как дела - Яндекс.Видео 1000 +привет братан как дела - Яндекс.Видео 1000 + 1000 +http://metric.ru/ 600 +http://metrica.yandex.com/ 655 +http://autometric.ru/ 667 +http://metris.ru/ 700 +http://metrika.ru/ 714 +привет как дела?... Херсон 1000 +привет как дела клип - Яндекс.Видео 1000 +привет 1000 +пап привет как дела - Яндекс.Видео 1000 +привет братан как дела - Яндекс.Видео 1000 + 1000 +http://metrika.ru/ 619 +http://metric.ru/ 700 +http://metris.ru/ 700 +http://autometric.ru/ 750 +http://metrica.yandex.com/ 793 +привет как дела?... Херсон 1000 +привет как дела клип - Яндекс.Видео 1000 +привет 1000 +пап привет как дела - Яндекс.Видео 1000 +привет братан как дела - Яндекс.Видео 1000 + 1000 +http://metric.ru/ 600 +http://autometric.ru/ 667 +http://metris.ru/ 700 +http://metrika.ru/ 714 +http://metrica.yandex.com/ 724 +привет как дела?... Херсон 1000 +привет как дела клип - Яндекс.Видео 1000 +привет 1000 +пап привет как дела - Яндекс.Видео 1000 +привет братан как дела - Яндекс.Видео 1000 + 1000 +http://metrica.yandex.com/ 714 +привет как дела?... Херсон 1000 +привет как дела клип - Яндекс.Видео 1000 +привет 1000 +пап привет как дела - Яндекс.Видео 1000 +привет братан как дела - Яндекс.Видео 1000 +http://metric.ru/ 1000 +http://autometric.ru/ 1000 +http://metris.ru/ 1000 +http://metrika.ru/ 1000 + 1000 +привет как дела клип - Яндекс.Видео 182 +пап привет как дела - Яндекс.Видео 354 +привет братан как дела - Яндекс.Видео 382 +привет как дела?... Херсон 649 +привет 838 +http://metric.ru/ 1000 +http://autometric.ru/ 1000 +http://metrica.yandex.com/ 1000 +http://metris.ru/ 1000 +http://metrika.ru/ 1000 + 1000 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +111 +111 +111 +111 +111 +429 +429 +429 +429 +429 +1000 +1000 +1000 +1000 +1000 +0 +0 +0 +0 +111 +429 +1000 +привет как дела?... Херсон 254 +пап привет как дела - Яндекс.Видео 398 +привет как дела клип - Яндекс.Видео 412 +привет братан как дела - Яндекс.Видео 461 +привет 471 +http://metric.ru/ 1000 +http://autometric.ru/ 1000 +http://metrica.yandex.com/ 1000 +http://metris.ru/ 1000 +http://metrika.ru/ 1000 + 1000 +привет как дела?... Херсон 343 +пап привет как дела - Яндекс.Видео 446 +привет 471 +привет как дела клип - Яндекс.Видео 482 +привет братан как дела - Яндекс.Видео 506 +http://metric.ru/ 1000 +http://autometric.ru/ 1000 +http://metrica.yandex.com/ 1000 +http://metris.ru/ 1000 +http://metrika.ru/ 1000 + 1000 +http://metrika.ru/ 579 +http://metric.ru/ 778 +http://metris.ru/ 778 +http://autometric.ru/ 818 +http://metrica.yandex.com/ 852 +привет как дела?... Херсон 1000 +привет как дела клип - Яндекс.Видео 1000 +привет 1000 +пап привет как дела - Яндекс.Видео 1000 +привет братан как дела - Яндекс.Видео 1000 + 1000 +http://metric.ru/ 667 +http://metrica.yandex.com/ 704 +http://autometric.ru/ 727 +http://metris.ru/ 778 +http://metrika.ru/ 789 +привет как дела?... Херсон 1000 +привет как дела клип - Яндекс.Видео 1000 +привет 1000 +пап привет как дела - Яндекс.Видео 1000 +привет братан как дела - Яндекс.Видео 1000 + 1000 +http://metrika.ru/ 684 +http://metric.ru/ 778 +http://metris.ru/ 778 +http://autometric.ru/ 818 +http://metrica.yandex.com/ 852 +привет как дела?... Херсон 1000 +привет как дела клип - Яндекс.Видео 1000 +привет 1000 +пап привет как дела - Яндекс.Видео 1000 +привет братан как дела - Яндекс.Видео 1000 + 1000 +http://metric.ru/ 667 +http://autometric.ru/ 727 +http://metrica.yandex.com/ 778 +http://metris.ru/ 778 +http://metrika.ru/ 789 +привет как дела?... Херсон 1000 +привет как дела клип - Яндекс.Видео 1000 +привет 1000 +пап привет как дела - Яндекс.Видео 1000 +привет братан как дела - Яндекс.Видео 1000 + 1000 +http://metrica.yandex.com/ 769 +привет как дела?... Херсон 1000 +привет как дела клип - Яндекс.Видео 1000 +привет 1000 +пап привет как дела - Яндекс.Видео 1000 +привет братан как дела - Яндекс.Видео 1000 +http://metric.ru/ 1000 +http://autometric.ru/ 1000 +http://metris.ru/ 1000 +http://metrika.ru/ 1000 + 1000 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +111 +111 +111 +111 +111 +600 +600 +600 +600 +600 +1000 +1000 +1000 +1000 +1000 +0 +0 +0 +0 +111 +600 +1000 +привет как дела?... Херсон 910 +пап привет как дела - Яндекс.Видео 928 +привет как дела клип - Яндекс.Видео 929 +привет братан как дела - Яндекс.Видео 955 +привет 1000 +http://metric.ru/ 1000 +http://autometric.ru/ 1000 +http://metrica.yandex.com/ 1000 +http://metris.ru/ 1000 +http://metrika.ru/ 1000 + 1000 +привет как дела?... Херсон 672 +пап привет как дела - Яндекс.Видео 735 +привет как дела клип - Яндекс.Видео 741 +привет братан как дела - Яндекс.Видео 753 +привет 1000 +http://metric.ru/ 1000 +http://autometric.ru/ 1000 +http://metrica.yandex.com/ 1000 +http://metris.ru/ 1000 +http://metrika.ru/ 1000 + 1000 +http://metrika.ru/ 579 +http://metric.ru/ 778 +http://metris.ru/ 778 +http://autometric.ru/ 818 +http://metrica.yandex.com/ 852 +привет как дела?... Херсон 1000 +привет как дела клип - Яндекс.Видео 1000 +привет 1000 +пап привет как дела - Яндекс.Видео 1000 +привет братан как дела - Яндекс.Видео 1000 + 1000 +http://metrika.ru/ 579 +http://metric.ru/ 778 +http://metris.ru/ 778 +http://autometric.ru/ 818 +http://metrica.yandex.com/ 852 +привет как дела?... Херсон 1000 +привет как дела клип - Яндекс.Видео 1000 +привет 1000 +пап привет как дела - Яндекс.Видео 1000 +привет братан как дела - Яндекс.Видео 1000 + 1000 +http://metric.ru/ 667 +http://metrica.yandex.com/ 704 +http://autometric.ru/ 727 +http://metris.ru/ 778 +http://metrika.ru/ 789 +привет как дела?... Херсон 1000 +привет как дела клип - Яндекс.Видео 1000 +привет 1000 +пап привет как дела - Яндекс.Видео 1000 +привет братан как дела - Яндекс.Видео 1000 + 1000 +http://metrika.ru/ 684 +http://metric.ru/ 778 +http://metris.ru/ 778 +http://autometric.ru/ 818 +http://metrica.yandex.com/ 852 +привет как дела?... Херсон 1000 +привет как дела клип - Яндекс.Видео 1000 +привет 1000 +пап привет как дела - Яндекс.Видео 1000 +привет братан как дела - Яндекс.Видео 1000 + 1000 +http://metric.ru/ 667 +http://autometric.ru/ 727 +http://metrica.yandex.com/ 778 +http://metris.ru/ 778 +http://metrika.ru/ 789 +привет как дела?... Херсон 1000 +привет как дела клип - Яндекс.Видео 1000 +привет 1000 +пап привет как дела - Яндекс.Видео 1000 +привет братан как дела - Яндекс.Видео 1000 + 1000 +http://metrica.yandex.com/ 769 +привет как дела?... Херсон 1000 +привет как дела клип - Яндекс.Видео 1000 +привет 1000 +пап привет как дела - Яндекс.Видео 1000 +привет братан как дела - Яндекс.Видео 1000 +http://metric.ru/ 1000 +http://autometric.ru/ 1000 +http://metris.ru/ 1000 +http://metrika.ru/ 1000 + 1000 diff --git a/dbms/tests/queries/0_stateless/00909_ngram_distance.sql b/dbms/tests/queries/0_stateless/00909_ngram_distance.sql new file mode 100644 index 00000000000..867e69f4fe7 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00909_ngram_distance.sql @@ -0,0 +1,106 @@ +select round(1000 * ngramDistanceUTF8(materialize(''), '')) from system.numbers limit 5; +select round(1000 * ngramDistanceUTF8(materialize('абв'), '')) from system.numbers limit 5; +select round(1000 * ngramDistanceUTF8(materialize(''), 'абв')) from system.numbers limit 5; +select round(1000 * ngramDistanceUTF8(materialize('абвгдеёжз'), 'абвгдеёжз')) from system.numbers limit 5; +select round(1000 * ngramDistanceUTF8(materialize('абвгдеёжз'), 'абвгдеёж')) from system.numbers limit 5; +select round(1000 * ngramDistanceUTF8(materialize('абвгдеёжз'), 'гдеёзд')) from system.numbers limit 5; +select round(1000 * ngramDistanceUTF8(materialize('абвгдеёжз'), 'ёёёёёёёё')) from system.numbers limit 5; + +select round(1000 * ngramDistanceUTF8('', '')); +select round(1000 * ngramDistanceUTF8('абв', '')); +select round(1000 * ngramDistanceUTF8('', 'абв')); +select round(1000 * ngramDistanceUTF8('абвгдеёжз', 'абвгдеёжз')); +select round(1000 * ngramDistanceUTF8('абвгдеёжз', 'абвгдеёж')); +select round(1000 * ngramDistanceUTF8('абвгдеёжз', 'гдеёзд')); +select round(1000 * ngramDistanceUTF8('абвгдеёжз', 'ёёёёёёёё')); + +drop table if exists test.test_distance; +create table test.test_distance (Title String) engine = Memory; +insert into test.test_distance values ('привет как дела?... Херсон'), ('привет как дела клип - Яндекс.Видео'), ('привет'), ('пап привет как дела - Яндекс.Видео'), ('привет братан как дела - Яндекс.Видео'), ('http://metric.ru/'), ('http://autometric.ru/'), ('http://metrica.yandex.com/'), ('http://metris.ru/'), ('http://metrika.ru/'), (''); + +SELECT Title, round(1000 * distance) FROM test.test_distance ORDER BY ngramDistanceUTF8(Title, 'привет как дела') as distance; +SELECT Title, round(1000 * distance) FROM test.test_distance ORDER BY ngramDistanceUTF8(Title, 'как привет дела') as distance; +SELECT Title, round(1000 * distance) FROM test.test_distance ORDER BY ngramDistanceUTF8(Title, 'metrika') as distance; +SELECT Title, round(1000 * distance) FROM test.test_distance ORDER BY ngramDistanceUTF8(Title, 'metrica') as distance; +SELECT Title, round(1000 * distance) FROM test.test_distance ORDER BY ngramDistanceUTF8(Title, 'metriks') as distance; +SELECT Title, round(1000 * distance) FROM test.test_distance ORDER BY ngramDistanceUTF8(Title, 'metrics') as distance; +SELECT Title, round(1000 * distance) FROM test.test_distance ORDER BY ngramDistanceUTF8(Title, 'yandex') as distance; + + +select round(1000 * ngramDistanceCaseInsensitiveUTF8(materialize(''), '')) from system.numbers limit 5; +select round(1000 * ngramDistanceCaseInsensitiveUTF8(materialize('абв'), '')) from system.numbers limit 5; +select round(1000 * ngramDistanceCaseInsensitiveUTF8(materialize(''), 'абв')) from system.numbers limit 5; +select round(1000 * ngramDistanceCaseInsensitiveUTF8(materialize('абвГДЕёжз'), 'АбвгдЕёжз')) from system.numbers limit 5; +select round(1000 * ngramDistanceCaseInsensitiveUTF8(materialize('аБВГдеёЖз'), 'АбвГдеёж')) from system.numbers limit 5; +select round(1000 * ngramDistanceCaseInsensitiveUTF8(materialize('абвгдеёжз'), 'гдеёЗД')) from system.numbers limit 5; +select round(1000 * ngramDistanceCaseInsensitiveUTF8(materialize('абвгдеёжз'), 'ЁЁЁЁЁЁЁЁ')) from system.numbers limit 5; + +select round(1000 * ngramDistanceCaseInsensitiveUTF8('', '')); +select round(1000 * ngramDistanceCaseInsensitiveUTF8('абв', '')); +select round(1000 * ngramDistanceCaseInsensitiveUTF8('', 'абв')); +select round(1000 * ngramDistanceCaseInsensitiveUTF8('абвГДЕёжз', 'АбвгдЕЁжз')); +select round(1000 * ngramDistanceCaseInsensitiveUTF8('аБВГдеёЖз', 'АбвГдеёж')); +select round(1000 * ngramDistanceCaseInsensitiveUTF8('абвгдеёжз', 'гдеёЗД')); +select round(1000 * ngramDistanceCaseInsensitiveUTF8('АБВГДеёжз', 'ЁЁЁЁЁЁЁЁ')); + +SELECT Title, round(1000 * distance) FROM test.test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, 'ПрИвЕт кАК ДЕЛа') as distance; +SELECT Title, round(1000 * distance) FROM test.test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, 'как ПРИВЕТ дела') as distance; +SELECT Title, round(1000 * distance) FROM test.test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, 'metrika') as distance; +SELECT Title, round(1000 * distance) FROM test.test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, 'Metrika') as distance; +SELECT Title, round(1000 * distance) FROM test.test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, 'mEtrica') as distance; +SELECT Title, round(1000 * distance) FROM test.test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, 'metriKS') as distance; +SELECT Title, round(1000 * distance) FROM test.test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, 'metrics') as distance; +SELECT Title, round(1000 * distance) FROM test.test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, 'YanDEX') as distance; +SELECT Title, round(1000 * distance) FROM test.test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, 'приВЕТ КАк ДеЛа КлИп - яндеКс.видео') as distance; + + +select round(1000 * ngramDistance(materialize(''), '')) from system.numbers limit 5; +select round(1000 * ngramDistance(materialize('abc'), '')) from system.numbers limit 5; +select round(1000 * ngramDistance(materialize(''), 'abc')) from system.numbers limit 5; +select round(1000 * ngramDistance(materialize('abcdefgh'), 'abcdefgh')) from system.numbers limit 5; +select round(1000 * ngramDistance(materialize('abcdefgh'), 'abcdefg')) from system.numbers limit 5; +select round(1000 * ngramDistance(materialize('abcdefgh'), 'defgh')) from system.numbers limit 5; +select round(1000 * ngramDistance(materialize('abcdefgh'), 'aaaaaaaa')) from system.numbers limit 5; + +select round(1000 * ngramDistance('', '')); +select round(1000 * ngramDistance('abc', '')); +select round(1000 * ngramDistance('', 'abc')); +select round(1000 * ngramDistance('abcdefgh', 'abcdefgh')); +select round(1000 * ngramDistance('abcdefgh', 'abcdefg')); +select round(1000 * ngramDistance('abcdefgh', 'defgh')); +select round(1000 * ngramDistance('abcdefgh', 'aaaaaaaa')); + +SELECT Title, round(1000 * distance) FROM test.test_distance ORDER BY ngramDistance(Title, 'привет как дела') as distance; +SELECT Title, round(1000 * distance) FROM test.test_distance ORDER BY ngramDistance(Title, 'как привет дела') as distance; +SELECT Title, round(1000 * distance) FROM test.test_distance ORDER BY ngramDistance(Title, 'metrika') as distance; +SELECT Title, round(1000 * distance) FROM test.test_distance ORDER BY ngramDistance(Title, 'metrica') as distance; +SELECT Title, round(1000 * distance) FROM test.test_distance ORDER BY ngramDistance(Title, 'metriks') as distance; +SELECT Title, round(1000 * distance) FROM test.test_distance ORDER BY ngramDistance(Title, 'metrics') as distance; +SELECT Title, round(1000 * distance) FROM test.test_distance ORDER BY ngramDistance(Title, 'yandex') as distance; + +select round(1000 * ngramDistanceCaseInsensitive(materialize(''), '')) from system.numbers limit 5; +select round(1000 * ngramDistanceCaseInsensitive(materialize('abc'), '')) from system.numbers limit 5; +select round(1000 * ngramDistanceCaseInsensitive(materialize(''), 'abc')) from system.numbers limit 5; +select round(1000 * ngramDistanceCaseInsensitive(materialize('abCdefgH'), 'Abcdefgh')) from system.numbers limit 5; +select round(1000 * ngramDistanceCaseInsensitive(materialize('abcdefgh'), 'abcdeFG')) from system.numbers limit 5; +select round(1000 * ngramDistanceCaseInsensitive(materialize('AAAAbcdefgh'), 'defgh')) from system.numbers limit 5; +select round(1000 * ngramDistanceCaseInsensitive(materialize('ABCdefgH'), 'aaaaaaaa')) from system.numbers limit 5; + +select round(1000 * ngramDistanceCaseInsensitive('', '')); +select round(1000 * ngramDistanceCaseInsensitive('abc', '')); +select round(1000 * ngramDistanceCaseInsensitive('', 'abc')); +select round(1000 * ngramDistanceCaseInsensitive('abCdefgH', 'Abcdefgh')); +select round(1000 * ngramDistanceCaseInsensitive('abcdefgh', 'abcdeFG')); +select round(1000 * ngramDistanceCaseInsensitive('AAAAbcdefgh', 'defgh')); +select round(1000 * ngramDistanceCaseInsensitive('ABCdefgH', 'aaaaaaaa')); + +SELECT Title, round(1000 * distance) FROM test.test_distance ORDER BY ngramDistanceCaseInsensitive(Title, 'ПрИвЕт кАК ДЕЛа') as distance; +SELECT Title, round(1000 * distance) FROM test.test_distance ORDER BY ngramDistanceCaseInsensitive(Title, 'как ПРИВЕТ дела') as distance; +SELECT Title, round(1000 * distance) FROM test.test_distance ORDER BY ngramDistanceCaseInsensitive(Title, 'metrika') as distance; +SELECT Title, round(1000 * distance) FROM test.test_distance ORDER BY ngramDistanceCaseInsensitive(Title, 'Metrika') as distance; +SELECT Title, round(1000 * distance) FROM test.test_distance ORDER BY ngramDistanceCaseInsensitive(Title, 'mEtrica') as distance; +SELECT Title, round(1000 * distance) FROM test.test_distance ORDER BY ngramDistanceCaseInsensitive(Title, 'metriKS') as distance; +SELECT Title, round(1000 * distance) FROM test.test_distance ORDER BY ngramDistanceCaseInsensitive(Title, 'metrics') as distance; +SELECT Title, round(1000 * distance) FROM test.test_distance ORDER BY ngramDistanceCaseInsensitive(Title, 'YanDEX') as distance; + +drop table if exists test.test_distance; diff --git a/dbms/tests/queries/0_stateless/00909_trigram_distance.reference b/dbms/tests/queries/0_stateless/00909_trigram_distance.reference deleted file mode 100644 index 14dba2a2dcf..00000000000 --- a/dbms/tests/queries/0_stateless/00909_trigram_distance.reference +++ /dev/null @@ -1,119 +0,0 @@ -0 -0 -0 -0 -0 -1000 -1000 -1000 -1000 -1000 -1000 -1000 -1000 -1000 -1000 -0 -0 -0 -0 -0 -77 -77 -77 -77 -77 -636 -636 -636 -636 -636 -1000 -1000 -1000 -1000 -1000 -0 -1000 -1000 -0 -77 -636 -1000 -привет как дела?... Херсон -пап привет как дела - Яндекс.Видео -привет как дела клип - Яндекс.Видео -привет братан как дела - Яндекс.Видео -привет -http://metric.ru/ -http://autometric.ru/ -http://metrica.yandex.com/ -http://metris.ru/ -http://metrika.ru/ - -привет как дела?... Херсон -пап привет как дела - Яндекс.Видео -привет -привет как дела клип - Яндекс.Видео -привет братан как дела - Яндекс.Видео -http://metric.ru/ -http://autometric.ru/ -http://metrica.yandex.com/ -http://metris.ru/ -http://metrika.ru/ - -http://metrika.ru/ -http://metric.ru/ -http://metris.ru/ -http://autometric.ru/ -http://metrica.yandex.com/ -привет как дела?... Херсон -привет как дела клип - Яндекс.Видео -привет -пап привет как дела - Яндекс.Видео -привет братан как дела - Яндекс.Видео - -http://metric.ru/ -http://metrica.yandex.com/ -http://autometric.ru/ -http://metris.ru/ -http://metrika.ru/ -привет как дела?... Херсон -привет как дела клип - Яндекс.Видео -привет -пап привет как дела - Яндекс.Видео -привет братан как дела - Яндекс.Видео - -http://metrika.ru/ -http://metric.ru/ -http://metris.ru/ -http://autometric.ru/ -http://metrica.yandex.com/ -привет как дела?... Херсон -привет как дела клип - Яндекс.Видео -привет -пап привет как дела - Яндекс.Видео -привет братан как дела - Яндекс.Видео - -http://metric.ru/ -http://autometric.ru/ -http://metris.ru/ -http://metrika.ru/ -http://metrica.yandex.com/ -привет как дела?... Херсон -привет как дела клип - Яндекс.Видео -привет -пап привет как дела - Яндекс.Видео -привет братан как дела - Яндекс.Видео - -http://metrica.yandex.com/ -привет как дела?... Херсон -привет как дела клип - Яндекс.Видео -привет -пап привет как дела - Яндекс.Видео -привет братан как дела - Яндекс.Видео -http://metric.ru/ -http://autometric.ru/ -http://metris.ru/ -http://metrika.ru/ - diff --git a/dbms/tests/queries/0_stateless/00909_trigram_distance.sql b/dbms/tests/queries/0_stateless/00909_trigram_distance.sql deleted file mode 100644 index ca6a18d2513..00000000000 --- a/dbms/tests/queries/0_stateless/00909_trigram_distance.sql +++ /dev/null @@ -1,29 +0,0 @@ -select round(1000 * trigramDistance(materialize(''), '')) from system.numbers limit 5; -select round(1000 * trigramDistance(materialize('абв'), '')) from system.numbers limit 5; -select round(1000 * trigramDistance(materialize(''), 'абв')) from system.numbers limit 5; -select round(1000 * trigramDistance(materialize('абвгдеёжз'), 'абвгдеёжз')) from system.numbers limit 5; -select round(1000 * trigramDistance(materialize('абвгдеёжз'), 'абвгдеёж')) from system.numbers limit 5; -select round(1000 * trigramDistance(materialize('абвгдеёжз'), 'гдеёзд')) from system.numbers limit 5; -select round(1000 * trigramDistance(materialize('абвгдеёжз'), 'ёёёёёёёё')) from system.numbers limit 5; - -select round(1000 * trigramDistance('', '')); -select round(1000 * trigramDistance('абв', '')); -select round(1000 * trigramDistance('', 'абв')); -select round(1000 * trigramDistance('абвгдеёжз', 'абвгдеёжз')); -select round(1000 * trigramDistance('абвгдеёжз', 'абвгдеёж')); -select round(1000 * trigramDistance('абвгдеёжз', 'гдеёзд')); -select round(1000 * trigramDistance('абвгдеёжз', 'ёёёёёёёё')); - -drop table if exists test.test_distance; -create table test.test_distance (Title String) engine = Memory; -insert into test.test_distance values ('привет как дела?... Херсон'), ('привет как дела клип - Яндекс.Видео'), ('привет'), ('пап привет как дела - Яндекс.Видео'), ('привет братан как дела - Яндекс.Видео'), ('http://metric.ru/'), ('http://autometric.ru/'), ('http://metrica.yandex.com/'), ('http://metris.ru/'), ('http://metrika.ru/'), (''); - -SELECT Title FROM test.test_distance ORDER BY trigramDistance(Title, 'привет как дела'); -SELECT Title FROM test.test_distance ORDER BY trigramDistance(Title, 'как привет дела'); -SELECT Title FROM test.test_distance ORDER BY trigramDistance(Title, 'metrika'); -SELECT Title FROM test.test_distance ORDER BY trigramDistance(Title, 'metrica'); -SELECT Title FROM test.test_distance ORDER BY trigramDistance(Title, 'metriks'); -SELECT Title FROM test.test_distance ORDER BY trigramDistance(Title, 'metrics'); -SELECT Title FROM test.test_distance ORDER BY trigramDistance(Title, 'yandex'); - -drop table if exists test.test_distance; diff --git a/dbms/tests/queries/0_stateless/00914_join_bgranvea.reference b/dbms/tests/queries/0_stateless/00914_join_bgranvea.reference new file mode 100644 index 00000000000..be94ace11f1 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00914_join_bgranvea.reference @@ -0,0 +1,4 @@ +b1 b1 +b1 b1 +b1 b1 +b1 b1 diff --git a/dbms/tests/queries/0_stateless/00914_join_bgranvea.sql b/dbms/tests/queries/0_stateless/00914_join_bgranvea.sql new file mode 100644 index 00000000000..4f7820efbbb --- /dev/null +++ b/dbms/tests/queries/0_stateless/00914_join_bgranvea.sql @@ -0,0 +1,15 @@ +USE test; + +DROP TABLE IF EXISTS table1; +DROP TABLE IF EXISTS table2; + +CREATE TABLE table1 (A String, B String, ts DateTime) ENGINE = MergeTree PARTITION BY toStartOfDay(ts) ORDER BY (ts, A, B); +CREATE TABLE table2 (B String, ts DateTime) ENGINE = MergeTree PARTITION BY toStartOfDay(ts) ORDER BY (ts, B); + +insert into table1 values('a1','b1','2019-02-05 16:50:00'),('a1','b1','2019-02-05 16:55:00'); +insert into table2 values('b1','2019-02-05 16:50:00'),('b1','2019-02-05 16:55:00'); + +SELECT t1.B, t2.B FROM table1 t1 ALL INNER JOIN table2 t2 ON t1.B = t2.B ORDER BY t1.B, t2.B; + +DROP TABLE table1; +DROP TABLE table2; diff --git a/dbms/tests/queries/0_stateless/00914_replicate.reference b/dbms/tests/queries/0_stateless/00914_replicate.reference new file mode 100644 index 00000000000..8932f7fcd7b --- /dev/null +++ b/dbms/tests/queries/0_stateless/00914_replicate.reference @@ -0,0 +1 @@ +[[\'a\']] diff --git a/dbms/tests/queries/0_stateless/00914_replicate.sql b/dbms/tests/queries/0_stateless/00914_replicate.sql new file mode 100644 index 00000000000..4147df0ad45 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00914_replicate.sql @@ -0,0 +1 @@ +SELECT CAST(replicate(['a'], [1]) AS String); diff --git a/dbms/tests/queries/0_stateless/00915_tuple_orantius.reference b/dbms/tests/queries/0_stateless/00915_tuple_orantius.reference new file mode 100644 index 00000000000..6b303cbce8b --- /dev/null +++ b/dbms/tests/queries/0_stateless/00915_tuple_orantius.reference @@ -0,0 +1 @@ +1 (1,2,3) 1 diff --git a/dbms/tests/queries/0_stateless/00915_tuple_orantius.sql b/dbms/tests/queries/0_stateless/00915_tuple_orantius.sql new file mode 100644 index 00000000000..938260c5123 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00915_tuple_orantius.sql @@ -0,0 +1 @@ +select 1 as x, (1,2,3) as y, x in y; diff --git a/dbms/tests/queries/0_stateless/00916_create_or_replace_view.reference b/dbms/tests/queries/0_stateless/00916_create_or_replace_view.reference new file mode 100644 index 00000000000..c8e912a9a45 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00916_create_or_replace_view.reference @@ -0,0 +1,2 @@ +CREATE VIEW test.t ( number UInt64) AS SELECT number FROM system.numbers +CREATE VIEW test.t ( next_number UInt64) AS SELECT number + 1 AS next_number FROM system.numbers diff --git a/dbms/tests/queries/0_stateless/00916_create_or_replace_view.sql b/dbms/tests/queries/0_stateless/00916_create_or_replace_view.sql new file mode 100644 index 00000000000..af658ac96f5 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00916_create_or_replace_view.sql @@ -0,0 +1,9 @@ +DROP TABLE IF EXISTS test.t; + +CREATE OR REPLACE VIEW test.t (number UInt64) AS SELECT number FROM system.numbers; +SHOW CREATE TABLE test.t; + +CREATE OR REPLACE VIEW test.t AS SELECT number+1 AS next_number FROM system.numbers; +SHOW CREATE TABLE test.t; + +DROP TABLE test.t; diff --git a/dbms/tests/queries/0_stateless/00916_join_using_duplicate_columns.reference b/dbms/tests/queries/0_stateless/00916_join_using_duplicate_columns.reference new file mode 100644 index 00000000000..be9a5a74a14 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00916_join_using_duplicate_columns.reference @@ -0,0 +1,5 @@ +1 +1 +1 +1 +1 1 diff --git a/dbms/tests/queries/0_stateless/00916_join_using_duplicate_columns.sql b/dbms/tests/queries/0_stateless/00916_join_using_duplicate_columns.sql new file mode 100644 index 00000000000..97cd1e8cac8 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00916_join_using_duplicate_columns.sql @@ -0,0 +1,8 @@ +SELECT * FROM (SELECT 1 AS x) ALL LEFT JOIN (SELECT 1 AS x) USING x; +SELECT * FROM (SELECT 1 AS x) ALL LEFT JOIN (SELECT 2 AS x) USING x; + +SELECT * FROM (SELECT 1 AS x) AS t1 ALL LEFT JOIN (SELECT 1 AS x) AS t2 USING x; +SELECT * FROM (SELECT 1 AS x) AS t1 ALL LEFT JOIN (SELECT 2 AS x) AS t2 USING x; + +SELECT * FROM (SELECT 1 AS x) AS t1 ALL LEFT JOIN (SELECT 1 AS x) AS t2 ON t1.x = t2.x; +-- (bug) SELECT * FROM (SELECT 1 AS x) AS t1 ALL LEFT JOIN (SELECT 2 AS x) AS t2 ON t1.x = t2.x; diff --git a/dbms/tests/queries/0_stateless/00917_multiple_joins_denny_crane.reference b/dbms/tests/queries/0_stateless/00917_multiple_joins_denny_crane.reference new file mode 100644 index 00000000000..38f7ecf84a4 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00917_multiple_joins_denny_crane.reference @@ -0,0 +1 @@ +CAT 2 diff --git a/dbms/tests/queries/0_stateless/00917_multiple_joins_denny_crane.sql b/dbms/tests/queries/0_stateless/00917_multiple_joins_denny_crane.sql new file mode 100644 index 00000000000..cf227ef0e8b --- /dev/null +++ b/dbms/tests/queries/0_stateless/00917_multiple_joins_denny_crane.sql @@ -0,0 +1,24 @@ +USE test; + +DROP TABLE IF EXISTS ANIMAL; + +CREATE TABLE ANIMAL ( ANIMAL Nullable(String) ) engine = TinyLog; +INSERT INTO ANIMAL (ANIMAL) VALUES ('CAT'), ('FISH'), ('DOG'), ('HORSE'), ('BIRD'); + +set enable_optimize_predicate_expression = 0; + +select * from ( +select x.b x, count(distinct x.c) ANIMAL +from ( +select a.ANIMAL a, 'CAT' b, c.ANIMAL c, d.ANIMAL d +from ANIMAL a join ANIMAL b on a.ANIMAL = b.ANIMAL + left outer join ANIMAL c on (b.ANIMAL = c.ANIMAL) + right outer join (select * from ANIMAL union all select * from ANIMAL + union all select * from ANIMAL) d on (a.ANIMAL = d.ANIMAL) +where d.ANIMAL <> 'CAT' and c.ANIMAL <>'DOG' and b.ANIMAL <> 'FISH') as x +where x.b >= 'CAT' +group by x.b +having ANIMAL >= 0) ANIMAL +where ANIMAL.ANIMAL >= 0; + +DROP TABLE ANIMAL; diff --git a/dbms/tests/queries/0_stateless/00918_has_unsufficient_type_check.reference b/dbms/tests/queries/0_stateless/00918_has_unsufficient_type_check.reference new file mode 100644 index 00000000000..7938dcdde86 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00918_has_unsufficient_type_check.reference @@ -0,0 +1,3 @@ +0 +1 +0 diff --git a/dbms/tests/queries/0_stateless/00918_has_unsufficient_type_check.sql b/dbms/tests/queries/0_stateless/00918_has_unsufficient_type_check.sql new file mode 100644 index 00000000000..f76fd446a8e --- /dev/null +++ b/dbms/tests/queries/0_stateless/00918_has_unsufficient_type_check.sql @@ -0,0 +1,3 @@ +SELECT hasAny([['Hello, world']], [[[]]]); +SELECT hasAny([['Hello, world']], [['Hello', 'world'], ['Hello, world']]); +SELECT hasAll([['Hello, world']], [['Hello', 'world'], ['Hello, world']]); diff --git a/dbms/tests/queries/0_stateless/00925_empty_replicated_merge_tree_optimize_final.reference b/dbms/tests/queries/0_stateless/00925_empty_replicated_merge_tree_optimize_final.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/00925_empty_replicated_merge_tree_optimize_final.sql b/dbms/tests/queries/0_stateless/00925_empty_replicated_merge_tree_optimize_final.sql new file mode 100644 index 00000000000..6903209c5c4 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00925_empty_replicated_merge_tree_optimize_final.sql @@ -0,0 +1,9 @@ +DROP TABLE IF EXISTS test.replicated_optimize1; +DROP TABLE IF EXISTS test.replicated_optimize2; +CREATE TABLE test.replicated_optimize1 (d Date, k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test/optimize', 'r1', d, k, 8192); +CREATE TABLE test.replicated_optimize2 (d Date, k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test/optimize', 'r2', d, k, 8192); + +OPTIMIZE TABLE test.replicated_optimize1 FINAL; + +DROP TABLE test.replicated_optimize1; +DROP TABLE test.replicated_optimize2; diff --git a/dbms/tests/queries/bugs/00816_concurrent_alter_column.sh b/dbms/tests/queries/bugs/00816_concurrent_alter_column.sh deleted file mode 100755 index 63e8e48e0bb..00000000000 --- a/dbms/tests/queries/bugs/00816_concurrent_alter_column.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash - -set -e - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. $CURDIR/../shell_config.sh - -echo "DROP TABLE IF EXISTS test.test" | ${CLICKHOUSE_CLIENT} -echo "CREATE TABLE test.test (ts DATETIME) ENGINE = MergeTree PARTITION BY toStartOfDay(ts) ORDER BY tuple()" | ${CLICKHOUSE_CLIENT} - -for i in {1..500}; do echo "ALTER TABLE test.test ADD COLUMN c$i DOUBLE;"; done | ${CLICKHOUSE_CLIENT} -n - -for i in {1..500}; do echo "ALTER TABLE test.test ADD COLUMN d DOUBLE" | ${CLICKHOUSE_CLIENT}; echo "ALTER TABLE test.test DROP COLUMN d" | ${CLICKHOUSE_CLIENT} -n; done & -for i in {1..500}; do echo "ALTER TABLE test.test ADD COLUMN e DOUBLE" | ${CLICKHOUSE_CLIENT}; echo "ALTER TABLE test.test DROP COLUMN e" | ${CLICKHOUSE_CLIENT} -n; done & - -wait - -echo "DROP TABLE test.test" | ${CLICKHOUSE_CLIENT} diff --git a/debian/changelog b/debian/changelog index d329555c1e1..e9bb4c1caa0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,5 @@ -clickhouse (19.3.4) unstable; urgency=low +clickhouse (19.5.1.1) unstable; urgency=low * Modified source code - -- Fri, 15 Feb 2019 14:50:36 +0300 + -- clickhouse-release Sat, 09 Mar 2019 10:45:02 +300 diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile index 2aa8451d73f..cbcc52c6f0d 100644 --- a/docker/client/Dockerfile +++ b/docker/client/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=19.3.4 +ARG version=19.5.1.1 RUN apt-get update \ && apt-get install --yes --no-install-recommends \ diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index 9e2384a4272..5fdf50750ee 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=19.3.4 +ARG version=19.5.1.1 ARG gosu_ver=1.10 RUN apt-get update \ diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile index 4e5132efbd9..315cb2f958b 100644 --- a/docker/test/Dockerfile +++ b/docker/test/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=19.3.4 +ARG version=19.5.1.1 RUN apt-get update && \ apt-get install -y apt-transport-https dirmngr && \ diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile index 34a64a24b1a..dcd7a0bcad0 100644 --- a/docker/test/stateless/Dockerfile +++ b/docker/test/stateless/Dockerfile @@ -20,7 +20,9 @@ RUN apt-get update -y \ netcat-openbsd \ telnet \ moreutils \ - brotli + brotli \ + gdb \ + lsof ENV TZ=Europe/Moscow @@ -42,4 +44,4 @@ CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \ echo "UBSAN_SYMBOLIZER_PATH=/usr/lib/llvm-6.0/bin/llvm-symbolizer" >> /etc/environment; \ echo "LLVM_SYMBOLIZER_PATH=/usr/lib/llvm-6.0/bin/llvm-symbolizer" >> /etc/environment; \ service zookeeper start; sleep 5; \ - service clickhouse-server start && sleep 5 && clickhouse-test --shard --zookeeper $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt + service clickhouse-server start && sleep 5 && clickhouse-test --shard --zookeeper $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt diff --git a/docs/en/data_types/fixedstring.md b/docs/en/data_types/fixedstring.md index d1d956a7df3..9725c71a779 100644 --- a/docs/en/data_types/fixedstring.md +++ b/docs/en/data_types/fixedstring.md @@ -1,8 +1,8 @@ # FixedString -A fixed-length string of `N` bytes (not characters or code points). +A fixed-length string of `N` bytes (neither characters nor code points). -To declare the column of `FixedString` type use the following syntax: +To declare a column of `FixedString` type, use the following syntax: ``` FixedString(N) @@ -10,31 +10,35 @@ To declare the column of `FixedString` type use the following syntax: Where `N` is a natural number. -It is efficient to use `FixedString` in case when data has the length of `N` bytes exactly. For example: IP addresses (`FixedString(16)` for IPv6), language codes (ru_RU, en_US ... ), currency codes (USD, RUB ... ), hashes (`FixedString(16)` for MD5, `FixedString(32)` for SHA256). In all other cases, you can get the loss of efficiency. To store hashes or IP addresses use their binary representation. To store UUID values use the [UUID](uuid.md) data type. +The `FixedString` type is efficient when data has the length of precisely `N` bytes. In all other cases, it is likely to reduce efficiency. + +Examples of the values that can be efficiently stored in `FixedString`-typed columns: + +- Binary representation of IP addresses (`FixedString(16)` for IPv6). +- Language codes (ru_RU, en_US ... ). +- Currency codes (USD, RUB ... ). +- Binary representation of hashes (`FixedString(16)` for MD5, `FixedString(32)` for SHA256). + +To store UUID values, use the [UUID](uuid.md) data type. When inserting the data, ClickHouse: -- Complements a string with null bytes if the string contains fewer bytes than `N`. +- Complements a string with null bytes if the string contains fewer than `N` bytes. - Throws the `Too large value for FixedString(N)` exception if the string contains more than `N` bytes. -When selecting the data, ClickHouse does not trim off the null bytes at the end of the string. If you use the `WHERE` clause, you should add null bytes manually to match the `FixedString` value. +When selecting the data, ClickHouse does not remove the null bytes at the end of the string. If you use the `WHERE` clause, you should add null bytes manually to match the `FixedString` value. The following example illustrates how to use the `WHERE` clause with `FixedString`. -Example: +Let's consider the following table with the single `FixedString(2)` column: ``` -SELECT * FROM FixedStringTable +┌─name──┐ +│ b │ +└───────┘ +``` -┌─a──┐ -│ aa │ -│ b │ -└────┘ - -SELECT * FROM FixedStringTable WHERE a = 'b' - -Ok. - -0 rows in set. Elapsed: 0.009 sec. +The query `SELECT * FROM FixedStringTable WHERE a = 'b'` does not return any data as a result. We should complement the filter pattern with null bytes. +``` SELECT * FROM FixedStringTable WHERE a = 'b\0' @@ -47,6 +51,6 @@ WHERE a = 'b\0' This behavior differs from MySQL behavior for the `CHAR` type (where strings are padded with spaces, and the spaces are removed for output). -Note that the length of the `FixedString` value is constant even if it is filled with null bytes only. The [empty](../query_language/functions/string_functions.md#string_functions-empty) functions returns `1` if the `FixedString` value contains the null bytes only. +Note that the length of the `FixedString(N)` value is constant. The [length](../query_language/functions/array_functions.md#array_functions-length) function returns `N` even if the `FixedString(N)` value is filled only with null bytes, but the [empty](../query_language/functions/string_functions.md#string_functions-empty) function returns `1` in this case. [Original article](https://clickhouse.yandex/docs/en/data_types/fixedstring/) diff --git a/docs/en/interfaces/cli.md b/docs/en/interfaces/cli.md index 632049b5901..b2b88870ddb 100644 --- a/docs/en/interfaces/cli.md +++ b/docs/en/interfaces/cli.md @@ -100,7 +100,7 @@ You can pass parameters to `clickhouse-client` (all parameters have a default va - Defined in the `-config-file` parameter. - `./clickhouse-client.xml` -- `\~/.clickhouse-client/config.xml` +- `~/.clickhouse-client/config.xml` - `/etc/clickhouse-client/config.xml` Example of a config file: diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index d6c569bc397..5d59815b25c 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -242,7 +242,7 @@ SELECT SearchPhrase, count() AS c FROM test.hits GROUP BY SearchPhrase WITH TOTA } ``` -The JSON is compatible with JavaScript. To ensure this, some characters are additionally escaped: the slash `/` is escaped as `\/`; alternative line breaks `U+2028` and `U+2029`, which break some browsers, are escaped as `\uXXXX`. ASCII control characters are escaped: backspace, form feed, line feed, carriage return, and horizontal tab are replaced with `\b`, `\f`, `\n`, `\r`, `\t` , as well as the remaining bytes in the 00-1F range using `\uXXXX` sequences. Invalid UTF-8 sequences are changed to the replacement character � so the output text will consist of valid UTF-8 sequences. For compatibility with JavaScript, Int64 and UInt64 integers are enclosed in double quotes by default. To remove the quotes, you can set the configuration parameter output_format_json_quote_64bit_integers to 0. +The JSON is compatible with JavaScript. To ensure this, some characters are additionally escaped: the slash `/` is escaped as `\/`; alternative line breaks `U+2028` and `U+2029`, which break some browsers, are escaped as `\uXXXX`. ASCII control characters are escaped: backspace, form feed, line feed, carriage return, and horizontal tab are replaced with `\b`, `\f`, `\n`, `\r`, `\t` , as well as the remaining bytes in the 00-1F range using `\uXXXX` sequences. Invalid UTF-8 sequences are changed to the replacement character � so the output text will consist of valid UTF-8 sequences. For compatibility with JavaScript, Int64 and UInt64 integers are enclosed in double quotes by default. To remove the quotes, you can set the configuration parameter [output_format_json_quote_64bit_integers](../operations/settings/settings.md#session_settings-output_format_json_quote_64bit_integers) to 0. `rows` – The total number of output rows. @@ -257,7 +257,7 @@ This format is only appropriate for outputting a query result, but not for parsi ClickHouse supports [NULL](../query_language/syntax.md), which is displayed as `null` in the JSON output. -See also the JSONEachRow format. +See also the [JSONEachRow](#jsoneachrow) format. ## JSONCompact {#jsoncompact} @@ -307,24 +307,73 @@ See also the `JSONEachRow` format. ## JSONEachRow {#jsoneachrow} -Outputs data as separate JSON objects for each row (newline delimited JSON). +When using this format, ClickHouse outputs rows as separated, newline delimited JSON objects, but the whole data is not a valid JSON. ```json -{"SearchPhrase":"","count()":"8267016"} -{"SearchPhrase": "bathroom interior design","count()": "2166"} -{"SearchPhrase":"yandex","count()":"1655"} -{"SearchPhrase":"2014 spring fashion","count()":"1549"} -{"SearchPhrase":"freeform photo","count()":"1480"} -{"SearchPhrase":"angelina jolie","count()":"1245"} -{"SearchPhrase":"omsk","count()":"1112"} -{"SearchPhrase":"photos of dog breeds","count()":"1091"} {"SearchPhrase":"curtain designs","count()":"1064"} {"SearchPhrase":"baku","count()":"1000"} +{"SearchPhrase":"","count":"8267016"} ``` -Unlike the JSON format, there is no substitution of invalid UTF-8 sequences. Any set of bytes can be output in the rows. This is necessary so that data can be formatted without losing any information. Values are escaped in the same way as for JSON. +When inserting the data, you should provide a separate JSON object for each row. -For parsing, any order is supported for the values of different columns. It is acceptable for some values to be omitted – they are treated as equal to their default values. In this case, zeros and blank rows are used as default values. Complex values that could be specified in the table are not supported as defaults, but it can be turned on by option `insert_sample_with_metadata=1`. Whitespace between elements is ignored. If a comma is placed after the objects, it is ignored. Objects don't necessarily have to be separated by new lines. +### Inserting the Data + +``` +INSERT INTO UserActivity FORMAT JSONEachRow {"PageViews":5, "UserID":"4324182021466249494", "Duration":146,"Sign":-1} {"UserID":"4324182021466249494","PageViews":6,"Duration":185,"Sign":1} +``` + +ClickHouse allows: + +- Any order of key-value pairs in the object. +- The omission of some values. + +ClickHouse ignores spaces between elements and commas after the objects. You can pass all the objects to a line. You do not have to separate them with line breaks. + +**Processing of omitted values** + +ClickHouse substitutes the omitted values with the default values of corresponding [data types](../data_types/index.md). + +In case of the `DEFAULT expr` is specified, ClickHouse uses different substitution rules depending on the [insert_sample_with_metadata](../operations/settings/settings.md#session_settings-insert_sample_with_metadata) setting. + +Let's consider the following table: + +``` +CREATE TABLE IF NOT EXISTS example_table +( + x UInt32, + a DEFAULT x * 2 +) ENGINE = Memory; +``` + +- If `insert_sample_with_metadata = 0`, then the default value for `x` and `a` equals `0` (as a default value for `UInt32` data type). +- If `insert_sample_with_metadata = 1`, then the default value for `x` equals `0`, but the default value of `a` equals `x * 2`. + +!!! note "Warning" + Use this option carefully, enabling it negatively affects the performance of the ClickHouse server. + +### Selecting the Data + +Let's consider the `UserActivity` table as an example: + +``` +┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ +│ 4324182021466249494 │ 5 │ 146 │ -1 │ +│ 4324182021466249494 │ 6 │ 185 │ 1 │ +└─────────────────────┴───────────┴──────────┴──────┘ +``` + +The query `SELECT * FROM UserActivity FORMAT JSONEachRow` returns: + +``` +{"UserID":"4324182021466249494","PageViews":5,"Duration":146,"Sign":-1} +{"UserID":"4324182021466249494","PageViews":6,"Duration":185,"Sign":1} +``` + +Unlike the [JSON](#json) format, there is no substitution of invalid UTF-8 sequences. Values are escaped in the same way as for `JSON`. + +!!! note "Note" + Any set of bytes can be output in the strings. Use the `JSONEachRow` format if you are sure that the data in the table can be formatted into JSON without losing any information. ## Native {#native} @@ -363,7 +412,7 @@ Rows are not escaped in Pretty* formats. Example is shown for the [PrettyCompact SELECT 'String with \'quotes\' and \t character' AS Escaping_test ``` -``` +``` ┌─Escaping_test────────────────────────┐ │ String with 'quotes' and character │ └──────────────────────────────────────┘ @@ -482,7 +531,7 @@ Row 1: x: 1 y: ᴺᵁᴸᴸ ``` -Rows are not escaped in Vertical format: +Rows are not escaped in Vertical format: ``` sql SELECT 'string with \'quotes\' and \t with some special \n characters' AS test FORMAT Vertical diff --git a/docs/en/operations/configuration_files.md b/docs/en/operations/configuration_files.md index 2184bb95122..1ee72ab188e 100644 --- a/docs/en/operations/configuration_files.md +++ b/docs/en/operations/configuration_files.md @@ -2,7 +2,7 @@ The main server config file is `config.xml`. It resides in the `/etc/clickhouse-server/` directory. -Individual settings can be overridden in the `*.xml` and `*.conf` files in the `conf.d` and `config.d` directories next to the config file. +Individual settings can be overridden in the `*.xml` and `*.conf` files in the `config.d` directory next to the config file. The `replace` or `remove` attributes can be specified for the elements of these config files. diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index fff660999e2..2c7e68e3716 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -81,9 +81,29 @@ If an error occurred while reading rows but the error counter is still less than If `input_format_allow_errors_ratio` is exceeded, ClickHouse throws an exception. -## insert_sample_with_metadata +## insert_sample_with_metadata {#session_settings-insert_sample_with_metadata} -For INSERT queries, specifies that the server need to send metadata about column defaults to the client. This will be used to calculate default expressions. Disabled by default. +Turns on/off the extended data exchange between a ClickHouse client and a ClickHouse server. The setting is applies for `INSERT` queries. + +When executing the `INSERT` query, ClickHouse client prepares data and sends it to the server for writing. During the preparation of the data, the client gets the table structure from the server. In some cases, the client needs more information than the server sends by default. Turn on the extended data exchange with `insert_sample_with_metadata = 1`. + +When the extended data exchange is enabled, the server sends the additional metadata along with the table structure. The composition of the metadata depends on the operation. + +Operations where you may need the extended data exchange enabled: + +- Inserting the data of the [JSONEachRow](../../interfaces/formats.md#jsoneachrow) format. + +For all other operations ClickHouse doesn't apply the setting. + +!!! note "Note" + The functionality of the extended data exchange consumes additional computing resources on the server and can reduce the performance. + +**Possible values** + +- 0 — Functionality is disabled. +- 1 — Functionality is enabled. + +**Default value:** 0. ## join_default_strictness {#settings-join_default_strictness} @@ -406,7 +426,7 @@ The results of compilation are saved in the build directory in the form of .so f If the value is true, running INSERT skips input data from columns with unknown names. Otherwise, this situation will generate an exception. It works for JSONEachRow and TSKV formats. -## output_format_json_quote_64bit_integers +## output_format_json_quote_64bit_integers {#session_settings-output_format_json_quote_64bit_integers} If the value is true, integers appear in quotes when using JSON\* Int64 and UInt64 formats (for compatibility with most JavaScript implementations); otherwise, integers are output without the quotes. diff --git a/docs/en/operations/system_tables.md b/docs/en/operations/system_tables.md index 34b44419cce..8387818043e 100644 --- a/docs/en/operations/system_tables.md +++ b/docs/en/operations/system_tables.md @@ -85,6 +85,22 @@ Columns: - `name`(`String`) – The name of the function. - `is_aggregate`(`UInt8`) — Whether the function is aggregate. +## system.graphite_retentions + +Contains information about parameters [graphite_rollup](server_settings/settings.md#server_settings-graphite_rollup) which use in tables with [\*GraphiteMergeTree](table_engines/graphitemergetree.md) engines. + +Столбцы: +- `config_name` (String) - `graphite_rollup` parameter name. +- `regexp` (String) - A pattern for the metric name. +- `function` (String) - The name of the aggregating function. +- `age` (UInt64) - The minimum age of the data in seconds. +- `precision` (UInt64) - How precisely to define the age of the data in seconds. +- `priority` (UInt16) - Pattern priority. +- `is_default` (UInt8) - Is pattern default or not. +- `Tables.database` (Array(String)) - Array of databases names of tables, which use `config_name` parameter. +- `Tables.table` (Array(String)) - Array of tables names, which use `config_name` parameter. + + ## system.merges Contains information about merges and part mutations currently in process for tables in the MergeTree family. diff --git a/docs/en/operations/table_engines/mergetree.md b/docs/en/operations/table_engines/mergetree.md index 53bb909b16d..2c9ff68c562 100644 --- a/docs/en/operations/table_engines/mergetree.md +++ b/docs/en/operations/table_engines/mergetree.md @@ -271,7 +271,7 @@ SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234 Stores extremes of the specified expression (if the expression is `tuple`, then it stores extremes for each element of `tuple`), uses stored info for skipping blocks of the data like the primary key. * `set(max_rows)` -Stores unique values of the specified expression (no more than `max_rows` rows), use them to check if the `WHERE` expression is not satisfiable on a block of the data. +Stores unique values of the specified expression (no more than `max_rows` rows, `max_rows=0` means "no limits"), use them to check if the `WHERE` expression is not satisfiable on a block of the data. ```sql INDEX sample_index (u64 * length(s)) TYPE minmax GRANULARITY 4 diff --git a/docs/en/query_language/functions/array_functions.md b/docs/en/query_language/functions/array_functions.md index fabd4dd35e5..0cbd03814ac 100644 --- a/docs/en/query_language/functions/array_functions.md +++ b/docs/en/query_language/functions/array_functions.md @@ -12,7 +12,7 @@ Returns 0 for an empty array, or 1 for a non-empty array. The result type is UInt8. The function also works for strings. -## length +## length {#array_functions-length} Returns the number of items in the array. The result type is UInt64. diff --git a/docs/en/query_language/functions/string_search_functions.md b/docs/en/query_language/functions/string_search_functions.md index b3b8b63d136..dce9917776c 100644 --- a/docs/en/query_language/functions/string_search_functions.md +++ b/docs/en/query_language/functions/string_search_functions.md @@ -70,5 +70,13 @@ For other regular expressions, the code is the same as for the 'match' function. The same thing as 'like', but negative. +## ngramDistance(haystack, needle) + +Calculates the 4-gram distance between `haystack` and `needle`: counts the symmetric difference between two multisets of 4-grams and normalizes it by the sum of their cardinalities. Returns float number from 0 to 1 -- the closer to zero, the more strings are similar to each other. If the `needle` is more than 32Kb, throws an exception. If some of the `haystack` strings are more than 32Kb, the distance is always one. + +For case-insensitive search or/and in UTF-8 format use functions `ngramDistanceCaseInsensitive, ngramDistanceUTF8, ngramDistanceCaseInsensitiveUTF8`. + +Notes: For UTF-8 case we use 3-gram distance. All these are not perfectly fair n-gram distances. We use 2-byte hashes to hash n-grams and then calculate the symmetric difference between these hash tables -- collisions may occur. With UTF-8 case-insensitive format we do not use fair `tolower` function -- we zero the 5-th bit (starting from zero) of each codepoint byte -- this works for Latin and mostly for all Cyrillic letters. + [Original article](https://clickhouse.yandex/docs/en/query_language/functions/string_search_functions/) diff --git a/docs/fa/query_language/functions/bitmap_functions.md b/docs/fa/query_language/functions/bitmap_functions.md new file mode 120000 index 00000000000..0a31d3d71d8 --- /dev/null +++ b/docs/fa/query_language/functions/bitmap_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/bitmap_functions.md \ No newline at end of file diff --git a/docs/ru/data_types/fixedstring.md b/docs/ru/data_types/fixedstring.md index 5c7fa19d506..1634a95bc67 100644 --- a/docs/ru/data_types/fixedstring.md +++ b/docs/ru/data_types/fixedstring.md @@ -1,11 +1,56 @@ -# FixedString(N) +# FixedString -Строка фиксированной длины N байт (не символов, не кодовых точек). N должно быть строго положительным натуральным числом. -При чтении сервером строки (например, при парсинге данных для INSERT), содержащей меньшее число байт, строка дополняется до N байт дописыванием нулевых байт справа. -При чтении сервером строки, содержащей большее число байт, выдаётся сообщение об ошибке. -При записи сервером строки (например, при выводе результата запроса SELECT), нулевые байты с конца строки не вырезаются, а выводятся. -Обратите внимание, как это поведение отличается от поведения MySQL для типа CHAR (строки дополняются пробелами, пробелы перед выводом вырезаются). +Строка фиксированной длины `N` байт (не символов, не кодовых точек). -С типом FixedString(N) умеет работать меньше функций, чем с типом String - то есть, он менее удобен в использовании. +Чтобы объявить столбец типа `FixedString`, используйте следующий синтаксис: + +``` + FixedString(N) +``` + +Где `N` — натуральное число. + +Тип `FixedString` эффективен, когда данные имеют длину ровно `N` байт. Во всех остальных случаях использование FixedString может привести к снижению эффективности. + +Примеры значений, которые можно эффективно хранить в столбцах типа `FixedString`: + +- Двоичное представление IP-адреса (`FixedString(16)` для IPv6). +- Коды языков (ru_RU, en_US ... ). +- Коды валют (USD, RUB ... ). +- Двоичное представление хэшей (`FixedString(16)` для MD5, `FixedString(32)` для SHA256). + +Для хранения значений UUID используйте тип данных [UUID](uuid.md). + +При вставке данных, ClickHouse: + +- Дополняет строку нулевыми байтами, если строка содержит меньше байтов, чем `N`. +- Генерирует исключение `Too large value for FixedString(N)`, если строка содержит более `N` байт. + +При выборе данных ClickHouse не обрезает нулевые байты в конце строки. Если вы используете секцию `WHERE`, то необходимо добавлять нулевые байты вручную, чтобы ClickHouse смог сопоставить выражение из фильтра значению `FixedString`. Следующий пример показывает, как использовать секцию `WHERE` с `FixedString`. + +Рассмотрим следующую таблицу с единственным столбцом типа `FixedString(2)`: + +``` +┌─name──┐ +│ b │ +└───────┘ +``` + +Запрос `SELECT * FROM FixedStringTable WHERE a = 'b'` не возвращает необходимых данных. Необходимо дополнить шаблон фильтра нулевыми байтами. + +``` +SELECT * FROM FixedStringTable +WHERE a = 'b\0' + +┌─a─┐ +│ b │ +└───┘ + +1 rows in set. Elapsed: 0.002 sec. +``` + +Это поведение отличается от поведения MySQL для типа `CHAR`, где строки дополняются пробелами, а пробелы перед выводом вырезаются. + +Обратите внимание, что длина значения `FixedString(N)` постоянна. Функция [length](../query_language/functions/array_functions.md#array_functions-length) возвращает `N` даже если значение `FixedString(N)` заполнено только нулевыми байтами, однако функция [empty](../query_language/functions/string_functions.md#string_functions-empty) в этом же случае возвращает `1`. [Оригинальная статья](https://clickhouse.yandex/docs/ru/data_types/fixedstring/) diff --git a/docs/ru/getting_started/index.md b/docs/ru/getting_started/index.md index 9dd85e93753..f2b589550ac 100644 --- a/docs/ru/getting_started/index.md +++ b/docs/ru/getting_started/index.md @@ -96,7 +96,7 @@ $ clickhouse-server --config-file=/etc/clickhouse-server/config.xml $ clickhouse-client ``` -По умолчанию он соединяется с localhost:9000, от имени пользователя `default` без пароля. Также клиент может быть использован для соединения с удалённым сервером с помощью аргемента `--host`. +По умолчанию он соединяется с localhost:9000, от имени пользователя `default` без пароля. Также клиент может быть использован для соединения с удалённым сервером с помощью аргумента `--host`. Терминал должен использовать кодировку UTF-8. diff --git a/docs/ru/interfaces/cli.md b/docs/ru/interfaces/cli.md index 4cb4742078e..2523bc0395c 100644 --- a/docs/ru/interfaces/cli.md +++ b/docs/ru/interfaces/cli.md @@ -103,7 +103,7 @@ cat file.csv | clickhouse-client --database=test --query="INSERT INTO test FORMA - Определенного параметром `--config-file`. - `./clickhouse-client.xml` -- `\~/.clickhouse-client/config.xml` +- `~/.clickhouse-client/config.xml` - `/etc/clickhouse-client/config.xml` Пример конфигурационного файла: diff --git a/docs/ru/operations/configuration_files.md b/docs/ru/operations/configuration_files.md index ebb95652d61..9514734678d 100644 --- a/docs/ru/operations/configuration_files.md +++ b/docs/ru/operations/configuration_files.md @@ -2,7 +2,7 @@ Основной конфигурационный файл сервера - `config.xml`. Он расположен в директории `/etc/clickhouse-server/`. -Отдельные настройки могут быть переопределены в файлах `*.xml` и `*.conf` из директорий `conf.d` и `config.d` рядом с конфигом. +Отдельные настройки могут быть переопределены в файлах `*.xml` и `*.conf` из директории `config.d` рядом с конфигом. У элементов этих конфигурационных файлов могут быть указаны атрибуты `replace` или `remove`. diff --git a/docs/ru/operations/system_tables.md b/docs/ru/operations/system_tables.md index 82aec59ec29..91d3991edcd 100644 --- a/docs/ru/operations/system_tables.md +++ b/docs/ru/operations/system_tables.md @@ -83,6 +83,23 @@ default_expression String - выражение для значения по ум - `name` (`String`) – Имя функции. - `is_aggregate` (`UInt8`) – Признак, является ли функция агрегатной. + +## system.graphite_retentions + +Содержит информацию о том, какие параметры [graphite_rollup](server_settings/settings.md#server_settings-graphite_rollup) используются в таблицах с движками [\*GraphiteMergeTree](table_engines/graphitemergetree.md). + +Столбцы: +- `config_name` (String) - Имя параметра, используемого для `graphite_rollup`. +- `regexp` (String) - Шаблон имени метрики. +- `function` (String) - Имя агрегирующей функции. +- `age` (UInt64) - Минимальный возраст данных в секундах. +- `precision` (UInt64) - Точность определения возраста данных в секундах. +- `priority` (UInt16) - Приоритет раздела pattern. +- `is_default` (UInt8) - Является ли раздел pattern дефолтным. +- `Tables.database` (Array(String)) - Массив имён баз данных таблиц, использующих параметр `config_name`. +- `Tables.table` (Array(String)) - Массив имён таблиц, использующих параметр `config_name`. + + ## system.merges Содержит информацию о производящихся прямо сейчас слияниях и мутациях кусков для таблиц семейства MergeTree. diff --git a/docs/ru/operations/table_engines/mergetree.md b/docs/ru/operations/table_engines/mergetree.md index 22df2d45e64..4318568edfb 100644 --- a/docs/ru/operations/table_engines/mergetree.md +++ b/docs/ru/operations/table_engines/mergetree.md @@ -260,7 +260,7 @@ SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234 Хранит минимум и максимум выражения (если выражение - `tuple`, то для каждого элемента `tuple`), используя их для пропуска блоков аналогично первичному ключу. * `set(max_rows)` -Хранит уникальные значения выражения на блоке в количестве не более `max_rows`, используя их для пропуска блоков, оценивая выполнимость `WHERE` выражения на хранимых данных. +Хранит уникальные значения выражения на блоке в количестве не более `max_rows` (если `max_rows = 0`, то ограничений нет), используя их для пропуска блоков, оценивая выполнимость `WHERE` выражения на хранимых данных. Примеры diff --git a/docs/ru/query_language/functions/array_functions.md b/docs/ru/query_language/functions/array_functions.md index 7cc4cb41b8e..406f8e01509 100644 --- a/docs/ru/query_language/functions/array_functions.md +++ b/docs/ru/query_language/functions/array_functions.md @@ -12,7 +12,7 @@ Тип результата - UInt8. Функция также работает для строк. -## length +## length {#array_functions-length} Возвращает количество элементов в массиве. Тип результата - UInt64. diff --git a/docs/ru/query_language/functions/string_search_functions.md b/docs/ru/query_language/functions/string_search_functions.md index a79ea043716..4b335cce34c 100644 --- a/docs/ru/query_language/functions/string_search_functions.md +++ b/docs/ru/query_language/functions/string_search_functions.md @@ -9,7 +9,7 @@ Для поиска без учета регистра используйте функцию `positionCaseInsensitive`. ## positionUTF8(haystack, needle) -Так же, как `position`, но позиция возвращается в кодовых точках Unicode. Работает при допущении, что строка содержит набор байт, представляющий текст в кодировке UTF-8. Если допущение не выполнено - то возвращает какой-нибудь результат (не кидает исключение). +Так же, как `position`, но позиция возвращается в кодовых точках Unicode. Работает при допущении, что строка содержит набор байт, представляющий текст в кодировке UTF-8. Если допущение не выполнено -- то возвращает какой-нибудь результат (не кидает исключение). Для поиска без учета регистра используйте функцию `positionCaseInsensitiveUTF8`. @@ -59,4 +59,12 @@ ## notLike(haystack, pattern), оператор haystack NOT LIKE pattern То же, что like, но с отрицанием. +## ngramDistance(haystack, needle) + +Вычисление 4-граммного расстояния между `haystack` и `needle`: считается симметрическая разность между двумя мультимножествами 4-грамм и нормализается на сумму их мощностей. Возвращает число float от 0 до 1 -- чем ближе к нулю, тем больше строки похожи друг на друга. Если `needle` больше чем 32КБ, кидается исключение. Если некоторые строки из `haystack` больше 32КБ, расстояние всегда равно единице. + +Для поиска без учета регистра и/или в формате UTF-8 используйте функции `ngramDistanceCaseInsensitive, ngramDistanceUTF8, ngramDistanceCaseInsensitiveUTF8`. + +Примечание: для случая UTF-8 мы используем триграммное расстояние. Вычисление n-граммного расстояния не совсем честное. Мы используем 2-х байтные хэши для хэширования n-грамм, а затем вычисляем симметричную разность между хэш таблицами -- могут возникнуть коллизии. В формате UTF-8 без учета регистра мы не используем честную функцию `tolower` -- мы обнуляем 5-й бит (нумерация с нуля) каждого байта кодовой точки -- это работает для латиницы и почти для всех кириллических букв. + [Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/string_search_functions/) diff --git a/docs/ru/query_language/insert_into.md b/docs/ru/query_language/insert_into.md index 6799eb71b64..631f56bde88 100644 --- a/docs/ru/query_language/insert_into.md +++ b/docs/ru/query_language/insert_into.md @@ -58,7 +58,7 @@ INSERT INTO [db.]table [(c1, c2, c3)] SELECT ... `INSERT` сортирует входящие данные по первичному ключу и разбивает их на партиции по месяцам. Если вы вставляете данные за разные месяцы вперемешку, то это может значительно снизить производительность запроса `INSERT`. Чтобы избежать этого: - Добавляйте данные достаточно большими пачками. Например, по 100 000 строк. -- Группируйте данные по месацам самостоятельно перед загрузкой в ClickHouse. +- Группируйте данные по месяцам самостоятельно перед загрузкой в ClickHouse. Снижения производительности не будет, если: diff --git a/docs/toc_en.yml b/docs/toc_en.yml index 886721763a1..a67bc9cd309 100644 --- a/docs/toc_en.yml +++ b/docs/toc_en.yml @@ -115,7 +115,7 @@ nav: - 'Working with Arrays': 'query_language/functions/array_functions.md' - 'Splitting and Merging Strings and Arrays': 'query_language/functions/splitting_merging_functions.md' - 'Bit': 'query_language/functions/bit_functions.md' - - 'Bitmap functions': 'query_language/functions/bitmap_functions.md' + - 'Bitmap': 'query_language/functions/bitmap_functions.md' - 'Hash': 'query_language/functions/hash_functions.md' - 'Generating Pseudo-Random Numbers': 'query_language/functions/random_functions.md' - 'Encoding': 'query_language/functions/encoding_functions.md' diff --git a/docs/toc_fa.yml b/docs/toc_fa.yml index 0b84040282a..67335934850 100644 --- a/docs/toc_fa.yml +++ b/docs/toc_fa.yml @@ -115,6 +115,7 @@ nav: - 'Working with Arrays': 'query_language/functions/array_functions.md' - 'Splitting and Merging Strings and Arrays': 'query_language/functions/splitting_merging_functions.md' - 'Bit': 'query_language/functions/bit_functions.md' + - 'Bitmap': 'query_language/functions/bitmap_functions.md' - 'Hash': 'query_language/functions/hash_functions.md' - 'Generating Pseudo-Random Numbers': 'query_language/functions/random_functions.md' - 'Encoding': 'query_language/functions/encoding_functions.md' diff --git a/docs/toc_zh.yml b/docs/toc_zh.yml index 64cb2446f41..81866ae7af8 100644 --- a/docs/toc_zh.yml +++ b/docs/toc_zh.yml @@ -114,6 +114,7 @@ nav: - 'Working with Arrays': 'query_language/functions/array_functions.md' - 'Splitting and Merging Strings and Arrays': 'query_language/functions/splitting_merging_functions.md' - 'Bit': 'query_language/functions/bit_functions.md' + - 'Bitmap': 'query_language/functions/bitmap_functions.md' - 'Hash': 'query_language/functions/hash_functions.md' - 'Generating Pseudo-Random Numbers': 'query_language/functions/random_functions.md' - 'Encoding': 'query_language/functions/encoding_functions.md' diff --git a/docs/zh/query_language/functions/bitmap_functions.md b/docs/zh/query_language/functions/bitmap_functions.md new file mode 120000 index 00000000000..0a31d3d71d8 --- /dev/null +++ b/docs/zh/query_language/functions/bitmap_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/bitmap_functions.md \ No newline at end of file diff --git a/libs/libcommon/include/common/constexpr_helpers.h b/libs/libcommon/include/common/constexpr_helpers.h index c6fcb0bb4db..2dad106a7a3 100644 --- a/libs/libcommon/include/common/constexpr_helpers.h +++ b/libs/libcommon/include/common/constexpr_helpers.h @@ -1,6 +1,7 @@ #pragma once #include +#include template inline constexpr bool static_in_v = std::disjunction_v...>; diff --git a/libs/libcommon/include/common/itoa.h b/libs/libcommon/include/common/itoa.h index bd49156f4b4..19575c1fbcc 100644 --- a/libs/libcommon/include/common/itoa.h +++ b/libs/libcommon/include/common/itoa.h @@ -30,6 +30,7 @@ #include #include #include +#include "likely.h" using int128_t = __int128; using uint128_t = unsigned __int128; diff --git a/libs/libcommon/include/common/mremap.h b/libs/libcommon/include/common/mremap.h index f569ff05d4e..31ca74da827 100644 --- a/libs/libcommon/include/common/mremap.h +++ b/libs/libcommon/include/common/mremap.h @@ -12,7 +12,8 @@ #define MREMAP_MAYMOVE 1 -void * mremap(void * old_address, +void * mremap( + void * old_address, size_t old_size, size_t new_size, int flags = 0, @@ -23,7 +24,8 @@ void * mremap(void * old_address, #endif -inline void * clickhouse_mremap(void * old_address, +inline void * clickhouse_mremap( + void * old_address, size_t old_size, size_t new_size, int flags = 0, @@ -32,7 +34,8 @@ inline void * clickhouse_mremap(void * old_address, [[maybe_unused]] int mmap_fd = -1, [[maybe_unused]] off_t mmap_offset = 0) { - return mremap(old_address, + return mremap( + old_address, old_size, new_size, flags diff --git a/libs/libdaemon/src/BaseDaemon.cpp b/libs/libdaemon/src/BaseDaemon.cpp index 005409bd08a..08bece59b51 100644 --- a/libs/libdaemon/src/BaseDaemon.cpp +++ b/libs/libdaemon/src/BaseDaemon.cpp @@ -700,12 +700,12 @@ static void checkRequiredInstructions(volatile InstructionFail & fail) #if __AVX2__ fail = InstructionFail::AVX2; - __asm__ volatile ("vpabsw %%ymm0, %%ymm0, %%ymm0" : : : "ymm0"); + __asm__ volatile ("vpabsw %%ymm0, %%ymm0" : : : "ymm0"); #endif #if __AVX512__ fail = InstructionFail::AVX512; - __asm__ volatile ("vpabsw %%zmm0, %%zmm0, %%zmm0" : : : "zmm0"); + __asm__ volatile ("vpabsw %%zmm0, %%zmm0" : : : "zmm0"); #endif fail = InstructionFail::NONE; diff --git a/libs/libglibc-compatibility/CMakeLists.txt b/libs/libglibc-compatibility/CMakeLists.txt index 3477e474c7c..5bf83b9263f 100644 --- a/libs/libglibc-compatibility/CMakeLists.txt +++ b/libs/libglibc-compatibility/CMakeLists.txt @@ -33,17 +33,3 @@ endif () add_library (glibc-compatibility ${GLIBC_COMPATIBILITY_SOURCES}) target_include_directories(glibc-compatibility PRIVATE libcxxabi) - -# glibc-compatibility does not depend on any libraries but is linked to all libraries implicitly. -# Avoid linking of the library to itself. -set_target_properties(glibc-compatibility PROPERTIES LINK_LIBRARIES "") - -# Garbage. Rough explanation: some libraries want to install itself and CMake forces us to also install the glibc-compatibility library. -install(TARGETS glibc-compatibility EXPORT CapnProtoTargets ARCHIVE DESTINATION "/tmp") -install(TARGETS glibc-compatibility EXPORT protobuf-targets ARCHIVE DESTINATION "/tmp") -install(TARGETS glibc-compatibility EXPORT double-conversionTargets ARCHIVE DESTINATION "/tmp") -install(TARGETS glibc-compatibility EXPORT SnappyTargets ARCHIVE DESTINATION "/tmp") - -if(ENABLE_TESTS) - add_subdirectory(tests) -endif() diff --git a/libs/libglibc-compatibility/musl/atomic_arch.h b/libs/libglibc-compatibility/musl/atomic_arch.h index da4e2037548..cb3bf8da7fd 100644 --- a/libs/libglibc-compatibility/musl/atomic_arch.h +++ b/libs/libglibc-compatibility/musl/atomic_arch.h @@ -1,3 +1,5 @@ +#include + #define a_cas a_cas static inline int a_cas(volatile int *p, int t, int s) { diff --git a/libs/libglibc-compatibility/tests/CMakeLists.txt b/libs/libglibc-compatibility/tests/CMakeLists.txt deleted file mode 100644 index a4f95187485..00000000000 --- a/libs/libglibc-compatibility/tests/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -include (${ClickHouse_SOURCE_DIR}/cmake/add_check.cmake) - -foreach (T longjmp siglongjmp) - add_executable (${T} ${T}.c) - target_link_libraries (${T} PRIVATE glibc-compatibility) - if (USE_LIBCXX) - target_link_libraries (${T} PRIVATE Threads::Threads) - endif () - set_target_properties (${T} PROPERTIES LINKER_LANGUAGE CXX) - add_check (${T}) -endforeach () diff --git a/libs/libglibc-compatibility/tests/longjmp.c b/libs/libglibc-compatibility/tests/longjmp.c deleted file mode 100644 index aa96e6276b4..00000000000 --- a/libs/libglibc-compatibility/tests/longjmp.c +++ /dev/null @@ -1,23 +0,0 @@ -#include - -int main() -{ - jmp_buf env; - int val; - volatile int count = 0; - val = setjmp(env); - ++count; - if (count == 1 && val != 0) - { - return 1; - } - if (count == 2 && val == 42) - { - return 0; - } - if (count == 1) - { - longjmp(env, 42); - } - return 1; -} diff --git a/libs/libglibc-compatibility/tests/siglongjmp.c b/libs/libglibc-compatibility/tests/siglongjmp.c deleted file mode 100644 index e4befb34259..00000000000 --- a/libs/libglibc-compatibility/tests/siglongjmp.c +++ /dev/null @@ -1,23 +0,0 @@ -#include - -int main() -{ - sigjmp_buf env; - int val; - volatile int count = 0; - val = sigsetjmp(env, 0); - ++count; - if (count == 1 && val != 0) - { - return 1; - } - if (count == 2 && val == 42) - { - return 0; - } - if (count == 1) - { - siglongjmp(env, 42); - } - return 1; -} diff --git a/utils/check-style/check-include b/utils/check-style/check-include index 4a02f488278..4c0e0c69e83 100755 --- a/utils/check-style/check-include +++ b/utils/check-style/check-include @@ -18,6 +18,7 @@ inc="-I. \ -I./contrib/libmetrohash/src \ -I./contrib/double-conversion \ -I./contrib/cityhash102/include \ +-I./contrib/croaring \ -I./contrib/murmurhash/include \ -I./contrib/zookeeper/src/c/include \ -I./contrib/zookeeper/src/c/generated \ @@ -54,6 +55,9 @@ inc="-I. \ -I./libs/consistent-hashing-sumbur \ -I./contrib/libhdfs3/include \ -I./contrib/base64/include \ +-I./contrib/protobuf/src \ +-I./contrib/cppkafka/include \ +-I./contrib/librdkafka-cmake/include \ -I./contrib/lz4/lib \ -I./dbms/src \ -I${BUILD_DIR}/dbms/src" @@ -64,5 +68,5 @@ if [ -z $1 ]; then else echo -n "$1 " echo -n `grep "#include" $1| wc -l` " " - echo "#include <$1> \n int main() {return 0;}" | time --format "%e %M" ${CXX:=g++-7} -c -std=c++1z $inc -x c++ - + echo "#include <$1>\nint main() {return 0;}" | time --format "%e %M" ${CXX:=g++-7} -c -std=c++1z $inc -x c++ - fi diff --git a/utils/zookeeper-cli/CMakeLists.txt b/utils/zookeeper-cli/CMakeLists.txt index 89db7922edd..550d0e855d8 100644 --- a/utils/zookeeper-cli/CMakeLists.txt +++ b/utils/zookeeper-cli/CMakeLists.txt @@ -1,5 +1,5 @@ add_executable(clickhouse-zookeeper-cli zookeeper-cli.cpp) -target_link_libraries(clickhouse-zookeeper-cli PRIVATE clickhouse_common_zookeeper ${LINE_EDITING_LIBS}) +target_link_libraries(clickhouse-zookeeper-cli PRIVATE clickhouse_common_zookeeper ${Poco_Foundation_LIBRARY} ${LINE_EDITING_LIBS}) if (READLINE_INCLUDE_DIR) target_include_directories (clickhouse-zookeeper-cli SYSTEM PRIVATE ${READLINE_INCLUDE_DIR}) endif () diff --git a/website/index.html b/website/index.html index 55badbf2363..9f59864002e 100644 --- a/website/index.html +++ b/website/index.html @@ -319,7 +319,7 @@
  • ClickHouse for Experimentation at Spotify
  • ClickHouse DB in DDoS mitigation
    • + rel="external nofollow" target="_blank">ClickHouse DB in DDoS mitigation at Qrator
  • Migrating to Yandex ClickHouse by LifeStreet (machine translation from Russian)