diff --git a/.gitignore b/.gitignore index 65b239a68ff..5dc058c97c0 100644 --- a/.gitignore +++ b/.gitignore @@ -205,6 +205,7 @@ configure-stamp *.bin *.mrk +*.mrk2 .dupload.conf diff --git a/.gitmodules b/.gitmodules index 56856bea6a0..567772bad60 100644 --- a/.gitmodules +++ b/.gitmodules @@ -67,6 +67,12 @@ [submodule "contrib/libgsasl"] path = contrib/libgsasl url = https://github.com/ClickHouse-Extras/libgsasl.git +[submodule "contrib/libcxx"] + path = contrib/libcxx + url = https://github.com/llvm-mirror/libcxx.git +[submodule "contrib/libcxxabi"] + path = contrib/libcxxabi + url = https://github.com/llvm-mirror/libcxxabi.git [submodule "contrib/snappy"] path = contrib/snappy url = https://github.com/google/snappy diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ed9fca1a4b..caf1f4fd7f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,20 @@ +## ClickHouse release 19.9.4.1, 2019-07-05 + +### Bug Fix +* Fix segfault in Delta codec which affects columns with values less than 32 bits size. The bug led to random memory corruption. [#5786](https://github.com/yandex/ClickHouse/pull/5786) ([alesapin](https://github.com/alesapin)) +* Fix rare bug in checking of part with LowCardinality column. [#5832](https://github.com/yandex/ClickHouse/pull/5832) ([alesapin](https://github.com/alesapin)) +* Fix segfault in TTL merge with non-physical columns in block. [#5819](https://github.com/yandex/ClickHouse/pull/5819) ([Anton Popov](https://github.com/CurtizJ)) +* Fix potential infinite sleeping of low-priority queries. [#5842](https://github.com/yandex/ClickHouse/pull/5842) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix how ClickHouse determines default time zone as UCT instead of UTC. [#5828](https://github.com/yandex/ClickHouse/pull/5828) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix bug about executing distributed DROP/ALTER/TRUNCATE/OPTIMIZE ON CLUSTER queries on follower replica before leader replica. Now they will be executed directly on leader replica. [#5757](https://github.com/yandex/ClickHouse/pull/5757) ([alesapin](https://github.com/alesapin)) +* Fix race condition, which cause that some queries may not appear in query_log instantly after SYSTEM FLUSH LOGS query. [#5685](https://github.com/yandex/ClickHouse/pull/5685) ([Anton Popov](https://github.com/CurtizJ)) +* Added missing support for constant arguments to `evalMLModel` function. [#5820](https://github.com/yandex/ClickHouse/pull/5820) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +## ClickHouse release 19.7.6.1, 2019-07-05 + +### Bug Fix +* Fix performance regression in some queries with JOIN. [#5192](https://github.com/yandex/ClickHouse/pull/5192) ([Winter Zhang](https://github.com/zhang2014)) + ## ClickHouse release 19.9.2.4, 2019-06-24 ### New Feature diff --git a/CHANGELOG_RU.md b/CHANGELOG_RU.md index 03585a85843..aada2462002 100644 --- a/CHANGELOG_RU.md +++ b/CHANGELOG_RU.md @@ -1,3 +1,15 @@ +## ClickHouse release 19.9.4.1, 2019-07-05 + +### Исправления ошибок +* Исправлен segmentation fault в кодеке сжатия Delta в колонках с величинами размером меньше 32 бит. Ошибка могла приводить к повреждениям памяти. [#5786](https://github.com/yandex/ClickHouse/pull/5786) ([alesapin](https://github.com/alesapin)) +* Исправлена ошибка в проверке кусков в LowCardinality колонках. [#5832](https://github.com/yandex/ClickHouse/pull/5832) ([alesapin](https://github.com/alesapin)) +* Исправлен segmentation fault при слиянии кусков с истекшим TTL в случае, когда в блоке присутствуют столбцы, не входящие в структуру таблицы. [#5819](https://github.com/yandex/ClickHouse/pull/5819) ([Anton Popov](https://github.com/CurtizJ)) +* Исправлена существовавшая возможность ухода в бесконечное ожидание на низко-приоритетных запросах. [#5842](https://github.com/yandex/ClickHouse/pull/5842) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Исправлена ошибка определения таймзоны по умолчанию (UCT вместо UTC). [#5828](https://github.com/yandex/ClickHouse/pull/5828) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Исправлена ошибка в распределенных запросах вида DROP/ALTER/TRUNCATE/OPTIMIZE ON CLUSTER. [#5757](https://github.com/yandex/ClickHouse/pull/5757) ([alesapin](https://github.com/alesapin)) +* Исправлена ошибка, которая при распределенных запросах могла привести к тому, что некоторые запросы не появлялись в query_log сразу после SYSTEM FLUSH LOGS запроса. [#5685](https://github.com/yandex/ClickHouse/pull/5685) ([Anton Popov](https://github.com/CurtizJ)) +* Добавлена отсутствовавшая поддержка константных аргументов для функции `evalMLModel`. [#5820](https://github.com/yandex/ClickHouse/pull/5820) ([alexey-milovidov](https://github.com/alexey-milovidov)) + ## ClickHouse release 19.9.2.4, 2019-06-24 ### Новые возможности @@ -47,6 +59,11 @@ * Исправлена сборка clickhouse как сабмодуля [#5574](https://github.com/yandex/ClickHouse/pull/5574) ([proller](https://github.com/proller)) * Улучшение теста производительности функций JSONExtract [#5444](https://github.com/yandex/ClickHouse/pull/5444) ([Vitaly Baranov](https://github.com/vitlibar)) +## ClickHouse release 19.7.6.1, 2019-07-05 + +### Исправления ошибок +* Исправлена просадка производительности в методе JOIN в некоторых видах запросов. [#5192](https://github.com/yandex/ClickHouse/pull/5192) ([Winter Zhang](https://github.com/zhang2014)) + ## ClickHouse release 19.8.3.8, 2019-06-11 ### Новые возможности diff --git a/CMakeLists.txt b/CMakeLists.txt index 0273abec108..a2cc5f15ac8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -104,7 +104,6 @@ if (COMPILER_GCC AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER "8.3.0") endif () if (COMPILER_CLANG) - # clang: warning: argument unused during compilation: '-stdlib=libc++' # clang: warning: argument unused during compilation: '-specs=/usr/share/dpkg/no-pie-compile.specs' [-Wunused-command-line-argument] set (COMMON_WARNING_FLAGS "${COMMON_WARNING_FLAGS} -Wno-unused-command-line-argument") endif () @@ -183,12 +182,18 @@ else () set (CXX_FLAGS_INTERNAL_COMPILER "-std=c++1z") endif () +if (COMPILER_GCC OR COMPILER_CLANG) + # Enable C++14 sized global deallocation functions. It should be enabled by setting -std=c++14 but I'm not sure. + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsized-deallocation") +endif () + option(WITH_COVERAGE "Build with coverage." 0) if(WITH_COVERAGE AND COMPILER_CLANG) set(COMPILER_FLAGS "${COMPILER_FLAGS} -fprofile-instr-generate -fcoverage-mapping") endif() if(WITH_COVERAGE AND COMPILER_GCC) set(COMPILER_FLAGS "${COMPILER_FLAGS} -fprofile-arcs -ftest-coverage") + set(COVERAGE_OPTION "-lgcov") endif() set (CMAKE_BUILD_COLOR_MAKEFILE ON) @@ -202,16 +207,72 @@ set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} -fn set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_C_FLAGS_ADD}") set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_C_FLAGS_ADD}") -include (cmake/use_libcxx.cmake) +# Uses MAKE_STATIC_LIBRARIES + + +option (UNBUNDLED "Try find all libraries in system. We recommend to avoid this mode for production builds, because we cannot guarantee exact versions and variants of libraries your system has installed. This mode exists for enthusiastic developers who search for trouble. Also it is useful for maintainers of OS packages." OFF) +if (UNBUNDLED) + set(NOT_UNBUNDLED 0) +else () + set(NOT_UNBUNDLED 1) +endif () +# Using system libs can cause lot of warnings in includes. +if (UNBUNDLED OR NOT (OS_LINUX OR APPLE) OR ARCH_32) + option (NO_WERROR "Disable -Werror compiler option" ON) +endif () + + +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_package (Threads) + +include (cmake/find_cxx.cmake) + +include (cmake/test_compiler.cmake) + +if (OS_LINUX AND COMPILER_CLANG AND USE_STATIC_LIBRARIES) + option (USE_LIBCXX "Use libc++ and libc++abi instead of libstdc++ (only make sense on Linux)" ${HAVE_LIBCXX}) + + if (USE_LIBCXX) + set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_LIBCPP_DEBUG=0") # More checks in debug build. + endif () +endif () + +if (USE_LIBCXX) + set (STATIC_STDLIB_FLAGS "") +else () + set (STATIC_STDLIB_FLAGS "-static-libgcc -static-libstdc++") +endif () + +if (MAKE_STATIC_LIBRARIES AND NOT APPLE AND NOT (COMPILER_CLANG AND OS_FREEBSD)) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${STATIC_STDLIB_FLAGS}") + + # Along with executables, we also build example of shared library for "library dictionary source"; and it also should be self-contained. + set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${STATIC_STDLIB_FLAGS}") +endif () + +if (USE_STATIC_LIBRARIES AND HAVE_NO_PIE) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FLAG_NO_PIE}") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FLAG_NO_PIE}") +endif () + +if (NOT SANITIZE) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-undefined") + set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-undefined") +endif() + include (cmake/find_unwind.cmake) if (USE_INTERNAL_UNWIND_LIBRARY) option (USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING "Use internal unwind library for exception handling" ${USE_STATIC_LIBRARIES}) endif () + # Set standard, system and compiler libraries explicitly. # This is intended for more control of what we are linking. +string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC) +set (CMAKE_POSTFIX_VARIABLE "CMAKE_${CMAKE_BUILD_TYPE_UC}_POSTFIX") + set (DEFAULT_LIBS "") if (OS_LINUX AND NOT UNBUNDLED AND (GLIBC_COMPATIBILITY OR USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING OR USE_LIBCXX)) # Note: this probably has no effect, but I'm not an expert in CMake. @@ -225,6 +286,8 @@ if (OS_LINUX AND NOT UNBUNDLED AND (GLIBC_COMPATIBILITY OR USE_INTERNAL_UNWIND_L set (BUILTINS_LIB_PATH "") if (COMPILER_CLANG) execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-${CMAKE_SYSTEM_PROCESSOR}.a OUTPUT_VARIABLE BUILTINS_LIB_PATH OUTPUT_STRIP_TRAILING_WHITESPACE) + else () + set (BUILTINS_LIB_PATH "-lgcc") endif () string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC) @@ -242,7 +305,7 @@ if (OS_LINUX AND NOT UNBUNDLED AND (GLIBC_COMPATIBILITY OR USE_INTERNAL_UNWIND_L if (USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING) # TODO: Allow to use non-static library as well. - set (EXCEPTION_HANDLING_LIBRARY "lib/libunwind${${CMAKE_POSTFIX_VARIABLE}}.a") + set (EXCEPTION_HANDLING_LIBRARY "${ClickHouse_BINARY_DIR}/contrib/libunwind-cmake/libunwind_static${${CMAKE_POSTFIX_VARIABLE}}.a") else () set (EXCEPTION_HANDLING_LIBRARY "-lgcc_eh") endif () @@ -250,9 +313,15 @@ if (OS_LINUX AND NOT UNBUNDLED AND (GLIBC_COMPATIBILITY OR USE_INTERNAL_UNWIND_L message (STATUS "Using exception handling library: ${EXCEPTION_HANDLING_LIBRARY}") if (USE_LIBCXX) - set (DEFAULT_LIBS "${DEFAULT_LIBS} -Wl,-Bstatic -lc++ -lc++abi ${EXCEPTION_HANDLING_LIBRARY} ${BUILTINS_LIB_PATH} -Wl,-Bdynamic") + if (USE_INTERNAL_LIBCXX_LIBRARY) + set (LIBCXX_LIBS "${ClickHouse_BINARY_DIR}/contrib/libcxx-cmake/libcxx_static${${CMAKE_POSTFIX_VARIABLE}}.a ${ClickHouse_BINARY_DIR}/contrib/libcxxabi-cmake/libcxxabi_static${${CMAKE_POSTFIX_VARIABLE}}.a") + else () + set (LIBCXX_LIBS "-lc++ -lc++abi") + endif () + + set (DEFAULT_LIBS "${DEFAULT_LIBS} -Wl,-Bstatic ${LIBCXX_LIBS} ${EXCEPTION_HANDLING_LIBRARY} ${BUILTINS_LIB_PATH} -Wl,-Bdynamic") else () - set (DEFAULT_LIBS "${DEFAULT_LIBS} -Wl,-Bstatic -lstdc++ ${EXCEPTION_HANDLING_LIBRARY} -lgcc ${BUILTINS_LIB_PATH} -Wl,-Bdynamic") + set (DEFAULT_LIBS "${DEFAULT_LIBS} -Wl,-Bstatic -lstdc++ ${EXCEPTION_HANDLING_LIBRARY} ${COVERAGE_OPTION} ${BUILTINS_LIB_PATH} -Wl,-Bdynamic") endif () # Linking with GLIBC prevents portability of binaries to older systems. @@ -279,6 +348,7 @@ endif () if (DEFAULT_LIBS) # Add default libs to all targets as the last dependency. set(CMAKE_CXX_STANDARD_LIBRARIES ${DEFAULT_LIBS}) + set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS}) endif () if (NOT MAKE_STATIC_LIBRARIES) @@ -338,11 +408,21 @@ if (UNBUNDLED OR NOT (OS_LINUX OR APPLE) OR ARCH_32) option (NO_WERROR "Disable -Werror compiler option" ON) endif () +if (USE_LIBCXX) + set (HAVE_LIBCXX 1) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") +endif() + +if (USE_LIBCXX AND USE_INTERNAL_LIBCXX_LIBRARY) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -nostdinc++ -isystem ${LIBCXX_INCLUDE_DIR} -isystem ${LIBCXXABI_INCLUDE_DIR}") +endif () + message (STATUS "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE} ; USE_STATIC_LIBRARIES=${USE_STATIC_LIBRARIES} MAKE_STATIC_LIBRARIES=${MAKE_STATIC_LIBRARIES} SPLIT_SHARED=${SPLIT_SHARED_LIBRARIES} UNBUNDLED=${UNBUNDLED} CCACHE=${CCACHE_FOUND} ${CCACHE_VERSION}") include(GNUInstallDirs) include (cmake/find_contrib_lib.cmake) +find_contrib_lib(double-conversion) # Must be before parquet include (cmake/find_ssl.cmake) include (cmake/lib_name.cmake) include (cmake/find_icu.cmake) @@ -377,16 +457,16 @@ include (cmake/find_pdqsort.cmake) include (cmake/find_hdfs3.cmake) # uses protobuf include (cmake/find_consistent-hashing.cmake) include (cmake/find_base64.cmake) +include (cmake/find_parquet.cmake) include (cmake/find_hyperscan.cmake) include (cmake/find_mimalloc.cmake) include (cmake/find_simdjson.cmake) include (cmake/find_rapidjson.cmake) + find_contrib_lib(cityhash) find_contrib_lib(farmhash) find_contrib_lib(metrohash) find_contrib_lib(btrie) -find_contrib_lib(double-conversion) -include (cmake/find_parquet.cmake) if (ENABLE_TESTS) include (cmake/find_gtest.cmake) @@ -416,8 +496,13 @@ if (GLIBC_COMPATIBILITY OR USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING) if (GLIBC_COMPATIBILITY) add_dependencies(${target_name} glibc-compatibility) endif () + + if (USE_LIBCXX AND USE_INTERNAL_LIBCXX_LIBRARY) + add_dependencies(${target_name} cxx_static cxxabi_static) + endif () + if (USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING) - add_dependencies(${target_name} ${UNWIND_LIBRARY}) + add_dependencies(${target_name} unwind_static) endif () endif () endfunction () @@ -458,4 +543,15 @@ if (GLIBC_COMPATIBILITY OR USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING) add_default_dependencies(boost_program_options_internal) add_default_dependencies(boost_system_internal) add_default_dependencies(boost_regex_internal) + add_default_dependencies(roaring) + add_default_dependencies(btrie) + add_default_dependencies(cpuid) + add_default_dependencies(mysqlclient) + add_default_dependencies(zlib) + add_default_dependencies(thrift) + add_default_dependencies(brotli) + add_default_dependencies(libprotobuf) + add_default_dependencies(base64) + add_default_dependencies(readpassphrase) + add_default_dependencies(unwind_static) endif () diff --git a/README.md b/README.md index 6538fbee0eb..3de9abdc333 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,6 @@ ClickHouse is an open-source column-oriented database management system that all * You can also [fill this form](https://forms.yandex.com/surveys/meet-yandex-clickhouse-team/) to meet Yandex ClickHouse team in person. ## Upcoming Events -* [ClickHouse Meetup in Minsk](https://yandex.ru/promo/metrica/clickhouse-minsk) on July 11. +* [ClickHouse Meetup in Saint Petersburg](https://yandex.ru/promo/clickhouse/saint-petersburg-2019) on July 27. * [ClickHouse Meetup in Shenzhen](https://www.huodongxing.com/event/3483759917300) on October 20. * [ClickHouse Meetup in Shanghai](https://www.huodongxing.com/event/4483760336000) on October 27. diff --git a/cmake/find_cxx.cmake b/cmake/find_cxx.cmake new file mode 100644 index 00000000000..2b2952f6efd --- /dev/null +++ b/cmake/find_cxx.cmake @@ -0,0 +1,26 @@ +if (NOT APPLE) + option (USE_INTERNAL_LIBCXX_LIBRARY "Set to FALSE to use system libcxx and libcxxabi libraries instead of bundled" ${NOT_UNBUNDLED}) +endif () + +if (USE_INTERNAL_LIBCXX_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libcxx/include/vector") + message (WARNING "submodule contrib/libcxx is missing. to fix try run: \n git submodule update --init --recursive") + set (USE_INTERNAL_LIBCXX_LIBRARY 0) +endif () + +if (USE_INTERNAL_LIBCXX_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libcxxabi/src") + message (WARNING "submodule contrib/libcxxabi is missing. to fix try run: \n git submodule update --init --recursive") + set (USE_INTERNAL_LIBCXXABI_LIBRARY 0) +endif () + +if (NOT USE_INTERNAL_LIBCXX_LIBRARY) + find_library (LIBCXX_LIBRARY c++) + find_library (LIBCXXABI_LIBRARY c++abi) +else () + set (LIBCXX_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcxx/include) + set (LIBCXXABI_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcxxabi/include) + set (LIBCXX_LIBRARY cxx_static) + set (LIBCXXABI_LIBRARY cxxabi_static) +endif () + +message (STATUS "Using libcxx: ${LIBCXX_LIBRARY}") +message (STATUS "Using libcxxabi: ${LIBCXXABI_LIBRARY}") diff --git a/cmake/find_mimalloc.cmake b/cmake/find_mimalloc.cmake index 6e3f24625b6..1820421379f 100644 --- a/cmake/find_mimalloc.cmake +++ b/cmake/find_mimalloc.cmake @@ -1,5 +1,5 @@ if (OS_LINUX AND NOT SANITIZE AND NOT ARCH_ARM AND NOT ARCH_32 AND NOT ARCH_PPC64LE) - option (ENABLE_MIMALLOC "Set to FALSE to disable usage of mimalloc for internal ClickHouse caches" ${NOT_UNBUNDLED}) + option (ENABLE_MIMALLOC "Set to FALSE to disable usage of mimalloc for internal ClickHouse caches" FALSE) endif () if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/mimalloc/include/mimalloc.h") @@ -8,6 +8,8 @@ if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/mimalloc/include/mimalloc.h") endif () if (ENABLE_MIMALLOC) + message (FATAL_ERROR "Mimalloc is not production ready. (Disable with cmake -D ENABLE_MIMALLOC=0). If you want to use mimalloc, you must manually remove this message.") + set (MIMALLOC_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/mimalloc/include) set (USE_MIMALLOC 1) set (MIMALLOC_LIBRARY mimalloc-static) diff --git a/cmake/find_poco.cmake b/cmake/find_poco.cmake index c4523ecfdf6..d9f779414d3 100644 --- a/cmake/find_poco.cmake +++ b/cmake/find_poco.cmake @@ -13,7 +13,10 @@ if (NOT DEFINED ENABLE_POCO_NETSSL OR ENABLE_POCO_NETSSL) list (APPEND POCO_COMPONENTS Crypto NetSSL) endif () if (NOT DEFINED ENABLE_POCO_MONGODB OR ENABLE_POCO_MONGODB) + set(ENABLE_POCO_MONGODB 1 CACHE BOOL "") list (APPEND POCO_COMPONENTS MongoDB) +else () + set(ENABLE_POCO_MONGODB 0 CACHE BOOL "") endif () # TODO: after new poco release with SQL library rename ENABLE_POCO_ODBC -> ENABLE_POCO_SQLODBC if (NOT DEFINED ENABLE_POCO_ODBC OR ENABLE_POCO_ODBC) @@ -37,6 +40,7 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY) set (ENABLE_DATA_MYSQL 0 CACHE BOOL "") set (ENABLE_DATA_POSTGRESQL 0 CACHE BOOL "") set (ENABLE_ENCODINGS 0 CACHE BOOL "") + set (ENABLE_MONGODB ${ENABLE_POCO_MONGODB} CACHE BOOL "" FORCE) # new after 2.0.0: set (POCO_ENABLE_ZIP 0 CACHE BOOL "") @@ -60,7 +64,7 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY) "${ClickHouse_SOURCE_DIR}/contrib/poco/Util/include/" ) - if (NOT DEFINED ENABLE_POCO_MONGODB OR ENABLE_POCO_MONGODB) + if (ENABLE_POCO_MONGODB) set (Poco_MongoDB_LIBRARY PocoMongoDB) set (Poco_MongoDB_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/poco/MongoDB/include/") endif () diff --git a/cmake/sanitize.cmake b/cmake/sanitize.cmake index 1d8ed9461eb..196a66e6845 100644 --- a/cmake/sanitize.cmake +++ b/cmake/sanitize.cmake @@ -6,28 +6,39 @@ if (SANITIZE) if (SANITIZE STREQUAL "address") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=address -fsanitize-address-use-after-scope") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=address -fsanitize-address-use-after-scope") - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address -fsanitize-address-use-after-scope") + if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address -fsanitize-address-use-after-scope") + endif() if (MAKE_STATIC_LIBRARIES AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libasan") endif () + elseif (SANITIZE STREQUAL "memory") - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=memory") - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=memory") - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=memory") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=memory -fsanitize-memory-track-origins") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=memory -fsanitize-memory-track-origins") + if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=memory") + endif() if (MAKE_STATIC_LIBRARIES AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libmsan") endif () + elseif (SANITIZE STREQUAL "thread") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=thread") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=thread") - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=thread") + if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=thread") + endif() if (MAKE_STATIC_LIBRARIES AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libtsan") endif () + elseif (SANITIZE STREQUAL "undefined") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=undefined -fno-sanitize-recover=all") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=undefined -fno-sanitize-recover=all") - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined") + if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined") + endif() if (MAKE_STATIC_LIBRARIES AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libubsan") endif () diff --git a/cmake/use_libcxx.cmake b/cmake/use_libcxx.cmake deleted file mode 100644 index 29ac9406fe0..00000000000 --- a/cmake/use_libcxx.cmake +++ /dev/null @@ -1,24 +0,0 @@ -# Uses MAKE_STATIC_LIBRARIES - - -set(THREADS_PREFER_PTHREAD_FLAG ON) -find_package (Threads) - -include (cmake/test_compiler.cmake) -include (cmake/arch.cmake) - -if (OS_LINUX AND COMPILER_CLANG) - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}") - - option (USE_LIBCXX "Use libc++ and libc++abi instead of libstdc++ (only make sense on Linux with Clang)" ${HAVE_LIBCXX}) - - if (USE_LIBCXX) - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") # Ok for clang6, for older can cause 'not used option' warning - set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_LIBCPP_DEBUG=0") # More checks in debug build. - endif () -endif () - -if (USE_STATIC_LIBRARIES AND HAVE_NO_PIE) - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FLAG_NO_PIE}") - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FLAG_NO_PIE}") -endif () diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index ba75615aadc..2cc8ae37806 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -10,6 +10,17 @@ endif () set_property(DIRECTORY PROPERTY EXCLUDE_FROM_ALL 1) + +if (USE_INTERNAL_UNWIND_LIBRARY) + add_subdirectory (libunwind-cmake) +endif () + +if (USE_LIBCXX AND USE_INTERNAL_LIBCXX_LIBRARY) + add_subdirectory(libcxx-cmake) + add_subdirectory(libcxxabi-cmake) +endif() + + if (USE_INTERNAL_BOOST_LIBRARY) add_subdirectory (boost-cmake) endif () @@ -29,8 +40,7 @@ if (USE_INTERNAL_RE2_LIBRARY) endif () if (USE_INTERNAL_DOUBLE_CONVERSION_LIBRARY) - set (BUILD_TESTING 0 CACHE INTERNAL "") - add_subdirectory (double-conversion) + add_subdirectory (double-conversion-cmake) endif () if (USE_INTERNAL_CITYHASH_LIBRARY) @@ -52,10 +62,6 @@ if (USE_INTERNAL_BTRIE_LIBRARY) add_subdirectory (libbtrie) endif () -if (USE_INTERNAL_UNWIND_LIBRARY) - add_subdirectory (libunwind) -endif () - if (USE_INTERNAL_ZLIB_LIBRARY) set (ZLIB_ENABLE_TESTS 0 CACHE INTERNAL "") set (SKIP_INSTALL_ALL 1 CACHE INTERNAL "") @@ -154,7 +160,7 @@ if (ENABLE_ODBC AND USE_INTERNAL_ODBC_LIBRARY) add_library(ODBC::ODBC ALIAS ${ODBC_LIBRARIES}) endif () -if (USE_INTERNAL_CAPNP_LIBRARY) +if (ENABLE_CAPNP AND USE_INTERNAL_CAPNP_LIBRARY) set (BUILD_TESTING 0 CACHE INTERNAL "") set (_save ${CMAKE_CXX_EXTENSIONS}) set (CMAKE_CXX_EXTENSIONS) @@ -251,7 +257,6 @@ if(USE_INTERNAL_GTEST_LIBRARY) add_subdirectory(${ClickHouse_SOURCE_DIR}/contrib/googletest/googletest ${CMAKE_CURRENT_BINARY_DIR}/googletest) # avoid problems with target_compile_definitions (gtest INTERFACE GTEST_HAS_POSIX_RE=0) - target_include_directories (gtest SYSTEM INTERFACE ${ClickHouse_SOURCE_DIR}/contrib/googletest/include) elseif(GTEST_SRC_DIR) add_subdirectory(${GTEST_SRC_DIR}/googletest ${CMAKE_CURRENT_BINARY_DIR}/googletest) target_compile_definitions(gtest INTERFACE GTEST_HAS_POSIX_RE=0) @@ -283,7 +288,7 @@ if (USE_INTERNAL_LLVM_LIBRARY) endif () if (USE_INTERNAL_LIBGSASL_LIBRARY) - add_subdirectory(libgsasl) + add_subdirectory(libgsasl) endif() if (USE_INTERNAL_LIBXML2_LIBRARY) diff --git a/contrib/double-conversion-cmake/CMakeLists.txt b/contrib/double-conversion-cmake/CMakeLists.txt new file mode 100644 index 00000000000..f91b0fb74c1 --- /dev/null +++ b/contrib/double-conversion-cmake/CMakeLists.txt @@ -0,0 +1,14 @@ +SET(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/double-conversion) + +add_library(double-conversion +${LIBRARY_DIR}/double-conversion/bignum.cc +${LIBRARY_DIR}/double-conversion/bignum-dtoa.cc +${LIBRARY_DIR}/double-conversion/cached-powers.cc +${LIBRARY_DIR}/double-conversion/diy-fp.cc +${LIBRARY_DIR}/double-conversion/double-conversion.cc +${LIBRARY_DIR}/double-conversion/fast-dtoa.cc +${LIBRARY_DIR}/double-conversion/fixed-dtoa.cc +${LIBRARY_DIR}/double-conversion/strtod.cc) + +target_include_directories(double-conversion SYSTEM PUBLIC "${LIBRARY_DIR}") + diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index 4840197c2fd..47f057c0559 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -15,7 +15,6 @@ ${JEMALLOC_SOURCE_DIR}/src/extent_mmap.c ${JEMALLOC_SOURCE_DIR}/src/hash.c ${JEMALLOC_SOURCE_DIR}/src/hook.c ${JEMALLOC_SOURCE_DIR}/src/jemalloc.c -${JEMALLOC_SOURCE_DIR}/src/jemalloc_cpp.cpp ${JEMALLOC_SOURCE_DIR}/src/large.c ${JEMALLOC_SOURCE_DIR}/src/log.c ${JEMALLOC_SOURCE_DIR}/src/malloc_io.c diff --git a/contrib/libcxx b/contrib/libcxx new file mode 160000 index 00000000000..9807685d51d --- /dev/null +++ b/contrib/libcxx @@ -0,0 +1 @@ +Subproject commit 9807685d51db467e097ad5eb8d5c2c16922794b2 diff --git a/contrib/libcxx-cmake/CMakeLists.txt b/contrib/libcxx-cmake/CMakeLists.txt new file mode 100644 index 00000000000..e9ca5e1e7cd --- /dev/null +++ b/contrib/libcxx-cmake/CMakeLists.txt @@ -0,0 +1,51 @@ +set(LIBCXX_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcxx) +#set(LIBCXX_BINARY_DIR ${ClickHouse_BINARY_DIR}/contrib/libcxx) + +set(SRCS +${LIBCXX_SOURCE_DIR}/src/optional.cpp +${LIBCXX_SOURCE_DIR}/src/variant.cpp +${LIBCXX_SOURCE_DIR}/src/chrono.cpp +${LIBCXX_SOURCE_DIR}/src/thread.cpp +${LIBCXX_SOURCE_DIR}/src/experimental/memory_resource.cpp +${LIBCXX_SOURCE_DIR}/src/iostream.cpp +${LIBCXX_SOURCE_DIR}/src/strstream.cpp +${LIBCXX_SOURCE_DIR}/src/ios.cpp +${LIBCXX_SOURCE_DIR}/src/future.cpp +${LIBCXX_SOURCE_DIR}/src/shared_mutex.cpp +${LIBCXX_SOURCE_DIR}/src/condition_variable.cpp +${LIBCXX_SOURCE_DIR}/src/hash.cpp +${LIBCXX_SOURCE_DIR}/src/string.cpp +${LIBCXX_SOURCE_DIR}/src/debug.cpp +#${LIBCXX_SOURCE_DIR}/src/support/win32/support.cpp +#${LIBCXX_SOURCE_DIR}/src/support/win32/locale_win32.cpp +#${LIBCXX_SOURCE_DIR}/src/support/win32/thread_win32.cpp +#${LIBCXX_SOURCE_DIR}/src/support/solaris/xlocale.cpp +${LIBCXX_SOURCE_DIR}/src/stdexcept.cpp +${LIBCXX_SOURCE_DIR}/src/utility.cpp +${LIBCXX_SOURCE_DIR}/src/any.cpp +${LIBCXX_SOURCE_DIR}/src/exception.cpp +${LIBCXX_SOURCE_DIR}/src/memory.cpp +${LIBCXX_SOURCE_DIR}/src/new.cpp +${LIBCXX_SOURCE_DIR}/src/valarray.cpp +${LIBCXX_SOURCE_DIR}/src/vector.cpp +${LIBCXX_SOURCE_DIR}/src/algorithm.cpp +${LIBCXX_SOURCE_DIR}/src/functional.cpp +${LIBCXX_SOURCE_DIR}/src/regex.cpp +${LIBCXX_SOURCE_DIR}/src/bind.cpp +${LIBCXX_SOURCE_DIR}/src/mutex.cpp +${LIBCXX_SOURCE_DIR}/src/charconv.cpp +${LIBCXX_SOURCE_DIR}/src/typeinfo.cpp +${LIBCXX_SOURCE_DIR}/src/locale.cpp +${LIBCXX_SOURCE_DIR}/src/filesystem/operations.cpp +${LIBCXX_SOURCE_DIR}/src/filesystem/int128_builtins.cpp +${LIBCXX_SOURCE_DIR}/src/filesystem/directory_iterator.cpp +${LIBCXX_SOURCE_DIR}/src/system_error.cpp +${LIBCXX_SOURCE_DIR}/src/random.cpp +) + +add_library(cxx_static ${SRCS}) + +target_include_directories(cxx_static PUBLIC ${LIBCXX_SOURCE_DIR}/include) +target_compile_definitions(cxx_static PRIVATE -D_LIBCPP_BUILDING_LIBRARY -DLIBCXX_BUILDING_LIBCXXABI) +target_compile_options(cxx_static PRIVATE -nostdinc++) + diff --git a/contrib/libcxxabi b/contrib/libcxxabi new file mode 160000 index 00000000000..d56efcc7a52 --- /dev/null +++ b/contrib/libcxxabi @@ -0,0 +1 @@ +Subproject commit d56efcc7a52739518dbe7df9e743073e00951fa1 diff --git a/contrib/libcxxabi-cmake/CMakeLists.txt b/contrib/libcxxabi-cmake/CMakeLists.txt new file mode 100644 index 00000000000..2abece86691 --- /dev/null +++ b/contrib/libcxxabi-cmake/CMakeLists.txt @@ -0,0 +1,34 @@ +set(LIBCXXABI_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcxxabi) +set(LIBCXX_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcxx) +#set(LIBCXXABI_BINARY_DIR ${ClickHouse_BINARY_DIR}/contrib/libcxxabi) + +set(SRCS +${LIBCXXABI_SOURCE_DIR}/src/stdlib_stdexcept.cpp +${LIBCXXABI_SOURCE_DIR}/src/cxa_virtual.cpp +${LIBCXXABI_SOURCE_DIR}/src/cxa_thread_atexit.cpp +${LIBCXXABI_SOURCE_DIR}/src/fallback_malloc.cpp +#${LIBCXXABI_SOURCE_DIR}/src/cxa_noexception.cpp +${LIBCXXABI_SOURCE_DIR}/src/cxa_guard.cpp +${LIBCXXABI_SOURCE_DIR}/src/cxa_default_handlers.cpp +${LIBCXXABI_SOURCE_DIR}/src/cxa_personality.cpp +${LIBCXXABI_SOURCE_DIR}/src/stdlib_exception.cpp +${LIBCXXABI_SOURCE_DIR}/src/abort_message.cpp +${LIBCXXABI_SOURCE_DIR}/src/cxa_demangle.cpp +${LIBCXXABI_SOURCE_DIR}/src/cxa_unexpected.cpp +${LIBCXXABI_SOURCE_DIR}/src/cxa_exception.cpp +${LIBCXXABI_SOURCE_DIR}/src/cxa_handlers.cpp +${LIBCXXABI_SOURCE_DIR}/src/cxa_exception_storage.cpp +${LIBCXXABI_SOURCE_DIR}/src/private_typeinfo.cpp +${LIBCXXABI_SOURCE_DIR}/src/stdlib_typeinfo.cpp +${LIBCXXABI_SOURCE_DIR}/src/cxa_aux_runtime.cpp +${LIBCXXABI_SOURCE_DIR}/src/cxa_vector.cpp +${LIBCXXABI_SOURCE_DIR}/src/stdlib_new_delete.cpp +) + +add_library(cxxabi_static ${SRCS}) + +target_include_directories(cxxabi_static PUBLIC ${LIBCXXABI_SOURCE_DIR}/include ${LIBCXX_SOURCE_DIR}/include) +target_compile_definitions(cxxabi_static PRIVATE -D_LIBCPP_BUILDING_LIBRARY) +target_compile_options(cxxabi_static PRIVATE -nostdinc++ -fno-sanitize=undefined) # If we don't disable UBSan, infinite recursion happens in dynamic_cast. + + diff --git a/contrib/librdkafka-cmake/config.h b/contrib/librdkafka-cmake/config.h index 403a79ea42e..bf67863ae7d 100644 --- a/contrib/librdkafka-cmake/config.h +++ b/contrib/librdkafka-cmake/config.h @@ -77,8 +77,6 @@ #define HAVE_PTHREAD_SETNAME_GNU 1 // python //#define HAVE_PYTHON 1 -// C11 threads -#if (__STDC_VERSION__ >= 201112L) && !defined(__STDC_NO_THREADS__) -# define WITH_C11THREADS 1 -#endif +// disable C11 threads for compatibility with old libc +#define WITH_C11THREADS 0 #endif /* _CONFIG_H_ */ diff --git a/contrib/libunwind-cmake/CMakeLists.txt b/contrib/libunwind-cmake/CMakeLists.txt new file mode 100644 index 00000000000..3d4cd319089 --- /dev/null +++ b/contrib/libunwind-cmake/CMakeLists.txt @@ -0,0 +1,31 @@ +set(LIBUNWIND_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libunwind) + +set(LIBUNWIND_CXX_SOURCES + ${LIBUNWIND_SOURCE_DIR}/src/libunwind.cpp + ${LIBUNWIND_SOURCE_DIR}/src/Unwind-EHABI.cpp + ${LIBUNWIND_SOURCE_DIR}/src/Unwind-seh.cpp) +if (APPLE) + set(LIBUNWIND_CXX_SOURCES ${LIBUNWIND_CXX_SOURCES} ${LIBUNWIND_SOURCE_DIR}/src/Unwind_AppleExtras.cpp) +endif () + +set(LIBUNWIND_C_SOURCES + ${LIBUNWIND_SOURCE_DIR}/src/UnwindLevel1.c + ${LIBUNWIND_SOURCE_DIR}/src/UnwindLevel1-gcc-ext.c + ${LIBUNWIND_SOURCE_DIR}/src/Unwind-sjlj.c) +set_source_files_properties(${LIBUNWIND_C_SOURCES} PROPERTIES COMPILE_FLAGS "-std=c99") + +set(LIBUNWIND_ASM_SOURCES + ${LIBUNWIND_SOURCE_DIR}/src/UnwindRegistersRestore.S + ${LIBUNWIND_SOURCE_DIR}/src/UnwindRegistersSave.S) +set_source_files_properties(${LIBUNWIND_ASM_SOURCES} PROPERTIES LANGUAGE C) + +set(LIBUNWIND_SOURCES + ${LIBUNWIND_CXX_SOURCES} + ${LIBUNWIND_C_SOURCES} + ${LIBUNWIND_ASM_SOURCES}) + +add_library(unwind_static ${LIBUNWIND_SOURCES}) + +target_include_directories(unwind_static PUBLIC ${LIBUNWIND_SOURCE_DIR}/include) +target_compile_definitions(unwind_static PRIVATE -D_LIBUNWIND_NO_HEAP=1 -D_DEBUG -D_LIBUNWIND_IS_NATIVE_ONLY) +target_compile_options(unwind_static PRIVATE -fno-exceptions -funwind-tables -fno-sanitize=all -nostdinc++ -fno-rtti) diff --git a/contrib/poco b/contrib/poco index ece721f1085..ea2516be366 160000 --- a/contrib/poco +++ b/contrib/poco @@ -1 +1 @@ -Subproject commit ece721f1085e3894cb5286e8560af84cd1445326 +Subproject commit ea2516be366a73a02a82b499ed4a7db1d40037e0 diff --git a/contrib/simdjson b/contrib/simdjson index 2151ad7f34c..3bd3116cf8f 160000 --- a/contrib/simdjson +++ b/contrib/simdjson @@ -1 +1 @@ -Subproject commit 2151ad7f34cf773a23f086e941d661f8a8873144 +Subproject commit 3bd3116cf8faf6d482dc31423b16533bfa2696f7 diff --git a/contrib/simdjson-cmake/CMakeLists.txt b/contrib/simdjson-cmake/CMakeLists.txt index 16a5dc1a791..bbb2d8e389f 100644 --- a/contrib/simdjson-cmake/CMakeLists.txt +++ b/contrib/simdjson-cmake/CMakeLists.txt @@ -11,8 +11,9 @@ set(SIMDJSON_SRC ${SIMDJSON_SRC_DIR}/stage2_build_tape.cpp ${SIMDJSON_SRC_DIR}/parsedjson.cpp ${SIMDJSON_SRC_DIR}/parsedjsoniterator.cpp + ${SIMDJSON_SRC_DIR}/simdjson.cpp ) add_library(${SIMDJSON_LIBRARY} ${SIMDJSON_SRC}) -target_include_directories(${SIMDJSON_LIBRARY} PUBLIC "${SIMDJSON_INCLUDE_DIR}") +target_include_directories(${SIMDJSON_LIBRARY} SYSTEM PUBLIC "${SIMDJSON_INCLUDE_DIR}") target_compile_options(${SIMDJSON_LIBRARY} PRIVATE -mavx2 -mbmi -mbmi2 -mpclmul) diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index 18c169211d9..849784e41bd 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -48,7 +48,7 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wshadow -Wshadow-uncaptured-local -Wextra-semi -Wcomma -Winconsistent-missing-destructor-override -Wunused-exception-parameter -Wcovered-switch-default -Wold-style-cast -Wrange-loop-analysis -Wunused-member-function -Wunreachable-code -Wunreachable-code-return -Wnewline-eof -Wembedded-directive -Wgnu-case-range -Wunused-macros -Wconditional-uninitialized -Wdeprecated -Wundef -Wreserved-id-macro -Wredundant-parens -Wzero-as-null-pointer-constant") if (WEVERYTHING) - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Weverything -Wno-c++98-compat -Wno-c++98-compat-pedantic -Wno-missing-noreturn -Wno-padded -Wno-switch-enum -Wno-shadow-field-in-constructor -Wno-deprecated-dynamic-exception-spec -Wno-float-equal -Wno-weak-vtables -Wno-shift-sign-overflow -Wno-sign-conversion -Wno-conversion -Wno-exit-time-destructors -Wno-undefined-func-template -Wno-documentation-unknown-command -Wno-missing-variable-declarations -Wno-unused-template -Wno-global-constructors -Wno-c99-extensions -Wno-missing-prototypes -Wno-weak-template-vtables -Wno-zero-length-array -Wno-gnu-anonymous-struct -Wno-nested-anon-types -Wno-double-promotion -Wno-disabled-macro-expansion -Wno-used-but-marked-unused -Wno-vla-extension -Wno-vla -Wno-packed") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Weverything -Wno-c++98-compat -Wno-c++98-compat-pedantic -Wno-padded -Wno-switch-enum -Wno-shadow-field-in-constructor -Wno-deprecated-dynamic-exception-spec -Wno-float-equal -Wno-weak-vtables -Wno-shift-sign-overflow -Wno-sign-conversion -Wno-conversion -Wno-exit-time-destructors -Wno-undefined-func-template -Wno-documentation-unknown-command -Wno-missing-variable-declarations -Wno-unused-template -Wno-global-constructors -Wno-c99-extensions -Wno-missing-prototypes -Wno-weak-template-vtables -Wno-zero-length-array -Wno-gnu-anonymous-struct -Wno-nested-anon-types -Wno-double-promotion -Wno-disabled-macro-expansion -Wno-vla-extension -Wno-vla -Wno-packed") # TODO Enable conversion, sign-conversion, double-promotion warnings. endif () @@ -102,6 +102,7 @@ add_headers_and_sources(clickhouse_common_io src/Common/HashTable) add_headers_and_sources(clickhouse_common_io src/IO) add_headers_and_sources(dbms src/Core) +add_headers_and_sources(dbms src/Compression/) add_headers_and_sources(dbms src/DataStreams) add_headers_and_sources(dbms src/DataTypes) add_headers_and_sources(dbms src/Databases) @@ -113,6 +114,18 @@ add_headers_and_sources(dbms src/Storages/Distributed) add_headers_and_sources(dbms src/Storages/MergeTree) add_headers_and_sources(dbms src/Client) add_headers_and_sources(dbms src/Formats) +add_headers_and_sources(dbms src/Processors) +add_headers_and_sources(dbms src/Processors/Executors) +add_headers_and_sources(dbms src/Processors/Formats) +add_headers_and_sources(dbms src/Processors/Formats/Impl) +add_headers_and_sources(dbms src/Processors/Transforms) +add_headers_and_sources(dbms src/Processors/Sources) +add_headers_only(dbms src/Server) + +if(USE_RDKAFKA) + add_headers_and_sources(dbms src/Storages/Kafka) +endif() + list (APPEND clickhouse_common_io_sources ${CONFIG_BUILD}) list (APPEND clickhouse_common_io_headers ${CONFIG_VERSION} ${CONFIG_COMMON}) @@ -222,6 +235,17 @@ target_link_libraries(clickhouse_common_io roaring ) +if(ZSTD_LIBRARY) + target_link_libraries(clickhouse_common_io PUBLIC ${ZSTD_LIBRARY}) +endif() + +if (USE_RDKAFKA) + target_link_libraries(dbms PRIVATE ${CPPKAFKA_LIBRARY} ${RDKAFKA_LIBRARY}) + if(NOT USE_INTERNAL_RDKAFKA_LIBRARY) + target_include_directories(dbms SYSTEM BEFORE PRIVATE ${RDKAFKA_INCLUDE_DIR}) + endif() +endif() + if(RE2_INCLUDE_DIR) target_include_directories(clickhouse_common_io SYSTEM BEFORE PUBLIC ${RE2_INCLUDE_DIR}) @@ -241,20 +265,22 @@ if(CPUINFO_LIBRARY) endif() target_link_libraries (dbms - PUBLIC - clickhouse_compression PRIVATE clickhouse_parsers clickhouse_common_config + clickhouse_common_zookeeper + string_utils # FIXME: not sure if it's private PUBLIC clickhouse_common_io PRIVATE clickhouse_dictionaries_embedded + ${LZ4_LIBRARY} PUBLIC ${MYSQLXX_LIBRARY} PRIVATE ${BTRIE_LIBRARIES} ${Boost_PROGRAM_OPTIONS_LIBRARY} + ${Boost_FILESYSTEM_LIBRARY} PUBLIC ${Boost_SYSTEM_LIBRARY} Threads::Threads @@ -263,6 +289,15 @@ target_link_libraries (dbms target_include_directories(dbms PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/src/Core/include) target_include_directories(clickhouse_common_io PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/src/Core/include) # uses some includes from core target_include_directories(dbms SYSTEM BEFORE PUBLIC ${PDQSORT_INCLUDE_DIR}) +target_include_directories(dbms SYSTEM PUBLIC ${PCG_RANDOM_INCLUDE_DIR}) + +if (NOT USE_INTERNAL_LZ4_LIBRARY) + target_include_directories(dbms SYSTEM BEFORE PRIVATE ${LZ4_INCLUDE_DIR}) +endif () +if (NOT USE_INTERNAL_ZSTD_LIBRARY AND ZSTD_INCLUDE_DIR) + target_include_directories(dbms SYSTEM BEFORE PRIVATE ${ZSTD_INCLUDE_DIR}) +endif () + if (NOT USE_INTERNAL_BOOST_LIBRARY) target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS}) @@ -318,10 +353,6 @@ if (USE_CAPNP) endif () endif () -if (USE_RDKAFKA) - target_link_libraries (dbms PRIVATE clickhouse_storage_kafka) -endif () - if (USE_PARQUET) target_link_libraries(dbms PRIVATE ${PARQUET_LIBRARY}) if (NOT USE_INTERNAL_PARQUET_LIBRARY OR USE_INTERNAL_PARQUET_LIBRARY_NATIVE_CMAKE) @@ -354,6 +385,7 @@ endif() if (USE_JEMALLOC) target_include_directories (dbms SYSTEM BEFORE PRIVATE ${JEMALLOC_INCLUDE_DIR}) # used in Interpreters/AsynchronousMetrics.cpp + target_include_directories (clickhouse_common_io SYSTEM BEFORE PRIVATE ${JEMALLOC_INCLUDE_DIR}) # new_delete.cpp endif () target_include_directories (dbms PUBLIC ${DBMS_INCLUDE_DIR} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/src/Formats/include) @@ -376,6 +408,10 @@ if (ENABLE_TESTS AND USE_GTEST) # attach all dbms gtest sources grep_gtest_sources(${ClickHouse_SOURCE_DIR}/dbms dbms_gtest_sources) add_executable(unit_tests_dbms ${dbms_gtest_sources}) + + # gtest framework has substandard code + target_compile_options(unit_tests_dbms PRIVATE -Wno-zero-as-null-pointer-constant -Wno-undef -Wno-sign-compare -Wno-used-but-marked-unused -Wno-missing-noreturn) + target_link_libraries(unit_tests_dbms PRIVATE ${GTEST_BOTH_LIBRARIES} clickhouse_functions clickhouse_parsers dbms clickhouse_common_zookeeper) add_check(unit_tests_dbms) endif () diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake index 77503018203..5714b5207b8 100644 --- a/dbms/cmake/version.cmake +++ b/dbms/cmake/version.cmake @@ -1,11 +1,11 @@ # This strings autochanged from release_lib.sh: -set(VERSION_REVISION 54423) +set(VERSION_REVISION 54424) set(VERSION_MAJOR 19) -set(VERSION_MINOR 11) -set(VERSION_PATCH 0) -set(VERSION_GITHASH badb6ab8310ed94e20f43ac9a9a227f7a2590009) -set(VERSION_DESCRIBE v19.11.0-testing) -set(VERSION_STRING 19.11.0) +set(VERSION_MINOR 12) +set(VERSION_PATCH 1) +set(VERSION_GITHASH a584f0ca6cb5df9b0d9baf1e2e1eaa7d12a20a44) +set(VERSION_DESCRIBE v19.12.1.1-prestable) +set(VERSION_STRING 19.12.1.1) # end of autochange set(VERSION_EXTRA "" CACHE STRING "") diff --git a/dbms/programs/benchmark/Benchmark.cpp b/dbms/programs/benchmark/Benchmark.cpp index 019080e2391..c69e9a54feb 100644 --- a/dbms/programs/benchmark/Benchmark.cpp +++ b/dbms/programs/benchmark/Benchmark.cpp @@ -61,6 +61,8 @@ public: randomize(randomize_), max_iterations(max_iterations_), max_time(max_time_), json_path(json_path_), settings(settings_), global_context(Context::createGlobal()), pool(concurrency) { + global_context.makeGlobalContext(); + std::cerr << std::fixed << std::setprecision(3); /// This is needed to receive blocks with columns of AggregateFunction data type diff --git a/dbms/programs/benchmark/CMakeLists.txt b/dbms/programs/benchmark/CMakeLists.txt index ccbefc0453a..58096985037 100644 --- a/dbms/programs/benchmark/CMakeLists.txt +++ b/dbms/programs/benchmark/CMakeLists.txt @@ -1,5 +1,5 @@ set(CLICKHOUSE_BENCHMARK_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Benchmark.cpp) -set(CLICKHOUSE_BENCHMARK_LINK PRIVATE clickhouse_aggregate_functions clickhouse_common_config clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY}) +set(CLICKHOUSE_BENCHMARK_LINK PRIVATE dbms clickhouse_aggregate_functions clickhouse_common_config ${Boost_PROGRAM_OPTIONS_LIBRARY}) set(CLICKHOUSE_BENCHMARK_INCLUDE SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR}) clickhouse_program_add(benchmark) diff --git a/dbms/programs/client/CMakeLists.txt b/dbms/programs/client/CMakeLists.txt index 06b142227d7..88e23a094d1 100644 --- a/dbms/programs/client/CMakeLists.txt +++ b/dbms/programs/client/CMakeLists.txt @@ -3,7 +3,7 @@ set(CLICKHOUSE_CLIENT_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/ConnectionParameters.cpp ) -set(CLICKHOUSE_CLIENT_LINK PRIVATE clickhouse_common_config clickhouse_functions clickhouse_aggregate_functions clickhouse_common_io ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY}) +set(CLICKHOUSE_CLIENT_LINK PRIVATE clickhouse_common_config clickhouse_functions clickhouse_aggregate_functions clickhouse_common_io clickhouse_parsers string_utils ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY}) set(CLICKHOUSE_CLIENT_INCLUDE SYSTEM PRIVATE ${READLINE_INCLUDE_DIR} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/include) include(CheckSymbolExists) diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index 2da1c4a987d..091a1ac063f 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -218,6 +218,7 @@ private: configReadClient(config(), home_path); + context.makeGlobalContext(); context.setApplicationType(Context::ApplicationType::CLIENT); /// settings and limits could be specified in config file, but passed settings has higher priority diff --git a/dbms/programs/compressor/CMakeLists.txt b/dbms/programs/compressor/CMakeLists.txt index 46fd4816ba2..c009bb55f76 100644 --- a/dbms/programs/compressor/CMakeLists.txt +++ b/dbms/programs/compressor/CMakeLists.txt @@ -1,7 +1,7 @@ # Also in utils set(CLICKHOUSE_COMPRESSOR_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Compressor.cpp) -set(CLICKHOUSE_COMPRESSOR_LINK PRIVATE clickhouse_compression clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY}) +set(CLICKHOUSE_COMPRESSOR_LINK PRIVATE dbms clickhouse_parsers ${Boost_PROGRAM_OPTIONS_LIBRARY}) #set(CLICKHOUSE_COMPRESSOR_INCLUDE SYSTEM PRIVATE ...) clickhouse_program_add(compressor) diff --git a/dbms/programs/copier/CMakeLists.txt b/dbms/programs/copier/CMakeLists.txt index 0aec381ebd5..85b2819ac7a 100644 --- a/dbms/programs/copier/CMakeLists.txt +++ b/dbms/programs/copier/CMakeLists.txt @@ -1,5 +1,5 @@ set(CLICKHOUSE_COPIER_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/ClusterCopier.cpp) -set(CLICKHOUSE_COPIER_LINK PRIVATE clickhouse_functions clickhouse_table_functions clickhouse_aggregate_functions clickhouse_dictionaries PUBLIC daemon) +set(CLICKHOUSE_COPIER_LINK PRIVATE clickhouse_common_zookeeper clickhouse_parsers clickhouse_functions clickhouse_table_functions clickhouse_aggregate_functions clickhouse_dictionaries string_utils PUBLIC daemon) set(CLICKHOUSE_COPIER_INCLUDE SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR}) clickhouse_program_add(copier) diff --git a/dbms/programs/copier/ClusterCopier.cpp b/dbms/programs/copier/ClusterCopier.cpp index e48ab92c4f8..43158dedd71 100644 --- a/dbms/programs/copier/ClusterCopier.cpp +++ b/dbms/programs/copier/ClusterCopier.cpp @@ -2171,10 +2171,10 @@ void ClusterCopierApp::mainImpl() << "revision " << ClickHouseRevision::get() << ")"); auto context = std::make_unique(Context::createGlobal()); + context->makeGlobalContext(); SCOPE_EXIT(context->shutdown()); context->setConfig(loaded_config.configuration); - context->setGlobalContext(*context); context->setApplicationType(Context::ApplicationType::LOCAL); context->setPath(process_path); diff --git a/dbms/programs/extract-from-config/CMakeLists.txt b/dbms/programs/extract-from-config/CMakeLists.txt index 4c01cd9c999..b82cbb966ae 100644 --- a/dbms/programs/extract-from-config/CMakeLists.txt +++ b/dbms/programs/extract-from-config/CMakeLists.txt @@ -1,5 +1,5 @@ set(CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/ExtractFromConfig.cpp) -set(CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK PRIVATE clickhouse_common_config clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY}) +set(CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK PRIVATE clickhouse_common_config clickhouse_common_io clickhouse_common_zookeeper ${Boost_PROGRAM_OPTIONS_LIBRARY}) #set(CLICKHOUSE_EXTRACT_FROM_CONFIG_INCLUDE SYSTEM PRIVATE ...) clickhouse_program_add(extract-from-config) diff --git a/dbms/programs/local/CMakeLists.txt b/dbms/programs/local/CMakeLists.txt index 41780936977..d066fd53277 100644 --- a/dbms/programs/local/CMakeLists.txt +++ b/dbms/programs/local/CMakeLists.txt @@ -1,5 +1,5 @@ set(CLICKHOUSE_LOCAL_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/LocalServer.cpp) -set(CLICKHOUSE_LOCAL_LINK PRIVATE clickhouse_dictionaries clickhouse_common_io clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Boost_PROGRAM_OPTIONS_LIBRARY}) +set(CLICKHOUSE_LOCAL_LINK PRIVATE clickhouse_storages_system clickhouse_dictionaries clickhouse_common_config clickhouse_common_io clickhouse_functions clickhouse_aggregate_functions clickhouse_parsers clickhouse_table_functions ${Boost_PROGRAM_OPTIONS_LIBRARY}) #set(CLICKHOUSE_LOCAL_INCLUDE SYSTEM PRIVATE ...) clickhouse_program_add(local) diff --git a/dbms/programs/local/LocalServer.cpp b/dbms/programs/local/LocalServer.cpp index 3e3b249fe82..bed55a0fc5f 100644 --- a/dbms/programs/local/LocalServer.cpp +++ b/dbms/programs/local/LocalServer.cpp @@ -131,7 +131,7 @@ try context = std::make_unique(Context::createGlobal()); - context->setGlobalContext(*context); + context->makeGlobalContext(); context->setApplicationType(Context::ApplicationType::LOCAL); tryInitPath(); @@ -275,8 +275,8 @@ void LocalServer::processQueries() if (!parse_res.second) throw Exception("Cannot parse and execute the following part of query: " + String(parse_res.first), ErrorCodes::SYNTAX_ERROR); - context->setSessionContext(*context); - context->setQueryContext(*context); + context->makeSessionContext(); + context->makeQueryContext(); context->setUser("default", "", Poco::Net::SocketAddress{}, ""); context->setCurrentQueryId(""); diff --git a/dbms/programs/obfuscator/Obfuscator.cpp b/dbms/programs/obfuscator/Obfuscator.cpp index 7488433d569..3c20510d481 100644 --- a/dbms/programs/obfuscator/Obfuscator.cpp +++ b/dbms/programs/obfuscator/Obfuscator.cpp @@ -1024,6 +1024,7 @@ try } Context context = Context::createGlobal(); + context.makeGlobalContext(); ReadBufferFromFileDescriptor file_in(STDIN_FILENO); WriteBufferFromFileDescriptor file_out(STDOUT_FILENO); diff --git a/dbms/programs/odbc-bridge/CMakeLists.txt b/dbms/programs/odbc-bridge/CMakeLists.txt index 90aad184f3e..060a36e9275 100644 --- a/dbms/programs/odbc-bridge/CMakeLists.txt +++ b/dbms/programs/odbc-bridge/CMakeLists.txt @@ -11,7 +11,7 @@ set(CLICKHOUSE_ODBC_BRIDGE_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/validateODBCConnectionString.cpp ) -set(CLICKHOUSE_ODBC_BRIDGE_LINK PRIVATE dbms clickhouse_common_io PUBLIC daemon) +set(CLICKHOUSE_ODBC_BRIDGE_LINK PRIVATE dbms clickhouse_parsers PUBLIC daemon) set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include) if (USE_POCO_SQLODBC) diff --git a/dbms/programs/odbc-bridge/ODBCBridge.cpp b/dbms/programs/odbc-bridge/ODBCBridge.cpp index aaacdfca826..cf265eb6abb 100644 --- a/dbms/programs/odbc-bridge/ODBCBridge.cpp +++ b/dbms/programs/odbc-bridge/ODBCBridge.cpp @@ -160,7 +160,7 @@ int ODBCBridge::main(const std::vector & /*args*/) http_params->setKeepAliveTimeout(keep_alive_timeout); context = std::make_shared(Context::createGlobal()); - context->setGlobalContext(*context); + context->makeGlobalContext(); auto server = Poco::Net::HTTPServer( new HandlerFactory("ODBCRequestHandlerFactory-factory", keep_alive_timeout, context), server_pool, socket, http_params); diff --git a/dbms/programs/performance-test/CMakeLists.txt b/dbms/programs/performance-test/CMakeLists.txt index c7eeaa45ab3..94e346c83cd 100644 --- a/dbms/programs/performance-test/CMakeLists.txt +++ b/dbms/programs/performance-test/CMakeLists.txt @@ -1,4 +1,4 @@ -set(CLICKHOUSE_PERFORMANCE_TEST_SOURCES +set(CLICKHOUSE_PERFORMANCE_TEST_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/JSONString.cpp ${CMAKE_CURRENT_SOURCE_DIR}/StopConditionsSet.cpp ${CMAKE_CURRENT_SOURCE_DIR}/TestStopConditions.cpp @@ -12,7 +12,7 @@ set(CLICKHOUSE_PERFORMANCE_TEST_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/PerformanceTestSuite.cpp ) -set(CLICKHOUSE_PERFORMANCE_TEST_LINK PRIVATE dbms clickhouse_common_io clickhouse_common_config ${Boost_PROGRAM_OPTIONS_LIBRARY}) +set(CLICKHOUSE_PERFORMANCE_TEST_LINK PRIVATE dbms clickhouse_common_config ${Boost_FILESYSTEM_LIBRARY} ${Boost_PROGRAM_OPTIONS_LIBRARY}) set(CLICKHOUSE_PERFORMANCE_TEST_INCLUDE SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR}) clickhouse_program_add(performance-test) diff --git a/dbms/programs/performance-test/ConfigPreprocessor.cpp b/dbms/programs/performance-test/ConfigPreprocessor.cpp index 3ea095a5175..5da9650454a 100644 --- a/dbms/programs/performance-test/ConfigPreprocessor.cpp +++ b/dbms/programs/performance-test/ConfigPreprocessor.cpp @@ -14,10 +14,15 @@ std::vector ConfigPreprocessor::processConfig( { std::vector result; - for (const auto & path : paths) + for (const auto & path_str : paths) { - result.emplace_back(XMLConfigurationPtr(new XMLConfiguration(path))); - result.back()->setString("path", Poco::Path(path).absolute().toString()); + auto test = XMLConfigurationPtr(new XMLConfiguration(path_str)); + result.push_back(test); + + const auto path = Poco::Path(path_str); + test->setString("path", path.absolute().toString()); + if (test->getString("name", "") == "") + test->setString("name", path.getBaseName()); } /// Leave tests: diff --git a/dbms/programs/performance-test/PerformanceTest.cpp b/dbms/programs/performance-test/PerformanceTest.cpp index c2d8d4f252c..a005fcb5fbb 100644 --- a/dbms/programs/performance-test/PerformanceTest.cpp +++ b/dbms/programs/performance-test/PerformanceTest.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -296,6 +297,47 @@ void PerformanceTest::runQueries( break; } } + + if (got_SIGINT) + { + return; + } + + // Pull memory usage data from query log. The log is normally filled in + // background, so we have to flush it synchronously here to see all the + // previous queries. + { + RemoteBlockInputStream flush_log(connection, "system flush logs", + {} /* header */, context); + flush_log.readPrefix(); + while (flush_log.read()); + flush_log.readSuffix(); + } + + for (auto & statistics : statistics_by_run) + { + RemoteBlockInputStream log_reader(connection, + "select memory_usage from system.query_log where type = 2 and query_id = '" + + statistics.query_id + "'", + {} /* header */, context); + + log_reader.readPrefix(); + Block block = log_reader.read(); + if (block.columns() == 0) + { + LOG_WARNING(log, "Query '" << statistics.query_id << "' is not found in query log."); + continue; + } + + assert(block.columns() == 1); + assert(block.getDataTypes()[0]->getName() == "UInt64"); + ColumnPtr column = block.getByPosition(0).column; + assert(column->size() == 1); + StringRef ref = column->getDataAt(0); + assert(ref.size == sizeof(UInt64)); + statistics.memory_usage = *reinterpret_cast(ref.data); + log_reader.readSuffix(); + } } diff --git a/dbms/programs/performance-test/PerformanceTestSuite.cpp b/dbms/programs/performance-test/PerformanceTestSuite.cpp index cfa7d202d1d..91314a0fbff 100644 --- a/dbms/programs/performance-test/PerformanceTestSuite.cpp +++ b/dbms/programs/performance-test/PerformanceTestSuite.cpp @@ -89,6 +89,7 @@ public: , input_files(input_files_) , log(&Poco::Logger::get("PerformanceTestSuite")) { + global_context.makeGlobalContext(); global_context.getSettingsRef().copyChangesFrom(cmd_settings); if (input_files.size() < 1) throw Exception("No tests were specified", ErrorCodes::BAD_ARGUMENTS); @@ -259,15 +260,12 @@ static std::vector getInputFiles(const po::variables_map & options, if (input_files.empty()) throw DB::Exception("Did not find any xml files", DB::ErrorCodes::BAD_ARGUMENTS); - else - LOG_INFO(log, "Found " << input_files.size() << " files"); } else { input_files = options["input-files"].as>(); - LOG_INFO(log, "Found " + std::to_string(input_files.size()) + " input files"); - std::vector collected_files; + std::vector collected_files; for (const std::string & filename : input_files) { fs::path file(filename); @@ -289,6 +287,8 @@ static std::vector getInputFiles(const po::variables_map & options, input_files = std::move(collected_files); } + + LOG_INFO(log, "Found " + std::to_string(input_files.size()) + " input files"); std::sort(input_files.begin(), input_files.end()); return input_files; } diff --git a/dbms/programs/performance-test/ReportBuilder.cpp b/dbms/programs/performance-test/ReportBuilder.cpp index 97d4874ca5d..4aa1933a209 100644 --- a/dbms/programs/performance-test/ReportBuilder.cpp +++ b/dbms/programs/performance-test/ReportBuilder.cpp @@ -157,6 +157,8 @@ std::string ReportBuilder::buildFullReport( runJSON.set("avg_bytes_per_second", statistics.avg_bytes_speed_value); } + runJSON.set("memory_usage", statistics.memory_usage); + run_infos.push_back(runJSON); } } diff --git a/dbms/programs/performance-test/TestStats.h b/dbms/programs/performance-test/TestStats.h index 5d70edc437c..b38ffa7386a 100644 --- a/dbms/programs/performance-test/TestStats.h +++ b/dbms/programs/performance-test/TestStats.h @@ -19,6 +19,7 @@ struct TestStats Stopwatch avg_bytes_speed_watch; bool last_query_was_cancelled = false; + std::string query_id; size_t queries = 0; @@ -49,6 +50,8 @@ struct TestStats size_t number_of_rows_speed_info_batches = 0; size_t number_of_bytes_speed_info_batches = 0; + UInt64 memory_usage = 0; + bool ready = false; // check if a query wasn't interrupted by SIGINT std::string exception; diff --git a/dbms/programs/performance-test/executeQuery.cpp b/dbms/programs/performance-test/executeQuery.cpp index f12808eac36..db82a48d0c1 100644 --- a/dbms/programs/performance-test/executeQuery.cpp +++ b/dbms/programs/performance-test/executeQuery.cpp @@ -2,9 +2,11 @@ #include #include #include +#include namespace DB { + namespace { @@ -36,7 +38,7 @@ void checkFulfilledConditionsAndUpdate( } } -} +} // anonymous namespace void executeQuery( Connection & connection, @@ -47,12 +49,18 @@ void executeQuery( Context & context, const Settings & settings) { + static const std::string query_id_prefix + = Poco::UUIDGenerator::defaultGenerator().create().toString() + "-"; + static int next_query_id = 1; + statistics.watch_per_query.restart(); statistics.last_query_was_cancelled = false; statistics.last_query_rows_read = 0; statistics.last_query_bytes_read = 0; + statistics.query_id = query_id_prefix + std::to_string(next_query_id++); RemoteBlockInputStream stream(connection, query, {}, context, &settings); + stream.setQueryId(statistics.query_id); stream.setProgressCallback( [&](const Progress & value) @@ -70,4 +78,5 @@ void executeQuery( statistics.setTotalTime(); } + } diff --git a/dbms/programs/server/CMakeLists.txt b/dbms/programs/server/CMakeLists.txt index fe260c1192d..72a2427ef3c 100644 --- a/dbms/programs/server/CMakeLists.txt +++ b/dbms/programs/server/CMakeLists.txt @@ -8,11 +8,17 @@ set(CLICKHOUSE_SERVER_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/RootRequestHandler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Server.cpp ${CMAKE_CURRENT_SOURCE_DIR}/TCPHandler.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/MySQLHandler.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/MySQLHandlerFactory.cpp ) -set(CLICKHOUSE_SERVER_LINK PRIVATE clickhouse_dictionaries clickhouse_common_io PUBLIC daemon PRIVATE clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Poco_Net_LIBRARY}) +if (USE_POCO_NETSSL) + set(CLICKHOUSE_SERVER_SOURCES + ${CLICKHOUSE_SERVER_SOURCES} + ${CMAKE_CURRENT_SOURCE_DIR}/MySQLHandler.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/MySQLHandlerFactory.cpp + ) +endif () + +set(CLICKHOUSE_SERVER_LINK PRIVATE clickhouse_dictionaries clickhouse_common_io clickhouse_common_config clickhouse_common_zookeeper clickhouse_parsers string_utils PUBLIC daemon PRIVATE clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Poco_Net_LIBRARY}) if (USE_POCO_NETSSL) set(CLICKHOUSE_SERVER_LINK ${CLICKHOUSE_SERVER_LINK} PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY}) endif () diff --git a/dbms/programs/server/HTTPHandler.cpp b/dbms/programs/server/HTTPHandler.cpp index 2349ab337f0..5db7ce1a3ae 100644 --- a/dbms/programs/server/HTTPHandler.cpp +++ b/dbms/programs/server/HTTPHandler.cpp @@ -211,7 +211,6 @@ void HTTPHandler::processQuery( Output & used_output) { Context context = server.context(); - context.setGlobalContext(server.context()); CurrentThread::QueryScope query_scope(context); diff --git a/dbms/programs/server/MySQLHandler.cpp b/dbms/programs/server/MySQLHandler.cpp index a935f13e82a..f9aa29ab0b5 100644 --- a/dbms/programs/server/MySQLHandler.cpp +++ b/dbms/programs/server/MySQLHandler.cpp @@ -18,13 +18,19 @@ #include #include +#include + namespace DB { + using namespace MySQLProtocol; + + using Poco::Net::SecureStreamSocket; using Poco::Net::SSLManager; + namespace ErrorCodes { extern const int MYSQL_CLIENT_INSUFFICIENT_CAPABILITIES; @@ -48,7 +54,7 @@ MySQLHandler::MySQLHandler(IServer & server_, const Poco::Net::StreamSocket & so void MySQLHandler::run() { connection_context = server.context(); - connection_context.setSessionContext(connection_context); + connection_context.makeSessionContext(); connection_context.setDefaultFormat("MySQLWire"); in = std::make_shared(socket()); diff --git a/dbms/programs/server/MySQLHandler.h b/dbms/programs/server/MySQLHandler.h index f55906f7428..ffbaf1d4a6b 100644 --- a/dbms/programs/server/MySQLHandler.h +++ b/dbms/programs/server/MySQLHandler.h @@ -4,7 +4,6 @@ #include #include #include -#include #include "IServer.h" diff --git a/dbms/programs/server/Server.cpp b/dbms/programs/server/Server.cpp index c40fa945cef..3d50466df84 100644 --- a/dbms/programs/server/Server.cpp +++ b/dbms/programs/server/Server.cpp @@ -187,7 +187,7 @@ int Server::main(const std::vector & /*args*/) * settings, available functions, data types, aggregate functions, databases... */ global_context = std::make_unique(Context::createGlobal()); - global_context->setGlobalContext(*global_context); + global_context->makeGlobalContext(); global_context->setApplicationType(Context::ApplicationType::SERVER); bool has_zookeeper = config().has("zookeeper"); @@ -533,10 +533,18 @@ int Server::main(const std::vector & /*args*/) if (!TaskStatsInfoGetter::checkPermissions()) { LOG_INFO(log, "It looks like the process has no CAP_NET_ADMIN capability, 'taskstats' performance statistics will be disabled." - " It could happen due to incorrect ClickHouse package installation." - " You could resolve the problem manually with 'sudo setcap cap_net_admin=+ep /usr/bin/clickhouse'." - " Note that it will not work on 'nosuid' mounted filesystems." - " It also doesn't work if you run clickhouse-server inside network namespace as it happens in some containers."); + " It could happen due to incorrect ClickHouse package installation." + " You could resolve the problem manually with 'sudo setcap cap_net_admin=+ep /usr/bin/clickhouse'." + " Note that it will not work on 'nosuid' mounted filesystems." + " It also doesn't work if you run clickhouse-server inside network namespace as it happens in some containers."); + } + + if (!hasLinuxCapability(CAP_SYS_NICE)) + { + LOG_INFO(log, "It looks like the process has no CAP_SYS_NICE capability, the setting 'os_thread_nice' will have no effect." + " It could happen due to incorrect ClickHouse package installation." + " You could resolve the problem manually with 'sudo setcap cap_sys_nice=+ep /usr/bin/clickhouse'." + " Note that it will not work on 'nosuid' mounted filesystems."); } #else LOG_INFO(log, "TaskStats is not implemented for this OS. IO accounting will be disabled."); @@ -738,6 +746,7 @@ int Server::main(const std::vector & /*args*/) if (config().has("mysql_port")) { +#if USE_POCO_NETSSL Poco::Net::ServerSocket socket; auto address = socket_bind_listen(socket, listen_host, config().getInt("mysql_port"), /* secure = */ true); socket.setReceiveTimeout(Poco::Timespan()); @@ -749,6 +758,10 @@ int Server::main(const std::vector & /*args*/) new Poco::Net::TCPServerParams)); LOG_INFO(log, "Listening for MySQL compatibility protocol: " + address.toString()); +#else + throw Exception{"SSL support for MySQL protocol is disabled because Poco library was built without NetSSL support.", + ErrorCodes::SUPPORT_IS_DISABLED}; +#endif } } catch (const Poco::Exception & e) diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index b436c213e7d..8debfd7d235 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -28,6 +28,9 @@ #include #include #include +#include + +#include #include "TCPHandler.h" @@ -54,7 +57,7 @@ void TCPHandler::runImpl() ThreadStatus thread_status; connection_context = server.context(); - connection_context.setSessionContext(connection_context); + connection_context.makeSessionContext(); Settings global_settings = connection_context.getSettings(); @@ -170,12 +173,13 @@ void TCPHandler::runImpl() send_exception_with_stack_trace = query_context->getSettingsRef().calculate_text_stack_trace; /// Should we send internal logs to client? + const auto client_logs_level = query_context->getSettingsRef().send_logs_level; if (client_revision >= DBMS_MIN_REVISION_WITH_SERVER_LOGS - && query_context->getSettingsRef().send_logs_level.value != LogsLevel::none) + && client_logs_level.value != LogsLevel::none) { state.logs_queue = std::make_shared(); - state.logs_queue->max_priority = Poco::Logger::parseLevel(query_context->getSettingsRef().send_logs_level.toString()); - CurrentThread::attachInternalTextLogsQueue(state.logs_queue); + state.logs_queue->max_priority = Poco::Logger::parseLevel(client_logs_level.toString()); + CurrentThread::attachInternalTextLogsQueue(state.logs_queue, client_logs_level.value); } query_context->setExternalTablesInitializer([&global_settings, this] (Context & context) @@ -207,6 +211,8 @@ void TCPHandler::runImpl() /// Does the request require receive data from client? if (state.need_receive_data_for_insert) processInsertQuery(global_settings); + else if (state.io.pipeline.initialized()) + processOrdinaryQueryWithProcessors(query_context->getSettingsRef().max_threads); else processOrdinaryQuery(); @@ -378,7 +384,10 @@ void TCPHandler::processInsertQuery(const Settings & global_settings) { const auto & db_and_table = query_context->getInsertionTable(); if (query_context->getSettingsRef().input_format_defaults_for_omitted_fields) - sendTableColumns(query_context->getTable(db_and_table.first, db_and_table.second)->getColumns()); + { + if (!db_and_table.second.empty()) + sendTableColumns(query_context->getTable(db_and_table.first, db_and_table.second)->getColumns()); + } } /// Send block to the client - table structure. @@ -447,9 +456,9 @@ void TCPHandler::processOrdinaryQuery() */ if (!block && !isQueryCancelled()) { - sendTotals(); - sendExtremes(); - sendProfileInfo(); + sendTotals(state.io.in->getTotals()); + sendExtremes(state.io.in->getExtremes()); + sendProfileInfo(state.io.in->getProfileInfo()); sendProgress(); sendLogs(); } @@ -465,6 +474,129 @@ void TCPHandler::processOrdinaryQuery() state.io.onFinish(); } +void TCPHandler::processOrdinaryQueryWithProcessors(size_t num_threads) +{ + auto & pipeline = state.io.pipeline; + + /// Send header-block, to allow client to prepare output format for data to send. + { + auto & header = pipeline.getHeader(); + + if (header) + sendData(header); + } + + auto lazy_format = std::make_shared(pipeline.getHeader()); + pipeline.setOutput(lazy_format); + + { + auto thread_group = CurrentThread::getGroup(); + ThreadPool pool(1); + auto executor = pipeline.execute(); + std::atomic_bool exception = false; + + pool.schedule([&]() + { + /// ThreadStatus thread_status; + + if (thread_group) + CurrentThread::attachTo(thread_group); + + SCOPE_EXIT( + if (thread_group) + CurrentThread::detachQueryIfNotDetached(); + ); + + CurrentMetrics::Increment query_thread_metric_increment{CurrentMetrics::QueryThread}; + setThreadName("QueryPipelineEx"); + + try + { + executor->execute(num_threads); + } + catch (...) + { + exception = true; + throw; + } + }); + + /// Wait in case of exception. Delete pipeline to release memory. + SCOPE_EXIT( + /// Clear queue in case if somebody is waiting lazy_format to push. + lazy_format->finish(); + lazy_format->clearQueue(); + + pool.wait(); + pipeline = QueryPipeline() + ); + + while (true) + { + Block block; + + while (true) + { + if (isQueryCancelled()) + { + /// A packet was received requesting to stop execution of the request. + executor->cancel(); + + break; + } + else + { + if (after_send_progress.elapsed() / 1000 >= query_context->getSettingsRef().interactive_delay) + { + /// Some time passed and there is a progress. + after_send_progress.restart(); + sendProgress(); + } + + sendLogs(); + + if ((block = lazy_format->getBlock(query_context->getSettingsRef().interactive_delay / 1000))) + break; + + if (lazy_format->isFinished()) + break; + + if (exception) + { + pool.wait(); + break; + } + } + } + + /** If data has run out, we will send the profiling data and total values to + * the last zero block to be able to use + * this information in the suffix output of stream. + * If the request was interrupted, then `sendTotals` and other methods could not be called, + * because we have not read all the data yet, + * and there could be ongoing calculations in other threads at the same time. + */ + if (!block && !isQueryCancelled()) + { + pool.wait(); + pipeline.finalize(); + + sendTotals(lazy_format->getTotals()); + sendExtremes(lazy_format->getExtremes()); + sendProfileInfo(lazy_format->getProfileInfo()); + sendProgress(); + sendLogs(); + } + + sendData(block); + if (!block) + break; + } + } + + state.io.onFinish(); +} + void TCPHandler::processTablesStatusRequest() { @@ -495,18 +627,16 @@ void TCPHandler::processTablesStatusRequest() } -void TCPHandler::sendProfileInfo() +void TCPHandler::sendProfileInfo(const BlockStreamProfileInfo & info) { writeVarUInt(Protocol::Server::ProfileInfo, *out); - state.io.in->getProfileInfo().write(*out); + info.write(*out); out->next(); } -void TCPHandler::sendTotals() +void TCPHandler::sendTotals(const Block & totals) { - const Block & totals = state.io.in->getTotals(); - if (totals) { initBlockOutput(totals); @@ -521,10 +651,8 @@ void TCPHandler::sendTotals() } -void TCPHandler::sendExtremes() +void TCPHandler::sendExtremes(const Block & extremes) { - Block extremes = state.io.in->getExtremes(); - if (extremes) { initBlockOutput(extremes); @@ -730,7 +858,7 @@ bool TCPHandler::receiveData() if (!(storage = query_context->tryGetExternalTable(external_table_name))) { NamesAndTypesList columns = block.getNamesAndTypesList(); - storage = StorageMemory::create(external_table_name, ColumnsDescription{columns}); + storage = StorageMemory::create("_external", external_table_name, ColumnsDescription{columns}); storage->startup(); query_context->addExternalTable(external_table_name, storage); } diff --git a/dbms/programs/server/TCPHandler.h b/dbms/programs/server/TCPHandler.h index 29b38765b0f..3cacd5fae95 100644 --- a/dbms/programs/server/TCPHandler.h +++ b/dbms/programs/server/TCPHandler.h @@ -145,6 +145,8 @@ private: /// Process a request that does not require the receiving of data blocks from the client void processOrdinaryQuery(); + void processOrdinaryQueryWithProcessors(size_t num_threads); + void processTablesStatusRequest(); void sendHello(); @@ -155,9 +157,9 @@ private: void sendProgress(); void sendLogs(); void sendEndOfStream(); - void sendProfileInfo(); - void sendTotals(); - void sendExtremes(); + void sendProfileInfo(const BlockStreamProfileInfo & info); + void sendTotals(const Block & totals); + void sendExtremes(const Block & extremes); /// Creates state.block_in/block_out for blocks read/write, depending on whether compression is enabled. void initBlockInput(); diff --git a/dbms/src/AggregateFunctions/AggregateFunctionMLMethod.cpp b/dbms/src/AggregateFunctions/AggregateFunctionMLMethod.cpp index af6971e3dae..9d82e6930ee 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionMLMethod.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionMLMethod.cpp @@ -45,11 +45,11 @@ namespace /// Such default parameters were picked because they did good on some tests, /// though it still requires to fit parameters to achieve better result - auto learning_rate = Float64(0.01); - auto l2_reg_coef = Float64(0.1); - UInt32 batch_size = 15; + auto learning_rate = Float64(1.0); + auto l2_reg_coef = Float64(0.5); + UInt64 batch_size = 15; - std::string weights_updater_name = "SGD"; + std::string weights_updater_name = "Adam"; std::unique_ptr gradient_computer; if (!parameters.empty()) @@ -62,12 +62,12 @@ namespace } if (parameters.size() > 2) { - batch_size = applyVisitor(FieldVisitorConvertToNumber(), parameters[2]); + batch_size = applyVisitor(FieldVisitorConvertToNumber(), parameters[2]); } if (parameters.size() > 3) { weights_updater_name = parameters[3].safeGet(); - if (weights_updater_name != "SGD" && weights_updater_name != "Momentum" && weights_updater_name != "Nesterov") + if (weights_updater_name != "SGD" && weights_updater_name != "Momentum" && weights_updater_name != "Nesterov" && weights_updater_name != "Adam") throw Exception("Invalid parameter for weights updater. The only supported are 'SGD', 'Momentum' and 'Nesterov'", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } @@ -106,8 +106,8 @@ void registerAggregateFunctionMLMethod(AggregateFunctionFactory & factory) LinearModelData::LinearModelData( Float64 learning_rate, Float64 l2_reg_coef, - UInt32 param_num, - UInt32 batch_capacity, + UInt64 param_num, + UInt64 batch_capacity, std::shared_ptr gradient_computer, std::shared_ptr weights_updater) : learning_rate(learning_rate) @@ -126,7 +126,7 @@ void LinearModelData::update_state() if (batch_size == 0) return; - weights_updater->update(batch_size, weights, bias, gradient_batch); + weights_updater->update(batch_size, weights, bias, learning_rate, gradient_batch); batch_size = 0; ++iter_num; gradient_batch.assign(gradient_batch.size(), Float64{0.0}); @@ -191,6 +191,7 @@ void LinearModelData::merge(const DB::LinearModelData & rhs) update_state(); /// can't update rhs state because it's constant + /// squared mean is more stable (in sence of quality of prediction) when two states with quietly different number of learning steps are merged Float64 frac = (static_cast(iter_num) * iter_num) / (iter_num * iter_num + rhs.iter_num * rhs.iter_num); for (size_t i = 0; i < weights.size(); ++i) @@ -210,7 +211,7 @@ void LinearModelData::add(const IColumn ** columns, size_t row_num) /// Here we have columns + 1 as first column corresponds to target value, and others - to features weights_updater->add_to_batch( - gradient_batch, *gradient_computer, weights, bias, learning_rate, l2_reg_coef, target, columns + 1, row_num); + gradient_batch, *gradient_computer, weights, bias, l2_reg_coef, target, columns + 1, row_num); ++batch_size; if (batch_size == batch_capacity) @@ -219,6 +220,90 @@ void LinearModelData::add(const IColumn ** columns, size_t row_num) } } +/// Weights updaters + +void Adam::write(WriteBuffer & buf) const +{ + writeBinary(average_gradient, buf); + writeBinary(average_squared_gradient, buf); +} + +void Adam::read(ReadBuffer & buf) +{ + readBinary(average_gradient, buf); + readBinary(average_squared_gradient, buf); +} + +void Adam::merge(const IWeightsUpdater & rhs, Float64 frac, Float64 rhs_frac) +{ + auto & adam_rhs = static_cast(rhs); + if (average_gradient.empty()) + { + if (!average_squared_gradient.empty() || + adam_rhs.average_gradient.size() != adam_rhs.average_squared_gradient.size()) + throw Exception("Average_gradient and average_squared_gradient must have same size", ErrorCodes::LOGICAL_ERROR); + + average_gradient.resize(adam_rhs.average_gradient.size(), Float64{0.0}); + average_squared_gradient.resize(adam_rhs.average_squared_gradient.size(), Float64{0.0}); + } + + for (size_t i = 0; i < average_gradient.size(); ++i) + { + average_gradient[i] = average_gradient[i] * frac + adam_rhs.average_gradient[i] * rhs_frac; + average_squared_gradient[i] = average_squared_gradient[i] * frac + adam_rhs.average_squared_gradient[i] * rhs_frac; + } + beta1_powered_ *= adam_rhs.beta1_powered_; + beta2_powered_ *= adam_rhs.beta2_powered_; +} + +void Adam::update(UInt64 batch_size, std::vector & weights, Float64 & bias, Float64 learning_rate, const std::vector & batch_gradient) +{ + if (average_gradient.empty()) + { + if (!average_squared_gradient.empty()) + throw Exception("Average_gradient and average_squared_gradient must have same size", ErrorCodes::LOGICAL_ERROR); + + average_gradient.resize(batch_gradient.size(), Float64{0.0}); + average_squared_gradient.resize(batch_gradient.size(), Float64{0.0}); + } + + for (size_t i = 0; i != average_gradient.size(); ++i) + { + Float64 normed_gradient = batch_gradient[i] / batch_size; + average_gradient[i] = beta1_ * average_gradient[i] + (1 - beta1_) * normed_gradient; + average_squared_gradient[i] = beta2_ * average_squared_gradient[i] + + (1 - beta2_) * normed_gradient * normed_gradient; + } + + for (size_t i = 0; i < weights.size(); ++i) + { + weights[i] += (learning_rate * average_gradient[i]) / + ((1 - beta1_powered_) * (sqrt(average_squared_gradient[i] / (1 - beta2_powered_)) + eps_)); + } + bias += (learning_rate * average_gradient[weights.size()]) / + ((1 - beta1_powered_) * (sqrt(average_squared_gradient[weights.size()] / (1 - beta2_powered_)) + eps_)); + + beta1_powered_ *= beta1_; + beta2_powered_ *= beta2_; +} + +void Adam::add_to_batch( + std::vector & batch_gradient, + IGradientComputer & gradient_computer, + const std::vector & weights, + Float64 bias, + Float64 l2_reg_coef, + Float64 target, + const IColumn ** columns, + size_t row_num) +{ + if (average_gradient.empty()) + { + average_gradient.resize(batch_gradient.size(), Float64{0.0}); + average_squared_gradient.resize(batch_gradient.size(), Float64{0.0}); + } + gradient_computer.compute(batch_gradient, weights, bias, l2_reg_coef, target, columns, row_num); +} void Nesterov::read(ReadBuffer & buf) { @@ -233,13 +318,16 @@ void Nesterov::write(WriteBuffer & buf) const void Nesterov::merge(const IWeightsUpdater & rhs, Float64 frac, Float64 rhs_frac) { auto & nesterov_rhs = static_cast(rhs); + if (accumulated_gradient.empty()) + accumulated_gradient.resize(nesterov_rhs.accumulated_gradient.size(), Float64{0.0}); + for (size_t i = 0; i < accumulated_gradient.size(); ++i) { accumulated_gradient[i] = accumulated_gradient[i] * frac + nesterov_rhs.accumulated_gradient[i] * rhs_frac; } } -void Nesterov::update(UInt32 batch_size, std::vector & weights, Float64 & bias, const std::vector & batch_gradient) +void Nesterov::update(UInt64 batch_size, std::vector & weights, Float64 & bias, Float64 learning_rate, const std::vector & batch_gradient) { if (accumulated_gradient.empty()) { @@ -248,7 +336,7 @@ void Nesterov::update(UInt32 batch_size, std::vector & weights, Float64 for (size_t i = 0; i < batch_gradient.size(); ++i) { - accumulated_gradient[i] = accumulated_gradient[i] * alpha_ + batch_gradient[i] / batch_size; + accumulated_gradient[i] = accumulated_gradient[i] * alpha_ + (learning_rate * batch_gradient[i]) / batch_size; } for (size_t i = 0; i < weights.size(); ++i) { @@ -262,7 +350,6 @@ void Nesterov::add_to_batch( IGradientComputer & gradient_computer, const std::vector & weights, Float64 bias, - Float64 learning_rate, Float64 l2_reg_coef, Float64 target, const IColumn ** columns, @@ -280,7 +367,7 @@ void Nesterov::add_to_batch( } auto shifted_bias = bias + accumulated_gradient[weights.size()] * alpha_; - gradient_computer.compute(batch_gradient, shifted_weights, shifted_bias, learning_rate, l2_reg_coef, target, columns, row_num); + gradient_computer.compute(batch_gradient, shifted_weights, shifted_bias, l2_reg_coef, target, columns, row_num); } void Momentum::read(ReadBuffer & buf) @@ -302,7 +389,7 @@ void Momentum::merge(const IWeightsUpdater & rhs, Float64 frac, Float64 rhs_frac } } -void Momentum::update(UInt32 batch_size, std::vector & weights, Float64 & bias, const std::vector & batch_gradient) +void Momentum::update(UInt64 batch_size, std::vector & weights, Float64 & bias, Float64 learning_rate, const std::vector & batch_gradient) { /// batch_size is already checked to be greater than 0 if (accumulated_gradient.empty()) @@ -312,7 +399,7 @@ void Momentum::update(UInt32 batch_size, std::vector & weights, Float64 for (size_t i = 0; i < batch_gradient.size(); ++i) { - accumulated_gradient[i] = accumulated_gradient[i] * alpha_ + batch_gradient[i] / batch_size; + accumulated_gradient[i] = accumulated_gradient[i] * alpha_ + (learning_rate * batch_gradient[i]) / batch_size; } for (size_t i = 0; i < weights.size(); ++i) { @@ -322,14 +409,14 @@ void Momentum::update(UInt32 batch_size, std::vector & weights, Float64 } void StochasticGradientDescent::update( - UInt32 batch_size, std::vector & weights, Float64 & bias, const std::vector & batch_gradient) + UInt64 batch_size, std::vector & weights, Float64 & bias, Float64 learning_rate, const std::vector & batch_gradient) { /// batch_size is already checked to be greater than 0 for (size_t i = 0; i < weights.size(); ++i) { - weights[i] += batch_gradient[i] / batch_size; + weights[i] += (learning_rate * batch_gradient[i]) / batch_size; } - bias += batch_gradient[weights.size()] / batch_size; + bias += (learning_rate * batch_gradient[weights.size()]) / batch_size; } void IWeightsUpdater::add_to_batch( @@ -337,15 +424,16 @@ void IWeightsUpdater::add_to_batch( IGradientComputer & gradient_computer, const std::vector & weights, Float64 bias, - Float64 learning_rate, Float64 l2_reg_coef, Float64 target, const IColumn ** columns, size_t row_num) { - gradient_computer.compute(batch_gradient, weights, bias, learning_rate, l2_reg_coef, target, columns, row_num); + gradient_computer.compute(batch_gradient, weights, bias, l2_reg_coef, target, columns, row_num); } +/// Gradient computers + void LogisticRegression::predict( ColumnVector::Container & container, Block & block, @@ -387,7 +475,6 @@ void LogisticRegression::compute( std::vector & batch_gradient, const std::vector & weights, Float64 bias, - Float64 learning_rate, Float64 l2_reg_coef, Float64 target, const IColumn ** columns, @@ -402,11 +489,11 @@ void LogisticRegression::compute( derivative *= target; derivative = exp(derivative); - batch_gradient[weights.size()] += learning_rate * target / (derivative + 1); + batch_gradient[weights.size()] += target / (derivative + 1); for (size_t i = 0; i < weights.size(); ++i) { auto value = (*columns[i]).getFloat64(row_num); - batch_gradient[i] += learning_rate * target * value / (derivative + 1) - 2 * learning_rate * l2_reg_coef * weights[i]; + batch_gradient[i] += target * value / (derivative + 1) - 2 * l2_reg_coef * weights[i]; } } @@ -459,7 +546,6 @@ void LinearRegression::compute( std::vector & batch_gradient, const std::vector & weights, Float64 bias, - Float64 learning_rate, Float64 l2_reg_coef, Float64 target, const IColumn ** columns, @@ -471,13 +557,13 @@ void LinearRegression::compute( auto value = (*columns[i]).getFloat64(row_num); derivative -= weights[i] * value; } - derivative *= (2 * learning_rate); + derivative *= 2; batch_gradient[weights.size()] += derivative; for (size_t i = 0; i < weights.size(); ++i) { auto value = (*columns[i]).getFloat64(row_num); - batch_gradient[i] += derivative * value - 2 * learning_rate * l2_reg_coef * weights[i]; + batch_gradient[i] += derivative * value - 2 * l2_reg_coef * weights[i]; } } diff --git a/dbms/src/AggregateFunctions/AggregateFunctionMLMethod.h b/dbms/src/AggregateFunctions/AggregateFunctionMLMethod.h index 90048924173..95ac64c21d8 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionMLMethod.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionMLMethod.h @@ -33,7 +33,6 @@ public: std::vector & batch_gradient, const std::vector & weights, Float64 bias, - Float64 learning_rate, Float64 l2_reg_coef, Float64 target, const IColumn ** columns, @@ -60,7 +59,6 @@ public: std::vector & batch_gradient, const std::vector & weights, Float64 bias, - Float64 learning_rate, Float64 l2_reg_coef, Float64 target, const IColumn ** columns, @@ -87,7 +85,6 @@ public: std::vector & batch_gradient, const std::vector & weights, Float64 bias, - Float64 learning_rate, Float64 l2_reg_coef, Float64 target, const IColumn ** columns, @@ -120,14 +117,18 @@ public: IGradientComputer & gradient_computer, const std::vector & weights, Float64 bias, - Float64 learning_rate, Float64 l2_reg_coef, Float64 target, const IColumn ** columns, size_t row_num); /// Updates current weights according to the gradient from the last mini-batch - virtual void update(UInt32 batch_size, std::vector & weights, Float64 & bias, const std::vector & gradient) = 0; + virtual void update( + UInt64 batch_size, + std::vector & weights, + Float64 & bias, + Float64 learning_rate, + const std::vector & gradient) = 0; /// Used during the merge of two states virtual void merge(const IWeightsUpdater &, Float64, Float64) {} @@ -143,7 +144,7 @@ public: class StochasticGradientDescent : public IWeightsUpdater { public: - void update(UInt32 batch_size, std::vector & weights, Float64 & bias, const std::vector & batch_gradient) override; + void update(UInt64 batch_size, std::vector & weights, Float64 & bias, Float64 learning_rate, const std::vector & batch_gradient) override; }; @@ -154,7 +155,7 @@ public: Momentum(Float64 alpha) : alpha_(alpha) {} - void update(UInt32 batch_size, std::vector & weights, Float64 & bias, const std::vector & batch_gradient) override; + void update(UInt64 batch_size, std::vector & weights, Float64 & bias, Float64 learning_rate, const std::vector & batch_gradient) override; virtual void merge(const IWeightsUpdater & rhs, Float64 frac, Float64 rhs_frac) override; @@ -180,13 +181,12 @@ public: IGradientComputer & gradient_computer, const std::vector & weights, Float64 bias, - Float64 learning_rate, Float64 l2_reg_coef, Float64 target, const IColumn ** columns, size_t row_num) override; - void update(UInt32 batch_size, std::vector & weights, Float64 & bias, const std::vector & batch_gradient) override; + void update(UInt64 batch_size, std::vector & weights, Float64 & bias, Float64 learning_rate, const std::vector & batch_gradient) override; virtual void merge(const IWeightsUpdater & rhs, Float64 frac, Float64 rhs_frac) override; @@ -195,11 +195,51 @@ public: void read(ReadBuffer & buf) override; private: - Float64 alpha_{0.1}; + const Float64 alpha_ = 0.9; std::vector accumulated_gradient; }; +class Adam : public IWeightsUpdater +{ +public: + Adam() + { + beta1_powered_ = beta1_; + beta2_powered_ = beta2_; + } + + void add_to_batch( + std::vector & batch_gradient, + IGradientComputer & gradient_computer, + const std::vector & weights, + Float64 bias, + Float64 l2_reg_coef, + Float64 target, + const IColumn ** columns, + size_t row_num) override; + + void update(UInt64 batch_size, std::vector & weights, Float64 & bias, Float64 learning_rate, const std::vector & batch_gradient) override; + + virtual void merge(const IWeightsUpdater & rhs, Float64 frac, Float64 rhs_frac) override; + + void write(WriteBuffer & buf) const override; + + void read(ReadBuffer & buf) override; + +private: + /// beta1 and beta2 hyperparameters have such recommended values + const Float64 beta1_ = 0.9; + const Float64 beta2_ = 0.999; + const Float64 eps_ = 0.000001; + Float64 beta1_powered_; + Float64 beta2_powered_; + + std::vector average_gradient; + std::vector average_squared_gradient; +}; + + /** LinearModelData is a class which manages current state of learning */ class LinearModelData @@ -210,8 +250,8 @@ public: LinearModelData( Float64 learning_rate, Float64 l2_reg_coef, - UInt32 param_num, - UInt32 batch_capacity, + UInt64 param_num, + UInt64 batch_capacity, std::shared_ptr gradient_computer, std::shared_ptr weights_updater); @@ -269,7 +309,7 @@ public: std::string weights_updater_name, Float64 learning_rate, Float64 l2_reg_coef, - UInt32 batch_size, + UInt64 batch_size, const DataTypes & arguments_types, const Array & params) : IAggregateFunctionDataHelper>(arguments_types, params) @@ -303,6 +343,8 @@ public: new_weights_updater = std::make_shared(); else if (weights_updater_name == "Nesterov") new_weights_updater = std::make_shared(); + else if (weights_updater_name == "Adam") + new_weights_updater = std::make_shared(); else throw Exception("Illegal name of weights updater (should have been checked earlier)", ErrorCodes::LOGICAL_ERROR); @@ -355,10 +397,10 @@ public: const char * getHeaderFilePath() const override { return __FILE__; } private: - UInt32 param_num; + UInt64 param_num; Float64 learning_rate; Float64 l2_reg_coef; - UInt32 batch_size; + UInt64 batch_size; std::shared_ptr gradient_computer; std::string weights_updater_name; }; @@ -371,4 +413,5 @@ struct NameLogisticRegression { static constexpr auto name = "stochasticLogisticRegression"; }; + } diff --git a/dbms/src/AggregateFunctions/AggregateFunctionQuantile.h b/dbms/src/AggregateFunctions/AggregateFunctionQuantile.h index 0748955f9ef..2e9ec914b99 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionQuantile.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionQuantile.h @@ -82,14 +82,6 @@ public: { if (!returns_many && levels.size() > 1) throw Exception("Aggregate function " + getName() + " require one parameter or less", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - - if constexpr (std::is_same_v>) - { - /// QuantileTiming only supports integers (it works only for unsigned integers but signed are also accepted for convenience). - if (!isInteger(argument_type)) - throw Exception("Argument for function " + std::string(Name::name) + " must be integer, but it has type " - + argument_type->getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } } String getName() const override { return Name::name; } @@ -111,16 +103,21 @@ public: void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override { - /// Out of range conversion may occur. This is Ok. + auto value = static_cast(*columns[0]).getData()[row_num]; - const auto & column = static_cast(*columns[0]); + if constexpr (std::is_same_v>) + { + /// QuantileTiming only supports integers. + if (isNaN(value) || value > std::numeric_limits::max() || value < std::numeric_limits::min()) + return; + } if constexpr (has_second_arg) this->data(place).add( - column.getData()[row_num], + value, columns[1]->getUInt(row_num)); else - this->data(place).add(column.getData()[row_num]); + this->data(place).add(value); } void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override diff --git a/dbms/src/AggregateFunctions/QuantileTiming.h b/dbms/src/AggregateFunctions/QuantileTiming.h index 131ca91dbbf..fbf4da725c0 100644 --- a/dbms/src/AggregateFunctions/QuantileTiming.h +++ b/dbms/src/AggregateFunctions/QuantileTiming.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include #include @@ -513,8 +512,6 @@ private: void mediumToLarge() { - CurrentMemoryTracker::alloc(sizeof(detail::QuantileTimingLarge)); - /// While the data is copied from medium, it is not possible to set `large` value (otherwise it will overwrite some data). detail::QuantileTimingLarge * tmp_large = new detail::QuantileTimingLarge; @@ -528,8 +525,6 @@ private: void tinyToLarge() { - CurrentMemoryTracker::alloc(sizeof(detail::QuantileTimingLarge)); - /// While the data is copied from `medium` it is not possible to set `large` value (otherwise it will overwrite some data). detail::QuantileTimingLarge * tmp_large = new detail::QuantileTimingLarge; @@ -562,8 +557,6 @@ public: else if (kind == Kind::Large) { delete large; - - CurrentMemoryTracker::free(sizeof(detail::QuantileTimingLarge)); } } diff --git a/dbms/src/AggregateFunctions/UniquesHashSet.h b/dbms/src/AggregateFunctions/UniquesHashSet.h index bc9a65c1bb6..d1df7b0df0d 100644 --- a/dbms/src/AggregateFunctions/UniquesHashSet.h +++ b/dbms/src/AggregateFunctions/UniquesHashSet.h @@ -126,20 +126,32 @@ private: { for (size_t i = 0; i < buf_size(); ++i) { - if (buf[i] && !good(buf[i])) + if (buf[i]) { - buf[i] = 0; - --m_size; + if (!good(buf[i])) + { + buf[i] = 0; + --m_size; + } + /** After removing the elements, there may have been room for items, + * which were placed further than necessary, due to a collision. + * You need to move them. + */ + else if (i != place(buf[i])) + { + HashValue x = buf[i]; + buf[i] = 0; + reinsertImpl(x); + } } } - /** After removing the elements, there may have been room for items, - * which were placed further than necessary, due to a collision. - * You need to move them. + /** We must process first collision resolution chain once again. + * Look at the comment in "resize" function. */ - for (size_t i = 0; i < buf_size(); ++i) + for (size_t i = 0; i < buf_size() && buf[i]; ++i) { - if (unlikely(buf[i] && i != place(buf[i]))) + if (i != place(buf[i])) { HashValue x = buf[i]; buf[i] = 0; diff --git a/dbms/src/CMakeLists.txt b/dbms/src/CMakeLists.txt index c5b661b5912..84755f7f280 100644 --- a/dbms/src/CMakeLists.txt +++ b/dbms/src/CMakeLists.txt @@ -12,5 +12,6 @@ add_subdirectory (Interpreters) add_subdirectory (AggregateFunctions) add_subdirectory (Client) add_subdirectory (TableFunctions) +add_subdirectory (Processors) add_subdirectory (Formats) add_subdirectory (Compression) diff --git a/dbms/src/Client/Connection.cpp b/dbms/src/Client/Connection.cpp index 9651ef54e1b..9cdda9fdf0d 100644 --- a/dbms/src/Client/Connection.cpp +++ b/dbms/src/Client/Connection.cpp @@ -73,7 +73,7 @@ void Connection::connect(const ConnectionTimeouts & timeouts) current_resolved_address = DNSResolver::instance().resolveAddress(host, port); - socket->connect(current_resolved_address, timeouts.connection_timeout); + socket->connect(*current_resolved_address, timeouts.connection_timeout); socket->setReceiveTimeout(timeouts.receive_timeout); socket->setSendTimeout(timeouts.send_timeout); socket->setNoDelay(true); @@ -533,12 +533,9 @@ void Connection::sendExternalTablesData(ExternalTablesData & data) LOG_DEBUG(log_wrapper.get(), msg.rdbuf()); } -Poco::Net::SocketAddress Connection::getResolvedAddress() const +std::optional Connection::getResolvedAddress() const { - if (connected) - return current_resolved_address; - - return DNSResolver::instance().resolveAddress(host, port); + return current_resolved_address; } @@ -595,7 +592,9 @@ Connection::Packet Connection::receivePacket() switch (res.type) { - case Protocol::Server::Data: + case Protocol::Server::Data: [[fallthrough]]; + case Protocol::Server::Totals: [[fallthrough]]; + case Protocol::Server::Extremes: res.block = receiveData(); return res; @@ -611,16 +610,6 @@ Connection::Packet Connection::receivePacket() res.profile_info = receiveProfileInfo(); return res; - case Protocol::Server::Totals: - /// Block with total values is passed in same form as ordinary block. The only difference is packed id. - res.block = receiveData(); - return res; - - case Protocol::Server::Extremes: - /// Same as above. - res.block = receiveData(); - return res; - case Protocol::Server::Log: res.block = receiveLogData(); return res; @@ -720,11 +709,14 @@ void Connection::initBlockLogsInput() void Connection::setDescription() { auto resolved_address = getResolvedAddress(); - description = host + ":" + toString(resolved_address.port()); - auto ip_address = resolved_address.host().toString(); + description = host + ":" + toString(port); - if (host != ip_address) - description += ", " + ip_address; + if (resolved_address) + { + auto ip_address = resolved_address->host().toString(); + if (host != ip_address) + description += ", " + ip_address; + } } diff --git a/dbms/src/Client/Connection.h b/dbms/src/Client/Connection.h index 2338e4c8965..03a771c257f 100644 --- a/dbms/src/Client/Connection.h +++ b/dbms/src/Client/Connection.h @@ -63,7 +63,7 @@ public: Poco::Timespan sync_request_timeout_ = Poco::Timespan(DBMS_DEFAULT_SYNC_REQUEST_TIMEOUT_SEC, 0)) : host(host_), port(port_), default_database(default_database_), - user(user_), password(password_), current_resolved_address(host, port), + user(user_), password(password_), client_name(client_name_), compression(compression_), secure(secure_), @@ -168,9 +168,6 @@ public: size_t outBytesCount() const { return out ? out->count() : 0; } size_t inBytesCount() const { return in ? in->count() : 0; } - /// Returns initially resolved address - Poco::Net::SocketAddress getResolvedAddress() const; - private: String host; UInt16 port; @@ -180,12 +177,15 @@ private: /// Address is resolved during the first connection (or the following reconnects) /// Use it only for logging purposes - Poco::Net::SocketAddress current_resolved_address; + std::optional current_resolved_address; /// For messages in log and in exceptions. String description; void setDescription(); + /// Returns resolved address if it was resolved. + std::optional getResolvedAddress() const; + String client_name; bool connected = false; diff --git a/dbms/src/Columns/Collator.cpp b/dbms/src/Columns/Collator.cpp index 3232893ee15..7e8cfba1aac 100644 --- a/dbms/src/Columns/Collator.cpp +++ b/dbms/src/Columns/Collator.cpp @@ -6,8 +6,8 @@ #include #else #ifdef __clang__ - #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wunused-private-field" + #pragma clang diagnostic ignored "-Wmissing-noreturn" #endif #endif @@ -26,7 +26,6 @@ namespace DB } } - Collator::Collator(const std::string & locale_) : locale(Poco::toLower(locale_)) { #if USE_ICU diff --git a/dbms/src/Columns/ColumnArray.cpp b/dbms/src/Columns/ColumnArray.cpp index b4382755ba3..b825342ee5f 100644 --- a/dbms/src/Columns/ColumnArray.cpp +++ b/dbms/src/Columns/ColumnArray.cpp @@ -178,11 +178,16 @@ StringRef ColumnArray::serializeValueIntoArena(size_t n, Arena & arena, char con char * pos = arena.allocContinue(sizeof(array_size), begin); memcpy(pos, &array_size, sizeof(array_size)); - size_t values_size = 0; - for (size_t i = 0; i < array_size; ++i) - values_size += getData().serializeValueIntoArena(offset + i, arena, begin).size; + StringRef res(pos, sizeof(array_size)); - return StringRef(begin, sizeof(array_size) + values_size); + for (size_t i = 0; i < array_size; ++i) + { + auto value_ref = getData().serializeValueIntoArena(offset + i, arena, begin); + res.data = value_ref.data - res.size; + res.size += value_ref.size; + } + + return res; } diff --git a/dbms/src/Columns/ColumnNullable.cpp b/dbms/src/Columns/ColumnNullable.cpp index 3ae692552e8..c5cdffba84d 100644 --- a/dbms/src/Columns/ColumnNullable.cpp +++ b/dbms/src/Columns/ColumnNullable.cpp @@ -103,12 +103,13 @@ StringRef ColumnNullable::serializeValueIntoArena(size_t n, Arena & arena, char auto pos = arena.allocContinue(s, begin); memcpy(pos, &arr[n], s); - size_t nested_size = 0; + if (arr[n]) + return StringRef(pos, s); - if (arr[n] == 0) - nested_size = getNestedColumn().serializeValueIntoArena(n, arena, begin).size; + auto nested_ref = getNestedColumn().serializeValueIntoArena(n, arena, begin); - return StringRef{begin, s + nested_size}; + /// serializeValueIntoArena may reallocate memory. Have to use ptr from nested_ref.data and move it back. + return StringRef(nested_ref.data - s, nested_ref.size + s); } const char * ColumnNullable::deserializeAndInsertFromArena(const char * pos) diff --git a/dbms/src/Columns/ColumnTuple.cpp b/dbms/src/Columns/ColumnTuple.cpp index f7a95c1ac9f..3ad7f007edf 100644 --- a/dbms/src/Columns/ColumnTuple.cpp +++ b/dbms/src/Columns/ColumnTuple.cpp @@ -142,11 +142,15 @@ void ColumnTuple::popBack(size_t n) StringRef ColumnTuple::serializeValueIntoArena(size_t n, Arena & arena, char const *& begin) const { - size_t values_size = 0; + StringRef res(begin, 0); for (auto & column : columns) - values_size += column->serializeValueIntoArena(n, arena, begin).size; + { + auto value_ref = column->serializeValueIntoArena(n, arena, begin); + res.data = value_ref.data - res.size; + res.size += value_ref.size; + } - return StringRef(begin, values_size); + return res; } const char * ColumnTuple::deserializeAndInsertFromArena(const char * pos) diff --git a/dbms/src/Columns/ColumnUnique.h b/dbms/src/Columns/ColumnUnique.h index febdeaafa95..0c5efd8058d 100644 --- a/dbms/src/Columns/ColumnUnique.h +++ b/dbms/src/Columns/ColumnUnique.h @@ -300,19 +300,19 @@ StringRef ColumnUnique::serializeValueIntoArena(size_t n, Arena & ar { if (is_nullable) { - const UInt8 null_flag = 1; - const UInt8 not_null_flag = 0; + static constexpr auto s = sizeof(UInt8); - auto pos = arena.allocContinue(sizeof(null_flag), begin); - auto & flag = (n == getNullValueIndex() ? null_flag : not_null_flag); - memcpy(pos, &flag, sizeof(flag)); + auto pos = arena.allocContinue(s, begin); + UInt8 flag = (n == getNullValueIndex() ? 1 : 0); + unalignedStore(pos, flag); - size_t nested_size = 0; + if (n == getNullValueIndex()) + return StringRef(pos, s); - if (n != getNullValueIndex()) - nested_size = column_holder->serializeValueIntoArena(n, arena, begin).size; + auto nested_ref = column_holder->serializeValueIntoArena(n, arena, begin); - return StringRef(pos, sizeof(null_flag) + nested_size); + /// serializeValueIntoArena may reallocate memory. Have to use ptr from nested_ref.data and move it back. + return StringRef(nested_ref.data - s, nested_ref.size + s); } return column_holder->serializeValueIntoArena(n, arena, begin); diff --git a/dbms/src/Columns/tests/CMakeLists.txt b/dbms/src/Columns/tests/CMakeLists.txt index 302c554a1fd..e69de29bb2d 100644 --- a/dbms/src/Columns/tests/CMakeLists.txt +++ b/dbms/src/Columns/tests/CMakeLists.txt @@ -1,4 +0,0 @@ -if(USE_GTEST) - add_executable(column_unique column_unique.cpp) - target_link_libraries(column_unique PRIVATE dbms ${GTEST_BOTH_LIBRARIES}) -endif() \ No newline at end of file diff --git a/dbms/src/Columns/tests/column_unique.cpp b/dbms/src/Columns/tests/gtest_column_unique.cpp similarity index 97% rename from dbms/src/Columns/tests/column_unique.cpp rename to dbms/src/Columns/tests/gtest_column_unique.cpp index 68b9367ee86..1d5ea5b4080 100644 --- a/dbms/src/Columns/tests/column_unique.cpp +++ b/dbms/src/Columns/tests/gtest_column_unique.cpp @@ -7,12 +7,6 @@ #include #include -#pragma GCC diagnostic ignored "-Wsign-compare" -#ifdef __clang__ -#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" -#pragma clang diagnostic ignored "-Wundef" -#endif - #include #include diff --git a/dbms/src/Common/Allocator.h b/dbms/src/Common/Allocator.h index abaa5927e3d..e9569673678 100644 --- a/dbms/src/Common/Allocator.h +++ b/dbms/src/Common/Allocator.h @@ -108,13 +108,92 @@ class AllocatorWithHint : Hint { protected: static constexpr bool clear_memory = clear_memory_; + static constexpr size_t small_memory_threshold = mmap_threshold; public: /// Allocate memory range. void * alloc(size_t size, size_t alignment = 0) { CurrentMemoryTracker::alloc(size); + return allocNoTrack(size, alignment); + } + /// Free memory range. + void free(void * buf, size_t size) + { + freeNoTrack(buf, size); + CurrentMemoryTracker::free(size); + } + + /** Enlarge memory range. + * Data from old range is moved to the beginning of new range. + * Address of memory range could change. + */ + void * realloc(void * buf, size_t old_size, size_t new_size, size_t alignment = 0) + { + if (old_size == new_size) + { + /// nothing to do. + /// BTW, it's not possible to change alignment while doing realloc. + } + else if (old_size < mmap_threshold && new_size < mmap_threshold && alignment <= MALLOC_MIN_ALIGNMENT) + { + /// Resize malloc'd memory region with no special alignment requirement. + CurrentMemoryTracker::realloc(old_size, new_size); + + void * new_buf = ::realloc(buf, new_size); + if (nullptr == new_buf) + DB::throwFromErrno("Allocator: Cannot realloc from " + formatReadableSizeWithBinarySuffix(old_size) + " to " + formatReadableSizeWithBinarySuffix(new_size) + ".", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); + + buf = new_buf; + if constexpr (clear_memory) + if (new_size > old_size) + memset(reinterpret_cast(buf) + old_size, 0, new_size - old_size); + } + else if (old_size >= mmap_threshold && new_size >= mmap_threshold) + { + /// Resize mmap'd memory region. + CurrentMemoryTracker::realloc(old_size, new_size); + + // On apple and freebsd self-implemented mremap used (common/mremap.h) + buf = clickhouse_mremap(buf, old_size, new_size, MREMAP_MAYMOVE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (MAP_FAILED == buf) + DB::throwFromErrno("Allocator: Cannot mremap memory chunk from " + formatReadableSizeWithBinarySuffix(old_size) + " to " + formatReadableSizeWithBinarySuffix(new_size) + ".", DB::ErrorCodes::CANNOT_MREMAP); + + /// No need for zero-fill, because mmap guarantees it. + } + else if (new_size < small_memory_threshold) + { + /// Small allocs that requires a copy. Assume there's enough memory in system. Call CurrentMemoryTracker once. + CurrentMemoryTracker::realloc(old_size, new_size); + + void * new_buf = allocNoTrack(new_size, alignment); + memcpy(new_buf, buf, std::min(old_size, new_size)); + freeNoTrack(buf, old_size); + buf = new_buf; + } + else + { + /// Big allocs that requires a copy. MemoryTracker is called inside 'alloc', 'free' methods. + + void * new_buf = alloc(new_size, alignment); + memcpy(new_buf, buf, std::min(old_size, new_size)); + free(buf, old_size); + buf = new_buf; + } + + return buf; + } + +protected: + static constexpr size_t getStackThreshold() + { + return 0; + } + +private: + void * allocNoTrack(size_t size, size_t alignment) + { void * buf; if (size >= mmap_threshold) @@ -149,15 +228,14 @@ public: if (0 != res) DB::throwFromErrno("Cannot allocate memory (posix_memalign) " + formatReadableSizeWithBinarySuffix(size) + ".", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY, res); - if (clear_memory) + if constexpr (clear_memory) memset(buf, 0, size); } } return buf; } - /// Free memory range. - void free(void * buf, size_t size) + void freeNoTrack(void * buf, size_t size) { if (size >= mmap_threshold) { @@ -168,63 +246,6 @@ public: { ::free(buf); } - - CurrentMemoryTracker::free(size); - } - - /** Enlarge memory range. - * Data from old range is moved to the beginning of new range. - * Address of memory range could change. - */ - void * realloc(void * buf, size_t old_size, size_t new_size, size_t alignment = 0) - { - if (old_size == new_size) - { - /// nothing to do. - /// BTW, it's not possible to change alignment while doing realloc. - } - else if (old_size < mmap_threshold && new_size < mmap_threshold && alignment <= MALLOC_MIN_ALIGNMENT) - { - /// Resize malloc'd memory region with no special alignment requirement. - CurrentMemoryTracker::realloc(old_size, new_size); - - void * new_buf = ::realloc(buf, new_size); - if (nullptr == new_buf) - DB::throwFromErrno("Allocator: Cannot realloc from " + formatReadableSizeWithBinarySuffix(old_size) + " to " + formatReadableSizeWithBinarySuffix(new_size) + ".", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); - - buf = new_buf; - if (clear_memory && new_size > old_size) - memset(reinterpret_cast(buf) + old_size, 0, new_size - old_size); - } - else if (old_size >= mmap_threshold && new_size >= mmap_threshold) - { - /// Resize mmap'd memory region. - CurrentMemoryTracker::realloc(old_size, new_size); - - // On apple and freebsd self-implemented mremap used (common/mremap.h) - buf = clickhouse_mremap(buf, old_size, new_size, MREMAP_MAYMOVE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (MAP_FAILED == buf) - DB::throwFromErrno("Allocator: Cannot mremap memory chunk from " + formatReadableSizeWithBinarySuffix(old_size) + " to " + formatReadableSizeWithBinarySuffix(new_size) + ".", DB::ErrorCodes::CANNOT_MREMAP); - - /// No need for zero-fill, because mmap guarantees it. - } - else - { - /// All other cases that requires a copy. MemoryTracker is called inside 'alloc', 'free' methods. - - void * new_buf = alloc(new_size, alignment); - memcpy(new_buf, buf, std::min(old_size, new_size)); - free(buf, old_size); - buf = new_buf; - } - - return buf; - } - -protected: - static constexpr size_t getStackThreshold() - { - return 0; } }; @@ -267,7 +288,7 @@ public: { if (size <= N) { - if (Base::clear_memory) + if constexpr (Base::clear_memory) memset(stack_memory, 0, N); return stack_memory; } diff --git a/dbms/src/Common/CombinedCardinalityEstimator.h b/dbms/src/Common/CombinedCardinalityEstimator.h index 824f0a8c018..e048e47cab5 100644 --- a/dbms/src/Common/CombinedCardinalityEstimator.h +++ b/dbms/src/Common/CombinedCardinalityEstimator.h @@ -3,7 +3,6 @@ #include #include #include -#include #include @@ -230,7 +229,6 @@ private: if (getContainerType() != details::ContainerType::SMALL) throw Poco::Exception("Internal error", ErrorCodes::LOGICAL_ERROR); - CurrentMemoryTracker::alloc(sizeof(Medium)); auto tmp_medium = std::make_unique(); for (const auto & x : small) @@ -247,7 +245,6 @@ private: if ((container_type != details::ContainerType::SMALL) && (container_type != details::ContainerType::MEDIUM)) throw Poco::Exception("Internal error", ErrorCodes::LOGICAL_ERROR); - CurrentMemoryTracker::alloc(sizeof(Large)); auto tmp_large = std::make_unique(); if (container_type == details::ContainerType::SMALL) @@ -277,15 +274,11 @@ private: { delete medium; medium = nullptr; - - CurrentMemoryTracker::free(sizeof(Medium)); } else if (container_type == details::ContainerType::LARGE) { delete large; large = nullptr; - - CurrentMemoryTracker::free(sizeof(Large)); } } diff --git a/dbms/src/Common/CpuId.h b/dbms/src/Common/CpuId.h index 1bae7407818..808502ba086 100644 --- a/dbms/src/Common/CpuId.h +++ b/dbms/src/Common/CpuId.h @@ -14,8 +14,9 @@ namespace DB namespace Cpu { -#if defined(__x86_64__) || defined(__i386__) -inline UInt64 _xgetbv(UInt32 xcr) noexcept +#if (defined(__x86_64__) || defined(__i386__)) +/// Our version is independent of -mxsave option, because we do dynamic dispatch. +inline UInt64 our_xgetbv(UInt32 xcr) noexcept { UInt32 eax; UInt32 edx; @@ -185,7 +186,7 @@ bool haveAVX() noexcept // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-optimization-manual.pdf // https://bugs.chromium.org/p/chromium/issues/detail?id=375968 return haveOSXSAVE() // implies haveXSAVE() - && (_xgetbv(0) & 6u) == 6u // XMM state and YMM state are enabled by OS + && (our_xgetbv(0) & 6u) == 6u // XMM state and YMM state are enabled by OS && ((CpuInfo(0x1).ecx >> 28) & 1u); // AVX bit #else return false; @@ -217,8 +218,8 @@ bool haveAVX512F() noexcept #if defined(__x86_64__) || defined(__i386__) // https://software.intel.com/en-us/articles/how-to-detect-knl-instruction-support return haveOSXSAVE() // implies haveXSAVE() - && (_xgetbv(0) & 6u) == 6u // XMM state and YMM state are enabled by OS - && ((_xgetbv(0) >> 5) & 7u) == 7u // ZMM state is enabled by OS + && (our_xgetbv(0) & 6u) == 6u // XMM state and YMM state are enabled by OS + && ((our_xgetbv(0) >> 5) & 7u) == 7u // ZMM state is enabled by OS && CpuInfo(0x0).eax >= 0x7 // leaf 7 is present && ((CpuInfo(0x7).ebx >> 16) & 1u); // AVX512F bit #else diff --git a/dbms/src/Common/CurrentThread.cpp b/dbms/src/Common/CurrentThread.cpp index 6c2c77dccd7..84d63a04b96 100644 --- a/dbms/src/Common/CurrentThread.cpp +++ b/dbms/src/Common/CurrentThread.cpp @@ -23,7 +23,7 @@ void CurrentThread::updatePerformanceCounters() { if (unlikely(!current_thread)) return; - get().updatePerformanceCounters(); + current_thread->updatePerformanceCounters(); } ThreadStatus & CurrentThread::get() @@ -36,35 +36,42 @@ ThreadStatus & CurrentThread::get() ProfileEvents::Counters & CurrentThread::getProfileEvents() { - return current_thread ? get().performance_counters : ProfileEvents::global_counters; + return current_thread ? current_thread->performance_counters : ProfileEvents::global_counters; } MemoryTracker * CurrentThread::getMemoryTracker() { if (unlikely(!current_thread)) return nullptr; - return &get().memory_tracker; + return ¤t_thread->memory_tracker; +} + +Int64 & CurrentThread::getUntrackedMemory() +{ + /// It assumes that (current_thread != nullptr) is already checked with getMemoryTracker() + return current_thread->untracked_memory; } void CurrentThread::updateProgressIn(const Progress & value) { if (unlikely(!current_thread)) return; - get().progress_in.incrementPiecewiseAtomically(value); + current_thread->progress_in.incrementPiecewiseAtomically(value); } void CurrentThread::updateProgressOut(const Progress & value) { if (unlikely(!current_thread)) return; - get().progress_out.incrementPiecewiseAtomically(value); + current_thread->progress_out.incrementPiecewiseAtomically(value); } -void CurrentThread::attachInternalTextLogsQueue(const std::shared_ptr & logs_queue) +void CurrentThread::attachInternalTextLogsQueue(const std::shared_ptr & logs_queue, + LogsLevel client_logs_level) { if (unlikely(!current_thread)) return; - get().attachInternalTextLogsQueue(logs_queue); + current_thread->attachInternalTextLogsQueue(logs_queue, client_logs_level); } std::shared_ptr CurrentThread::getInternalTextLogsQueue() @@ -73,10 +80,10 @@ std::shared_ptr CurrentThread::getInternalTextLogsQueue() if (unlikely(!current_thread)) return nullptr; - if (get().getCurrentState() == ThreadStatus::ThreadState::Died) + if (current_thread->getCurrentState() == ThreadStatus::ThreadState::Died) return nullptr; - return get().getInternalTextLogsQueue(); + return current_thread->getInternalTextLogsQueue(); } ThreadGroupStatusPtr CurrentThread::getGroup() @@ -84,7 +91,7 @@ ThreadGroupStatusPtr CurrentThread::getGroup() if (unlikely(!current_thread)) return nullptr; - return get().getThreadGroup(); + return current_thread->getThreadGroup(); } } diff --git a/dbms/src/Common/CurrentThread.h b/dbms/src/Common/CurrentThread.h index 49b46721008..685ac879530 100644 --- a/dbms/src/Common/CurrentThread.h +++ b/dbms/src/Common/CurrentThread.h @@ -3,6 +3,7 @@ #include #include +#include #include @@ -38,7 +39,8 @@ public: static ThreadGroupStatusPtr getGroup(); /// A logs queue used by TCPHandler to pass logs to a client - static void attachInternalTextLogsQueue(const std::shared_ptr & logs_queue); + static void attachInternalTextLogsQueue(const std::shared_ptr & logs_queue, + LogsLevel client_logs_level); static std::shared_ptr getInternalTextLogsQueue(); /// Makes system calls to update ProfileEvents that contain info from rusage and taskstats @@ -46,6 +48,7 @@ public: static ProfileEvents::Counters & getProfileEvents(); static MemoryTracker * getMemoryTracker(); + static Int64 & getUntrackedMemory(); /// Update read and write rows (bytes) statistics (used in system.query_thread_log) static void updateProgressIn(const Progress & value); @@ -69,7 +72,7 @@ public: static void finalizePerformanceCounters(); /// Returns a non-empty string if the thread is attached to a query - static const std::string & getQueryId(); + static StringRef getQueryId(); /// Non-master threads call this method in destructor automatically static void detachQuery(); diff --git a/dbms/src/Common/ErrorCodes.cpp b/dbms/src/Common/ErrorCodes.cpp index 17bf5d0a0e5..a3b788c230f 100644 --- a/dbms/src/Common/ErrorCodes.cpp +++ b/dbms/src/Common/ErrorCodes.cpp @@ -433,6 +433,7 @@ namespace ErrorCodes extern const int UNKNOWN_QUERY_PARAMETER = 456; extern const int BAD_QUERY_PARAMETER = 457; extern const int CANNOT_UNLINK = 458; + extern const int CANNOT_SET_THREAD_PRIORITY = 459; extern const int KEEPER_EXCEPTION = 999; extern const int POCO_EXCEPTION = 1000; diff --git a/dbms/src/Common/EventCounter.h b/dbms/src/Common/EventCounter.h new file mode 100644 index 00000000000..edd552ace83 --- /dev/null +++ b/dbms/src/Common/EventCounter.h @@ -0,0 +1,47 @@ +#pragma once + +#include +#include +#include + + +/** Allow to subscribe for multiple events and wait for them one by one in arbitrary order. + */ +class EventCounter +{ +private: + size_t events_happened = 0; + size_t events_waited = 0; + + mutable std::mutex mutex; + std::condition_variable condvar; + +public: + void notify() + { + { + std::lock_guard lock(mutex); + ++events_happened; + } + condvar.notify_all(); + } + + void wait() + { + std::unique_lock lock(mutex); + condvar.wait(lock, [&]{ return events_happened > events_waited; }); + ++events_waited; + } + + template + bool waitFor(Duration && duration) + { + std::unique_lock lock(mutex); + if (condvar.wait(lock, std::forward(duration), [&]{ return events_happened > events_waited; })) + { + ++events_waited; + return true; + } + return false; + } +}; diff --git a/dbms/src/Common/FileChecker.cpp b/dbms/src/Common/FileChecker.cpp index d196c703e36..5256b683653 100644 --- a/dbms/src/Common/FileChecker.cpp +++ b/dbms/src/Common/FileChecker.cpp @@ -42,35 +42,40 @@ void FileChecker::update(const Files::const_iterator & begin, const Files::const save(); } -bool FileChecker::check() const +CheckResults FileChecker::check() const { /** Read the files again every time you call `check` - so as not to violate the constancy. * `check` method is rarely called. */ + + CheckResults results; Map local_map; load(local_map, files_info_path); if (local_map.empty()) - return true; + return {}; for (const auto & name_size : local_map) { - Poco::File file(Poco::Path(files_info_path).parent().toString() + "/" + name_size.first); + Poco::Path path = Poco::Path(files_info_path).parent().toString() + "/" + name_size.first; + Poco::File file(path); if (!file.exists()) { - LOG_ERROR(log, "File " << file.path() << " doesn't exist"); - return false; + results.emplace_back(path.getFileName(), false, "File " + file.path() + " doesn't exist"); + break; } + size_t real_size = file.getSize(); if (real_size != name_size.second) { - LOG_ERROR(log, "Size of " << file.path() << " is wrong. Size is " << real_size << " but should be " << name_size.second); - return false; + results.emplace_back(path.getFileName(), false, "Size of " + file.path() + " is wrong. Size is " + toString(real_size) + " but should be " + toString(name_size.second)); + break; } + results.emplace_back(path.getFileName(), true, ""); } - return true; + return results; } void FileChecker::initialize() diff --git a/dbms/src/Common/FileChecker.h b/dbms/src/Common/FileChecker.h index 26167826888..bd69867b11c 100644 --- a/dbms/src/Common/FileChecker.h +++ b/dbms/src/Common/FileChecker.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB @@ -24,7 +25,7 @@ public: void update(const Files::const_iterator & begin, const Files::const_iterator & end); /// Check the files whose parameters are specified in sizes.json - bool check() const; + CheckResults check() const; private: void initialize(); diff --git a/dbms/src/Common/HyperLogLogWithSmallSetOptimization.h b/dbms/src/Common/HyperLogLogWithSmallSetOptimization.h index 836fbda222e..548b745cb6e 100644 --- a/dbms/src/Common/HyperLogLogWithSmallSetOptimization.h +++ b/dbms/src/Common/HyperLogLogWithSmallSetOptimization.h @@ -4,7 +4,6 @@ #include #include -#include namespace DB @@ -39,8 +38,6 @@ private: void toLarge() { - CurrentMemoryTracker::alloc(sizeof(Large)); - /// At the time of copying data from `tiny`, setting the value of `large` is still not possible (otherwise it will overwrite some data). Large * tmp_large = new Large; @@ -56,11 +53,7 @@ public: ~HyperLogLogWithSmallSetOptimization() { if (isLarge()) - { delete large; - - CurrentMemoryTracker::free(sizeof(Large)); - } } void insert(Key value) diff --git a/dbms/src/Common/MemoryTracker.cpp b/dbms/src/Common/MemoryTracker.cpp index bc324be4904..b3d661d95ee 100644 --- a/dbms/src/Common/MemoryTracker.cpp +++ b/dbms/src/Common/MemoryTracker.cpp @@ -1,3 +1,5 @@ +#include + #include "MemoryTracker.h" #include #include @@ -17,6 +19,8 @@ namespace DB static constexpr size_t log_peak_memory_usage_every = 1ULL << 30; +/// Each thread could new/delete memory in range of (-untracked_memory_limit, untracked_memory_limit) without access to common counters. +static constexpr Int64 untracked_memory_limit = 4 * 1024 * 1024; MemoryTracker::~MemoryTracker() @@ -85,6 +89,9 @@ void MemoryTracker::alloc(Int64 size) { free(size); + /// Prevent recursion. Exception::ctor -> std::string -> new[] -> MemoryTracker::alloc + auto untrack_lock = blocker.cancel(); + std::stringstream message; message << "Memory tracker"; if (description) @@ -100,6 +107,9 @@ void MemoryTracker::alloc(Int64 size) { free(size); + /// Prevent recursion. Exception::ctor -> std::string -> new[] -> MemoryTracker::alloc + auto untrack_lock = blocker.cancel(); + std::stringstream message; message << "Memory limit"; if (description) @@ -191,19 +201,41 @@ namespace CurrentMemoryTracker void alloc(Int64 size) { if (auto memory_tracker = DB::CurrentThread::getMemoryTracker()) - memory_tracker->alloc(size); + { + Int64 & untracked = DB::CurrentThread::getUntrackedMemory(); + untracked += size; + if (untracked > untracked_memory_limit) + { + /// Zero untracked before track. If tracker throws out-of-limit we would be able to alloc up to untracked_memory_limit bytes + /// more. It could be usefull for enlarge Exception message in rethrow logic. + Int64 tmp = untracked; + untracked = 0; + memory_tracker->alloc(tmp); + } + } } void realloc(Int64 old_size, Int64 new_size) { - if (auto memory_tracker = DB::CurrentThread::getMemoryTracker()) - memory_tracker->alloc(new_size - old_size); + Int64 addition = new_size - old_size; + if (addition > 0) + alloc(addition); + else + free(-addition); } void free(Int64 size) { if (auto memory_tracker = DB::CurrentThread::getMemoryTracker()) - memory_tracker->free(size); + { + Int64 & untracked = DB::CurrentThread::getUntrackedMemory(); + untracked -= size; + if (untracked < -untracked_memory_limit) + { + memory_tracker->free(-untracked); + untracked = 0; + } + } } } diff --git a/dbms/src/Common/MemoryTracker.h b/dbms/src/Common/MemoryTracker.h index 9f439c7550c..4ce0ac262fa 100644 --- a/dbms/src/Common/MemoryTracker.h +++ b/dbms/src/Common/MemoryTracker.h @@ -45,7 +45,11 @@ public: void realloc(Int64 old_size, Int64 new_size) { - alloc(new_size - old_size); + Int64 addition = new_size - old_size; + if (addition > 0) + alloc(addition); + else + free(-addition); } /** This function should be called after memory deallocation. diff --git a/dbms/src/Common/MiAllocator.cpp b/dbms/src/Common/MiAllocator.cpp index 456609374ee..cafa6c135f7 100644 --- a/dbms/src/Common/MiAllocator.cpp +++ b/dbms/src/Common/MiAllocator.cpp @@ -5,15 +5,33 @@ #include "MiAllocator.h" #include +#include +#include +#include + namespace DB { +namespace ErrorCodes +{ + extern const int CANNOT_ALLOCATE_MEMORY; +} void * MiAllocator::alloc(size_t size, size_t alignment) { + void * ptr; if (alignment == 0) - return mi_malloc(size); + { + ptr = mi_malloc(size); + if (!ptr) + DB::throwFromErrno("MiAllocator: Cannot allocate in mimalloc " + formatReadableSizeWithBinarySuffix(size) + ".", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); + } else - return mi_malloc_aligned(size, alignment); + { + ptr = mi_malloc_aligned(size, alignment); + if (!ptr) + DB::throwFromErrno("MiAllocator: Cannot allocate in mimalloc (mi_malloc_aligned) " + formatReadableSizeWithBinarySuffix(size) + " with alignment " + toString(alignment) + ".", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); + } + return ptr; } void MiAllocator::free(void * buf, size_t) @@ -32,10 +50,21 @@ void * MiAllocator::realloc(void * old_ptr, size_t, size_t new_size, size_t alig return nullptr; } - if (alignment == 0) - return mi_realloc(old_ptr, alignment); + void * ptr; - return mi_realloc_aligned(old_ptr, new_size, alignment); + if (alignment == 0) + { + ptr = mi_realloc(old_ptr, alignment); + if (!ptr) + DB::throwFromErrno("MiAllocator: Cannot reallocate in mimalloc " + formatReadableSizeWithBinarySuffix(size) + ".", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); + } + else + { + ptr = mi_realloc_aligned(old_ptr, new_size, alignment); + if (!ptr) + DB::throwFromErrno("MiAllocator: Cannot reallocate in mimalloc (mi_realloc_aligned) " + formatReadableSizeWithBinarySuffix(size) + " with alignment " + toString(alignment) + ".", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); + } + return ptr; } } diff --git a/dbms/src/Common/Stopwatch.h b/dbms/src/Common/Stopwatch.h index d6508b23b44..7bfaccc72b2 100644 --- a/dbms/src/Common/Stopwatch.h +++ b/dbms/src/Common/Stopwatch.h @@ -189,3 +189,16 @@ private: Timestamp stop_ts; bool is_running = false; }; + + +template +class StopwatchGuard : public TStopwatch +{ +public: + explicit StopwatchGuard(UInt64 & elapsed_ns_) : elapsed_ns(elapsed_ns_) {} + + ~StopwatchGuard() { elapsed_ns += TStopwatch::elapsedNanoseconds(); } + +private: + UInt64 & elapsed_ns; +}; diff --git a/dbms/src/Common/TaskStatsInfoGetter.cpp b/dbms/src/Common/TaskStatsInfoGetter.cpp index 35a68c5a90c..b361161483a 100644 --- a/dbms/src/Common/TaskStatsInfoGetter.cpp +++ b/dbms/src/Common/TaskStatsInfoGetter.cpp @@ -34,6 +34,11 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } +// Replace NLMSG_OK with explicit casts since that system macro contains signedness bugs which are not going to be fixed. +static inline bool is_nlmsg_ok(const struct nlmsghdr * const nlh, const ssize_t len) +{ + return len >= static_cast(sizeof(*nlh)) && nlh->nlmsg_len >= sizeof(*nlh) && static_cast(len) >= nlh->nlmsg_len; +} namespace { @@ -128,7 +133,7 @@ struct NetlinkMessage if (header.nlmsg_type == NLMSG_ERROR) throw Exception("Can't receive Netlink response: error " + std::to_string(error.error), ErrorCodes::NETLINK_ERROR); - if (!NLMSG_OK((&header), bytes_received)) + if (!is_nlmsg_ok(&header, bytes_received)) throw Exception("Can't receive Netlink response: wrong number of bytes received", ErrorCodes::NETLINK_ERROR); } }; diff --git a/dbms/src/Common/ThreadStatus.cpp b/dbms/src/Common/ThreadStatus.cpp index c2e415ab363..e5fe5d6f23b 100644 --- a/dbms/src/Common/ThreadStatus.cpp +++ b/dbms/src/Common/ThreadStatus.cpp @@ -50,6 +50,19 @@ ThreadStatus::ThreadStatus() ThreadStatus::~ThreadStatus() { + try + { + if (untracked_memory > 0) + memory_tracker.alloc(untracked_memory); + else + memory_tracker.free(-untracked_memory); + } + catch (const DB::Exception &) + { + /// It's a minor tracked memory leak here (not the memory itself but it's counter). + /// We've already allocated a little bit more then the limit and cannot track it in the thread memory tracker or its parent. + } + if (deleter) deleter(); current_thread = nullptr; @@ -117,7 +130,8 @@ void ThreadStatus::assertState(const std::initializer_list & permitted_stat throw Exception(ss.str(), ErrorCodes::LOGICAL_ERROR); } -void ThreadStatus::attachInternalTextLogsQueue(const InternalTextLogsQueuePtr & logs_queue) +void ThreadStatus::attachInternalTextLogsQueue(const InternalTextLogsQueuePtr & logs_queue, + LogsLevel client_logs_level) { logs_queue_ptr = logs_queue; @@ -126,6 +140,7 @@ void ThreadStatus::attachInternalTextLogsQueue(const InternalTextLogsQueuePtr & std::lock_guard lock(thread_group->mutex); thread_group->logs_queue_ptr = logs_queue; + thread_group->client_logs_level = client_logs_level; } } diff --git a/dbms/src/Common/ThreadStatus.h b/dbms/src/Common/ThreadStatus.h index f92d604b372..fcfb3e0e19f 100644 --- a/dbms/src/Common/ThreadStatus.h +++ b/dbms/src/Common/ThreadStatus.h @@ -1,8 +1,11 @@ #pragma once +#include #include #include +#include + #include #include @@ -61,6 +64,8 @@ public: UInt32 master_thread_number = 0; Int32 master_thread_os_id = -1; + LogsLevel client_logs_level = LogsLevel::none; + String query; }; @@ -85,10 +90,14 @@ public: UInt32 thread_number = 0; /// Linux's PID (or TGID) (the same id is shown by ps util) Int32 os_thread_id = -1; + /// Also called "nice" value. If it was changed to non-zero (when attaching query) - will be reset to zero when query is detached. + Int32 os_thread_priority = 0; /// TODO: merge them into common entity ProfileEvents::Counters performance_counters{VariableContext::Thread}; MemoryTracker memory_tracker{VariableContext::Thread}; + /// Small amount of untracked memory (per thread atomic-less counter) + Int64 untracked_memory = 0; /// Statistics of read and write rows/bytes Progress progress_in; @@ -114,7 +123,7 @@ public: return thread_state.load(std::memory_order_relaxed); } - const std::string & getQueryId() const; + StringRef getQueryId() const; /// Starts new query and create new thread group for it, current thread becomes master thread of the query void initializeQuery(); @@ -127,7 +136,8 @@ public: return thread_state == Died ? nullptr : logs_queue_ptr.lock(); } - void attachInternalTextLogsQueue(const InternalTextLogsQueuePtr & logs_queue); + void attachInternalTextLogsQueue(const InternalTextLogsQueuePtr & logs_queue, + LogsLevel client_logs_level); /// Sets query context for current thread and its thread group /// NOTE: query_context have to be alive until detachQuery() is called diff --git a/dbms/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp b/dbms/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp index 489ac04c5e4..2f332c7039d 100644 --- a/dbms/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp +++ b/dbms/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp @@ -5,11 +5,6 @@ #include #include -#pragma GCC diagnostic ignored "-Wsign-compare" -#ifdef __clang__ - #pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" - #pragma clang diagnostic ignored "-Wundef" -#endif #include #include diff --git a/dbms/src/Common/formatIPv6.cpp b/dbms/src/Common/formatIPv6.cpp index f8100557ba2..9ec6c427ab4 100644 --- a/dbms/src/Common/formatIPv6.cpp +++ b/dbms/src/Common/formatIPv6.cpp @@ -153,7 +153,7 @@ void formatIPv6(const unsigned char * src, char *& dst, UInt8 zeroed_tail_bytes_ } /// Was it a trailing run of 0x00's? - if (best.base != -1 && size_t(best.base + best.len) == words.size()) + if (best.base != -1 && size_t(best.base) + size_t(best.len) == words.size()) *dst++ = ':'; *dst++ = '\0'; diff --git a/dbms/src/Common/isLocalAddress.h b/dbms/src/Common/isLocalAddress.h index 81039dff68e..63de5e000a9 100644 --- a/dbms/src/Common/isLocalAddress.h +++ b/dbms/src/Common/isLocalAddress.h @@ -23,9 +23,7 @@ namespace DB * - the routing rules that affect which network interface we go to the specified address are not checked. */ bool isLocalAddress(const Poco::Net::SocketAddress & address, UInt16 clickhouse_port); - bool isLocalAddress(const Poco::Net::SocketAddress & address); - bool isLocalAddress(const Poco::Net::IPAddress & address); /// Returns number of different bytes in hostnames, used for load balancing diff --git a/dbms/src/Common/new_delete.cpp b/dbms/src/Common/new_delete.cpp new file mode 100644 index 00000000000..aff708135e1 --- /dev/null +++ b/dbms/src/Common/new_delete.cpp @@ -0,0 +1,143 @@ +#include + +#include +#include +#include + +/// Replace default new/delete with memory tracking versions. +/// @sa https://en.cppreference.com/w/cpp/memory/new/operator_new +/// https://en.cppreference.com/w/cpp/memory/new/operator_delete +#if NOT_UNBUNDLED + +namespace Memory +{ + +ALWAYS_INLINE void trackMemory(std::size_t size) +{ +#if USE_JEMALLOC + /// The nallocx() function allocates no memory, but it performs the same size computation as the mallocx() function + /// @note je_mallocx() != je_malloc(). It's expected they don't differ much in allocation logic. + if (likely(size != 0)) + CurrentMemoryTracker::alloc(nallocx(size, 0)); +#else + CurrentMemoryTracker::alloc(size); +#endif +} + +ALWAYS_INLINE bool trackMemoryNoExept(std::size_t size) noexcept +{ + try + { + trackMemory(size); + } + catch (...) + { + return false; + } + + return true; +} + +ALWAYS_INLINE void untrackMemory(void * ptr [[maybe_unused]], std::size_t size [[maybe_unused]] = 0) noexcept +{ + try + { +#if USE_JEMALLOC + /// @note It's also possible to use je_malloc_usable_size() here. + if (likely(ptr != nullptr)) + CurrentMemoryTracker::free(sallocx(ptr, 0)); +#else + if (size) + CurrentMemoryTracker::free(size); +#endif + } + catch (...) + {} +} + +} + +/// new + +void * operator new(std::size_t size) +{ + Memory::trackMemory(size); + return Memory::newImpl(size); +} + +void * operator new[](std::size_t size) +{ + Memory::trackMemory(size); + return Memory::newImpl(size); +} + +void * operator new(std::size_t size, const std::nothrow_t &) noexcept +{ + if (likely(Memory::trackMemoryNoExept(size))) + return Memory::newNoExept(size); + return nullptr; +} + +void * operator new[](std::size_t size, const std::nothrow_t &) noexcept +{ + if (likely(Memory::trackMemoryNoExept(size))) + return Memory::newNoExept(size); + return nullptr; +} + +/// delete + +/// C++17 std 21.6.2.1 (11) +/// If a function without a size parameter is defined, the program should also define the corresponding function with a size parameter. +/// If a function with a size parameter is defined, the program shall also define the corresponding version without the size parameter. + +/// cppreference: +/// It's unspecified whether size-aware or size-unaware version is called when deleting objects of +/// incomplete type and arrays of non-class and trivially-destructible class types. + +void operator delete(void * ptr) noexcept +{ + Memory::untrackMemory(ptr); + Memory::deleteImpl(ptr); +} + +void operator delete[](void * ptr) noexcept +{ + Memory::untrackMemory(ptr); + Memory::deleteImpl(ptr); +} + +void operator delete(void * ptr, std::size_t size) noexcept +{ + Memory::untrackMemory(ptr, size); + Memory::deleteSized(ptr, size); +} + +void operator delete[](void * ptr, std::size_t size) noexcept +{ + Memory::untrackMemory(ptr, size); + Memory::deleteSized(ptr, size); +} + +#else + +/// new + +void * operator new(std::size_t size) { return Memory::newImpl(size); } +void * operator new[](std::size_t size) { return Memory::newImpl(size); } + +void * operator new(std::size_t size, const std::nothrow_t &) noexcept { return Memory::newNoExept(size); } +void * operator new[](std::size_t size, const std::nothrow_t &) noexcept { return Memory::newNoExept(size); } + +/// delete + +void operator delete(void * ptr) noexcept { Memory::deleteImpl(ptr); } +void operator delete[](void * ptr) noexcept { Memory::deleteImpl(ptr); } + +void operator delete(void * ptr, const std::nothrow_t &) noexcept { Memory::deleteImpl(ptr); } +void operator delete[](void * ptr, const std::nothrow_t &) noexcept { Memory::deleteImpl(ptr); } + +void operator delete(void * ptr, std::size_t size) noexcept { Memory::deleteSized(ptr, size); } +void operator delete[](void * ptr, std::size_t size) noexcept { Memory::deleteSized(ptr, size); } + +#endif diff --git a/dbms/src/Common/tests/CMakeLists.txt b/dbms/src/Common/tests/CMakeLists.txt index 23b1614e704..83d69b2c8f2 100644 --- a/dbms/src/Common/tests/CMakeLists.txt +++ b/dbms/src/Common/tests/CMakeLists.txt @@ -23,10 +23,10 @@ add_executable (small_table small_table.cpp) target_link_libraries (small_table PRIVATE clickhouse_common_io) add_executable (parallel_aggregation parallel_aggregation.cpp) -target_link_libraries (parallel_aggregation PRIVATE clickhouse_compression clickhouse_common_io) +target_link_libraries (parallel_aggregation PRIVATE dbms) add_executable (parallel_aggregation2 parallel_aggregation2.cpp) -target_link_libraries (parallel_aggregation2 PRIVATE clickhouse_compression clickhouse_common_io) +target_link_libraries (parallel_aggregation2 PRIVATE dbms) add_executable (int_hashes_perf int_hashes_perf.cpp AvalancheTest.cpp Random.cpp) target_link_libraries (int_hashes_perf PRIVATE clickhouse_common_io) @@ -42,7 +42,7 @@ add_executable (radix_sort radix_sort.cpp) target_link_libraries (radix_sort PRIVATE clickhouse_common_io) add_executable (arena_with_free_lists arena_with_free_lists.cpp) -target_link_libraries (arena_with_free_lists PRIVATE clickhouse_compression clickhouse_common_io) +target_link_libraries (arena_with_free_lists PRIVATE dbms) add_executable (pod_array pod_array.cpp) target_link_libraries (pod_array PRIVATE clickhouse_common_io) @@ -62,7 +62,7 @@ target_link_libraries (space_saving PRIVATE clickhouse_common_io) add_executable (integer_hash_tables_and_hashes integer_hash_tables_and_hashes.cpp) target_include_directories (integer_hash_tables_and_hashes SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) -target_link_libraries (integer_hash_tables_and_hashes PRIVATE clickhouse_compression clickhouse_common_io) +target_link_libraries (integer_hash_tables_and_hashes PRIVATE dbms) add_executable (allocator allocator.cpp) target_link_libraries (allocator PRIVATE clickhouse_common_io) @@ -75,3 +75,6 @@ target_link_libraries (cow_compositions PRIVATE clickhouse_common_io) add_executable (stopwatch stopwatch.cpp) target_link_libraries (stopwatch PRIVATE clickhouse_common_io) + +add_executable (mi_malloc_test mi_malloc_test.cpp) +target_link_libraries (mi_malloc_test PRIVATE clickhouse_common_io) diff --git a/dbms/src/Common/tests/gtest_rw_lock.cpp b/dbms/src/Common/tests/gtest_rw_lock.cpp index 6e9b91dc048..68927c8bc4a 100644 --- a/dbms/src/Common/tests/gtest_rw_lock.cpp +++ b/dbms/src/Common/tests/gtest_rw_lock.cpp @@ -1,8 +1,3 @@ -#pragma GCC diagnostic ignored "-Wsign-compare" -#ifdef __clang__ - #pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" - #pragma clang diagnostic ignored "-Wundef" -#endif #include #include diff --git a/dbms/src/Common/tests/gtest_shell_command.cpp b/dbms/src/Common/tests/gtest_shell_command.cpp index 2378cda2ee7..79b8309d0fa 100644 --- a/dbms/src/Common/tests/gtest_shell_command.cpp +++ b/dbms/src/Common/tests/gtest_shell_command.cpp @@ -9,11 +9,6 @@ #include #include -#pragma GCC diagnostic ignored "-Wsign-compare" -#ifdef __clang__ - #pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" - #pragma clang diagnostic ignored "-Wundef" -#endif #include diff --git a/dbms/src/Common/tests/gtest_thread_pool_concurrent_wait.cpp b/dbms/src/Common/tests/gtest_thread_pool_concurrent_wait.cpp index 1e38e418a22..213e70ce3dd 100644 --- a/dbms/src/Common/tests/gtest_thread_pool_concurrent_wait.cpp +++ b/dbms/src/Common/tests/gtest_thread_pool_concurrent_wait.cpp @@ -1,10 +1,5 @@ #include -#pragma GCC diagnostic ignored "-Wsign-compare" -#ifdef __clang__ - #pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" - #pragma clang diagnostic ignored "-Wundef" -#endif #include /** Reproduces bug in ThreadPool. diff --git a/dbms/src/Common/tests/gtest_thread_pool_limit.cpp b/dbms/src/Common/tests/gtest_thread_pool_limit.cpp index 2bd38f34d10..c18ff2e38ee 100644 --- a/dbms/src/Common/tests/gtest_thread_pool_limit.cpp +++ b/dbms/src/Common/tests/gtest_thread_pool_limit.cpp @@ -2,11 +2,6 @@ #include #include -#pragma GCC diagnostic ignored "-Wsign-compare" -#ifdef __clang__ - #pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" - #pragma clang diagnostic ignored "-Wundef" -#endif #include /// Test for thread self-removal when number of free threads in pool is too large. diff --git a/dbms/src/Common/tests/gtest_thread_pool_loop.cpp b/dbms/src/Common/tests/gtest_thread_pool_loop.cpp index 80b7b94d988..63d4114b867 100644 --- a/dbms/src/Common/tests/gtest_thread_pool_loop.cpp +++ b/dbms/src/Common/tests/gtest_thread_pool_loop.cpp @@ -2,11 +2,6 @@ #include #include -#pragma GCC diagnostic ignored "-Wsign-compare" -#ifdef __clang__ - #pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" - #pragma clang diagnostic ignored "-Wundef" -#endif #include diff --git a/dbms/src/Common/tests/gtest_thread_pool_schedule_exception.cpp b/dbms/src/Common/tests/gtest_thread_pool_schedule_exception.cpp index 001d9c30b27..52091a1ea7f 100644 --- a/dbms/src/Common/tests/gtest_thread_pool_schedule_exception.cpp +++ b/dbms/src/Common/tests/gtest_thread_pool_schedule_exception.cpp @@ -2,11 +2,6 @@ #include #include -#pragma GCC diagnostic ignored "-Wsign-compare" -#ifdef __clang__ - #pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" - #pragma clang diagnostic ignored "-Wundef" -#endif #include diff --git a/dbms/src/Common/tests/gtest_unescapeForFileName.cpp b/dbms/src/Common/tests/gtest_unescapeForFileName.cpp index b7e045b4318..33194a6dade 100644 --- a/dbms/src/Common/tests/gtest_unescapeForFileName.cpp +++ b/dbms/src/Common/tests/gtest_unescapeForFileName.cpp @@ -1,10 +1,5 @@ #include -#pragma GCC diagnostic ignored "-Wsign-compare" -#ifdef __clang__ - #pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" - #pragma clang diagnostic ignored "-Wundef" -#endif #include diff --git a/dbms/src/Common/tests/mi_malloc_test.cpp b/dbms/src/Common/tests/mi_malloc_test.cpp new file mode 100644 index 00000000000..d9ee75fba6e --- /dev/null +++ b/dbms/src/Common/tests/mi_malloc_test.cpp @@ -0,0 +1,118 @@ +/** In addition to ClickHouse (Apache 2) license, this file can be also used under MIT license: + +MIT License + +Copyright (c) 2019 Yandex LLC, Alexey Milovidov + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +*/ + +#include +#include +#include +#include +#include +#include + +#include + +//#undef USE_MIMALLOC +//#define USE_MIMALLOC 0 + +#if USE_MIMALLOC + +#include +#define malloc mi_malloc +#define free mi_free + +#else + +#include + +#endif + + +size_t total_size{0}; + +struct Allocation +{ + void * ptr = nullptr; + size_t size = 0; + + Allocation() {} + + Allocation(size_t size) + : size(size) + { + ptr = malloc(size); + if (!ptr) + throw std::runtime_error("Cannot allocate memory"); + total_size += size; + } + + ~Allocation() + { + if (ptr) + { + free(ptr); + total_size -= size; + } + ptr = nullptr; + } + + Allocation(const Allocation &) = delete; + + Allocation(Allocation && rhs) + { + ptr = rhs.ptr; + size = rhs.size; + rhs.ptr = nullptr; + rhs.size = 0; + } +}; + + +int main(int, char **) +{ + std::vector allocations; + + constexpr size_t limit = 100000000; + constexpr size_t min_alloc_size = 65536; + constexpr size_t max_alloc_size = 10000000; + + std::mt19937 rng; + auto distribution = std::uniform_int_distribution(min_alloc_size, max_alloc_size); + + size_t total_allocations = 0; + + while (true) + { + size_t size = distribution(rng); + + while (total_size + size > limit) + allocations.pop_back(); + + allocations.emplace_back(size); + + ++total_allocations; + if (total_allocations % (1ULL << 20) == 0) + std::cerr << "Total allocations: " << total_allocations << "\n"; + } +} diff --git a/dbms/src/Common/tests/parallel_aggregation2.cpp b/dbms/src/Common/tests/parallel_aggregation2.cpp index 5bee292f58d..a2b26e82420 100644 --- a/dbms/src/Common/tests/parallel_aggregation2.cpp +++ b/dbms/src/Common/tests/parallel_aggregation2.cpp @@ -34,7 +34,7 @@ struct AggregateIndependent { results.reserve(num_threads); for (size_t i = 0; i < num_threads; ++i) - results.emplace_back(new Map); + results.emplace_back(std::make_unique()); for (size_t i = 0; i < num_threads; ++i) { @@ -77,7 +77,7 @@ struct AggregateIndependentWithSequentialKeysOptimization { results.reserve(num_threads); for (size_t i = 0; i < num_threads; ++i) - results.emplace_back(new Map); + results.emplace_back(std::make_unique()); for (size_t i = 0; i < num_threads; ++i) { diff --git a/dbms/src/Compression/CMakeLists.txt b/dbms/src/Compression/CMakeLists.txt index 0032186205a..36cab0b3590 100644 --- a/dbms/src/Compression/CMakeLists.txt +++ b/dbms/src/Compression/CMakeLists.txt @@ -1,21 +1,3 @@ -include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake) -add_headers_and_sources(clickhouse_compression .) -add_library(clickhouse_compression ${clickhouse_compression_headers} ${clickhouse_compression_sources}) -target_link_libraries(clickhouse_compression PRIVATE clickhouse_parsers clickhouse_common_io ${LZ4_LIBRARY} ${CITYHASH_LIBRARIES}) -if(ZSTD_LIBRARY) - target_link_libraries(clickhouse_compression PRIVATE ${ZSTD_LIBRARY}) -endif() - -target_include_directories(clickhouse_compression PUBLIC ${DBMS_INCLUDE_DIR}) -target_include_directories(clickhouse_compression SYSTEM PUBLIC ${PCG_RANDOM_INCLUDE_DIR}) - -if (NOT USE_INTERNAL_LZ4_LIBRARY) - target_include_directories(clickhouse_compression SYSTEM BEFORE PRIVATE ${LZ4_INCLUDE_DIR}) -endif () -if (NOT USE_INTERNAL_ZSTD_LIBRARY AND ZSTD_INCLUDE_DIR) - target_include_directories(clickhouse_compression SYSTEM BEFORE PRIVATE ${ZSTD_INCLUDE_DIR}) -endif () - if(ENABLE_TESTS) add_subdirectory(tests) endif() diff --git a/dbms/src/Compression/CompressionCodecDoubleDelta.cpp b/dbms/src/Compression/CompressionCodecDoubleDelta.cpp index 80d363e0ec5..d3830dc9fdb 100644 --- a/dbms/src/Compression/CompressionCodecDoubleDelta.cpp +++ b/dbms/src/Compression/CompressionCodecDoubleDelta.cpp @@ -12,6 +12,7 @@ #include #include #include +#include namespace DB { @@ -24,28 +25,23 @@ extern const int CANNOT_DECOMPRESS; namespace { -UInt32 getDeltaTypeByteSize(UInt8 data_bytes_size) + +Int64 getMaxValueForByteSize(UInt8 byte_size) { - // both delta and double delta can be twice the size of data item, but not less than 32 bits and not more that 64. - return std::min(64/8, std::max(32/8, data_bytes_size * 2)); -} - -UInt32 getCompressedHeaderSize(UInt8 data_bytes_size) -{ - const UInt8 items_count_size = 4; - - return items_count_size + data_bytes_size + getDeltaTypeByteSize(data_bytes_size); -} - -UInt32 getCompressedDataSize(UInt8 data_bytes_size, UInt32 uncompressed_size) -{ - const UInt32 items_count = uncompressed_size / data_bytes_size; - - // 11111 + max 64 bits of double delta. - const UInt32 max_item_size_bits = 5 + getDeltaTypeByteSize(data_bytes_size) * 8; - - // + 8 is to round up to next byte. - return (items_count * max_item_size_bits + 8) / 8; + switch (byte_size) + { + case sizeof(UInt8): + return std::numeric_limits::max(); + case sizeof(UInt16): + return std::numeric_limits::max(); + case sizeof(UInt32): + return std::numeric_limits::max(); + case sizeof(UInt64): + return std::numeric_limits::max(); + default: + assert(false && "only 1, 2, 4 and 8 data sizes are supported"); + } + __builtin_unreachable(); } struct WriteSpec @@ -55,8 +51,10 @@ struct WriteSpec const UInt8 data_bits; }; +const std::array DELTA_SIZES{7, 9, 12, 32, 64}; + template -WriteSpec getWriteSpec(const T & value) +WriteSpec getDeltaWriteSpec(const T & value) { if (value > -63 && value < 64) { @@ -80,27 +78,60 @@ WriteSpec getWriteSpec(const T & value) } } -template +WriteSpec getDeltaMaxWriteSpecByteSize(UInt8 data_bytes_size) +{ + return getDeltaWriteSpec(getMaxValueForByteSize(data_bytes_size)); +} + +UInt32 getCompressedHeaderSize(UInt8 data_bytes_size) +{ + const UInt8 items_count_size = 4; + const UInt8 first_delta_bytes_size = data_bytes_size; + + return items_count_size + data_bytes_size + first_delta_bytes_size; +} + +UInt32 getCompressedDataSize(UInt8 data_bytes_size, UInt32 uncompressed_size) +{ + const UInt32 items_count = uncompressed_size / data_bytes_size; + const auto double_delta_write_spec = getDeltaMaxWriteSpecByteSize(data_bytes_size); + + const UInt32 max_item_size_bits = double_delta_write_spec.prefix_bits + double_delta_write_spec.data_bits; + + // + 8 is to round up to next byte. + auto result = (items_count * max_item_size_bits + 7) / 8; + + return result; +} + +template UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest) { - static_assert(std::is_unsigned_v && std::is_signed_v, "T must be unsigned, while DeltaType must be signed integer type."); - using UnsignedDeltaType = typename std::make_unsigned::type; + // Since only unsinged int has granted 2-compliment overflow handling, we are doing math here on unsigned types. + // To simplify and booletproof code, we operate enforce ValueType to be unsigned too. + static_assert(std::is_unsigned_v, "ValueType must be unsigned."); + using UnsignedDeltaType = ValueType; - if (source_size % sizeof(T) != 0) - throw Exception("Cannot compress, data size " + toString(source_size) + " is not aligned to " + toString(sizeof(T)), ErrorCodes::CANNOT_COMPRESS); + // We use signed delta type to turn huge unsigned values into smaller signed: + // ffffffff => -1 + using SignedDeltaType = typename std::make_signed::type; + + if (source_size % sizeof(ValueType) != 0) + throw Exception("Cannot compress, data size " + toString(source_size) + + " is not aligned to " + toString(sizeof(ValueType)), ErrorCodes::CANNOT_COMPRESS); const char * source_end = source + source_size; - const UInt32 items_count = source_size / sizeof(T); + const UInt32 items_count = source_size / sizeof(ValueType); unalignedStore(dest, items_count); dest += sizeof(items_count); - T prev_value{}; - DeltaType prev_delta{}; + ValueType prev_value{}; + UnsignedDeltaType prev_delta{}; if (source < source_end) { - prev_value = unalignedLoad(source); - unalignedStore(dest, prev_value); + prev_value = unalignedLoad(source); + unalignedStore(dest, prev_value); source += sizeof(prev_value); dest += sizeof(prev_value); @@ -108,24 +139,26 @@ UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest) if (source < source_end) { - const T curr_value = unalignedLoad(source); - prev_delta = static_cast(curr_value - prev_value); - unalignedStore(dest, prev_delta); + const ValueType curr_value = unalignedLoad(source); + + prev_delta = curr_value - prev_value; + unalignedStore(dest, prev_delta); source += sizeof(curr_value); dest += sizeof(prev_delta); prev_value = curr_value; } - WriteBuffer buffer(dest, getCompressedDataSize(sizeof(T), source_size - sizeof(T)*2)); + WriteBuffer buffer(dest, getCompressedDataSize(sizeof(ValueType), source_size - sizeof(ValueType)*2)); BitWriter writer(buffer); - for (; source < source_end; source += sizeof(T)) + int item = 2; + for (; source < source_end; source += sizeof(ValueType), ++item) { - const T curr_value = unalignedLoad(source); + const ValueType curr_value = unalignedLoad(source); - const DeltaType delta = static_cast(curr_value - prev_value); - const DeltaType double_delta = delta - prev_delta; + const UnsignedDeltaType delta = curr_value - prev_value; + const UnsignedDeltaType double_delta = delta - prev_delta; prev_delta = delta; prev_value = curr_value; @@ -136,9 +169,11 @@ UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest) } else { - const auto sign = std::signbit(double_delta); - const auto abs_value = static_cast(std::abs(double_delta)); - const auto write_spec = getWriteSpec(double_delta); + const SignedDeltaType signed_dd = static_cast(double_delta); + const auto sign = std::signbit(signed_dd); + // -1 shirnks dd down to fit into number of bits, and there can't be 0, so it is OK. + const auto abs_value = static_cast(std::abs(signed_dd) - 1); + const auto write_spec = getDeltaWriteSpec(signed_dd); writer.writeBits(write_spec.prefix_bits, write_spec.prefix); writer.writeBits(1, sign); @@ -151,22 +186,25 @@ UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest) return sizeof(items_count) + sizeof(prev_value) + sizeof(prev_delta) + buffer.count(); } -template +template void decompressDataForType(const char * source, UInt32 source_size, char * dest) { - static_assert(std::is_unsigned_v && std::is_signed_v, "T must be unsigned, while DeltaType must be signed integer type."); + static_assert(std::is_unsigned_v, "ValueType must be unsigned."); + using UnsignedDeltaType = ValueType; + using SignedDeltaType = typename std::make_signed::type; + const char * source_end = source + source_size; const UInt32 items_count = unalignedLoad(source); source += sizeof(items_count); - T prev_value{}; - DeltaType prev_delta{}; + ValueType prev_value{}; + UnsignedDeltaType prev_delta{}; if (source < source_end) { - prev_value = unalignedLoad(source); - unalignedStore(dest, prev_value); + prev_value = unalignedLoad(source); + unalignedStore(dest, prev_value); source += sizeof(prev_value); dest += sizeof(prev_value); @@ -174,9 +212,9 @@ void decompressDataForType(const char * source, UInt32 source_size, char * dest) if (source < source_end) { - prev_delta = unalignedLoad(source); - prev_value = prev_value + static_cast(prev_delta); - unalignedStore(dest, prev_value); + prev_delta = unalignedLoad(source); + prev_value = prev_value + static_cast(prev_delta); + unalignedStore(dest, prev_value); source += sizeof(prev_delta); dest += sizeof(prev_value); @@ -189,32 +227,35 @@ void decompressDataForType(const char * source, UInt32 source_size, char * dest) // we have to keep track of items to avoid reading more that there is. for (UInt32 items_read = 2; items_read < items_count && !reader.eof(); ++items_read) { - DeltaType double_delta = 0; + UnsignedDeltaType double_delta = 0; if (reader.readBit() == 1) { - const UInt8 data_sizes[] = {6, 8, 11, 31, 63}; UInt8 i = 0; - for (; i < sizeof(data_sizes) - 1; ++i) + for (; i < sizeof(DELTA_SIZES) - 1; ++i) { const auto next_bit = reader.readBit(); if (next_bit == 0) + { break; + } } const UInt8 sign = reader.readBit(); - double_delta = static_cast(reader.readBits(data_sizes[i])); + SignedDeltaType signed_dd = static_cast(reader.readBits(DELTA_SIZES[i] - 1) + 1); if (sign) { - double_delta *= -1; + signed_dd *= -1; } + double_delta = static_cast(signed_dd); } // else if first bit is zero, no need to read more data. - const T curr_value = prev_value + static_cast(prev_delta + double_delta); - unalignedStore(dest, curr_value); + const UnsignedDeltaType delta = double_delta + prev_delta; + const ValueType curr_value = prev_value + delta; + unalignedStore(dest, curr_value); dest += sizeof(curr_value); - prev_delta = static_cast(curr_value - prev_value); + prev_delta = curr_value - prev_value; prev_value = curr_value; } } @@ -267,19 +308,20 @@ UInt32 CompressionCodecDoubleDelta::doCompressData(const char * source, UInt32 s memcpy(&dest[2], source, bytes_to_skip); size_t start_pos = 2 + bytes_to_skip; UInt32 compressed_size = 0; + switch (data_bytes_size) { case 1: - compressed_size = compressDataForType(&source[bytes_to_skip], source_size - bytes_to_skip, &dest[start_pos]); + compressed_size = compressDataForType(&source[bytes_to_skip], source_size - bytes_to_skip, &dest[start_pos]); break; case 2: - compressed_size = compressDataForType(&source[bytes_to_skip], source_size - bytes_to_skip, &dest[start_pos]); + compressed_size = compressDataForType(&source[bytes_to_skip], source_size - bytes_to_skip, &dest[start_pos]); break; case 4: - compressed_size = compressDataForType(&source[bytes_to_skip], source_size - bytes_to_skip, &dest[start_pos]); + compressed_size = compressDataForType(&source[bytes_to_skip], source_size - bytes_to_skip, &dest[start_pos]); break; case 8: - compressed_size = compressDataForType(&source[bytes_to_skip], source_size - bytes_to_skip, &dest[start_pos]); + compressed_size = compressDataForType(&source[bytes_to_skip], source_size - bytes_to_skip, &dest[start_pos]); break; } @@ -296,16 +338,16 @@ void CompressionCodecDoubleDelta::doDecompressData(const char * source, UInt32 s switch (bytes_size) { case 1: - decompressDataForType(&source[2 + bytes_to_skip], source_size_no_header, &dest[bytes_to_skip]); + decompressDataForType(&source[2 + bytes_to_skip], source_size_no_header, &dest[bytes_to_skip]); break; case 2: - decompressDataForType(&source[2 + bytes_to_skip], source_size_no_header, &dest[bytes_to_skip]); + decompressDataForType(&source[2 + bytes_to_skip], source_size_no_header, &dest[bytes_to_skip]); break; case 4: - decompressDataForType(&source[2 + bytes_to_skip], source_size_no_header, &dest[bytes_to_skip]); + decompressDataForType(&source[2 + bytes_to_skip], source_size_no_header, &dest[bytes_to_skip]); break; case 8: - decompressDataForType(&source[2 + bytes_to_skip], source_size_no_header, &dest[bytes_to_skip]); + decompressDataForType(&source[2 + bytes_to_skip], source_size_no_header, &dest[bytes_to_skip]); break; } } diff --git a/dbms/src/Compression/LZ4_decompress_faster.cpp b/dbms/src/Compression/LZ4_decompress_faster.cpp index 0d65a06b098..65ffdb2173b 100644 --- a/dbms/src/Compression/LZ4_decompress_faster.cpp +++ b/dbms/src/Compression/LZ4_decompress_faster.cpp @@ -154,7 +154,7 @@ inline void copyOverlap8(UInt8 * op, const UInt8 *& match, const size_t offset) */ inline void copyOverlap8Shuffle(UInt8 * op, const UInt8 *& match, const size_t offset) { -#ifdef __SSSE3__ +#if defined(__SSSE3__) && !defined(MEMORY_SANITIZER) static constexpr UInt8 __attribute__((__aligned__(8))) masks[] = { @@ -268,7 +268,7 @@ inline void copyOverlap16(UInt8 * op, const UInt8 *& match, const size_t offset) inline void copyOverlap16Shuffle(UInt8 * op, const UInt8 *& match, const size_t offset) { -#ifdef __SSSE3__ +#if defined(__SSSE3__) && !defined(MEMORY_SANITIZER) static constexpr UInt8 __attribute__((__aligned__(16))) masks[] = { diff --git a/dbms/src/Compression/tests/CMakeLists.txt b/dbms/src/Compression/tests/CMakeLists.txt index 50f212f18c3..3cfc0ccb7dc 100644 --- a/dbms/src/Compression/tests/CMakeLists.txt +++ b/dbms/src/Compression/tests/CMakeLists.txt @@ -1,5 +1,5 @@ add_executable (compressed_buffer compressed_buffer.cpp) -target_link_libraries (compressed_buffer PRIVATE clickhouse_compression clickhouse_common_io) +target_link_libraries (compressed_buffer PRIVATE dbms) add_executable (cached_compressed_read_buffer cached_compressed_read_buffer.cpp) -target_link_libraries (cached_compressed_read_buffer PRIVATE clickhouse_compression clickhouse_common_io) +target_link_libraries (cached_compressed_read_buffer PRIVATE dbms) diff --git a/dbms/src/Compression/tests/gtest_compressionCodec.cpp b/dbms/src/Compression/tests/gtest_compressionCodec.cpp index e1413ccd7bd..2b2f6927ed4 100644 --- a/dbms/src/Compression/tests/gtest_compressionCodec.cpp +++ b/dbms/src/Compression/tests/gtest_compressionCodec.cpp @@ -8,23 +8,18 @@ #include +#include #include #include #include -#include -#include -#include -#include -#include #include -#include +#include +#include +#include +#include + #include -#pragma GCC diagnostic ignored "-Wsign-compare" -#ifdef __clang__ -#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" -#pragma clang diagnostic ignored "-Wundef" -#endif #include using namespace DB; @@ -119,36 +114,71 @@ template result = ::testing::AssertionFailure(); } - result << "mismatching " << sizeof(T) << "-byte item #" << i - << "\nexpected: " << bin(left_value) - << "\ngot : " << bin(right_value) - << std::endl; - - if (++mismatching_items >= MAX_MISMATCHING_ITEMS) + if (++mismatching_items <= MAX_MISMATCHING_ITEMS) { - result << "..." << std::endl; - break; + result << "mismatching " << sizeof(T) << "-byte item #" << i + << "\nexpected: " << bin(left_value) << " (0x" << std::hex << left_value << ")" + << "\ngot : " << bin(right_value) << " (0x" << std::hex << right_value << ")" + << std::endl; + if (mismatching_items == MAX_MISMATCHING_ITEMS) + { + result << "..." << std::endl; + } } } } + if (mismatching_items > 0) + { + result << "\ntotal mismatching items:" << mismatching_items << " of " << size; + } return result; } struct CodecTestParam { + std::string type_name; std::vector source_data; UInt8 data_byte_size; + double min_compression_ratio; std::string case_name; + + // to allow setting ratio after building with complex builder functions. + CodecTestParam && setRatio(const double & ratio) && + { + this->min_compression_ratio = ratio; + return std::move(*this); + } }; +CodecTestParam operator+(CodecTestParam && left, CodecTestParam && right) +{ + assert(left.type_name == right.type_name); + assert(left.data_byte_size == right.data_byte_size); + + std::vector data(std::move(left.source_data)); + data.insert(data.end(), right.source_data.begin(), right.source_data.end()); + + return CodecTestParam{ + left.type_name, + std::move(data), + left.data_byte_size, + std::min(left.min_compression_ratio, right.min_compression_ratio), + left.case_name + " + " + right.case_name + }; +} + std::ostream & operator<<(std::ostream & ostr, const CodecTestParam & param) { return ostr << "name: " << param.case_name + << "\ntype name:" << param.type_name << "\nbyte size: " << static_cast(param.data_byte_size) << "\ndata size: " << param.source_data.size(); } +// compression ratio < 1.0 means that codec output is smaller than input. +const double DEFAULT_MIN_COMPRESSION_RATIO = 1.0; + template CodecTestParam makeParam(Args && ... args) { @@ -162,11 +192,11 @@ CodecTestParam makeParam(Args && ... args) write_pos += sizeof(v); } - return CodecTestParam{std::move(data), sizeof(T), - (boost::format("%1% %2%") % (sizeof(T) * std::size(vals)) % " predefined values").str()}; + return CodecTestParam{type_name(), std::move(data), sizeof(T), DEFAULT_MIN_COMPRESSION_RATIO, + (boost::format("%1% values of %2%") % std::size(vals) % type_name()).str()}; } -template +template CodecTestParam generateParam(Generator gen, const char* gen_name) { static_assert (End >= Begin, "End must be not less than Begin"); @@ -181,8 +211,8 @@ CodecTestParam generateParam(Generator gen, const char* gen_name) write_pos += sizeof(v); } - return CodecTestParam{std::move(data), sizeof(T), - (boost::format("%1% from %2% (%3% => %4%)") % type_name() % gen_name % Begin % End).str()}; + return CodecTestParam{type_name(), std::move(data), sizeof(T), DEFAULT_MIN_COMPRESSION_RATIO, + (boost::format("%1% values of %2% from %3%") % (End - Begin) % type_name() % gen_name).str()}; } void TestTranscoding(ICompressionCodec * codec, const CodecTestParam & param) @@ -216,6 +246,13 @@ void TestTranscoding(ICompressionCodec * codec, const CodecTestParam & param) default: FAIL() << "Invalid data_byte_size: " << param.data_byte_size; } + const auto header_size = codec->getHeaderSize(); + const auto compression_ratio = (encoded_size - header_size) / (source_data.size() * 1.0); + + ASSERT_LE(compression_ratio, param.min_compression_ratio) + << "\n\tdecoded size: " << source_data.size() + << "\n\tencoded size: " << encoded_size + << "(no header: " << encoded_size - header_size << ")"; } class CodecTest : public ::testing::TestWithParam @@ -230,20 +267,34 @@ public: TEST_P(CodecTest, DoubleDelta) { - const auto & param = GetParam(); + auto param = GetParam(); auto codec = std::make_unique(param.data_byte_size); + if (param.type_name == type_name() || param.type_name == type_name()) + { + // dd doesn't work great with many cases of integers and may result in very poor compression rate. + param.min_compression_ratio *= 1.5; + } TestTranscoding(codec.get(), param); } TEST_P(CodecTest, Gorilla) { - const auto & param = GetParam(); + auto param = GetParam(); auto codec = std::make_unique(param.data_byte_size); + if (param.type_name == type_name() || param.type_name == type_name() + || param.type_name == type_name() || param.type_name == type_name()) + { + // gorilla doesn't work great with many cases of integers and may result in very poor compression rate. + param.min_compression_ratio *= 1.5; + } TestTranscoding(codec.get(), param); } +// Here we use generators to produce test payload for codecs. +// Generator is a callable that should produce output value of the same type as input value. + auto SameValueGenerator = [](auto value) { return [=](auto i) @@ -261,30 +312,44 @@ auto SequentialGenerator = [](auto stride = 1) }; }; +// Generator that helps debugging output of other generators +// by logging every output value alongside iteration index and input. +//auto LoggingProxyGenerator = [](auto other_generator, const char * name, std::ostream & ostr, const int limit = std::numeric_limits::max()) +//{ +// ostr << "\n\nValues from " << name << ":\n"; +// auto count = std::make_shared(0); +// return [&, count](auto i) +// { +// using ValueType = decltype(i); +// const auto ret = static_cast(other_generator(i)); +// if (++(*count) < limit) +// { +// ostr << "\t" << *count << " : " << i << " => " << ret << "\n"; +// } + +// return ret; +// }; +//}; + template struct MonotonicGenerator { MonotonicGenerator(T stride = 1, size_t max_step = 10) - : prev_value{}, + : prev_value(0), stride(stride), max_step(max_step) {} template - U operator()(U i) + U operator()(U) { - if (!prev_value.has_value()) - { - prev_value = i * stride; - } - - const U result = *prev_value + static_cast(stride * (rand() % max_step)); + const U result = prev_value + static_cast(stride * (rand() % max_step)); prev_value = result; return result; } - std::optional prev_value; + T prev_value; const T stride; const size_t max_step; }; @@ -301,25 +366,45 @@ auto MinMaxGenerator = [](auto i) } }; -auto RandomGenerator = [](auto i) {return static_cast(rand());}; +template +struct RandomGenerator +{ + RandomGenerator(T seed = 0, T value_cap = std::numeric_limits::max()) + : e(seed), + value_cap(value_cap) + { + } + + template + U operator()(U i) + { + return static_cast(distribution(e) % value_cap); + } + +private: + std::default_random_engine e; + std::uniform_int_distribution distribution; + const T value_cap; +}; auto RandomishGenerator = [](auto i) { return static_cast(sin(static_cast(i) * i) * i); }; -INSTANTIATE_TEST_CASE_P(Basic, +// helper macro to produce human-friendly test case name +#define G(generator) generator, #generator + +INSTANTIATE_TEST_CASE_P(Mixed, CodecTest, ::testing::Values( - makeParam(1, 2, 3, 4), - makeParam(1, 2, 3, 4), - makeParam(1.1, 2.2, 3.3, 4.4), - makeParam(1.1, 2.2, 3.3, 4.4) + generateParam(G(MinMaxGenerator)) + generateParam(G(SequentialGenerator(1))).setRatio(1), + generateParam(G(MinMaxGenerator)) + generateParam(G(SequentialGenerator(1))).setRatio(1), + generateParam(G(MinMaxGenerator)) + generateParam(G(SequentialGenerator(1))).setRatio(1), + generateParam(G(MinMaxGenerator)) + generateParam(G(SequentialGenerator(1))).setRatio(1) ), ); -#define G(generator) generator, #generator - INSTANTIATE_TEST_CASE_P(Same, CodecTest, ::testing::Values( @@ -359,18 +444,20 @@ INSTANTIATE_TEST_CASE_P(Monotonic, INSTANTIATE_TEST_CASE_P(Random, CodecTest, ::testing::Values( - generateParam(G(RandomGenerator)), - generateParam(G(RandomGenerator)) + generateParam(G(RandomGenerator(0, 1000'000'000))).setRatio(1.2), + generateParam(G(RandomGenerator(0, 1000'000'000))).setRatio(1.1) ), ); -INSTANTIATE_TEST_CASE_P(RandomLike, +INSTANTIATE_TEST_CASE_P(Randomish, CodecTest, ::testing::Values( - generateParam(G(RandomishGenerator)), - generateParam(G(RandomishGenerator)), - generateParam(G(RandomishGenerator)), - generateParam(G(RandomishGenerator)) + generateParam(G(RandomishGenerator)).setRatio(1.1), + generateParam(G(RandomishGenerator)).setRatio(1.1), + generateParam(G(RandomishGenerator)).setRatio(1.1), + generateParam(G(RandomishGenerator)).setRatio(1.1), + generateParam(G(RandomishGenerator)).setRatio(1.1), + generateParam(G(RandomishGenerator)).setRatio(1.1) ), ); diff --git a/dbms/src/Core/Block.cpp b/dbms/src/Core/Block.cpp index 89aaf2e806c..9f8b66b1778 100644 --- a/dbms/src/Core/Block.cpp +++ b/dbms/src/Core/Block.cpp @@ -370,6 +370,11 @@ Block Block::cloneWithColumns(const Columns & columns) const Block res; size_t num_columns = data.size(); + + if (num_columns != columns.size()) + throw Exception("Cannot clone block with columns because block has " + toString(num_columns) + " columns, " + "but " + toString(columns.size()) + " columns given.", ErrorCodes::LOGICAL_ERROR); + for (size_t i = 0; i < num_columns; ++i) res.insert({ columns[i], data[i].type, data[i].name }); diff --git a/dbms/src/Core/Defines.h b/dbms/src/Core/Defines.h index bb39c495087..1214b742803 100644 --- a/dbms/src/Core/Defines.h +++ b/dbms/src/Core/Defines.h @@ -56,7 +56,7 @@ #define DBMS_MIN_REVISION_WITH_LOW_CARDINALITY_TYPE 54405 -#define DBMS_MIN_REVISION_WITH_CLIENT_WRITE_INFO 54421 +#define DBMS_MIN_REVISION_WITH_CLIENT_WRITE_INFO 54420 /// Version of ClickHouse TCP protocol. Set to git tag with latest protocol change. #define DBMS_TCP_PROTOCOL_VERSION 54226 @@ -108,6 +108,14 @@ #define THREAD_SANITIZER 1 #endif +#if defined(__has_feature) + #if __has_feature(memory_sanitizer) + #define MEMORY_SANITIZER 1 + #endif +#elif defined(__MEMORY_SANITIZER__) + #define MEMORY_SANITIZER 1 +#endif + /// Explicitly allow undefined behaviour for certain functions. Use it as a function attribute. /// It is useful in case when compiler cannot see (and exploit) it, but UBSan can. /// Example: multiplication of signed integers with possibility of overflow when both sides are from user input. @@ -131,3 +139,7 @@ /// This number is only used for distributed version compatible. /// It could be any magic number. #define DBMS_DISTRIBUTED_SENDS_MAGIC_NUMBER 0xCAFECABE + +/// A macro for suppressing warnings about unused variables or function results. +/// Useful for structured bindings which have no standard way to declare this. +#define UNUSED(X) (void) (X) diff --git a/dbms/src/Core/ExternalTable.cpp b/dbms/src/Core/ExternalTable.cpp index 65bff362aa7..e1e059a3b63 100644 --- a/dbms/src/Core/ExternalTable.cpp +++ b/dbms/src/Core/ExternalTable.cpp @@ -160,7 +160,7 @@ void ExternalTablesHandler::handlePart(const Poco::Net::MessageHeader & header, /// Create table NamesAndTypesList columns = sample_block.getNamesAndTypesList(); - StoragePtr storage = StorageMemory::create(data.second, ColumnsDescription{columns}); + StoragePtr storage = StorageMemory::create("_external", data.second, ColumnsDescription{columns}); storage->startup(); context.addExternalTable(data.second, storage); BlockOutputStreamPtr output = storage->write(ASTPtr(), context); diff --git a/dbms/src/Core/Field.h b/dbms/src/Core/Field.h index 832b45cd6f8..156f272312b 100644 --- a/dbms/src/Core/Field.h +++ b/dbms/src/Core/Field.h @@ -717,8 +717,8 @@ class WriteBuffer; /// It is assumed that all elements of the array have the same type. void readBinary(Array & x, ReadBuffer & buf); -inline void readText(Array &, ReadBuffer &) { throw Exception("Cannot read Array.", ErrorCodes::NOT_IMPLEMENTED); } -inline void readQuoted(Array &, ReadBuffer &) { throw Exception("Cannot read Array.", ErrorCodes::NOT_IMPLEMENTED); } +[[noreturn]] inline void readText(Array &, ReadBuffer &) { throw Exception("Cannot read Array.", ErrorCodes::NOT_IMPLEMENTED); } +[[noreturn]] inline void readQuoted(Array &, ReadBuffer &) { throw Exception("Cannot read Array.", ErrorCodes::NOT_IMPLEMENTED); } /// It is assumed that all elements of the array have the same type. /// Also write size and type into buf. UInt64 and Int64 is written in variadic size form @@ -726,16 +726,16 @@ void writeBinary(const Array & x, WriteBuffer & buf); void writeText(const Array & x, WriteBuffer & buf); -inline void writeQuoted(const Array &, WriteBuffer &) { throw Exception("Cannot write Array quoted.", ErrorCodes::NOT_IMPLEMENTED); } +[[noreturn]] inline void writeQuoted(const Array &, WriteBuffer &) { throw Exception("Cannot write Array quoted.", ErrorCodes::NOT_IMPLEMENTED); } void readBinary(Tuple & x, ReadBuffer & buf); -inline void readText(Tuple &, ReadBuffer &) { throw Exception("Cannot read Tuple.", ErrorCodes::NOT_IMPLEMENTED); } -inline void readQuoted(Tuple &, ReadBuffer &) { throw Exception("Cannot read Tuple.", ErrorCodes::NOT_IMPLEMENTED); } +[[noreturn]] inline void readText(Tuple &, ReadBuffer &) { throw Exception("Cannot read Tuple.", ErrorCodes::NOT_IMPLEMENTED); } +[[noreturn]] inline void readQuoted(Tuple &, ReadBuffer &) { throw Exception("Cannot read Tuple.", ErrorCodes::NOT_IMPLEMENTED); } void writeBinary(const Tuple & x, WriteBuffer & buf); void writeText(const Tuple & x, WriteBuffer & buf); -inline void writeQuoted(const Tuple &, WriteBuffer &) { throw Exception("Cannot write Tuple quoted.", ErrorCodes::NOT_IMPLEMENTED); } +[[noreturn]] inline void writeQuoted(const Tuple &, WriteBuffer &) { throw Exception("Cannot write Tuple quoted.", ErrorCodes::NOT_IMPLEMENTED); } } diff --git a/dbms/src/Core/MySQLProtocol.cpp b/dbms/src/Core/MySQLProtocol.cpp index f727ae3df54..529aac27074 100644 --- a/dbms/src/Core/MySQLProtocol.cpp +++ b/dbms/src/Core/MySQLProtocol.cpp @@ -30,7 +30,7 @@ String PacketSender::packetToText(String payload) uint64_t readLengthEncodedNumber(std::istringstream & ss) { - char c; + char c{}; uint64_t buf = 0; ss.get(c); auto cc = static_cast(c); diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index 1a5408f414f..4df07df60f6 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -126,12 +126,14 @@ struct Settings : public SettingsCollection M(SettingUInt64, mark_cache_min_lifetime, 10000, "If the maximum size of mark_cache is exceeded, delete only records older than mark_cache_min_lifetime seconds.") \ \ M(SettingFloat, max_streams_to_max_threads_ratio, 1, "Allows you to use more sources than the number of threads - to more evenly distribute work across threads. It is assumed that this is a temporary solution, since it will be possible in the future to make the number of sources equal to the number of threads, but for each source to dynamically select available work for itself.") \ + M(SettingFloat, max_streams_multiplier_for_merge_tables, 5, "Ask more streams when reading from Merge table. Streams will be spread across tables that Merge table will use. This allows more even distribution of work across threads and especially helpful when merged tables differ in size.") \ \ M(SettingString, network_compression_method, "LZ4", "Allows you to select the method of data compression when writing.") \ \ M(SettingInt64, network_zstd_compression_level, 1, "Allows you to select the level of ZSTD compression.") \ \ M(SettingUInt64, priority, 0, "Priority of the query. 1 - the highest, higher value - lower priority; 0 - do not use priorities.") \ + M(SettingInt64, os_thread_priority, 0, "If non zero - set corresponding 'nice' value for query processing threads. Can be used to adjust query priority for OS scheduler.") \ \ M(SettingBool, log_queries, 0, "Log requests and write the log to the system table.") \ \ @@ -168,7 +170,7 @@ struct Settings : public SettingsCollection M(SettingBool, input_format_skip_unknown_fields, false, "Skip columns with unknown names from input data (it works for JSONEachRow, CSVWithNames, TSVWithNames and TSKV formats).") \ M(SettingBool, input_format_with_names_use_header, false, "For TSVWithNames and CSVWithNames input formats this controls whether format parser is to assume that column data appear in the input exactly as they are specified in the header.") \ M(SettingBool, input_format_import_nested_json, false, "Map nested JSON data to nested tables (it works for JSONEachRow format).") \ - M(SettingBool, input_format_defaults_for_omitted_fields, false, "For input data calculate default expressions for omitted fields (it works for JSONEachRow format).") \ + M(SettingBool, input_format_defaults_for_omitted_fields, true, "For input data calculate default expressions for omitted fields (it works for JSONEachRow format).") \ \ M(SettingBool, input_format_values_interpret_expressions, true, "For Values format: if field could not be parsed by streaming parser, run SQL parser and try to interpret it as SQL expression.") \ \ @@ -324,10 +326,17 @@ struct Settings : public SettingsCollection M(SettingBool, external_table_functions_use_nulls, true, "If it is set to true, external table functions will implicitly use Nullable type if needed. Otherwise NULLs will be substituted with default values. Currently supported only for 'mysql' table function.") \ M(SettingBool, allow_experimental_data_skipping_indices, false, "If it is set to true, data skipping indices can be used in CREATE TABLE/ALTER TABLE queries.") \ \ - M(SettingBool, allow_hyperscan, 1, "Allow functions that use Hyperscan library. Disable to avoid potentially long compilation times and excessive resource usage.") \ - M(SettingBool, allow_simdjson, 1, "Allow using simdjson library in 'JSON*' functions if AVX2 instructions are available. If disabled rapidjson will be used.") \ + M(SettingBool, experimental_use_processors, false, "Use processors pipeline.") \ \ - M(SettingUInt64, max_partitions_per_insert_block, 100, "Limit maximum number of partitions in single INSERTed block. Zero means unlimited. Throw exception if the block contains too many partitions. This setting is a safety threshold, because using large number of partitions is a common misconception.") + M(SettingBool, allow_hyperscan, true, "Allow functions that use Hyperscan library. Disable to avoid potentially long compilation times and excessive resource usage.") \ + M(SettingBool, allow_simdjson, true, "Allow using simdjson library in 'JSON*' functions if AVX2 instructions are available. If disabled rapidjson will be used.") \ + \ + M(SettingUInt64, max_partitions_per_insert_block, 100, "Limit maximum number of partitions in single INSERTed block. Zero means unlimited. Throw exception if the block contains too many partitions. This setting is a safety threshold, because using large number of partitions is a common misconception.") \ + M(SettingBool, check_query_single_value_result, true, "Return check query result as single 1/0 value") \ + \ + /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \ + \ + M(SettingBool, allow_experimental_low_cardinality_type, true, "Obsolete setting, does nothing. Will be removed after 2019-08-13") DECLARE_SETTINGS_COLLECTION(LIST_OF_SETTINGS) diff --git a/dbms/src/Core/SettingsCommon.h b/dbms/src/Core/SettingsCommon.h index b30b5b50c68..08064096133 100644 --- a/dbms/src/Core/SettingsCommon.h +++ b/dbms/src/Core/SettingsCommon.h @@ -275,7 +275,7 @@ namespace details { static void serializeName(const StringRef & name, WriteBuffer & buf); static String deserializeName(ReadBuffer & buf); - static void throwNameNotFound(const StringRef & name); + [[noreturn]] static void throwNameNotFound(const StringRef & name); }; } diff --git a/dbms/src/Core/SortCursor.h b/dbms/src/Core/SortCursor.h index 5a49209cb71..3a2b64cd8b6 100644 --- a/dbms/src/Core/SortCursor.h +++ b/dbms/src/Core/SortCursor.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -47,26 +48,44 @@ struct SortCursorImpl reset(block); } + SortCursorImpl(const Columns & columns, const SortDescription & desc_, size_t order_ = 0) + : desc(desc_), sort_columns_size(desc.size()), order(order_), need_collation(desc.size()) + { + for (auto & column_desc : desc) + { + if (!column_desc.column_name.empty()) + throw Exception("SortDesctiption should contain column position if SortCursor was used without header.", + ErrorCodes::LOGICAL_ERROR); + } + reset(columns, {}); + } + bool empty() const { return rows == 0; } /// Set the cursor to the beginning of the new block. void reset(const Block & block) + { + reset(block.getColumns(), block); + } + + /// Set the cursor to the beginning of the new block. + void reset(const Columns & columns, const Block & block) { all_columns.clear(); sort_columns.clear(); - size_t num_columns = block.columns(); + size_t num_columns = columns.size(); for (size_t j = 0; j < num_columns; ++j) - all_columns.push_back(block.safeGetByPosition(j).column.get()); + all_columns.push_back(columns[j].get()); for (size_t j = 0, size = desc.size(); j < size; ++j) { - size_t column_number = !desc[j].column_name.empty() - ? block.getPositionByName(desc[j].column_name) - : desc[j].column_number; - - sort_columns.push_back(block.safeGetByPosition(column_number).column.get()); + auto & column_desc = desc[j]; + size_t column_number = !column_desc.column_name.empty() + ? block.getPositionByName(column_desc.column_name) + : column_desc.column_number; + sort_columns.push_back(columns[column_number].get()); need_collation[j] = desc[j].collator != nullptr && typeid_cast(sort_columns.back()); /// TODO Nullable(String) has_collation |= need_collation[j]; diff --git a/dbms/src/DataStreams/BlockIO.h b/dbms/src/DataStreams/BlockIO.h index 913dbdbe5c7..e7c1d41e845 100644 --- a/dbms/src/DataStreams/BlockIO.h +++ b/dbms/src/DataStreams/BlockIO.h @@ -4,6 +4,8 @@ #include +#include + namespace DB { @@ -25,6 +27,8 @@ struct BlockIO BlockOutputStreamPtr out; BlockInputStreamPtr in; + QueryPipeline pipeline; + /// Callbacks for query logging could be set here. std::function finish_callback; std::function exception_callback; @@ -54,6 +58,7 @@ struct BlockIO process_list_entry = rhs.process_list_entry; in = rhs.in; out = rhs.out; + pipeline = rhs.pipeline; finish_callback = rhs.finish_callback; exception_callback = rhs.exception_callback; diff --git a/dbms/src/DataStreams/BlockStreamProfileInfo.cpp b/dbms/src/DataStreams/BlockStreamProfileInfo.cpp index 0277c19ae42..5f83f419861 100644 --- a/dbms/src/DataStreams/BlockStreamProfileInfo.cpp +++ b/dbms/src/DataStreams/BlockStreamProfileInfo.cpp @@ -71,6 +71,9 @@ void BlockStreamProfileInfo::update(Block & block) void BlockStreamProfileInfo::collectInfosForStreamsWithName(const char * name, BlockStreamProfileInfos & res) const { + if (!parent) + return; + if (parent->getName() == name) { res.push_back(this); diff --git a/dbms/src/DataStreams/BlockStreamProfileInfo.h b/dbms/src/DataStreams/BlockStreamProfileInfo.h index ebc83204451..6b60fa95e24 100644 --- a/dbms/src/DataStreams/BlockStreamProfileInfo.h +++ b/dbms/src/DataStreams/BlockStreamProfileInfo.h @@ -50,6 +50,13 @@ struct BlockStreamProfileInfo /// If skip_block_size_info if true, then rows, bytes and block fields are ignored. void setFrom(const BlockStreamProfileInfo & rhs, bool skip_block_size_info); + /// Only for Processors. + void setRowsBeforeLimit(size_t rows_before_limit_) + { + applied_limit = true; + rows_before_limit = rows_before_limit_; + } + private: void calculateRowsBeforeLimit() const; diff --git a/dbms/src/DataStreams/InputStreamFromASTInsertQuery.cpp b/dbms/src/DataStreams/InputStreamFromASTInsertQuery.cpp index aa3faaede28..d8f536c6ace 100644 --- a/dbms/src/DataStreams/InputStreamFromASTInsertQuery.cpp +++ b/dbms/src/DataStreams/InputStreamFromASTInsertQuery.cpp @@ -51,7 +51,7 @@ InputStreamFromASTInsertQuery::InputStreamFromASTInsertQuery( res_stream = context.getInputFormat(format, *input_buffer_contacenated, header, context.getSettings().max_insert_block_size); - if (context.getSettingsRef().input_format_defaults_for_omitted_fields) + if (context.getSettingsRef().input_format_defaults_for_omitted_fields && !ast_insert_query->table.empty()) { StoragePtr storage = context.getTable(ast_insert_query->database, ast_insert_query->table); auto column_defaults = storage->getColumns().getDefaults(); diff --git a/dbms/src/DataStreams/LimitBlockInputStream.cpp b/dbms/src/DataStreams/LimitBlockInputStream.cpp index b1b8f5c95d1..1348a223000 100644 --- a/dbms/src/DataStreams/LimitBlockInputStream.cpp +++ b/dbms/src/DataStreams/LimitBlockInputStream.cpp @@ -23,7 +23,7 @@ Block LimitBlockInputStream::readImpl() Block res; UInt64 rows = 0; - /// pos - how many lines were read, including the last read block + /// pos - how many rows were read, including the last read block if (pos >= offset + limit) { @@ -46,7 +46,7 @@ Block LimitBlockInputStream::readImpl() pos += rows; } while (pos <= offset); - /// give away the whole block + /// return the whole block if (pos >= offset + rows && pos <= offset + limit) return res; @@ -61,7 +61,7 @@ Block LimitBlockInputStream::readImpl() static_cast(limit) + static_cast(offset) - static_cast(pos) + static_cast(rows))); for (size_t i = 0; i < res.columns(); ++i) - res.safeGetByPosition(i).column = res.safeGetByPosition(i).column->cut(start, length); + res.getByPosition(i).column = res.getByPosition(i).column->cut(start, length); // TODO: we should provide feedback to child-block, so it will know how many rows are actually consumed. // It's crucial for streaming engines like Kafka. diff --git a/dbms/src/DataStreams/RemoteBlockInputStream.cpp b/dbms/src/DataStreams/RemoteBlockInputStream.cpp index 740e60ffb09..eb576075f80 100644 --- a/dbms/src/DataStreams/RemoteBlockInputStream.cpp +++ b/dbms/src/DataStreams/RemoteBlockInputStream.cpp @@ -292,7 +292,7 @@ void RemoteBlockInputStream::sendQuery() established = true; auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(settings); - multiplexed_connections->sendQuery(timeouts, query, "", stage, &context.getClientInfo(), true); + multiplexed_connections->sendQuery(timeouts, query, query_id, stage, &context.getClientInfo(), true); established = false; sent_query = true; diff --git a/dbms/src/DataStreams/RemoteBlockInputStream.h b/dbms/src/DataStreams/RemoteBlockInputStream.h index 3cef2099030..af8d79c324c 100644 --- a/dbms/src/DataStreams/RemoteBlockInputStream.h +++ b/dbms/src/DataStreams/RemoteBlockInputStream.h @@ -46,6 +46,11 @@ public: ~RemoteBlockInputStream() override; + /// Set the query_id. For now, used by performance test to later find the query + /// in the server query_log. Must be called before sending the query to the + /// server. + void setQueryId(const std::string& query_id_) { assert(!sent_query); query_id = query_id_; } + /// Specify how we allocate connections on a shard. void setPoolMode(PoolMode pool_mode_) { pool_mode = pool_mode_; } @@ -95,6 +100,7 @@ private: std::unique_ptr multiplexed_connections; const String query; + String query_id = ""; Context context; /// Temporary tables needed to be sent to remote servers diff --git a/dbms/src/DataStreams/RemoteBlockOutputStream.cpp b/dbms/src/DataStreams/RemoteBlockOutputStream.cpp index 2f93d88ac7c..a95ea174541 100644 --- a/dbms/src/DataStreams/RemoteBlockOutputStream.cpp +++ b/dbms/src/DataStreams/RemoteBlockOutputStream.cpp @@ -50,6 +50,11 @@ RemoteBlockOutputStream::RemoteBlockOutputStream(Connection & connection_, if (auto log_queue = CurrentThread::getInternalTextLogsQueue()) log_queue->pushBlock(std::move(packet.block)); } + else if (Protocol::Server::TableColumns == packet.type) + { + /// Server could attach ColumnsDescription in front of stream for column defaults. There's no need to pass it through cause + /// client's already got this information for remote table. Ignore. + } else throw NetException("Unexpected packet from server (expected Data or Exception, got " + String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER); diff --git a/dbms/src/DataStreams/TTLBlockInputStream.cpp b/dbms/src/DataStreams/TTLBlockInputStream.cpp index 1e765f8bb3c..11999b894aa 100644 --- a/dbms/src/DataStreams/TTLBlockInputStream.cpp +++ b/dbms/src/DataStreams/TTLBlockInputStream.cpp @@ -40,8 +40,10 @@ TTLBlockInputStream::TTLBlockInputStream( auto it = column_defaults.find(name); if (it != column_defaults.end()) - default_expr_list->children.emplace_back( - setAlias(it->second.expression, it->first)); + { + auto expression = it->second.expression->clone(); + default_expr_list->children.emplace_back(setAlias(expression, it->first)); + } } else new_ttl_infos.columns_ttl.emplace(name, ttl_info); diff --git a/dbms/src/DataStreams/processConstants.cpp b/dbms/src/DataStreams/processConstants.cpp index d945474c0a2..6129c600bdf 100644 --- a/dbms/src/DataStreams/processConstants.cpp +++ b/dbms/src/DataStreams/processConstants.cpp @@ -9,7 +9,7 @@ void removeConstantsFromBlock(Block & block) size_t i = 0; while (i < columns) { - if (isColumnConst(*block.getByPosition(i).column)) + if (block.getByPosition(i).column && isColumnConst(*block.getByPosition(i).column)) { block.erase(i); --columns; @@ -22,13 +22,14 @@ void removeConstantsFromBlock(Block & block) void removeConstantsFromSortDescription(const Block & header, SortDescription & description) { + /// Note: This code is not correct if column description contains column numbers instead of column names. + /// Hopefully, everywhere where it is used, column description contains names. description.erase(std::remove_if(description.begin(), description.end(), [&](const SortColumnDescription & elem) { - if (!elem.column_name.empty()) - return isColumnConst(*header.getByName(elem.column_name).column); - else - return isColumnConst(*header.safeGetByPosition(elem.column_number).column); + auto & column = !elem.column_name.empty() ? header.getByName(elem.column_name) + : header.safeGetByPosition(elem.column_number); + return column.column && isColumnConst(*column.column); }), description.end()); } @@ -41,7 +42,7 @@ void enrichBlockWithConstants(Block & block, const Block & header) for (size_t i = 0; i < columns; ++i) { const auto & col_type_name = header.getByPosition(i); - if (isColumnConst(*col_type_name.column)) + if (col_type_name.column && isColumnConst(*col_type_name.column)) block.insert(i, {col_type_name.column->cloneResized(rows), col_type_name.type, col_type_name.name}); } } diff --git a/dbms/src/DataStreams/tests/collapsing_sorted_stream.cpp b/dbms/src/DataStreams/tests/collapsing_sorted_stream.cpp index a43b7d347eb..43205a9e7b5 100644 --- a/dbms/src/DataStreams/tests/collapsing_sorted_stream.cpp +++ b/dbms/src/DataStreams/tests/collapsing_sorted_stream.cpp @@ -68,6 +68,7 @@ try CollapsingFinalBlockInputStream collapsed(inputs, descr, "Sign"); Context context = Context::createGlobal(); + context.makeGlobalContext(); WriteBufferFromFileDescriptor out_buf(STDERR_FILENO); BlockOutputStreamPtr output = context.getOutputFormat("TabSeparated", out_buf, block1); diff --git a/dbms/src/DataStreams/tests/expression_stream.cpp b/dbms/src/DataStreams/tests/expression_stream.cpp index 3cbce14649d..b63d9d1f3b5 100644 --- a/dbms/src/DataStreams/tests/expression_stream.cpp +++ b/dbms/src/DataStreams/tests/expression_stream.cpp @@ -35,6 +35,7 @@ try ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0); Context context = Context::createGlobal(); + context.makeGlobalContext(); NamesAndTypesList source_columns = {{"number", std::make_shared()}}; auto syntax_result = SyntaxAnalyzer(context, {}).analyze(ast, source_columns); diff --git a/dbms/src/DataStreams/tests/filter_stream.cpp b/dbms/src/DataStreams/tests/filter_stream.cpp index ed12c09dc99..4199ba40b53 100644 --- a/dbms/src/DataStreams/tests/filter_stream.cpp +++ b/dbms/src/DataStreams/tests/filter_stream.cpp @@ -40,6 +40,7 @@ try std::cerr << std::endl; Context context = Context::createGlobal(); + context.makeGlobalContext(); NamesAndTypesList source_columns = {{"number", std::make_shared()}}; auto syntax_result = SyntaxAnalyzer(context, {}).analyze(ast, source_columns); diff --git a/dbms/src/DataStreams/tests/gtest_blocks_size_merging_streams.cpp b/dbms/src/DataStreams/tests/gtest_blocks_size_merging_streams.cpp index 9cbe82111a2..57cd68d16d8 100644 --- a/dbms/src/DataStreams/tests/gtest_blocks_size_merging_streams.cpp +++ b/dbms/src/DataStreams/tests/gtest_blocks_size_merging_streams.cpp @@ -1,9 +1,3 @@ -#pragma GCC diagnostic ignored "-Wsign-compare" -#ifdef __clang__ -#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" -#pragma clang diagnostic ignored "-Wundef" -#endif - #include #include #include diff --git a/dbms/src/DataStreams/tests/union_stream2.cpp b/dbms/src/DataStreams/tests/union_stream2.cpp index 284e66b91a9..3eb1927f80a 100644 --- a/dbms/src/DataStreams/tests/union_stream2.cpp +++ b/dbms/src/DataStreams/tests/union_stream2.cpp @@ -23,6 +23,7 @@ int main(int, char **) try { Context context = Context::createGlobal(); + context.makeGlobalContext(); Settings settings = context.getSettings(); context.setPath("./"); diff --git a/dbms/src/DataTypes/DataTypeEnum.cpp b/dbms/src/DataTypes/DataTypeEnum.cpp index a784f1502e4..36a26540cda 100644 --- a/dbms/src/DataTypes/DataTypeEnum.cpp +++ b/dbms/src/DataTypes/DataTypeEnum.cpp @@ -350,9 +350,20 @@ Field DataTypeEnum::castToValue(const Field & value_or_name) const template class DataTypeEnum; template class DataTypeEnum; +static void checkASTStructure(const ASTPtr & child) +{ + const auto * func = child->as(); + if (!func + || func->name != "equals" + || func->parameters + || !func->arguments + || func->arguments->children.size() != 2) + throw Exception("Elements of Enum data type must be of form: 'name' = number, where name is string literal and number is an integer", + ErrorCodes::UNEXPECTED_AST_STRUCTURE); +} template -static DataTypePtr create(const ASTPtr & arguments) +static DataTypePtr createExact(const ASTPtr & arguments) { if (!arguments || arguments->children.empty()) throw Exception("Enum data type cannot be empty", ErrorCodes::EMPTY_DATA_PASSED); @@ -365,15 +376,9 @@ static DataTypePtr create(const ASTPtr & arguments) /// Children must be functions 'equals' with string literal as left argument and numeric literal as right argument. for (const ASTPtr & child : arguments->children) { - const auto * func = child->as(); - if (!func - || func->name != "equals" - || func->parameters - || !func->arguments - || func->arguments->children.size() != 2) - throw Exception("Elements of Enum data type must be of form: 'name' = number, where name is string literal and number is an integer", - ErrorCodes::UNEXPECTED_AST_STRUCTURE); + checkASTStructure(child); + const auto * func = child->as(); const auto * name_literal = func->arguments->children[0]->as(); const auto * value_literal = func->arguments->children[1]->as(); @@ -397,11 +402,38 @@ static DataTypePtr create(const ASTPtr & arguments) return std::make_shared(values); } +static DataTypePtr create(const ASTPtr & arguments) +{ + if (!arguments || arguments->children.empty()) + throw Exception("Enum data type cannot be empty", ErrorCodes::EMPTY_DATA_PASSED); + + /// Children must be functions 'equals' with string literal as left argument and numeric literal as right argument. + for (const ASTPtr & child : arguments->children) + { + checkASTStructure(child); + + const auto * func = child->as(); + const auto * value_literal = func->arguments->children[1]->as(); + + if (!value_literal + || (value_literal->value.getType() != Field::Types::UInt64 && value_literal->value.getType() != Field::Types::Int64)) + throw Exception("Elements of Enum data type must be of form: 'name' = number, where name is string literal and number is an integer", + ErrorCodes::UNEXPECTED_AST_STRUCTURE); + + Int64 value = value_literal->value.get(); + + if (value > std::numeric_limits::max() || value < std::numeric_limits::min()) + return createExact(arguments); + } + + return createExact(arguments); +} void registerDataTypeEnum(DataTypeFactory & factory) { - factory.registerDataType("Enum8", create>); - factory.registerDataType("Enum16", create>); + factory.registerDataType("Enum8", createExact>); + factory.registerDataType("Enum16", createExact>); + factory.registerDataType("Enum", create); } } diff --git a/dbms/src/DataTypes/DataTypesDecimal.cpp b/dbms/src/DataTypes/DataTypesDecimal.cpp index 9dd811e7aec..920d7b360bd 100644 --- a/dbms/src/DataTypes/DataTypesDecimal.cpp +++ b/dbms/src/DataTypes/DataTypesDecimal.cpp @@ -241,7 +241,7 @@ static DataTypePtr create(const ASTPtr & arguments) } template -static DataTypePtr createExect(const ASTPtr & arguments) +static DataTypePtr createExact(const ASTPtr & arguments) { if (!arguments || arguments->children.size() != 1) throw Exception("Decimal data type family must have exactly two arguments: precision and scale", @@ -260,9 +260,9 @@ static DataTypePtr createExect(const ASTPtr & arguments) void registerDataTypeDecimal(DataTypeFactory & factory) { - factory.registerDataType("Decimal32", createExect, DataTypeFactory::CaseInsensitive); - factory.registerDataType("Decimal64", createExect, DataTypeFactory::CaseInsensitive); - factory.registerDataType("Decimal128", createExect, DataTypeFactory::CaseInsensitive); + factory.registerDataType("Decimal32", createExact, DataTypeFactory::CaseInsensitive); + factory.registerDataType("Decimal64", createExact, DataTypeFactory::CaseInsensitive); + factory.registerDataType("Decimal128", createExact, DataTypeFactory::CaseInsensitive); factory.registerDataType("Decimal", create, DataTypeFactory::CaseInsensitive); factory.registerAlias("DEC", "Decimal", DataTypeFactory::CaseInsensitive); diff --git a/dbms/src/DataTypes/tests/gtest_data_type_get_common_type.cpp b/dbms/src/DataTypes/tests/gtest_data_type_get_common_type.cpp index d2782ae9179..8ad8e955e75 100644 --- a/dbms/src/DataTypes/tests/gtest_data_type_get_common_type.cpp +++ b/dbms/src/DataTypes/tests/gtest_data_type_get_common_type.cpp @@ -4,11 +4,6 @@ #include -#pragma GCC diagnostic ignored "-Wsign-compare" -#ifdef __clang__ - #pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" - #pragma clang diagnostic ignored "-Wundef" -#endif #include diff --git a/dbms/src/Databases/DatabaseDictionary.cpp b/dbms/src/Databases/DatabaseDictionary.cpp index b11f4de88b8..5e69b447a1d 100644 --- a/dbms/src/Databases/DatabaseDictionary.cpp +++ b/dbms/src/Databases/DatabaseDictionary.cpp @@ -27,7 +27,7 @@ DatabaseDictionary::DatabaseDictionary(const String & name_) { } -void DatabaseDictionary::loadTables(Context &, ThreadPool *, bool) +void DatabaseDictionary::loadTables(Context &, bool) { } @@ -52,7 +52,7 @@ Tables DatabaseDictionary::listTables(const Context & context, const FilterByNam auto dict_name = dict_ptr->getName(); const DictionaryStructure & dictionary_structure = dict_ptr->getStructure(); auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure); - tables[dict_name] = StorageDictionary::create(dict_name, ColumnsDescription{columns}, context, true, dict_name); + tables[dict_name] = StorageDictionary::create(getDatabaseName(), dict_name, ColumnsDescription{columns}, context, true, dict_name); } return tables; } @@ -73,7 +73,7 @@ StoragePtr DatabaseDictionary::tryGetTable( { const DictionaryStructure & dictionary_structure = dict_ptr->getStructure(); auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure); - return StorageDictionary::create(table_name, ColumnsDescription{columns}, context, true, table_name); + return StorageDictionary::create(getDatabaseName(), table_name, ColumnsDescription{columns}, context, true, table_name); } return {}; diff --git a/dbms/src/Databases/DatabaseDictionary.h b/dbms/src/Databases/DatabaseDictionary.h index 92b52ffd4a0..1a3b815ae07 100644 --- a/dbms/src/Databases/DatabaseDictionary.h +++ b/dbms/src/Databases/DatabaseDictionary.h @@ -33,7 +33,6 @@ public: void loadTables( Context & context, - ThreadPool * thread_pool, bool has_force_restore_data_flag) override; bool isTableExist( diff --git a/dbms/src/Databases/DatabaseMemory.cpp b/dbms/src/Databases/DatabaseMemory.cpp index 3eea0bc666a..e84d5a95078 100644 --- a/dbms/src/Databases/DatabaseMemory.cpp +++ b/dbms/src/Databases/DatabaseMemory.cpp @@ -18,7 +18,6 @@ DatabaseMemory::DatabaseMemory(String name_) void DatabaseMemory::loadTables( Context & /*context*/, - ThreadPool * /*thread_pool*/, bool /*has_force_restore_data_flag*/) { /// Nothing to load. diff --git a/dbms/src/Databases/DatabaseMemory.h b/dbms/src/Databases/DatabaseMemory.h index fe7cc783ba3..325fde0437f 100644 --- a/dbms/src/Databases/DatabaseMemory.h +++ b/dbms/src/Databases/DatabaseMemory.h @@ -25,7 +25,6 @@ public: void loadTables( Context & context, - ThreadPool * thread_pool, bool has_force_restore_data_flag) override; void createTable( diff --git a/dbms/src/Databases/DatabaseMySQL.cpp b/dbms/src/Databases/DatabaseMySQL.cpp index af8b129d3b0..e9dd5c0dacc 100644 --- a/dbms/src/Databases/DatabaseMySQL.cpp +++ b/dbms/src/Databases/DatabaseMySQL.cpp @@ -230,7 +230,7 @@ DatabaseMySQL::MySQLStorageInfo DatabaseMySQL::createStorageInfo( const String & table_name, const NamesAndTypesList & columns_name_and_type, const UInt64 & table_modification_time) const { const auto & mysql_table = StorageMySQL::create( - table_name, std::move(mysql_pool), mysql_database_name, table_name, + database_name, table_name, std::move(mysql_pool), mysql_database_name, table_name, false, "", ColumnsDescription{columns_name_and_type}, global_context); const auto & create_table_query = std::make_shared(); diff --git a/dbms/src/Databases/DatabaseMySQL.h b/dbms/src/Databases/DatabaseMySQL.h index 7ce836d6a64..f0fad5b8e5c 100644 --- a/dbms/src/Databases/DatabaseMySQL.h +++ b/dbms/src/Databases/DatabaseMySQL.h @@ -46,7 +46,7 @@ public: throw Exception("MySQL database engine does not support detach table.", ErrorCodes::NOT_IMPLEMENTED); } - void loadTables(Context &, ThreadPool *, bool) override + void loadTables(Context &, bool) override { /// do nothing } diff --git a/dbms/src/Databases/DatabaseOrdinary.cpp b/dbms/src/Databases/DatabaseOrdinary.cpp index 9fa7d1b1196..25b3eb652b5 100644 --- a/dbms/src/Databases/DatabaseOrdinary.cpp +++ b/dbms/src/Databases/DatabaseOrdinary.cpp @@ -119,7 +119,6 @@ DatabaseOrdinary::DatabaseOrdinary(String name_, const String & metadata_path_, void DatabaseOrdinary::loadTables( Context & context, - ThreadPool * thread_pool, bool has_force_restore_data_flag) { using FileNames = std::vector; @@ -161,96 +160,68 @@ void DatabaseOrdinary::loadTables( */ std::sort(file_names.begin(), file_names.end()); - size_t total_tables = file_names.size(); + const size_t total_tables = file_names.size(); LOG_INFO(log, "Total " << total_tables << " tables."); AtomicStopwatch watch; std::atomic tables_processed {0}; - Poco::Event all_tables_processed; - ExceptionHandler exception_handler; - auto task_function = [&](const String & table) + auto loadOneTable = [&](const String & table) { - SCOPE_EXIT( - if (++tables_processed == total_tables) - all_tables_processed.set() - ); + loadTable(context, metadata_path, *this, name, data_path, table, has_force_restore_data_flag); /// Messages, so that it's not boring to wait for the server to load for a long time. - if ((tables_processed + 1) % PRINT_MESSAGE_EACH_N_TABLES == 0 + if (++tables_processed % PRINT_MESSAGE_EACH_N_TABLES == 0 || watch.compareAndRestart(PRINT_MESSAGE_EACH_N_SECONDS)) { LOG_INFO(log, std::fixed << std::setprecision(2) << tables_processed * 100.0 / total_tables << "%"); watch.restart(); } - - loadTable(context, metadata_path, *this, name, data_path, table, has_force_restore_data_flag); }; - for (const auto & filename : file_names) - { - auto task = createExceptionHandledJob(std::bind(task_function, filename), exception_handler); + ThreadPool pool(SettingMaxThreads().getAutoValue()); - if (thread_pool) - thread_pool->schedule(task); - else - task(); + for (const auto & file_name : file_names) + { + pool.schedule([&]() { loadOneTable(file_name); }); } - if (thread_pool) - all_tables_processed.wait(); - - exception_handler.throwIfException(); + pool.wait(); /// After all tables was basically initialized, startup them. - startupTables(thread_pool); + startupTables(pool); } -void DatabaseOrdinary::startupTables(ThreadPool * thread_pool) +void DatabaseOrdinary::startupTables(ThreadPool & thread_pool) { LOG_INFO(log, "Starting up tables."); - AtomicStopwatch watch; - std::atomic tables_processed {0}; - size_t total_tables = tables.size(); - Poco::Event all_tables_processed; - ExceptionHandler exception_handler; - + const size_t total_tables = tables.size(); if (!total_tables) return; - auto task_function = [&](const StoragePtr & table) - { - SCOPE_EXIT( - if (++tables_processed == total_tables) - all_tables_processed.set() - ); + AtomicStopwatch watch; + std::atomic tables_processed {0}; - if ((tables_processed + 1) % PRINT_MESSAGE_EACH_N_TABLES == 0 + auto startupOneTable = [&](const StoragePtr & table) + { + table->startup(); + + if (++tables_processed % PRINT_MESSAGE_EACH_N_TABLES == 0 || watch.compareAndRestart(PRINT_MESSAGE_EACH_N_SECONDS)) { LOG_INFO(log, std::fixed << std::setprecision(2) << tables_processed * 100.0 / total_tables << "%"); watch.restart(); } - - table->startup(); }; - for (const auto & name_storage : tables) + for (const auto & table : tables) { - auto task = createExceptionHandledJob(std::bind(task_function, name_storage.second), exception_handler); - - if (thread_pool) - thread_pool->schedule(task); - else - task(); + thread_pool.schedule([&]() { startupOneTable(table.second); }); } - if (thread_pool) - all_tables_processed.wait(); - - exception_handler.throwIfException(); + thread_pool.wait(); } diff --git a/dbms/src/Databases/DatabaseOrdinary.h b/dbms/src/Databases/DatabaseOrdinary.h index 887bf101d62..369f78d36ba 100644 --- a/dbms/src/Databases/DatabaseOrdinary.h +++ b/dbms/src/Databases/DatabaseOrdinary.h @@ -19,7 +19,6 @@ public: void loadTables( Context & context, - ThreadPool * thread_pool, bool has_force_restore_data_flag) override; void createTable( @@ -73,7 +72,7 @@ private: const String data_path; Poco::Logger * log; - void startupTables(ThreadPool * thread_pool); + void startupTables(ThreadPool & thread_pool); ASTPtr getCreateTableQueryImpl(const Context & context, const String & table_name, bool throw_on_error) const; }; diff --git a/dbms/src/Databases/IDatabase.h b/dbms/src/Databases/IDatabase.h index ea344c712d3..6136be6c98b 100644 --- a/dbms/src/Databases/IDatabase.h +++ b/dbms/src/Databases/IDatabase.h @@ -56,11 +56,10 @@ public: /// Get name of database engine. virtual String getEngineName() const = 0; - /// Load a set of existing tables. If thread_pool is specified, use it. + /// Load a set of existing tables. /// You can call only once, right after the object is created. virtual void loadTables( Context & context, - ThreadPool * thread_pool, bool has_force_restore_data_flag) = 0; /// Check the existence of the table. diff --git a/dbms/src/Dictionaries/CacheDictionary.h b/dbms/src/Dictionaries/CacheDictionary.h index c6607196af5..cc613d0d96b 100644 --- a/dbms/src/Dictionaries/CacheDictionary.h +++ b/dbms/src/Dictionaries/CacheDictionary.h @@ -30,8 +30,6 @@ public: const DictionaryLifetime dict_lifetime, const size_t size); - std::exception_ptr getCreationException() const override { return {}; } - std::string getName() const override { return name; } std::string getTypeName() const override { return "Cache"; } @@ -62,8 +60,6 @@ public: const DictionaryStructure & getStructure() const override { return dict_struct; } - std::chrono::time_point getCreationTime() const override { return creation_time; } - bool isInjective(const std::string & attribute_name) const override { return dict_struct.attributes[&getAttribute(attribute_name) - attributes.data()].injective; @@ -284,8 +280,6 @@ private: mutable std::atomic element_count{0}; mutable std::atomic hit_count{0}; mutable std::atomic query_count{0}; - - const std::chrono::time_point creation_time = std::chrono::system_clock::now(); }; } diff --git a/dbms/src/Dictionaries/ClickHouseDictionarySource.cpp b/dbms/src/Dictionaries/ClickHouseDictionarySource.cpp index 3703d11d832..cd977d18ad0 100644 --- a/dbms/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/dbms/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -74,6 +74,8 @@ ClickHouseDictionarySource::ClickHouseDictionarySource( { /// We should set user info even for the case when the dictionary is loaded in-process (without TCP communication). context.setUser(user, password, Poco::Net::SocketAddress("127.0.0.1", 0), {}); + /// Processors are not supported here yet. + context.getSettingsRef().experimental_use_processors = false; } diff --git a/dbms/src/Dictionaries/ClickHouseDictionarySource.h b/dbms/src/Dictionaries/ClickHouseDictionarySource.h index 2603f24fa0f..991782b1549 100644 --- a/dbms/src/Dictionaries/ClickHouseDictionarySource.h +++ b/dbms/src/Dictionaries/ClickHouseDictionarySource.h @@ -27,6 +27,7 @@ public: /// copy-constructor is provided in order to support cloneability ClickHouseDictionarySource(const ClickHouseDictionarySource & other); + ClickHouseDictionarySource & operator=(const ClickHouseDictionarySource &) = delete; BlockInputStreamPtr loadAll() override; diff --git a/dbms/src/Dictionaries/ComplexKeyCacheDictionary.h b/dbms/src/Dictionaries/ComplexKeyCacheDictionary.h index 2c080d68b7b..ffac807c04c 100644 --- a/dbms/src/Dictionaries/ComplexKeyCacheDictionary.h +++ b/dbms/src/Dictionaries/ComplexKeyCacheDictionary.h @@ -50,8 +50,6 @@ public: std::string getKeyDescription() const { return key_description; } - std::exception_ptr getCreationException() const override { return {}; } - std::string getName() const override { return name; } std::string getTypeName() const override { return "ComplexKeyCache"; } @@ -86,8 +84,6 @@ public: const DictionaryStructure & getStructure() const override { return dict_struct; } - std::chrono::time_point getCreationTime() const override { return creation_time; } - bool isInjective(const std::string & attribute_name) const override { return dict_struct.attributes[&getAttribute(attribute_name) - attributes.data()].injective; diff --git a/dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp b/dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp index 606e10f2412..39ef9124061 100644 --- a/dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp +++ b/dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp @@ -29,18 +29,8 @@ ComplexKeyHashedDictionary::ComplexKeyHashedDictionary( , saved_block{std::move(saved_block)} { createAttributes(); - - try - { - loadData(); - calculateBytesAllocated(); - } - catch (...) - { - creation_exception = std::current_exception(); - } - - creation_time = std::chrono::system_clock::now(); + loadData(); + calculateBytesAllocated(); } #define DECLARE(TYPE) \ diff --git a/dbms/src/Dictionaries/ComplexKeyHashedDictionary.h b/dbms/src/Dictionaries/ComplexKeyHashedDictionary.h index b9aaa42a829..54ee8627f9b 100644 --- a/dbms/src/Dictionaries/ComplexKeyHashedDictionary.h +++ b/dbms/src/Dictionaries/ComplexKeyHashedDictionary.h @@ -32,8 +32,6 @@ public: std::string getKeyDescription() const { return key_description; } - std::exception_ptr getCreationException() const override { return creation_exception; } - std::string getName() const override { return name; } std::string getTypeName() const override { return "ComplexKeyHashed"; } @@ -61,8 +59,6 @@ public: const DictionaryStructure & getStructure() const override { return dict_struct; } - std::chrono::time_point getCreationTime() const override { return creation_time; } - bool isInjective(const std::string & attribute_name) const override { return dict_struct.attributes[&getAttribute(attribute_name) - attributes.data()].injective; @@ -255,10 +251,6 @@ private: size_t bucket_count = 0; mutable std::atomic query_count{0}; - std::chrono::time_point creation_time; - - std::exception_ptr creation_exception; - BlockPtr saved_block; }; diff --git a/dbms/src/Dictionaries/ExecutableDictionarySource.h b/dbms/src/Dictionaries/ExecutableDictionarySource.h index c5ace9ab78f..9816161a70e 100644 --- a/dbms/src/Dictionaries/ExecutableDictionarySource.h +++ b/dbms/src/Dictionaries/ExecutableDictionarySource.h @@ -23,6 +23,7 @@ public: const Context & context); ExecutableDictionarySource(const ExecutableDictionarySource & other); + ExecutableDictionarySource & operator=(const ExecutableDictionarySource &) = delete; BlockInputStreamPtr loadAll() override; diff --git a/dbms/src/Dictionaries/FlatDictionary.cpp b/dbms/src/Dictionaries/FlatDictionary.cpp index 628178c8542..b7b70748c01 100644 --- a/dbms/src/Dictionaries/FlatDictionary.cpp +++ b/dbms/src/Dictionaries/FlatDictionary.cpp @@ -36,18 +36,8 @@ FlatDictionary::FlatDictionary( , saved_block{std::move(saved_block)} { createAttributes(); - - try - { - loadData(); - calculateBytesAllocated(); - } - catch (...) - { - creation_exception = std::current_exception(); - } - - creation_time = std::chrono::system_clock::now(); + loadData(); + calculateBytesAllocated(); } diff --git a/dbms/src/Dictionaries/FlatDictionary.h b/dbms/src/Dictionaries/FlatDictionary.h index 2a00de6f754..de14cc3dc1a 100644 --- a/dbms/src/Dictionaries/FlatDictionary.h +++ b/dbms/src/Dictionaries/FlatDictionary.h @@ -29,8 +29,6 @@ public: bool require_nonempty, BlockPtr saved_block = nullptr); - std::exception_ptr getCreationException() const override { return creation_exception; } - std::string getName() const override { return name; } std::string getTypeName() const override { return "Flat"; } @@ -58,8 +56,6 @@ public: const DictionaryStructure & getStructure() const override { return dict_struct; } - std::chrono::time_point getCreationTime() const override { return creation_time; } - bool isInjective(const std::string & attribute_name) const override { return dict_struct.attributes[&getAttribute(attribute_name) - attributes.data()].injective; @@ -244,10 +240,6 @@ private: size_t bucket_count = 0; mutable std::atomic query_count{0}; - std::chrono::time_point creation_time; - - std::exception_ptr creation_exception; - BlockPtr saved_block; }; diff --git a/dbms/src/Dictionaries/HTTPDictionarySource.h b/dbms/src/Dictionaries/HTTPDictionarySource.h index 92980e183e3..78fe5193533 100644 --- a/dbms/src/Dictionaries/HTTPDictionarySource.h +++ b/dbms/src/Dictionaries/HTTPDictionarySource.h @@ -26,6 +26,7 @@ public: const Context & context); HTTPDictionarySource(const HTTPDictionarySource & other); + HTTPDictionarySource & operator=(const HTTPDictionarySource &) = delete; BlockInputStreamPtr loadAll() override; diff --git a/dbms/src/Dictionaries/HashedDictionary.cpp b/dbms/src/Dictionaries/HashedDictionary.cpp index d67d6dcf9a2..413cfadec39 100644 --- a/dbms/src/Dictionaries/HashedDictionary.cpp +++ b/dbms/src/Dictionaries/HashedDictionary.cpp @@ -30,18 +30,8 @@ HashedDictionary::HashedDictionary( , saved_block{std::move(saved_block)} { createAttributes(); - - try - { - loadData(); - calculateBytesAllocated(); - } - catch (...) - { - creation_exception = std::current_exception(); - } - - creation_time = std::chrono::system_clock::now(); + loadData(); + calculateBytesAllocated(); } diff --git a/dbms/src/Dictionaries/HashedDictionary.h b/dbms/src/Dictionaries/HashedDictionary.h index b0605f26bad..92875f27cf3 100644 --- a/dbms/src/Dictionaries/HashedDictionary.h +++ b/dbms/src/Dictionaries/HashedDictionary.h @@ -28,8 +28,6 @@ public: bool require_nonempty, BlockPtr saved_block = nullptr); - std::exception_ptr getCreationException() const override { return creation_exception; } - std::string getName() const override { return name; } std::string getTypeName() const override { return "Hashed"; } @@ -57,8 +55,6 @@ public: const DictionaryStructure & getStructure() const override { return dict_struct; } - std::chrono::time_point getCreationTime() const override { return creation_time; } - bool isInjective(const std::string & attribute_name) const override { return dict_struct.attributes[&getAttribute(attribute_name) - attributes.data()].injective; @@ -248,10 +244,6 @@ private: size_t bucket_count = 0; mutable std::atomic query_count{0}; - std::chrono::time_point creation_time; - - std::exception_ptr creation_exception; - BlockPtr saved_block; }; diff --git a/dbms/src/Dictionaries/LibraryDictionarySource.h b/dbms/src/Dictionaries/LibraryDictionarySource.h index a667b286be3..d09e5eee691 100644 --- a/dbms/src/Dictionaries/LibraryDictionarySource.h +++ b/dbms/src/Dictionaries/LibraryDictionarySource.h @@ -35,6 +35,7 @@ public: Block & sample_block); LibraryDictionarySource(const LibraryDictionarySource & other); + LibraryDictionarySource & operator=(const LibraryDictionarySource &) = delete; ~LibraryDictionarySource() override; diff --git a/dbms/src/Dictionaries/MySQLDictionarySource.h b/dbms/src/Dictionaries/MySQLDictionarySource.h index 30447a49aea..cfc45f42bb3 100644 --- a/dbms/src/Dictionaries/MySQLDictionarySource.h +++ b/dbms/src/Dictionaries/MySQLDictionarySource.h @@ -37,6 +37,7 @@ public: /// copy-constructor is provided in order to support cloneability MySQLDictionarySource(const MySQLDictionarySource & other); + MySQLDictionarySource & operator=(const MySQLDictionarySource &) = delete; BlockInputStreamPtr loadAll() override; diff --git a/dbms/src/Dictionaries/RangeHashedDictionary.cpp b/dbms/src/Dictionaries/RangeHashedDictionary.cpp index ac509b4d1e5..05f29e05c42 100644 --- a/dbms/src/Dictionaries/RangeHashedDictionary.cpp +++ b/dbms/src/Dictionaries/RangeHashedDictionary.cpp @@ -80,18 +80,8 @@ RangeHashedDictionary::RangeHashedDictionary( , require_nonempty(require_nonempty) { createAttributes(); - - try - { - loadData(); - calculateBytesAllocated(); - } - catch (...) - { - creation_exception = std::current_exception(); - } - - creation_time = std::chrono::system_clock::now(); + loadData(); + calculateBytesAllocated(); } diff --git a/dbms/src/Dictionaries/RangeHashedDictionary.h b/dbms/src/Dictionaries/RangeHashedDictionary.h index b54c88de4e8..a02b1377db5 100644 --- a/dbms/src/Dictionaries/RangeHashedDictionary.h +++ b/dbms/src/Dictionaries/RangeHashedDictionary.h @@ -24,8 +24,6 @@ public: const DictionaryLifetime dict_lifetime, bool require_nonempty); - std::exception_ptr getCreationException() const override { return creation_exception; } - std::string getName() const override { return dictionary_name; } std::string getTypeName() const override { return "RangeHashed"; } @@ -53,8 +51,6 @@ public: const DictionaryStructure & getStructure() const override { return dict_struct; } - std::chrono::time_point getCreationTime() const override { return creation_time; } - bool isInjective(const std::string & attribute_name) const override { return dict_struct.attributes[&getAttribute(attribute_name) - attributes.data()].injective; @@ -227,10 +223,6 @@ private: size_t element_count = 0; size_t bucket_count = 0; mutable std::atomic query_count{0}; - - std::chrono::time_point creation_time; - - std::exception_ptr creation_exception; }; } diff --git a/dbms/src/Dictionaries/TrieDictionary.h b/dbms/src/Dictionaries/TrieDictionary.h index f434ebbc77d..a873f7bdd16 100644 --- a/dbms/src/Dictionaries/TrieDictionary.h +++ b/dbms/src/Dictionaries/TrieDictionary.h @@ -33,8 +33,6 @@ public: std::string getKeyDescription() const { return key_description; } - std::exception_ptr getCreationException() const override { return creation_exception; } - std::string getName() const override { return name; } std::string getTypeName() const override { return "Trie"; } @@ -62,8 +60,6 @@ public: const DictionaryStructure & getStructure() const override { return dict_struct; } - std::chrono::time_point getCreationTime() const override { return creation_time; } - bool isInjective(const std::string & attribute_name) const override { return dict_struct.attributes[&getAttribute(attribute_name) - attributes.data()].injective; diff --git a/dbms/src/Dictionaries/XDBCDictionarySource.h b/dbms/src/Dictionaries/XDBCDictionarySource.h index 5197300c341..253f802d8fd 100644 --- a/dbms/src/Dictionaries/XDBCDictionarySource.h +++ b/dbms/src/Dictionaries/XDBCDictionarySource.h @@ -36,6 +36,7 @@ public: /// copy-constructor is provided in order to support cloneability XDBCDictionarySource(const XDBCDictionarySource & other); + XDBCDictionarySource & operator=(const XDBCDictionarySource &) = delete; BlockInputStreamPtr loadAll() override; diff --git a/dbms/src/Formats/FormatFactory.cpp b/dbms/src/Formats/FormatFactory.cpp index 3ae0bf3b6de..4fae140abee 100644 --- a/dbms/src/Formats/FormatFactory.cpp +++ b/dbms/src/Formats/FormatFactory.cpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB @@ -26,6 +27,50 @@ const FormatFactory::Creators & FormatFactory::getCreators(const String & name) throw Exception("Unknown format " + name, ErrorCodes::UNKNOWN_FORMAT); } +const FormatFactory::ProcessorCreators & FormatFactory::getProcessorCreators(const String & name) const +{ + auto it = processors_dict.find(name); + if (processors_dict.end() != it) + return it->second; + throw Exception("Unknown format " + name, ErrorCodes::UNKNOWN_FORMAT); +} + +static FormatSettings getInputFormatSetting(const Settings & settings) +{ + FormatSettings format_settings; + format_settings.csv.delimiter = settings.format_csv_delimiter; + format_settings.csv.allow_single_quotes = settings.format_csv_allow_single_quotes; + format_settings.csv.allow_double_quotes = settings.format_csv_allow_double_quotes; + format_settings.csv.empty_as_default = settings.input_format_defaults_for_omitted_fields; + format_settings.values.interpret_expressions = settings.input_format_values_interpret_expressions; + format_settings.with_names_use_header = settings.input_format_with_names_use_header; + format_settings.skip_unknown_fields = settings.input_format_skip_unknown_fields; + format_settings.import_nested_json = settings.input_format_import_nested_json; + format_settings.date_time_input_format = settings.date_time_input_format; + format_settings.input_allow_errors_num = settings.input_format_allow_errors_num; + format_settings.input_allow_errors_ratio = settings.input_format_allow_errors_ratio; + + return format_settings; +} + +static FormatSettings getOutputFormatSetting(const Settings & settings) +{ + FormatSettings format_settings; + format_settings.json.quote_64bit_integers = settings.output_format_json_quote_64bit_integers; + format_settings.json.quote_denormals = settings.output_format_json_quote_denormals; + format_settings.json.escape_forward_slashes = settings.output_format_json_escape_forward_slashes; + format_settings.csv.delimiter = settings.format_csv_delimiter; + format_settings.csv.allow_single_quotes = settings.format_csv_allow_single_quotes; + format_settings.csv.allow_double_quotes = settings.format_csv_allow_double_quotes; + format_settings.pretty.max_rows = settings.output_format_pretty_max_rows; + format_settings.pretty.max_column_pad_width = settings.output_format_pretty_max_column_pad_width; + format_settings.pretty.color = settings.output_format_pretty_color; + format_settings.write_statistics = settings.output_format_write_statistics; + format_settings.parquet.row_group_size = settings.output_format_parquet_row_group_size; + + return format_settings; +} + BlockInputStreamPtr FormatFactory::getInput( const String & name, @@ -41,19 +86,7 @@ BlockInputStreamPtr FormatFactory::getInput( throw Exception("Format " + name + " is not suitable for input", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT); const Settings & settings = context.getSettingsRef(); - - FormatSettings format_settings; - format_settings.csv.delimiter = settings.format_csv_delimiter; - format_settings.csv.allow_single_quotes = settings.format_csv_allow_single_quotes; - format_settings.csv.allow_double_quotes = settings.format_csv_allow_double_quotes; - format_settings.csv.empty_as_default = settings.input_format_defaults_for_omitted_fields; - format_settings.values.interpret_expressions = settings.input_format_values_interpret_expressions; - format_settings.with_names_use_header = settings.input_format_with_names_use_header; - format_settings.skip_unknown_fields = settings.input_format_skip_unknown_fields; - format_settings.import_nested_json = settings.input_format_import_nested_json; - format_settings.date_time_input_format = settings.date_time_input_format; - format_settings.input_allow_errors_num = settings.input_format_allow_errors_num; - format_settings.input_allow_errors_ratio = settings.input_format_allow_errors_ratio; + FormatSettings format_settings = getInputFormatSetting(settings); return input_getter( buf, sample, context, max_block_size, rows_portion_size, callback ? callback : ReadCallback(), format_settings); @@ -67,19 +100,7 @@ BlockOutputStreamPtr FormatFactory::getOutput(const String & name, WriteBuffer & throw Exception("Format " + name + " is not suitable for output", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT); const Settings & settings = context.getSettingsRef(); - - FormatSettings format_settings; - format_settings.json.quote_64bit_integers = settings.output_format_json_quote_64bit_integers; - format_settings.json.quote_denormals = settings.output_format_json_quote_denormals; - format_settings.json.escape_forward_slashes = settings.output_format_json_escape_forward_slashes; - format_settings.csv.delimiter = settings.format_csv_delimiter; - format_settings.csv.allow_single_quotes = settings.format_csv_allow_single_quotes; - format_settings.csv.allow_double_quotes = settings.format_csv_allow_double_quotes; - format_settings.pretty.max_rows = settings.output_format_pretty_max_rows; - format_settings.pretty.max_column_pad_width = settings.output_format_pretty_max_column_pad_width; - format_settings.pretty.color = settings.output_format_pretty_color; - format_settings.write_statistics = settings.output_format_write_statistics; - format_settings.parquet.row_group_size = settings.output_format_parquet_row_group_size; + FormatSettings format_settings = getOutputFormatSetting(settings); /** Materialization is needed, because formats can use the functions `IDataType`, * which only work with full columns. @@ -89,12 +110,46 @@ BlockOutputStreamPtr FormatFactory::getOutput(const String & name, WriteBuffer & } +InputFormatPtr FormatFactory::getInputFormat(const String & name, ReadBuffer & buf, const Block & sample, const Context & context, UInt64 max_block_size) const +{ + const auto & input_getter = getProcessorCreators(name).first; + if (!input_getter) + throw Exception("Format " + name + " is not suitable for input", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT); + + const Settings & settings = context.getSettingsRef(); + FormatSettings format_settings = getInputFormatSetting(settings); + + RowInputFormatParams params; + params.max_block_size = max_block_size; + params.allow_errors_num = format_settings.input_allow_errors_num; + params.allow_errors_ratio = format_settings.input_allow_errors_ratio; + + return input_getter(buf, sample, context, params, format_settings); +} + + +OutputFormatPtr FormatFactory::getOutputFormat(const String & name, WriteBuffer & buf, const Block & sample, const Context & context) const +{ + const auto & output_getter = getProcessorCreators(name).second; + if (!output_getter) + throw Exception("Format " + name + " is not suitable for output", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT); + + const Settings & settings = context.getSettingsRef(); + FormatSettings format_settings = getOutputFormatSetting(settings); + + /** TODO: Materialization is needed, because formats can use the functions `IDataType`, + * which only work with full columns. + */ + return output_getter(buf, sample, context, format_settings); +} + + void FormatFactory::registerInputFormat(const String & name, InputCreator input_creator) { auto & target = dict[name].first; if (target) throw Exception("FormatFactory: Input format " + name + " is already registered", ErrorCodes::LOGICAL_ERROR); - target = input_creator; + target = std::move(input_creator); } void FormatFactory::registerOutputFormat(const String & name, OutputCreator output_creator) @@ -102,7 +157,23 @@ void FormatFactory::registerOutputFormat(const String & name, OutputCreator outp auto & target = dict[name].second; if (target) throw Exception("FormatFactory: Output format " + name + " is already registered", ErrorCodes::LOGICAL_ERROR); - target = output_creator; + target = std::move(output_creator); +} + +void FormatFactory::registerInputFormatProcessor(const String & name, InputProcessorCreator input_creator) +{ + auto & target = processors_dict[name].first; + if (target) + throw Exception("FormatFactory: Input format " + name + " is already registered", ErrorCodes::LOGICAL_ERROR); + target = std::move(input_creator); +} + +void FormatFactory::registerOutputFormatProcessor(const String & name, OutputProcessorCreator output_creator) +{ + auto & target = processors_dict[name].second; + if (target) + throw Exception("FormatFactory: Output format " + name + " is already registered", ErrorCodes::LOGICAL_ERROR); + target = std::move(output_creator); } @@ -127,6 +198,25 @@ void registerOutputFormatParquet(FormatFactory & factory); void registerInputFormatProtobuf(FormatFactory & factory); void registerOutputFormatProtobuf(FormatFactory & factory); +void registerInputFormatProcessorNative(FormatFactory & factory); +void registerOutputFormatProcessorNative(FormatFactory & factory); +void registerInputFormatProcessorRowBinary(FormatFactory & factory); +void registerOutputFormatProcessorRowBinary(FormatFactory & factory); +void registerInputFormatProcessorTabSeparated(FormatFactory & factory); +void registerOutputFormatProcessorTabSeparated(FormatFactory & factory); +void registerInputFormatProcessorValues(FormatFactory & factory); +void registerOutputFormatProcessorValues(FormatFactory & factory); +void registerInputFormatProcessorCSV(FormatFactory & factory); +void registerOutputFormatProcessorCSV(FormatFactory & factory); +void registerInputFormatProcessorTSKV(FormatFactory & factory); +void registerOutputFormatProcessorTSKV(FormatFactory & factory); +void registerInputFormatProcessorJSONEachRow(FormatFactory & factory); +void registerOutputFormatProcessorJSONEachRow(FormatFactory & factory); +void registerInputFormatProcessorParquet(FormatFactory & factory); +void registerOutputFormatProcessorParquet(FormatFactory & factory); +void registerInputFormatProcessorProtobuf(FormatFactory & factory); +void registerOutputFormatProcessorProtobuf(FormatFactory & factory); + /// Output only (presentational) formats. void registerOutputFormatPretty(FormatFactory & factory); @@ -141,9 +231,22 @@ void registerOutputFormatODBCDriver2(FormatFactory & factory); void registerOutputFormatNull(FormatFactory & factory); void registerOutputFormatMySQLWire(FormatFactory & factory); +void registerOutputFormatProcessorPretty(FormatFactory & factory); +void registerOutputFormatProcessorPrettyCompact(FormatFactory & factory); +void registerOutputFormatProcessorPrettySpace(FormatFactory & factory); +void registerOutputFormatProcessorVertical(FormatFactory & factory); +void registerOutputFormatProcessorJSON(FormatFactory & factory); +void registerOutputFormatProcessorJSONCompact(FormatFactory & factory); +void registerOutputFormatProcessorXML(FormatFactory & factory); +void registerOutputFormatProcessorODBCDriver(FormatFactory & factory); +void registerOutputFormatProcessorODBCDriver2(FormatFactory & factory); +void registerOutputFormatProcessorNull(FormatFactory & factory); +void registerOutputFormatProcessorMySQLWrite(FormatFactory & factory); + /// Input only formats. void registerInputFormatCapnProto(FormatFactory & factory); +void registerInputFormatProcessorCapnProto(FormatFactory & factory); FormatFactory::FormatFactory() @@ -168,6 +271,28 @@ FormatFactory::FormatFactory() registerInputFormatParquet(*this); registerOutputFormatParquet(*this); + registerOutputFormatMySQLWire(*this); + + registerInputFormatProcessorNative(*this); + registerOutputFormatProcessorNative(*this); + registerInputFormatProcessorRowBinary(*this); + registerOutputFormatProcessorRowBinary(*this); + registerInputFormatProcessorTabSeparated(*this); + registerOutputFormatProcessorTabSeparated(*this); + registerInputFormatProcessorValues(*this); + registerOutputFormatProcessorValues(*this); + registerInputFormatProcessorCSV(*this); + registerOutputFormatProcessorCSV(*this); + registerInputFormatProcessorTSKV(*this); + registerOutputFormatProcessorTSKV(*this); + registerInputFormatProcessorJSONEachRow(*this); + registerOutputFormatProcessorJSONEachRow(*this); + registerInputFormatProcessorProtobuf(*this); + registerOutputFormatProcessorProtobuf(*this); + registerInputFormatProcessorCapnProto(*this); + registerInputFormatProcessorParquet(*this); + registerOutputFormatProcessorParquet(*this); + registerOutputFormatPretty(*this); registerOutputFormatPrettyCompact(*this); registerOutputFormatPrettySpace(*this); @@ -178,7 +303,18 @@ FormatFactory::FormatFactory() registerOutputFormatODBCDriver(*this); registerOutputFormatODBCDriver2(*this); registerOutputFormatNull(*this); - registerOutputFormatMySQLWire(*this); + + registerOutputFormatProcessorPretty(*this); + registerOutputFormatProcessorPrettyCompact(*this); + registerOutputFormatProcessorPrettySpace(*this); + registerOutputFormatProcessorVertical(*this); + registerOutputFormatProcessorJSON(*this); + registerOutputFormatProcessorJSONCompact(*this); + registerOutputFormatProcessorXML(*this); + registerOutputFormatProcessorODBCDriver(*this); + registerOutputFormatProcessorODBCDriver2(*this); + registerOutputFormatProcessorNull(*this); + registerOutputFormatProcessorMySQLWrite(*this); } } diff --git a/dbms/src/Formats/FormatFactory.h b/dbms/src/Formats/FormatFactory.h index 9c8b87e7d8b..5cc1fcecb93 100644 --- a/dbms/src/Formats/FormatFactory.h +++ b/dbms/src/Formats/FormatFactory.h @@ -19,6 +19,18 @@ struct FormatSettings; class ReadBuffer; class WriteBuffer; +class IProcessor; +using ProcessorPtr = std::shared_ptr; + +class IInputFormat; +class IOutputFormat; + +struct RowInputFormatParams; + +using InputFormatPtr = std::shared_ptr; +using OutputFormatPtr = std::shared_ptr; + + /** Allows to create an IBlockInputStream or IBlockOutputStream by the name of the format. * Note: format and compression are independent things. */ @@ -45,9 +57,24 @@ private: const Context & context, const FormatSettings & settings)>; + using InputProcessorCreator = std::function; + + using OutputProcessorCreator = std::function; + using Creators = std::pair; + using ProcessorCreators = std::pair; using FormatsDictionary = std::unordered_map; + using FormatProcessorsDictionary = std::unordered_map; public: BlockInputStreamPtr getInput( @@ -62,10 +89,19 @@ public: BlockOutputStreamPtr getOutput(const String & name, WriteBuffer & buf, const Block & sample, const Context & context) const; + InputFormatPtr getInputFormat(const String & name, ReadBuffer & buf, + const Block & sample, const Context & context, UInt64 max_block_size) const; + + OutputFormatPtr getOutputFormat(const String & name, WriteBuffer & buf, + const Block & sample, const Context & context) const; + /// Register format by its name. void registerInputFormat(const String & name, InputCreator input_creator); void registerOutputFormat(const String & name, OutputCreator output_creator); + void registerInputFormatProcessor(const String & name, InputProcessorCreator input_creator); + void registerOutputFormatProcessor(const String & name, OutputProcessorCreator output_creator); + const FormatsDictionary & getAllFormats() const { return dict; @@ -73,11 +109,13 @@ public: private: FormatsDictionary dict; + FormatProcessorsDictionary processors_dict; FormatFactory(); friend class ext::singleton; const Creators & getCreators(const String & name) const; + const ProcessorCreators & getProcessorCreators(const String & name) const; }; } diff --git a/dbms/src/Formats/ProtobufReader.cpp b/dbms/src/Formats/ProtobufReader.cpp index 2c0ea1251f4..ac45c1d94ee 100644 --- a/dbms/src/Formats/ProtobufReader.cpp +++ b/dbms/src/Formats/ProtobufReader.cpp @@ -41,11 +41,12 @@ namespace constexpr UInt64 END_OF_GROUP = static_cast(-2); Int64 decodeZigZag(UInt64 n) { return static_cast((n >> 1) ^ (~(n & 1) + 1)); } +} - void unknownFormat() - { - throw Exception("Protobuf messages are corrupted or don't match the provided schema. Please note that Protobuf stream is length-delimited: every message is prefixed by its length in varint.", ErrorCodes::UNKNOWN_PROTOBUF_FORMAT); - } + +[[noreturn]] void ProtobufReader::SimpleReader::throwUnknownFormat() +{ + throw Exception("Protobuf messages are corrupted or don't match the provided schema. Please note that Protobuf stream is length-delimited: every message is prefixed by its length in varint.", ErrorCodes::UNKNOWN_PROTOBUF_FORMAT); } @@ -67,7 +68,10 @@ bool ProtobufReader::SimpleReader::startMessage() if (unlikely(in.eof())) return false; size_t size_of_message = readVarint(); + if (size_of_message == 0) + throwUnknownFormat(); current_message_end = cursor + size_of_message; + root_message_end = current_message_end; } else { @@ -91,7 +95,7 @@ void ProtobufReader::SimpleReader::endMessage() else if (unlikely(cursor > current_message_end)) { if (!parent_message_ends.empty()) - unknownFormat(); + throwUnknownFormat(); moveCursorBackward(cursor - current_message_end); } current_message_end = REACHED_END; @@ -141,7 +145,7 @@ bool ProtobufReader::SimpleReader::readFieldNumber(UInt32 & field_number) UInt64 varint = readVarint(); if (unlikely(varint & (static_cast(0xFFFFFFFF) << 32))) - unknownFormat(); + throwUnknownFormat(); UInt32 key = static_cast(varint); field_number = (key >> 3); WireType wire_type = static_cast(key & 0x07); @@ -171,7 +175,7 @@ bool ProtobufReader::SimpleReader::readFieldNumber(UInt32 & field_number) case GROUP_END: { if (current_message_end != END_OF_GROUP) - unknownFormat(); + throwUnknownFormat(); current_message_end = REACHED_END; return false; } @@ -181,7 +185,7 @@ bool ProtobufReader::SimpleReader::readFieldNumber(UInt32 & field_number) return true; } } - unknownFormat(); + throwUnknownFormat(); __builtin_unreachable(); } @@ -257,7 +261,7 @@ void ProtobufReader::SimpleReader::ignore(UInt64 num_bytes) void ProtobufReader::SimpleReader::moveCursorBackward(UInt64 num_bytes) { if (in.offset() < num_bytes) - unknownFormat(); + throwUnknownFormat(); in.position() -= num_bytes; cursor -= num_bytes; } @@ -294,7 +298,7 @@ UInt64 ProtobufReader::SimpleReader::continueReadingVarint(UInt64 first_byte) PROTOBUF_READER_READ_VARINT_BYTE(10) #undef PROTOBUF_READER_READ_VARINT_BYTE - unknownFormat(); + throwUnknownFormat(); __builtin_unreachable(); } @@ -327,7 +331,7 @@ void ProtobufReader::SimpleReader::ignoreVarint() PROTOBUF_READER_IGNORE_VARINT_BYTE(10) #undef PROTOBUF_READER_IGNORE_VARINT_BYTE - unknownFormat(); + throwUnknownFormat(); } void ProtobufReader::SimpleReader::ignoreGroup() @@ -371,11 +375,10 @@ void ProtobufReader::SimpleReader::ignoreGroup() break; } } - unknownFormat(); + throwUnknownFormat(); } } - // Implementation for a converter from any protobuf field type to any DB data type. class ProtobufReader::ConverterBaseImpl : public ProtobufReader::IConverter { diff --git a/dbms/src/Formats/ProtobufReader.h b/dbms/src/Formats/ProtobufReader.h index 9fc4b5cb8e3..41088e54ac3 100644 --- a/dbms/src/Formats/ProtobufReader.h +++ b/dbms/src/Formats/ProtobufReader.h @@ -97,10 +97,19 @@ private: bool readUInt(UInt64 & value); template bool readFixed(T & value); bool readStringInto(PaddedPODArray & str); - bool ALWAYS_INLINE maybeCanReadValue() const { return field_end != REACHED_END; } + + bool ALWAYS_INLINE maybeCanReadValue() const + { + if (field_end == REACHED_END) + return false; + if (cursor < root_message_end) + return true; + + throwUnknownFormat(); + } private: - void readBinary(void* data, size_t size); + void readBinary(void * data, size_t size); void ignore(UInt64 num_bytes); void moveCursorBackward(UInt64 num_bytes); @@ -119,6 +128,8 @@ private: void ignoreVarint(); void ignoreGroup(); + [[noreturn]] static void throwUnknownFormat(); + static constexpr UInt64 REACHED_END = 0; ReadBuffer & in; @@ -126,6 +137,8 @@ private: std::vector parent_message_ends; UInt64 current_message_end; UInt64 field_end; + + UInt64 root_message_end; }; class IConverter diff --git a/dbms/src/Functions/CMakeLists.txt b/dbms/src/Functions/CMakeLists.txt index a584bd14a7d..13b294197cf 100644 --- a/dbms/src/Functions/CMakeLists.txt +++ b/dbms/src/Functions/CMakeLists.txt @@ -11,6 +11,7 @@ add_library(clickhouse_functions ${clickhouse_functions_sources}) target_link_libraries(clickhouse_functions PUBLIC clickhouse_dictionaries + clickhouse_dictionaries_embedded dbms ${CONSISTENT_HASHING_LIBRARY} consistent-hashing-sumbur @@ -19,6 +20,9 @@ target_link_libraries(clickhouse_functions ${METROHASH_LIBRARIES} murmurhash ${BASE64_LIBRARY} + + PRIVATE + ${Boost_FILESYSTEM_LIBRARY} ) if (OPENSSL_CRYPTO_LIBRARY) diff --git a/dbms/src/Functions/CRC32.cpp b/dbms/src/Functions/CRC32.cpp index 95b93701783..80e0f163571 100644 --- a/dbms/src/Functions/CRC32.cpp +++ b/dbms/src/Functions/CRC32.cpp @@ -41,7 +41,7 @@ struct CRC32Impl } } - static void array(const ColumnString::Offsets & /*offsets*/, PaddedPODArray & /*res*/) + [[noreturn]] static void array(const ColumnString::Offsets & /*offsets*/, PaddedPODArray & /*res*/) { throw Exception("Cannot apply function CRC32 to Array argument", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } diff --git a/dbms/src/Functions/CustomWeekTransforms.h b/dbms/src/Functions/CustomWeekTransforms.h new file mode 100644 index 00000000000..88d341e5906 --- /dev/null +++ b/dbms/src/Functions/CustomWeekTransforms.h @@ -0,0 +1,137 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include + +/// The default mode value to use for the WEEK() function +#define DEFAULT_WEEK_MODE 0 + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int ILLEGAL_COLUMN; +} + +/** + * CustomWeek Transformations. + */ + +static inline UInt32 dateIsNotSupported(const char * name) +{ + throw Exception("Illegal type Date of argument for function " + std::string(name), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); +} + +/// This factor transformation will say that the function is monotone everywhere. +struct ZeroTransform +{ + static inline UInt16 execute(UInt32, UInt8, const DateLUTImpl &) { return 0; } + static inline UInt16 execute(UInt16, UInt8, const DateLUTImpl &) { return 0; } +}; + +struct ToWeekImpl +{ + static constexpr auto name = "toWeek"; + + static inline UInt8 execute(UInt32 t, UInt8 week_mode, const DateLUTImpl & time_zone) + { + YearWeek yw = time_zone.toYearWeek(time_zone.toDayNum(t), week_mode); + return yw.second; + } + static inline UInt8 execute(UInt16 d, UInt8 week_mode, const DateLUTImpl & time_zone) + { + YearWeek yw = time_zone.toYearWeek(DayNum(d), week_mode); + return yw.second; + } + + using FactorTransform = ZeroTransform; +}; + +struct ToYearWeekImpl +{ + static constexpr auto name = "toYearWeek"; + + static inline UInt32 execute(UInt32 t, UInt8 week_mode, const DateLUTImpl & time_zone) + { + YearWeek yw = time_zone.toYearWeek(time_zone.toDayNum(t), week_mode | static_cast(WeekModeFlag::YEAR)); + return yw.first * 100 + yw.second; + } + static inline UInt32 execute(UInt16 d, UInt8 week_mode, const DateLUTImpl & time_zone) + { + YearWeek yw = time_zone.toYearWeek(DayNum(d), week_mode | static_cast(WeekModeFlag::YEAR)); + return yw.first * 100 + yw.second; + } + + using FactorTransform = ZeroTransform; +}; + +struct ToStartOfWeekImpl +{ + static constexpr auto name = "toStartOfWeek"; + + static inline UInt16 execute(UInt32 t, UInt8 week_mode, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t), week_mode); + } + static inline UInt16 execute(UInt16 d, UInt8 week_mode, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfWeek(DayNum(d), week_mode); + } + + using FactorTransform = ZeroTransform; +}; + +template +struct Transformer +{ + static void + vector(const PaddedPODArray & vec_from, PaddedPODArray & vec_to, UInt8 week_mode, const DateLUTImpl & time_zone) + { + size_t size = vec_from.size(); + vec_to.resize(size); + + for (size_t i = 0; i < size; ++i) + vec_to[i] = Transform::execute(vec_from[i], week_mode, time_zone); + } +}; + + +template +struct CustomWeekTransformImpl +{ + static void execute(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) + { + using Op = Transformer; + + UInt8 week_mode = DEFAULT_WEEK_MODE; + if (arguments.size() > 1) + { + if (const auto week_mode_column = checkAndGetColumnConst(block.getByPosition(arguments[1]).column.get())) + week_mode = week_mode_column->getValue(); + } + + const DateLUTImpl & time_zone = extractTimeZoneFromFunctionArguments(block, arguments, 2, 0); + const ColumnPtr source_col = block.getByPosition(arguments[0]).column; + if (const auto * sources = checkAndGetColumn>(source_col.get())) + { + auto col_to = ColumnVector::create(); + Op::vector(sources->getData(), col_to->getData(), week_mode, time_zone); + block.getByPosition(result).column = std::move(col_to); + } + else + { + throw Exception( + "Illegal column " + block.getByPosition(arguments[0]).column->getName() + " of first argument of function " + + Transform::name, + ErrorCodes::ILLEGAL_COLUMN); + } + } +}; + +} diff --git a/dbms/src/Functions/FunctionCustomWeekToSomething.h b/dbms/src/Functions/FunctionCustomWeekToSomething.h new file mode 100644 index 00000000000..1a4cc26a456 --- /dev/null +++ b/dbms/src/Functions/FunctionCustomWeekToSomething.h @@ -0,0 +1,153 @@ +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + + +/// See CustomWeekTransforms.h +template +class FunctionCustomWeekToSomething : public IFunction +{ +public: + static constexpr auto name = Transform::name; + static FunctionPtr create(const Context &) { return std::make_shared(); } + + String getName() const override { return name; } + + bool isVariadic() const override { return true; } + size_t getNumberOfArguments() const override { return 0; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + if (arguments.size() == 1) + { + if (!isDateOrDateTime(arguments[0].type)) + throw Exception( + "Illegal type " + arguments[0].type->getName() + " of argument of function " + getName() + + ". Should be a date or a date with time", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + else if (arguments.size() == 2) + { + if (!isDateOrDateTime(arguments[0].type)) + throw Exception( + "Illegal type " + arguments[0].type->getName() + " of argument of function " + getName() + + ". Should be a date or a date with time", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (!isUInt8(arguments[1].type)) + throw Exception( + "Function " + getName() + + " supports 1 or 2 or 3 arguments. The 1st argument " + "must be of type Date or DateTime. The 2nd argument (optional) must be " + "a constant UInt8 with week mode. The 3nd argument (optional) must be " + "a constant string with timezone name", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + else if (arguments.size() == 3) + { + if (!isDateOrDateTime(arguments[0].type)) + throw Exception( + "Illegal type " + arguments[0].type->getName() + " of argument of function " + getName() + + ". Should be a date or a date with time", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (!isUInt8(arguments[1].type)) + throw Exception( + "Function " + getName() + + " supports 1 or 2 or 3 arguments. The 1st argument " + "must be of type Date or DateTime. The 2nd argument (optional) must be " + "a constant UInt8 with week mode. The 3nd argument (optional) must be " + "a constant string with timezone name", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (!isString(arguments[2].type)) + throw Exception( + "Function " + getName() + + " supports 1 or 2 or 3 arguments. The 1st argument " + "must be of type Date or DateTime. The 2nd argument (optional) must be " + "a constant UInt8 with week mode. The 3nd argument (optional) must be " + "a constant string with timezone name", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (isDate(arguments[0].type) && std::is_same_v) + throw Exception( + "The timezone argument of function " + getName() + " is allowed only when the 1st argument has the type DateTime", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + else + throw Exception( + "Number of arguments for function " + getName() + " doesn't match: passed " + toString(arguments.size()) + + ", should be 1 or 2 or 3", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + return std::make_shared(); + } + + bool useDefaultImplementationForConstants() const override { return true; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2}; } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + { + const IDataType * from_type = block.getByPosition(arguments[0]).type.get(); + WhichDataType which(from_type); + + if (which.isDate()) + CustomWeekTransformImpl::execute( + block, arguments, result, input_rows_count); + else if (which.isDateTime()) + CustomWeekTransformImpl::execute( + block, arguments, result, input_rows_count); + else + throw Exception( + "Illegal type " + block.getByPosition(arguments[0]).type->getName() + " of argument of function " + getName(), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + + + bool hasInformationAboutMonotonicity() const override { return true; } + + Monotonicity getMonotonicityForRange(const IDataType & type, const Field & left, const Field & right) const override + { + IFunction::Monotonicity is_monotonic{true}; + IFunction::Monotonicity is_not_monotonic; + + if (std::is_same_v) + { + is_monotonic.is_always_monotonic = true; + return is_monotonic; + } + + /// This method is called only if the function has one argument. Therefore, we do not care about the non-local time zone. + const DateLUTImpl & date_lut = DateLUT::instance(); + + if (left.isNull() || right.isNull()) + return is_not_monotonic; + + /// The function is monotonous on the [left, right] segment, if the factor transformation returns the same values for them. + + if (checkAndGetDataType(&type)) + { + return Transform::FactorTransform::execute(UInt16(left.get()), DEFAULT_WEEK_MODE, date_lut) + == Transform::FactorTransform::execute(UInt16(right.get()), DEFAULT_WEEK_MODE, date_lut) + ? is_monotonic + : is_not_monotonic; + } + else + { + return Transform::FactorTransform::execute(UInt32(left.get()), DEFAULT_WEEK_MODE, date_lut) + == Transform::FactorTransform::execute(UInt32(right.get()), DEFAULT_WEEK_MODE, date_lut) + ? is_monotonic + : is_not_monotonic; + } + } +}; + +} diff --git a/dbms/src/Functions/FunctionsCoding.h b/dbms/src/Functions/FunctionsCoding.h index 8734bbc03ed..ed34863765d 100644 --- a/dbms/src/Functions/FunctionsCoding.h +++ b/dbms/src/Functions/FunctionsCoding.h @@ -1548,13 +1548,15 @@ public: ColumnString::Chars & vec_res_upper_range = col_res_upper_range->getChars(); vec_res_upper_range.resize(input_rows_count * IPV6_BINARY_LENGTH); + static constexpr UInt8 max_cidr_mask = IPV6_BINARY_LENGTH * 8; + for (size_t offset = 0; offset < input_rows_count; ++offset) { const size_t offset_ipv6 = offset * IPV6_BINARY_LENGTH; UInt8 cidr = col_const_cidr_in ? col_const_cidr_in->getValue() : col_cidr_in->getData()[offset]; - + cidr = std::min(cidr, max_cidr_mask); applyCIDRMask(&vec_in[offset_ipv6], &vec_res_lower_range[offset_ipv6], &vec_res_upper_range[offset_ipv6], cidr); } diff --git a/dbms/src/Functions/FunctionsVisitParam.h b/dbms/src/Functions/FunctionsVisitParam.h index 41a49dfd908..353dda930ef 100644 --- a/dbms/src/Functions/FunctionsVisitParam.h +++ b/dbms/src/Functions/FunctionsVisitParam.h @@ -100,7 +100,7 @@ struct ExtractRaw for (auto extract_begin = pos; pos != end; ++pos) { - if (*pos == current_expect_end) + if (current_expect_end && *pos == current_expect_end) { expects_end.pop_back(); current_expect_end = expects_end.empty() ? 0 : expects_end.back(); @@ -192,7 +192,7 @@ struct ExtractParamImpl /// We check that the entry does not pass through the boundaries of strings. if (pos + needle.size() < begin + offsets[i]) - res[i] = ParamExtractor::extract(pos + needle.size(), begin + offsets[i]); + res[i] = ParamExtractor::extract(pos + needle.size(), begin + offsets[i] - 1); /// don't include terminating zero else res[i] = 0; diff --git a/dbms/src/Functions/GatherUtils/CMakeLists.txt b/dbms/src/Functions/GatherUtils/CMakeLists.txt index 20c92352c57..bcb02051774 100644 --- a/dbms/src/Functions/GatherUtils/CMakeLists.txt +++ b/dbms/src/Functions/GatherUtils/CMakeLists.txt @@ -1,7 +1,7 @@ include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake) add_headers_and_sources(clickhouse_functions_gatherutils .) add_library(clickhouse_functions_gatherutils ${clickhouse_functions_gatherutils_sources} ${clickhouse_functions_gatherutils_headers}) -target_link_libraries(clickhouse_functions_gatherutils PRIVATE clickhouse_common_io) +target_link_libraries(clickhouse_functions_gatherutils PRIVATE dbms) if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" OR CMAKE_BUILD_TYPE_UC STREQUAL "MINSIZEREL") # Won't generate debug info for files with heavy template instantiation to achieve faster linking and lower size. diff --git a/dbms/src/Functions/RapidJSONParser.h b/dbms/src/Functions/RapidJSONParser.h index 9ffa355b9a9..c88d61fb69d 100644 --- a/dbms/src/Functions/RapidJSONParser.h +++ b/dbms/src/Functions/RapidJSONParser.h @@ -22,8 +22,10 @@ struct RapidJSONParser bool parse(const StringRef & json) { - document.Parse(json.data); - return !document.HasParseError(); + rapidjson::MemoryStream ms(json.data, json.size); + rapidjson::EncodedInputStream, rapidjson::MemoryStream> is(ms); + document.ParseStream(is); + return !document.HasParseError() && (ms.Tell() == json.size); } struct Iterator diff --git a/dbms/src/Functions/SimdJSONParser.h b/dbms/src/Functions/SimdJSONParser.h index ef43e571635..dfa28ccfd56 100644 --- a/dbms/src/Functions/SimdJSONParser.h +++ b/dbms/src/Functions/SimdJSONParser.h @@ -7,18 +7,8 @@ #include #include -#ifdef __clang__ - #pragma clang diagnostic push - #pragma clang diagnostic ignored "-Wold-style-cast" - #pragma clang diagnostic ignored "-Wnewline-eof" -#endif - #include -#ifdef __clang__ - #pragma clang diagnostic pop -#endif - namespace DB { @@ -42,7 +32,7 @@ struct SimdJSONParser bool parse(const StringRef & json) { return !json_parse(json.data, json.size, pj); } - using Iterator = ParsedJson::iterator; + using Iterator = simdjson::ParsedJson::iterator; Iterator getRoot() { return Iterator{pj}; } static bool isInt64(const Iterator & it) { return it.is_integer(); } @@ -143,7 +133,7 @@ struct SimdJSONParser } private: - ParsedJson pj; + simdjson::ParsedJson pj; }; } diff --git a/dbms/src/Functions/URL/CMakeLists.txt b/dbms/src/Functions/URL/CMakeLists.txt index 5bad6dc89cb..54221b43cf6 100644 --- a/dbms/src/Functions/URL/CMakeLists.txt +++ b/dbms/src/Functions/URL/CMakeLists.txt @@ -1,8 +1,8 @@ include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake) add_headers_and_sources(clickhouse_functions_url .) add_library(clickhouse_functions_url ${clickhouse_functions_url_sources} ${clickhouse_functions_url_headers}) -target_link_libraries(clickhouse_functions_url PRIVATE clickhouse_common_io) -target_include_directories(clickhouse_functions_url PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/../include) # ${CMAKE_CURRENT_BINARY_DIR}/include +target_link_libraries(clickhouse_functions_url PRIVATE dbms) +target_include_directories(clickhouse_functions_url PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/../include) # ${CMAKE_CURRENT_BINARY_DIR}/include if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" OR CMAKE_BUILD_TYPE_UC STREQUAL "MINSIZEREL") # Won't generate debug info for files with heavy template instantiation to achieve faster linking and lower size. diff --git a/dbms/src/Functions/URL/decodeURLComponent.cpp b/dbms/src/Functions/URL/decodeURLComponent.cpp index ee60498af69..008e3b0645f 100644 --- a/dbms/src/Functions/URL/decodeURLComponent.cpp +++ b/dbms/src/Functions/URL/decodeURLComponent.cpp @@ -89,7 +89,7 @@ struct DecodeURLComponentImpl res_data.resize(res_offset); } - static void vector_fixed(const ColumnString::Chars &, size_t, ColumnString::Chars &) + [[noreturn]] static void vector_fixed(const ColumnString::Chars &, size_t, ColumnString::Chars &) { throw Exception("Column of type FixedString is not supported by URL functions", ErrorCodes::ILLEGAL_COLUMN); } diff --git a/dbms/src/Functions/array/CMakeLists.txt b/dbms/src/Functions/array/CMakeLists.txt index 59ae693bb67..f826db1e817 100644 --- a/dbms/src/Functions/array/CMakeLists.txt +++ b/dbms/src/Functions/array/CMakeLists.txt @@ -1,7 +1,7 @@ include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake) add_headers_and_sources(clickhouse_functions_array .) add_library(clickhouse_functions_array ${clickhouse_functions_array_sources} ${clickhouse_functions_array_headers}) -target_link_libraries(clickhouse_functions_array PRIVATE clickhouse_common_io clickhouse_functions_gatherutils) +target_link_libraries(clickhouse_functions_array PRIVATE dbms clickhouse_functions_gatherutils) if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" OR CMAKE_BUILD_TYPE_UC STREQUAL "MINSIZEREL") # Won't generate debug info for files with heavy template instantiation to achieve faster linking and lower size. diff --git a/dbms/src/Functions/bitTest.cpp b/dbms/src/Functions/bitTest.cpp index 1818f7af479..ebb52a4dacf 100644 --- a/dbms/src/Functions/bitTest.cpp +++ b/dbms/src/Functions/bitTest.cpp @@ -1,5 +1,7 @@ #include #include +#include + namespace DB { @@ -10,7 +12,7 @@ struct BitTestImpl using ResultType = UInt8; template - static inline Result apply(A a, B b) + NO_SANITIZE_UNDEFINED static inline Result apply(A a, B b) { return (typename NumberTraits::ToInteger::Type(a) >> typename NumberTraits::ToInteger::Type(b)) & 1; } diff --git a/dbms/src/Functions/gcd.cpp b/dbms/src/Functions/gcd.cpp index 0373799bd74..d48318ce115 100644 --- a/dbms/src/Functions/gcd.cpp +++ b/dbms/src/Functions/gcd.cpp @@ -1,6 +1,7 @@ #include #include -#include +#include + namespace DB { @@ -15,7 +16,7 @@ struct GCDImpl { throwIfDivisionLeadsToFPE(typename NumberTraits::ToInteger::Type(a), typename NumberTraits::ToInteger::Type(b)); throwIfDivisionLeadsToFPE(typename NumberTraits::ToInteger::Type(b), typename NumberTraits::ToInteger::Type(a)); - return boost::integer::gcd( + return std::gcd( typename NumberTraits::ToInteger::Type(a), typename NumberTraits::ToInteger::Type(b)); } diff --git a/dbms/src/Functions/isValidUTF8.cpp b/dbms/src/Functions/isValidUTF8.cpp index 09b7b81030e..ff3c4466115 100644 --- a/dbms/src/Functions/isValidUTF8.cpp +++ b/dbms/src/Functions/isValidUTF8.cpp @@ -309,7 +309,7 @@ SOFTWARE. res[i] = isValidUTF8(data.data() + i * n, n); } - static void array(const ColumnString::Offsets &, PaddedPODArray &) + [[noreturn]] static void array(const ColumnString::Offsets &, PaddedPODArray &) { throw Exception("Cannot apply function isValidUTF8 to Array argument", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } diff --git a/dbms/src/Functions/lcm.cpp b/dbms/src/Functions/lcm.cpp index bf2b7e1e79f..a76519a6603 100644 --- a/dbms/src/Functions/lcm.cpp +++ b/dbms/src/Functions/lcm.cpp @@ -1,6 +1,7 @@ #include #include -#include +#include + namespace DB { @@ -15,7 +16,7 @@ struct LCMImpl { throwIfDivisionLeadsToFPE(typename NumberTraits::ToInteger::Type(a), typename NumberTraits::ToInteger::Type(b)); throwIfDivisionLeadsToFPE(typename NumberTraits::ToInteger::Type(b), typename NumberTraits::ToInteger::Type(a)); - return boost::integer::lcm( + return std::lcm( typename NumberTraits::ToInteger::Type(a), typename NumberTraits::ToInteger::Type(b)); } diff --git a/dbms/src/Functions/lengthUTF8.cpp b/dbms/src/Functions/lengthUTF8.cpp index ba2214fc9d7..2549e32b8f7 100644 --- a/dbms/src/Functions/lengthUTF8.cpp +++ b/dbms/src/Functions/lengthUTF8.cpp @@ -48,7 +48,7 @@ struct LengthUTF8Impl } } - static void array(const ColumnString::Offsets &, PaddedPODArray &) + [[noreturn]] static void array(const ColumnString::Offsets &, PaddedPODArray &) { throw Exception("Cannot apply function lengthUTF8 to Array argument", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } diff --git a/dbms/src/Functions/registerFunctionsDateTime.cpp b/dbms/src/Functions/registerFunctionsDateTime.cpp index 9540edaa6e4..adb2b240c91 100644 --- a/dbms/src/Functions/registerFunctionsDateTime.cpp +++ b/dbms/src/Functions/registerFunctionsDateTime.cpp @@ -16,6 +16,7 @@ void registerFunctionToStartOfDay(FunctionFactory &); void registerFunctionToMonday(FunctionFactory &); void registerFunctionToISOWeek(FunctionFactory &); void registerFunctionToISOYear(FunctionFactory &); +void registerFunctionToCustomWeek(FunctionFactory &); void registerFunctionToStartOfMonth(FunctionFactory &); void registerFunctionToStartOfQuarter(FunctionFactory &); void registerFunctionToStartOfYear(FunctionFactory &); @@ -79,6 +80,7 @@ void registerFunctionsDateTime(FunctionFactory & factory) registerFunctionToMonday(factory); registerFunctionToISOWeek(factory); registerFunctionToISOYear(factory); + registerFunctionToCustomWeek(factory); registerFunctionToStartOfMonth(factory); registerFunctionToStartOfQuarter(factory); registerFunctionToStartOfYear(factory); diff --git a/dbms/src/Functions/reverseUTF8.cpp b/dbms/src/Functions/reverseUTF8.cpp index 188b48bf7dc..261aa6935e2 100644 --- a/dbms/src/Functions/reverseUTF8.cpp +++ b/dbms/src/Functions/reverseUTF8.cpp @@ -61,7 +61,7 @@ struct ReverseUTF8Impl } } - static void vector_fixed(const ColumnString::Chars &, size_t, ColumnString::Chars &) + [[noreturn]] static void vector_fixed(const ColumnString::Chars &, size_t, ColumnString::Chars &) { throw Exception("Cannot apply function reverseUTF8 to fixed string.", ErrorCodes::ILLEGAL_COLUMN); } diff --git a/dbms/src/Functions/toCustomWeek.cpp b/dbms/src/Functions/toCustomWeek.cpp new file mode 100644 index 00000000000..5ba0b8e8b2a --- /dev/null +++ b/dbms/src/Functions/toCustomWeek.cpp @@ -0,0 +1,25 @@ +#include +#include +#include +#include +#include + + +namespace DB +{ +using FunctionToWeek = FunctionCustomWeekToSomething; +using FunctionToYearWeek = FunctionCustomWeekToSomething; +using FunctionToStartOfWeek = FunctionCustomWeekToSomething; + +void registerFunctionToCustomWeek(FunctionFactory & factory) +{ + factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); + + /// Compatibility aliases for mysql. + factory.registerAlias("week", "toWeek", FunctionFactory::CaseInsensitive); + factory.registerAlias("yearweek", "toYearWeek", FunctionFactory::CaseInsensitive); +} + +} diff --git a/dbms/src/Functions/toValidUTF8.cpp b/dbms/src/Functions/toValidUTF8.cpp index a1dc78c7915..63e3bdab21d 100644 --- a/dbms/src/Functions/toValidUTF8.cpp +++ b/dbms/src/Functions/toValidUTF8.cpp @@ -124,7 +124,7 @@ struct ToValidUTF8Impl write_buffer.finish(); } - static void vector_fixed(const ColumnString::Chars &, size_t, ColumnString::Chars &) + [[noreturn]] static void vector_fixed(const ColumnString::Chars &, size_t, ColumnString::Chars &) { throw Exception("Column of type FixedString is not supported by toValidUTF8 function", ErrorCodes::ILLEGAL_COLUMN); } diff --git a/dbms/src/IO/BitHelpers.h b/dbms/src/IO/BitHelpers.h index 3652dd0057a..1947d9d99ba 100644 --- a/dbms/src/IO/BitHelpers.h +++ b/dbms/src/IO/BitHelpers.h @@ -150,7 +150,6 @@ public: const UInt64 mask = maskLowBits(to_write); v &= mask; -// assert(v <= 255); bits_buffer <<= to_write; bits_buffer |= v; diff --git a/dbms/src/IO/DoubleConverter.h b/dbms/src/IO/DoubleConverter.h index e05226333dd..0c46024d65e 100644 --- a/dbms/src/IO/DoubleConverter.h +++ b/dbms/src/IO/DoubleConverter.h @@ -34,9 +34,11 @@ class DoubleConverter DoubleConverter() = default; public: - /** @todo Add commentary on how this constant is deduced. - * e.g. it's minus sign, integral zero, decimal point, up to 5 leading zeros and kBase10MaximalLength digits. */ - static constexpr auto MAX_REPRESENTATION_LENGTH = 26; + /// Sign (1 byte) + DigitsBeforePoint + point (1 byte) + DigitsAfterPoint + zero byte. + /// See comment to DoubleToStringConverter::ToFixed method for explanation. + static constexpr auto MAX_REPRESENTATION_LENGTH = + 1 + double_conversion::DoubleToStringConverter::kMaxFixedDigitsBeforePoint + + 1 + double_conversion::DoubleToStringConverter::kMaxFixedDigitsAfterPoint + 1; using BufferType = char[MAX_REPRESENTATION_LENGTH]; static const auto & instance() diff --git a/dbms/src/IO/HDFSCommon.cpp b/dbms/src/IO/HDFSCommon.cpp index 34cef18068f..0f1a58942d6 100644 --- a/dbms/src/IO/HDFSCommon.cpp +++ b/dbms/src/IO/HDFSCommon.cpp @@ -2,6 +2,7 @@ #if USE_HDFS #include + namespace DB { namespace ErrorCodes @@ -9,12 +10,13 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; extern const int NETWORK_ERROR; } + HDFSBuilderPtr createHDFSBuilder(const Poco::URI & uri) { auto & host = uri.getHost(); auto port = uri.getPort(); auto & path = uri.getPath(); - if (host.empty() || port == 0 || path.empty()) + if (host.empty() || path.empty()) throw Exception("Illegal HDFS URI: " + uri.toString(), ErrorCodes::BAD_ARGUMENTS); HDFSBuilderPtr builder(hdfsNewBuilder()); @@ -25,6 +27,18 @@ HDFSBuilderPtr createHDFSBuilder(const Poco::URI & uri) hdfsBuilderConfSetStr(builder.get(), "input.write.timeout", "60000"); // 1 min hdfsBuilderConfSetStr(builder.get(), "input.connect.timeout", "60000"); // 1 min + std::string user_info = uri.getUserInfo(); + if (!user_info.empty() && user_info.front() != ':') + { + std::string user; + size_t delim_pos = user_info.find(":"); + if (delim_pos != std::string::npos) + user = user_info.substr(0, delim_pos); + else + user = user_info; + + hdfsBuilderSetUserName(builder.get(), user.c_str()); + } hdfsBuilderSetNameNode(builder.get(), host.c_str()); hdfsBuilderSetNameNodePort(builder.get(), port); return builder; diff --git a/dbms/src/IO/ReadHelpers.h b/dbms/src/IO/ReadHelpers.h index a24c1b4546c..e1f0480a7a9 100644 --- a/dbms/src/IO/ReadHelpers.h +++ b/dbms/src/IO/ReadHelpers.h @@ -676,7 +676,7 @@ inline void readText(String & x, ReadBuffer & buf) { readEscapedString(x, buf); inline void readText(LocalDate & x, ReadBuffer & buf) { readDateText(x, buf); } inline void readText(LocalDateTime & x, ReadBuffer & buf) { readDateTimeText(x, buf); } inline void readText(UUID & x, ReadBuffer & buf) { readUUIDText(x, buf); } -inline void readText(UInt128 &, ReadBuffer &) +[[noreturn]] inline void readText(UInt128 &, ReadBuffer &) { /** Because UInt128 isn't a natural type, without arithmetic operator and only use as an intermediary type -for UUID- * it should never arrive here. But because we used the DataTypeNumber class we should have at least a definition of it. @@ -755,7 +755,7 @@ inline void readCSV(String & x, ReadBuffer & buf, const FormatSettings::CSV & se inline void readCSV(LocalDate & x, ReadBuffer & buf) { readCSVSimple(x, buf); } inline void readCSV(LocalDateTime & x, ReadBuffer & buf) { readCSVSimple(x, buf); } inline void readCSV(UUID & x, ReadBuffer & buf) { readCSVSimple(x, buf); } -inline void readCSV(UInt128 &, ReadBuffer &) +[[noreturn]] inline void readCSV(UInt128 &, ReadBuffer &) { /** Because UInt128 isn't a natural type, without arithmetic operator and only use as an intermediary type -for UUID- * it should never arrive here. But because we used the DataTypeNumber class we should have at least a definition of it. diff --git a/dbms/src/IO/WriteHelpers.h b/dbms/src/IO/WriteHelpers.h index b494f863421..073888c8712 100644 --- a/dbms/src/IO/WriteHelpers.h +++ b/dbms/src/IO/WriteHelpers.h @@ -830,7 +830,7 @@ inline void writeCSV(const String & x, WriteBuffer & buf) { writeCSVString<>(x, inline void writeCSV(const LocalDate & x, WriteBuffer & buf) { writeDoubleQuoted(x, buf); } inline void writeCSV(const LocalDateTime & x, WriteBuffer & buf) { writeDoubleQuoted(x, buf); } inline void writeCSV(const UUID & x, WriteBuffer & buf) { writeDoubleQuoted(x, buf); } -inline void writeCSV(const UInt128, WriteBuffer &) +[[noreturn]] inline void writeCSV(const UInt128, WriteBuffer &) { /** Because UInt128 isn't a natural type, without arithmetic operator and only use as an intermediary type -for UUID- * it should never arrive here. But because we used the DataTypeNumber class we should have at least a definition of it. diff --git a/dbms/src/IO/tests/CMakeLists.txt b/dbms/src/IO/tests/CMakeLists.txt index 127dc45d9bb..1c804b29c04 100644 --- a/dbms/src/IO/tests/CMakeLists.txt +++ b/dbms/src/IO/tests/CMakeLists.txt @@ -26,7 +26,7 @@ add_executable (read_escaped_string read_escaped_string.cpp) target_link_libraries (read_escaped_string PRIVATE clickhouse_common_io) add_executable (async_write async_write.cpp) -target_link_libraries (async_write PRIVATE clickhouse_compression clickhouse_common_io) +target_link_libraries (async_write PRIVATE dbms) add_executable (parse_int_perf parse_int_perf.cpp) target_link_libraries (parse_int_perf PRIVATE clickhouse_common_io) diff --git a/dbms/src/IO/tests/gtest_aio_seek_back_after_eof.cpp b/dbms/src/IO/tests/gtest_aio_seek_back_after_eof.cpp index 382182e8cbf..85643d2a78b 100644 --- a/dbms/src/IO/tests/gtest_aio_seek_back_after_eof.cpp +++ b/dbms/src/IO/tests/gtest_aio_seek_back_after_eof.cpp @@ -1,10 +1,5 @@ #if defined(__linux__) || defined(__FreeBSD__) -#pragma GCC diagnostic ignored "-Wsign-compare" -#ifdef __clang__ -#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" -#pragma clang diagnostic ignored "-Wundef" -#endif #include #include diff --git a/dbms/src/IO/tests/gtest_bit_io.cpp b/dbms/src/IO/tests/gtest_bit_io.cpp index abb5a53e346..85df2580783 100644 --- a/dbms/src/IO/tests/gtest_bit_io.cpp +++ b/dbms/src/IO/tests/gtest_bit_io.cpp @@ -16,12 +16,6 @@ #include #include -#pragma GCC diagnostic ignored "-Wsign-compare" -#ifdef __clang__ -#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" -#pragma clang diagnostic ignored "-Wundef" -#endif - #include using namespace DB; diff --git a/dbms/src/IO/tests/gtest_cascade_and_memory_write_buffer.cpp b/dbms/src/IO/tests/gtest_cascade_and_memory_write_buffer.cpp index ecf4054e122..aa95ab7cb58 100644 --- a/dbms/src/IO/tests/gtest_cascade_and_memory_write_buffer.cpp +++ b/dbms/src/IO/tests/gtest_cascade_and_memory_write_buffer.cpp @@ -1,8 +1,3 @@ -#pragma GCC diagnostic ignored "-Wsign-compare" -#ifdef __clang__ - #pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" - #pragma clang diagnostic ignored "-Wundef" -#endif #include #include diff --git a/dbms/src/IO/tests/read_buffer_aio.cpp b/dbms/src/IO/tests/read_buffer_aio.cpp index 7ed07b0930c..81d04da79f2 100644 --- a/dbms/src/IO/tests/read_buffer_aio.cpp +++ b/dbms/src/IO/tests/read_buffer_aio.cpp @@ -18,7 +18,7 @@ void prepare2(std::string & filename, std::string & buf); void prepare3(std::string & filename, std::string & buf); void prepare4(std::string & filename, std::string & buf); std::string createTmpFile(); -void die(const std::string & msg); +[[noreturn]] void die(const std::string & msg); void runTest(unsigned int num, const std::function & func); bool test1(const std::string & filename); diff --git a/dbms/src/IO/tests/write_buffer_aio.cpp b/dbms/src/IO/tests/write_buffer_aio.cpp index df8dde6b190..4e76a91639c 100644 --- a/dbms/src/IO/tests/write_buffer_aio.cpp +++ b/dbms/src/IO/tests/write_buffer_aio.cpp @@ -14,7 +14,7 @@ namespace namespace fs = boost::filesystem; void run(); -void die(const std::string & msg); +[[noreturn]] void die(const std::string & msg); void runTest(unsigned int num, const std::function & func); std::string createTmpFile(); std::string generateString(size_t n); diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index e16eca87c69..b3c0aa87f8a 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -2053,7 +2053,6 @@ void Aggregator::mergeStream(const BlockInputStreamPtr & stream, AggregatedDataV * Then the calculations can be parallelized by buckets. * We decompose the blocks to the bucket numbers indicated in them. */ - using BucketToBlocks = std::map; BucketToBlocks bucket_to_blocks; /// Read all the data. @@ -2071,11 +2070,22 @@ void Aggregator::mergeStream(const BlockInputStreamPtr & stream, AggregatedDataV bucket_to_blocks[block.info.bucket_num].emplace_back(std::move(block)); } - LOG_TRACE(log, "Read " << total_input_blocks << " blocks of partially aggregated data, total " << total_input_rows << " rows."); + LOG_TRACE(log, "Read " << total_input_blocks << " blocks of partially aggregated data, total " << total_input_rows + << " rows."); + mergeBlocks(bucket_to_blocks, result, max_threads); +} + +void Aggregator::mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVariants & result, size_t max_threads) +{ if (bucket_to_blocks.empty()) return; + UInt64 total_input_rows = 0; + for (auto & bucket : bucket_to_blocks) + for (auto & block : bucket.second) + total_input_rows += block.rows(); + /** `minus one` means the absence of information about the bucket * - in the case of single-level aggregation, as well as for blocks with "overflowing" values. * If there is at least one block with a bucket number greater or equal than zero, then there was a two-level aggregation. diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index 44ab6e594a2..983f34c1933 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -857,6 +857,10 @@ public: */ void mergeStream(const BlockInputStreamPtr & stream, AggregatedDataVariants & result, size_t max_threads); + using BucketToBlocks = std::map; + /// Merge partially aggregated blocks separated to buckets into one data structure. + void mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVariants & result, size_t max_threads); + /// Merge several partially aggregated blocks into one. /// Precondition: for all blocks block.info.is_overflows flag must be the same. /// (either all blocks are from overflow data or none blocks are). diff --git a/dbms/src/Interpreters/AsynchronousMetrics.cpp b/dbms/src/Interpreters/AsynchronousMetrics.cpp index 28a6d5dfb4d..621c27d6be8 100644 --- a/dbms/src/Interpreters/AsynchronousMetrics.cpp +++ b/dbms/src/Interpreters/AsynchronousMetrics.cpp @@ -159,10 +159,14 @@ void AsynchronousMetrics::update() size_t max_part_count_for_partition = 0; + size_t number_of_databases = databases.size(); + size_t total_number_of_tables = 0; + for (const auto & db : databases) { for (auto iterator = db.second->getIterator(context); iterator->isValid(); iterator->next()) { + ++total_number_of_tables; auto & table = iterator->table(); StorageMergeTree * table_merge_tree = dynamic_cast(table.get()); StorageReplicatedMergeTree * table_replicated_merge_tree = dynamic_cast(table.get()); @@ -213,6 +217,9 @@ void AsynchronousMetrics::update() set("ReplicasMaxRelativeDelay", max_relative_delay); set("MaxPartCountForPartition", max_part_count_for_partition); + + set("NumberOfDatabases", number_of_databases); + set("NumberOfTables", total_number_of_tables); } #if USE_TCMALLOC diff --git a/dbms/src/Interpreters/CatBoostModel.cpp b/dbms/src/Interpreters/CatBoostModel.cpp index 7f98e5131ae..3e6e66b5c3f 100644 --- a/dbms/src/Interpreters/CatBoostModel.cpp +++ b/dbms/src/Interpreters/CatBoostModel.cpp @@ -504,20 +504,6 @@ std::shared_ptr getCatBoostWrapperHolder(const std::string & CatBoostModel::CatBoostModel(std::string name_, std::string model_path_, std::string lib_path_, const ExternalLoadableLifetime & lifetime) : name(std::move(name_)), model_path(std::move(model_path_)), lib_path(std::move(lib_path_)), lifetime(lifetime) -{ - try - { - init(); - } - catch (...) - { - creation_exception = std::current_exception(); - } - - creation_time = std::chrono::system_clock::now(); -} - -void CatBoostModel::init() { api_provider = getCatBoostWrapperHolder(lib_path); api = &api_provider->getAPI(); diff --git a/dbms/src/Interpreters/CatBoostModel.h b/dbms/src/Interpreters/CatBoostModel.h index 6f613ad0f24..541dd111c82 100644 --- a/dbms/src/Interpreters/CatBoostModel.h +++ b/dbms/src/Interpreters/CatBoostModel.h @@ -68,9 +68,6 @@ public: std::shared_ptr clone() const override; - std::chrono::time_point getCreationTime() const override { return creation_time; } - std::exception_ptr getCreationException() const override { return creation_exception; } - private: std::string name; std::string model_path; @@ -85,9 +82,6 @@ private: size_t cat_features_count; size_t tree_count; - std::chrono::time_point creation_time; - std::exception_ptr creation_exception; - void init(); }; diff --git a/dbms/src/Interpreters/Cluster.cpp b/dbms/src/Interpreters/Cluster.cpp index 986f99e0aaf..3c7c9bbe9da 100644 --- a/dbms/src/Interpreters/Cluster.cpp +++ b/dbms/src/Interpreters/Cluster.cpp @@ -29,9 +29,9 @@ namespace /// Default shard weight. static constexpr UInt32 default_weight = 1; -inline bool isLocal(const Cluster::Address & address, const Poco::Net::SocketAddress & resolved_address, UInt16 clickhouse_port) +inline bool isLocalImpl(const Cluster::Address & address, const Poco::Net::SocketAddress & resolved_address, UInt16 clickhouse_port) { - /// If there is replica, for which: + /// If there is replica, for which: /// - its port is the same that the server is listening; /// - its host is resolved to set of addresses, one of which is the same as one of addresses of network interfaces of the server machine*; /// then we must go to this shard without any inter-process communication. @@ -48,10 +48,31 @@ inline bool isLocal(const Cluster::Address & address, const Poco::Net::SocketAdd /// Implementation of Cluster::Address class +std::optional Cluster::Address::getResolvedAddress() const +{ + try + { + return DNSResolver::instance().resolveAddress(host_name, port); + } + catch (...) + { + /// Failure in DNS resolution in cluster initialization is Ok. + tryLogCurrentException("Cluster"); + return {}; + } +} + + +bool Cluster::Address::isLocal(UInt16 clickhouse_port) const +{ + if (auto resolved = getResolvedAddress()) + return isLocalImpl(*this, *resolved, clickhouse_port); + return false; +} + + Cluster::Address::Address(const Poco::Util::AbstractConfiguration & config, const String & config_prefix) { - UInt16 clickhouse_port = static_cast(config.getInt("tcp_port", 0)); - host_name = config.getString(config_prefix + ".host"); port = static_cast(config.getInt(config_prefix + ".port")); if (config.has(config_prefix + ".user")) @@ -60,10 +81,9 @@ Cluster::Address::Address(const Poco::Util::AbstractConfiguration & config, cons user = config.getString(config_prefix + ".user", "default"); password = config.getString(config_prefix + ".password", ""); default_database = config.getString(config_prefix + ".default_database", ""); - initially_resolved_address = DNSResolver::instance().resolveAddress(host_name, port); - is_local = isLocal(*this, initially_resolved_address, clickhouse_port); secure = config.getBool(config_prefix + ".secure", false) ? Protocol::Secure::Enable : Protocol::Secure::Disable; compression = config.getBool(config_prefix + ".compression", true) ? Protocol::Compression::Enable : Protocol::Compression::Disable; + is_local = isLocal(config.getInt("tcp_port", 0)); } @@ -74,9 +94,7 @@ Cluster::Address::Address(const String & host_port_, const String & user_, const host_name = parsed_host_port.first; port = parsed_host_port.second; secure = secure_ ? Protocol::Secure::Enable : Protocol::Secure::Disable; - - initially_resolved_address = DNSResolver::instance().resolveAddress(parsed_host_port.first, parsed_host_port.second); - is_local = isLocal(*this, initially_resolved_address, clickhouse_port); + is_local = isLocal(clickhouse_port); } diff --git a/dbms/src/Interpreters/Cluster.h b/dbms/src/Interpreters/Cluster.h index 06714da5cef..e778c9bcf6f 100644 --- a/dbms/src/Interpreters/Cluster.h +++ b/dbms/src/Interpreters/Cluster.h @@ -60,7 +60,7 @@ public: /// This database is selected when no database is specified for Distributed table String default_database; /// The locality is determined at the initialization, and is not changed even if DNS is changed - bool is_local; + bool is_local = false; bool user_specified = false; Protocol::Compression compression = Protocol::Compression::Enable; @@ -84,17 +84,14 @@ public: String toFullString() const; static Address fromFullString(const String & address_full_string); - /// Returns initially resolved address - Poco::Net::SocketAddress getResolvedAddress() const - { - return initially_resolved_address; - } + /// Returns resolved address if it does resolve. + std::optional getResolvedAddress() const; auto tuple() const { return std::tie(host_name, port, secure, user, password, default_database); } bool operator==(const Address & other) const { return tuple() == other.tuple(); } private: - Poco::Net::SocketAddress initially_resolved_address; + bool isLocal(UInt16 clickhouse_port) const; }; using Addresses = std::vector
; diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 2909de8a60b..bf779ada6b4 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -1779,6 +1779,11 @@ BlockOutputStreamPtr Context::getOutputFormat(const String & name, WriteBuffer & return FormatFactory::instance().getOutput(name, buf, sample, *this); } +OutputFormatPtr Context::getOutputFormatProcessor(const String & name, WriteBuffer & buf, const Block & sample) const +{ + return FormatFactory::instance().getOutputFormat(name, buf, sample, *this); +} + time_t Context::getUptimeSeconds() const { diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 45f4006b0cd..44547f97ef2 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -74,6 +74,8 @@ class ShellCommand; class ICompressionCodec; class SettingsConstraints; +class IOutputFormat; +using OutputFormatPtr = std::shared_ptr; #if USE_EMBEDDED_COMPILER @@ -133,7 +135,7 @@ private: Tables table_function_results; /// Temporary tables obtained by execution of table functions. Keyed by AST tree id. Context * query_context = nullptr; Context * session_context = nullptr; /// Session context or nullptr. Could be equal to this. - Context * global_context = nullptr; /// Global context or nullptr. Could be equal to this. + Context * global_context = nullptr; /// Global context. Could be equal to this. UInt64 session_close_cycle = 0; bool session_is_used = false; @@ -289,6 +291,8 @@ public: BlockInputStreamPtr getInputFormat(const String & name, ReadBuffer & buf, const Block & sample, UInt64 max_block_size) const; BlockOutputStreamPtr getOutputFormat(const String & name, WriteBuffer & buf, const Block & sample) const; + OutputFormatPtr getOutputFormatProcessor(const String & name, WriteBuffer & buf, const Block & sample) const; + InterserverIOHandler & getInterserverIOHandler(); /// How other servers can access this for downloading replicated data. @@ -344,7 +348,10 @@ public: void setQueryContext(Context & context_) { query_context = &context_; } void setSessionContext(Context & context_) { session_context = &context_; } - void setGlobalContext(Context & context_) { global_context = &context_; } + + void makeQueryContext() { query_context = this; } + void makeSessionContext() { session_context = this; } + void makeGlobalContext() { global_context = this; } const Settings & getSettingsRef() const { return settings; } Settings & getSettingsRef() { return settings; } diff --git a/dbms/src/Interpreters/DDLWorker.cpp b/dbms/src/Interpreters/DDLWorker.cpp index e1cea632a37..1a4113a0d9c 100644 --- a/dbms/src/Interpreters/DDLWorker.cpp +++ b/dbms/src/Interpreters/DDLWorker.cpp @@ -506,8 +506,9 @@ void DDLWorker::parseQueryAndResolveHost(DDLTask & task) { const Cluster::Address & address = shards[shard_num][replica_num]; - if (isLocalAddress(address.getResolvedAddress(), context.getTCPPort()) - || (context.getTCPPortSecure() && isLocalAddress(address.getResolvedAddress(), *context.getTCPPortSecure()))) + if (auto resolved = address.getResolvedAddress(); + resolved && (isLocalAddress(*resolved, context.getTCPPort()) + || (context.getTCPPortSecure() && isLocalAddress(*resolved, *context.getTCPPortSecure())))) { if (found_via_resolving) { diff --git a/dbms/src/Interpreters/DNSCacheUpdater.cpp b/dbms/src/Interpreters/DNSCacheUpdater.cpp index 66686dd9010..90c27ea2854 100644 --- a/dbms/src/Interpreters/DNSCacheUpdater.cpp +++ b/dbms/src/Interpreters/DNSCacheUpdater.cpp @@ -16,7 +16,6 @@ DNSCacheUpdater::DNSCacheUpdater(Context & context_, Int32 update_period_seconds void DNSCacheUpdater::run() { - watch.restart(); auto & resolver = DNSResolver::instance(); /// Reload cluster config if IP of any host has been changed since last update. @@ -34,8 +33,12 @@ void DNSCacheUpdater::run() } } - auto interval_ms = std::max(0, update_period_seconds * 1000 - static_cast(watch.elapsedMilliseconds())); - task_handle->scheduleAfter(interval_ms); + /** DNS resolution may take a while and by this reason, actual update period will be longer than update_period_seconds. + * We intentionally allow this "drift" for two reasons: + * - automatically throttle when DNS requests take longer time; + * - add natural randomization on huge clusters - avoid sending all requests at the same moment of time from different servers. + */ + task_handle->scheduleAfter(update_period_seconds * 1000); } void DNSCacheUpdater::start() diff --git a/dbms/src/Interpreters/DNSCacheUpdater.h b/dbms/src/Interpreters/DNSCacheUpdater.h index 63f6dba7285..52577ad43f6 100644 --- a/dbms/src/Interpreters/DNSCacheUpdater.h +++ b/dbms/src/Interpreters/DNSCacheUpdater.h @@ -24,7 +24,6 @@ private: BackgroundSchedulePool & pool; BackgroundSchedulePoolTaskHolder task_handle; - Stopwatch watch; }; diff --git a/dbms/src/Interpreters/ExpressionActions.cpp b/dbms/src/Interpreters/ExpressionActions.cpp index 79f9eb60711..43592c2616a 100644 --- a/dbms/src/Interpreters/ExpressionActions.cpp +++ b/dbms/src/Interpreters/ExpressionActions.cpp @@ -740,22 +740,27 @@ void ExpressionActions::execute(Block & block, bool dry_run) const } } +bool ExpressionActions::hasTotalsInJoin() const +{ + bool has_totals_in_join = false; + for (const auto & action : actions) + { + if (action.join && action.join->hasTotals()) + { + has_totals_in_join = true; + break; + } + } + + return has_totals_in_join; +} + void ExpressionActions::executeOnTotals(Block & block) const { /// If there is `totals` in the subquery for JOIN, but we do not have totals, then take the block with the default values instead of `totals`. if (!block) { - bool has_totals_in_join = false; - for (const auto & action : actions) - { - if (action.join && action.join->hasTotals()) - { - has_totals_in_join = true; - break; - } - } - - if (has_totals_in_join) + if (hasTotalsInJoin()) { for (const auto & name_and_type : input_columns) { diff --git a/dbms/src/Interpreters/ExpressionActions.h b/dbms/src/Interpreters/ExpressionActions.h index e8755d5f4ea..68cc3e73b17 100644 --- a/dbms/src/Interpreters/ExpressionActions.h +++ b/dbms/src/Interpreters/ExpressionActions.h @@ -223,6 +223,9 @@ public: /// Execute the expression on the block. The block must contain all the columns returned by getRequiredColumns. void execute(Block & block, bool dry_run = false) const; + /// Check if joined subquery has totals. + bool hasTotalsInJoin() const; + /** Execute the expression on the block of total values. * Almost the same as `execute`. The difference is only when JOIN is executed. */ diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp index a1345cd7ba9..e8143f3c41a 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp +++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp @@ -144,7 +144,11 @@ void ExpressionAnalyzer::analyzeAggregation() { getRootActions(array_join_expression_list, true, temp_actions); addMultipleArrayJoinAction(temp_actions, is_array_join_left); - array_join_columns = temp_actions->getSampleBlock().getNamesAndTypesList(); + + array_join_columns.clear(); + for (auto & column : temp_actions->getSampleBlock().getNamesAndTypesList()) + if (syntax->array_join_result_to_source.count(column.name)) + array_join_columns.emplace_back(column); } const ASTTablesInSelectQueryElement * join = select_query->join(); @@ -275,12 +279,14 @@ void ExpressionAnalyzer::tryMakeSetForIndexFromSubquery(const ASTPtr & subquery_ SetPtr set = std::make_shared(settings.size_limits_for_set, true); set->setHeader(res.in->getHeader()); + res.in->readPrefix(); while (Block block = res.in->read()) { /// If the limits have been exceeded, give up and let the default subquery processing actions take place. if (!set->insertFromBlock(block)) return; } + res.in->readSuffix(); prepared_sets[set_key] = std::move(set); } @@ -1034,7 +1040,28 @@ void ExpressionAnalyzer::collectUsedColumns() /// You need to read at least one column to find the number of rows. if (select_query && required.empty()) - required.insert(ExpressionActions::getSmallestColumn(source_columns)); + { + /// We will find a column with minimum compressed size. Because it is the column that is cheapest to read. + size_t min_data_compressed = 0; + String min_column_name; + if (storage) + { + auto column_sizes = storage->getColumnSizes(); + for (auto & [column_name, column_size] : column_sizes) + { + if (min_data_compressed == 0 || min_data_compressed > column_size.data_compressed) + { + min_data_compressed = column_size.data_compressed; + min_column_name = column_name; + } + } + } + if (min_data_compressed > 0) + required.insert(min_column_name); + else + /// If we have no information about columns sizes, choose a column of minimum size of its data type. + required.insert(ExpressionActions::getSmallestColumn(source_columns)); + } NameSet unknown_required_source_columns = required; diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.h b/dbms/src/Interpreters/ExpressionAnalyzer.h index 6bba6860f6f..52c43dda90f 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.h +++ b/dbms/src/Interpreters/ExpressionAnalyzer.h @@ -249,7 +249,7 @@ private: void assertAggregation() const; /** - * Create Set from a subuquery or a table expression in the query. The created set is suitable for using the index. + * Create Set from a subquery or a table expression in the query. The created set is suitable for using the index. * The set will not be created if its size hits the limit. */ void tryMakeSetForIndexFromSubquery(const ASTPtr & subquery_or_table_name); diff --git a/dbms/src/Interpreters/ExternalLoader.cpp b/dbms/src/Interpreters/ExternalLoader.cpp index c34193f71e4..c7b3d202a28 100644 --- a/dbms/src/Interpreters/ExternalLoader.cpp +++ b/dbms/src/Interpreters/ExternalLoader.cpp @@ -219,7 +219,7 @@ class ExternalLoader::LoadingDispatcher : private boost::noncopyable { public: /// Called to load or reload an object. - using CreateObjectFunction = std::function; /// Called after loading/reloading an object to calculate the time of the next update. @@ -528,16 +528,48 @@ public: /// The function doesn't touch the objects which were never tried to load. void reloadOutdated() { - std::lock_guard lock{mutex}; - TimePoint now = std::chrono::system_clock::now(); - for (auto & [name, info] : infos) - if ((now >= info.next_update_time) && !info.loading() && info.was_loading()) + std::unordered_map is_modified_map; + + { + std::lock_guard lock{mutex}; + TimePoint now = std::chrono::system_clock::now(); + for (const auto & name_and_info : infos) { - if (info.loaded() && !is_object_modified(info.object)) - info.next_update_time = calculate_next_update_time(info.object, info.error_count); - else - startLoading(name, info); + const auto & info = name_and_info.second; + if ((now >= info.next_update_time) && !info.loading() && info.was_loading()) + is_modified_map.emplace(info.object, true); } + } + + /// The `mutex` should be unlocked while we're calling the function is_object_modified(). + for (auto & [object, is_modified_flag] : is_modified_map) + { + try + { + is_modified_flag = is_object_modified(object); + } + catch (...) + { + tryLogCurrentException(log, "Could not check if " + type_name + " '" + object->getName() + "' was modified"); + } + } + + { + std::lock_guard lock{mutex}; + TimePoint now = std::chrono::system_clock::now(); + for (auto & [name, info] : infos) + if ((now >= info.next_update_time) && !info.loading() && info.was_loading()) + { + auto it = is_modified_map.find(info.object); + if (it == is_modified_map.end()) + continue; /// Object has been just added, it can be simply omitted from this update of outdated. + bool is_modified_flag = it->second; + if (info.loaded() && !is_modified_flag) + info.next_update_time = calculate_next_update_time(info.object, info.error_count); + else + startLoading(name, info); + } + } } private: @@ -751,7 +783,7 @@ private: std::exception_ptr new_exception; try { - std::tie(new_object, new_exception) = create_object(name, config, config_changed, previous_version); + new_object = create_object(name, config, config_changed, previous_version); } catch (...) { @@ -760,8 +792,6 @@ private: if (!new_object && !new_exception) throw Exception("No object created and no exception raised for " + type_name, ErrorCodes::LOGICAL_ERROR); - if (new_object && new_exception) - new_object = nullptr; /// Calculate a new update time. TimePoint next_update_time; @@ -1120,17 +1150,13 @@ void ExternalLoader::reload(bool load_never_loading) loading_dispatcher->reload(load_never_loading); } -ExternalLoader::ObjectWithException ExternalLoader::createObject( +ExternalLoader::LoadablePtr ExternalLoader::createObject( const String & name, const ObjectConfig & config, bool config_changed, const LoadablePtr & previous_version) const { if (previous_version && !config_changed) - { - auto new_object = previous_version->clone(); - return {new_object, new_object->getCreationException()}; - } + return previous_version->clone(); - auto new_object = create(name, *config.config, config.key_in_config); - return {new_object, new_object->getCreationException()}; + return create(name, *config.config, config.key_in_config); } ExternalLoader::TimePoint ExternalLoader::calculateNextUpdateTime(const LoadablePtr & loaded_object, size_t error_count) const diff --git a/dbms/src/Interpreters/ExternalLoader.h b/dbms/src/Interpreters/ExternalLoader.h index da999bfe21a..4c94b8d69cd 100644 --- a/dbms/src/Interpreters/ExternalLoader.h +++ b/dbms/src/Interpreters/ExternalLoader.h @@ -186,10 +186,8 @@ protected: private: struct ObjectConfig; - using ObjectWithException = std::pair; - ObjectWithException - createObject(const String & name, const ObjectConfig & config, bool config_changed, const LoadablePtr & previous_version) const; + LoadablePtr createObject(const String & name, const ObjectConfig & config, bool config_changed, const LoadablePtr & previous_version) const; TimePoint calculateNextUpdateTime(const LoadablePtr & loaded_object, size_t error_count) const; class ConfigFilesReader; diff --git a/dbms/src/Interpreters/GlobalSubqueriesVisitor.h b/dbms/src/Interpreters/GlobalSubqueriesVisitor.h index 4b94a804858..229fa00a59f 100644 --- a/dbms/src/Interpreters/GlobalSubqueriesVisitor.h +++ b/dbms/src/Interpreters/GlobalSubqueriesVisitor.h @@ -92,7 +92,7 @@ public: Block sample = interpreter->getSampleBlock(); NamesAndTypesList columns = sample.getNamesAndTypesList(); - StoragePtr external_storage = StorageMemory::create(external_table_name, ColumnsDescription{columns}); + StoragePtr external_storage = StorageMemory::create("_external", external_table_name, ColumnsDescription{columns}); external_storage->startup(); /** We replace the subquery with the name of the temporary table. diff --git a/dbms/src/Interpreters/IExternalLoadable.h b/dbms/src/Interpreters/IExternalLoadable.h index 7b875f02060..f8725a67989 100644 --- a/dbms/src/Interpreters/IExternalLoadable.h +++ b/dbms/src/Interpreters/IExternalLoadable.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include #include @@ -41,10 +40,6 @@ public: virtual bool isModified() const = 0; /// Returns new object with the same configuration. Is used to update modified object when lifetime exceeded. virtual std::shared_ptr clone() const = 0; - - virtual std::chrono::time_point getCreationTime() const = 0; - - virtual std::exception_ptr getCreationException() const = 0; }; } diff --git a/dbms/src/Interpreters/IInterpreter.h b/dbms/src/Interpreters/IInterpreter.h index b51e773a17b..e1090061cf3 100644 --- a/dbms/src/Interpreters/IInterpreter.h +++ b/dbms/src/Interpreters/IInterpreter.h @@ -2,6 +2,7 @@ #include +#include namespace DB { @@ -17,6 +18,10 @@ public: */ virtual BlockIO execute() = 0; + virtual QueryPipeline executeWithProcessors() { throw Exception("executeWithProcessors not implemented", ErrorCodes::NOT_IMPLEMENTED); } + + virtual bool canExecuteWithProcessors() const { return false; } + virtual ~IInterpreter() {} }; diff --git a/dbms/src/Interpreters/InternalTextLogsQueue.cpp b/dbms/src/Interpreters/InternalTextLogsQueue.cpp index a27838a4203..6028514f11f 100644 --- a/dbms/src/Interpreters/InternalTextLogsQueue.cpp +++ b/dbms/src/Interpreters/InternalTextLogsQueue.cpp @@ -20,13 +20,13 @@ Block InternalTextLogsQueue::getSampleBlock() { return Block { {std::make_shared(), "event_time"}, - {std::make_shared(), "event_time_microseconds"}, - {std::make_shared(), "host_name"}, - {std::make_shared(), "query_id"}, - {std::make_shared(), "thread_number"}, - {std::make_shared(), "priority"}, - {std::make_shared(), "source"}, - {std::make_shared(), "text"} + {std::make_shared(), "event_time_microseconds"}, + {std::make_shared(), "host_name"}, + {std::make_shared(), "query_id"}, + {std::make_shared(), "thread_number"}, + {std::make_shared(), "priority"}, + {std::make_shared(), "source"}, + {std::make_shared(), "text"} }; } diff --git a/dbms/src/Interpreters/InterpreterCheckQuery.cpp b/dbms/src/Interpreters/InterpreterCheckQuery.cpp index c99c74fa33a..8326a627e6c 100644 --- a/dbms/src/Interpreters/InterpreterCheckQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCheckQuery.cpp @@ -4,13 +4,30 @@ #include #include #include +#include #include #include +#include namespace DB { +namespace +{ + +NamesAndTypes getBlockStructure() +{ + return { + {"part_path", std::make_shared()}, + {"is_passed", std::make_shared()}, + {"message", std::make_shared()}, + }; +} + +} + + InterpreterCheckQuery::InterpreterCheckQuery(const ASTPtr & query_ptr_, const Context & context_) : query_ptr(query_ptr_), context(context_) { @@ -19,18 +36,43 @@ InterpreterCheckQuery::InterpreterCheckQuery(const ASTPtr & query_ptr_, const Co BlockIO InterpreterCheckQuery::execute() { - const auto & alter = query_ptr->as(); - const String & table_name = alter.table; - String database_name = alter.database.empty() ? context.getCurrentDatabase() : alter.database; + const auto & check = query_ptr->as(); + const String & table_name = check.table; + String database_name = check.database.empty() ? context.getCurrentDatabase() : check.database; StoragePtr table = context.getTable(database_name, table_name); + auto check_results = table->checkData(query_ptr, context); - auto column = ColumnUInt8::create(); - column->insertValue(UInt64(table->checkData())); - result = Block{{ std::move(column), std::make_shared(), "result" }}; + Block block; + if (context.getSettingsRef().check_query_single_value_result) + { + bool result = std::all_of(check_results.begin(), check_results.end(), [] (const CheckResult & res) { return res.success; }); + auto column = ColumnUInt8::create(); + column->insertValue(UInt64(result)); + block = Block{{std::move(column), std::make_shared(), "result"}}; + } + else + { + auto block_structure = getBlockStructure(); + auto path_column = block_structure[0].type->createColumn(); + auto is_passed_column = block_structure[1].type->createColumn(); + auto message_column = block_structure[2].type->createColumn(); + + for (const auto & check_result : check_results) + { + path_column->insert(check_result.fs_path); + is_passed_column->insert(static_cast(check_result.success)); + message_column->insert(check_result.failure_message); + } + + block = Block({ + {std::move(path_column), block_structure[0].type, block_structure[0].name}, + {std::move(is_passed_column), block_structure[1].type, block_structure[1].name}, + {std::move(message_column), block_structure[2].type, block_structure[2].name}}); + } BlockIO res; - res.in = std::make_shared(result); + res.in = std::make_shared(block); return res; } diff --git a/dbms/src/Interpreters/InterpreterCheckQuery.h b/dbms/src/Interpreters/InterpreterCheckQuery.h index ca6af4af7bc..c667ca74c22 100644 --- a/dbms/src/Interpreters/InterpreterCheckQuery.h +++ b/dbms/src/Interpreters/InterpreterCheckQuery.h @@ -21,7 +21,6 @@ private: ASTPtr query_ptr; const Context & context; - Block result; }; } diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index 7853e0c0841..19d27a3851a 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -159,7 +159,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) if (need_write_metadata) Poco::File(metadata_file_tmp_path).renameTo(metadata_file_path); - database->loadTables(context, thread_pool, has_force_restore_data_flag); + database->loadTables(context, has_force_restore_data_flag); } catch (...) { @@ -625,7 +625,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) insert->select = create.select->clone(); if (create.temporary && !context.getSessionContext().hasQueryContext()) - context.getSessionContext().setQueryContext(context.getSessionContext()); + context.getSessionContext().makeQueryContext(); return InterpreterInsertQuery(insert, create.temporary ? context.getSessionContext() : context, diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.h b/dbms/src/Interpreters/InterpreterCreateQuery.h index 82ba09fb474..68e24c50ccc 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.h +++ b/dbms/src/Interpreters/InterpreterCreateQuery.h @@ -31,11 +31,6 @@ public: static ASTPtr formatIndices(const IndicesDescription & indices); - void setDatabaseLoadingThreadpool(ThreadPool & thread_pool_) - { - thread_pool = &thread_pool_; - } - void setForceRestoreData(bool has_force_restore_data_flag_) { has_force_restore_data_flag = has_force_restore_data_flag_; @@ -61,9 +56,6 @@ private: ASTPtr query_ptr; Context & context; - /// Using while loading database. - ThreadPool * thread_pool = nullptr; - /// Skip safety threshold when loading tables. bool has_force_restore_data_flag = false; /// Is this an internal query - not from the user. diff --git a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp index d037f87a857..e6da9b63d7b 100644 --- a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp @@ -101,6 +101,9 @@ BlockInputStreamPtr InterpreterDescribeQuery::executeImpl() for (const auto & column : columns) { + if (column.is_virtual) + continue; + res_columns[0]->insert(column.name); res_columns[1]->insert(column.type->getName()); diff --git a/dbms/src/Interpreters/InterpreterKillQueryQuery.cpp b/dbms/src/Interpreters/InterpreterKillQueryQuery.cpp index 89339668088..4e4120c5580 100644 --- a/dbms/src/Interpreters/InterpreterKillQueryQuery.cpp +++ b/dbms/src/Interpreters/InterpreterKillQueryQuery.cpp @@ -265,6 +265,11 @@ Block InterpreterKillQueryQuery::getSelectResult(const String & columns, const S if (where_expression) select_query += " WHERE " + queryToString(where_expression); + auto use_processors = context.getSettingsRef().experimental_use_processors; + context.getSettingsRef().experimental_use_processors = false; + + SCOPE_EXIT(context.getSettingsRef().experimental_use_processors = use_processors); + BlockIO block_io = executeQuery(select_query, context, true); Block res = block_io.in->read(); diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index bab04df70ab..d6e46fbedb3 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -58,6 +58,26 @@ #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + namespace DB { @@ -356,6 +376,13 @@ BlockInputStreams InterpreterSelectQuery::executeWithMultipleStreams() return pipeline.streams; } +QueryPipeline InterpreterSelectQuery::executeWithProcessors() +{ + QueryPipeline query_pipeline; + executeImpl(query_pipeline, input, options.only_analyze); + return query_pipeline; +} + InterpreterSelectQuery::AnalysisResult InterpreterSelectQuery::analyzeExpressions(QueryProcessingStage::Enum from_stage, bool dry_run, const FilterInfoPtr & filter_info) { @@ -555,7 +582,8 @@ InterpreterSelectQuery::analyzeExpressions(QueryProcessingStage::Enum from_stage } -void InterpreterSelectQuery::executeImpl(Pipeline & pipeline, const BlockInputStreamPtr & prepared_input, bool dry_run) +template +void InterpreterSelectQuery::executeImpl(TPipeline & pipeline, const BlockInputStreamPtr & prepared_input, bool dry_run) { /** Streams of data. When the query is executed in parallel, we have several data streams. * If there is no GROUP BY, then perform all operations before ORDER BY and LIMIT in parallel, then @@ -567,6 +595,8 @@ void InterpreterSelectQuery::executeImpl(Pipeline & pipeline, const BlockInputSt * then perform the remaining operations with one resulting stream. */ + constexpr bool pipeline_with_processors = std::is_same::value; + /// Now we will compose block streams that perform the necessary actions. auto & query = getSelectQuery(); const Settings & settings = context.getSettingsRef(); @@ -611,21 +641,42 @@ void InterpreterSelectQuery::executeImpl(Pipeline & pipeline, const BlockInputSt if (dry_run) { - pipeline.streams.emplace_back(std::make_shared(source_header)); + if constexpr (pipeline_with_processors) + pipeline.init({std::make_shared(source_header)}); + else + pipeline.streams.emplace_back(std::make_shared(source_header)); + expressions = analyzeExpressions(QueryProcessingStage::FetchColumns, true, filter_info); if (storage && expressions.filter_info && expressions.prewhere_info) throw Exception("PREWHERE is not supported if the table is filtered by row-level security expression", ErrorCodes::ILLEGAL_PREWHERE); if (expressions.prewhere_info) - pipeline.streams.back() = std::make_shared( + { + if constexpr (pipeline_with_processors) + pipeline.addSimpleTransform([&](const Block & header) + { + return std::make_shared( + header, + expressions.prewhere_info->prewhere_actions, + expressions.prewhere_info->prewhere_column_name, + expressions.prewhere_info->remove_prewhere_column); + }); + else + pipeline.streams.back() = std::make_shared( pipeline.streams.back(), expressions.prewhere_info->prewhere_actions, expressions.prewhere_info->prewhere_column_name, expressions.prewhere_info->remove_prewhere_column); + } } else { if (prepared_input) - pipeline.streams.push_back(prepared_input); + { + if constexpr (pipeline_with_processors) + pipeline.init({std::make_shared(prepared_input)}); + else + pipeline.streams.push_back(prepared_input); + } expressions = analyzeExpressions(from_stage, false, filter_info); @@ -662,25 +713,78 @@ void InterpreterSelectQuery::executeImpl(Pipeline & pipeline, const BlockInputSt { if (expressions.filter_info) { - pipeline.transform([&](auto & stream) + if constexpr (pipeline_with_processors) { - stream = std::make_shared( - stream, - expressions.filter_info->actions, - expressions.filter_info->column_name, - expressions.filter_info->do_remove_column); - }); + pipeline.addSimpleTransform([&](const Block & block, QueryPipeline::StreamType stream_type) -> ProcessorPtr + { + if (stream_type == QueryPipeline::StreamType::Totals) + return nullptr; + + return std::make_shared( + block, + expressions.filter_info->actions, + expressions.filter_info->column_name, + expressions.filter_info->do_remove_column); + }); + } + else + { + pipeline.transform([&](auto & stream) + { + stream = std::make_shared( + stream, + expressions.filter_info->actions, + expressions.filter_info->column_name, + expressions.filter_info->do_remove_column); + }); + } } if (expressions.hasJoin()) { + Block header_before_join; + + if constexpr (pipeline_with_processors) + { + header_before_join = pipeline.getHeader(); + + /// In case joined subquery has totals, and we don't, add default chunk to totals. + bool default_totals = false; + if (!pipeline.hasTotals()) + { + pipeline.addDefaultTotals(); + default_totals = true; + } + + pipeline.addSimpleTransform([&](const Block & header, QueryPipeline::StreamType type) + { + bool on_totals = type == QueryPipeline::StreamType::Totals; + return std::make_shared(header, expressions.before_join, on_totals, default_totals); + }); + } + else + { + header_before_join = pipeline.firstStream()->getHeader(); + /// Applies to all sources except stream_with_non_joined_data. + for (auto & stream : pipeline.streams) + stream = std::make_shared(stream, expressions.before_join); + } + const auto & join = query.join()->table_join->as(); if (isRightOrFull(join.kind)) - pipeline.stream_with_non_joined_data = expressions.before_join->createStreamWithNonJoinedDataIfFullOrRightJoin( - pipeline.firstStream()->getHeader(), settings.max_block_size); + { + auto stream = expressions.before_join->createStreamWithNonJoinedDataIfFullOrRightJoin( + header_before_join, settings.max_block_size); - for (auto & stream : pipeline.streams) /// Applies to all sources except stream_with_non_joined_data. - stream = std::make_shared(stream, expressions.before_join); + if constexpr (pipeline_with_processors) + { + auto source = std::make_shared(std::move(stream)); + pipeline.addDelayedStream(source); + } + else + pipeline.stream_with_non_joined_data = std::move(stream); + + } } if (expressions.has_where) @@ -810,13 +914,18 @@ void InterpreterSelectQuery::executeImpl(Pipeline & pipeline, const BlockInputSt if (need_second_distinct_pass || query.limitLength() || query.limitBy() - || pipeline.stream_with_non_joined_data) + || pipeline.hasDelayedStream()) { need_merge_streams = true; } if (need_merge_streams) - executeUnion(pipeline); + { + if constexpr (pipeline_with_processors) + pipeline.resize(1); + else + executeUnion(pipeline); + } /** If there was more than one stream, * then DISTINCT needs to be performed once again after merging all streams. @@ -888,10 +997,13 @@ static UInt64 getLimitForSorting(const ASTSelectQuery & query, const Context & c } +template void InterpreterSelectQuery::executeFetchColumns( - QueryProcessingStage::Enum processing_stage, Pipeline & pipeline, + QueryProcessingStage::Enum processing_stage, TPipeline & pipeline, const PrewhereInfoPtr & prewhere_info, const Names & columns_to_remove_after_prewhere) { + constexpr bool pipeline_with_processors = std::is_same::value; + auto & query = getSelectQuery(); const Settings & settings = context.getSettingsRef(); @@ -1105,7 +1217,7 @@ void InterpreterSelectQuery::executeFetchColumns( throw Exception("Setting 'max_block_size' cannot be zero", ErrorCodes::PARAMETER_OUT_OF_BOUND); /// Initialize the initial data streams to which the query transforms are superimposed. Table or subquery or prepared input? - if (!pipeline.streams.empty()) + if (pipeline.initialized()) { /// Prepared input. } @@ -1127,7 +1239,11 @@ void InterpreterSelectQuery::executeFetchColumns( interpreter_subquery->ignoreWithTotals(); } - pipeline.streams = interpreter_subquery->executeWithMultipleStreams(); + if constexpr (pipeline_with_processors) + /// Just use pipeline from subquery. + pipeline = interpreter_subquery->executeWithProcessors(); + else + pipeline.streams = interpreter_subquery->executeWithMultipleStreams(); } else if (storage) { @@ -1146,27 +1262,23 @@ void InterpreterSelectQuery::executeFetchColumns( query_info.sets = query_analyzer->getPreparedSets(); query_info.prewhere_info = prewhere_info; - pipeline.streams = storage->read(required_columns, query_info, context, processing_stage, max_block_size, max_streams); + auto streams = storage->read(required_columns, query_info, context, processing_stage, max_block_size, max_streams); - if (pipeline.streams.empty()) + if (streams.empty()) { - pipeline.streams = {std::make_shared(storage->getSampleBlockForColumns(required_columns))}; + streams = {std::make_shared(storage->getSampleBlockForColumns(required_columns))}; if (query_info.prewhere_info) - pipeline.transform([&](auto & stream) - { - stream = std::make_shared( - stream, - prewhere_info->prewhere_actions, - prewhere_info->prewhere_column_name, - prewhere_info->remove_prewhere_column); - }); + streams.back() = std::make_shared( + streams.back(), + prewhere_info->prewhere_actions, + prewhere_info->prewhere_column_name, + prewhere_info->remove_prewhere_column); + } - pipeline.transform([&](auto & stream) - { + for (auto & stream : streams) stream->addTableLock(table_lock); - }); /// Set the limits and quota for reading data, the speed and time of the query. { @@ -1194,15 +1306,52 @@ void InterpreterSelectQuery::executeFetchColumns( QuotaForIntervals & quota = context.getQuota(); - pipeline.transform([&](auto & stream) + for (auto & stream : streams) { if (!options.ignore_limits) stream->setLimits(limits); if (options.to_stage == QueryProcessingStage::Complete) stream->setQuota(quota); - }); + } } + + if constexpr (pipeline_with_processors) + { + /// Unify streams. They must have same headers. + if (streams.size() > 1) + { + /// Unify streams in case they have different headers. + auto first_header = streams.at(0)->getHeader(); + for (size_t i = 1; i < streams.size(); ++i) + { + auto & stream = streams[i]; + auto header = stream->getHeader(); + auto mode = ConvertingBlockInputStream::MatchColumnsMode::Name; + if (!blocksHaveEqualStructure(first_header, header)) + stream = std::make_shared(context, stream, first_header, mode); + } + } + + Processors sources; + sources.reserve(streams.size()); + + for (auto & stream : streams) + { + bool force_add_agg_info = processing_stage == QueryProcessingStage::WithMergeableState; + auto source = std::make_shared(stream, force_add_agg_info); + + if (processing_stage == QueryProcessingStage::Complete) + source->addTotalsPort(); + + sources.emplace_back(std::move(source)); + + } + + pipeline.init(std::move(sources)); + } + else + pipeline.streams = std::move(streams); } else throw Exception("Logical error in InterpreterSelectQuery: nowhere to read", ErrorCodes::LOGICAL_ERROR); @@ -1210,10 +1359,20 @@ void InterpreterSelectQuery::executeFetchColumns( /// Aliases in table declaration. if (processing_stage == QueryProcessingStage::FetchColumns && alias_actions) { - pipeline.transform([&](auto & stream) + if constexpr (pipeline_with_processors) { - stream = std::make_shared(stream, alias_actions); - }); + pipeline.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header, alias_actions); + }); + } + else + { + pipeline.transform([&](auto & stream) + { + stream = std::make_shared(stream, alias_actions); + }); + } } } @@ -1226,6 +1385,13 @@ void InterpreterSelectQuery::executeWhere(Pipeline & pipeline, const ExpressionA }); } +void InterpreterSelectQuery::executeWhere(QueryPipeline & pipeline, const ExpressionActionsPtr & expression, bool remove_fiter) +{ + pipeline.addSimpleTransform([&](const Block & block) + { + return std::make_shared(block, expression, getSelectQuery().where()->getColumnName(), remove_fiter); + }); +} void InterpreterSelectQuery::executeAggregation(Pipeline & pipeline, const ExpressionActionsPtr & expression, bool overflow_row, bool final) { @@ -1294,6 +1460,76 @@ void InterpreterSelectQuery::executeAggregation(Pipeline & pipeline, const Expre } +void InterpreterSelectQuery::executeAggregation(QueryPipeline & pipeline, const ExpressionActionsPtr & expression, bool overflow_row, bool final) +{ + pipeline.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header, expression); + }); + + Names key_names; + AggregateDescriptions aggregates; + query_analyzer->getAggregateInfo(key_names, aggregates); + + Block header_before_aggregation = pipeline.getHeader(); + ColumnNumbers keys; + for (const auto & name : key_names) + keys.push_back(header_before_aggregation.getPositionByName(name)); + for (auto & descr : aggregates) + if (descr.arguments.empty()) + for (const auto & name : descr.argument_names) + descr.arguments.push_back(header_before_aggregation.getPositionByName(name)); + + const Settings & settings = context.getSettingsRef(); + + /** Two-level aggregation is useful in two cases: + * 1. Parallel aggregation is done, and the results should be merged in parallel. + * 2. An aggregation is done with store of temporary data on the disk, and they need to be merged in a memory efficient way. + */ + bool allow_to_use_two_level_group_by = pipeline.getNumMainStreams() > 1 || settings.max_bytes_before_external_group_by != 0; + + Aggregator::Params params(header_before_aggregation, keys, aggregates, + overflow_row, settings.max_rows_to_group_by, settings.group_by_overflow_mode, + settings.compile ? &context.getCompiler() : nullptr, settings.min_count_to_compile, + allow_to_use_two_level_group_by ? settings.group_by_two_level_threshold : SettingUInt64(0), + allow_to_use_two_level_group_by ? settings.group_by_two_level_threshold_bytes : SettingUInt64(0), + settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set, + context.getTemporaryPath(), settings.max_threads); + + auto transform_params = std::make_shared(params, final); + + pipeline.dropTotalsIfHas(); + + /// If there are several sources, then we perform parallel aggregation + if (pipeline.getNumMainStreams() > 1) + { + pipeline.resize(max_streams); + + auto many_data = std::make_shared(max_streams); + auto merge_threads = settings.aggregation_memory_efficient_merge_threads + ? static_cast(settings.aggregation_memory_efficient_merge_threads) + : static_cast(settings.max_threads); + + size_t counter = 0; + pipeline.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header, transform_params, many_data, counter++, max_streams, merge_threads); + }); + + pipeline.resize(1); + } + else + { + pipeline.resize(1); + + pipeline.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header, transform_params); + }); + } +} + + void InterpreterSelectQuery::executeMergeAggregated(Pipeline & pipeline, bool overflow_row, bool final) { Names key_names; @@ -1345,6 +1581,67 @@ void InterpreterSelectQuery::executeMergeAggregated(Pipeline & pipeline, bool ov } } +void InterpreterSelectQuery::executeMergeAggregated(QueryPipeline & pipeline, bool overflow_row, bool final) +{ + Names key_names; + AggregateDescriptions aggregates; + query_analyzer->getAggregateInfo(key_names, aggregates); + + Block header_before_merge = pipeline.getHeader(); + + ColumnNumbers keys; + for (const auto & name : key_names) + keys.push_back(header_before_merge.getPositionByName(name)); + + /** There are two modes of distributed aggregation. + * + * 1. In different threads read from the remote servers blocks. + * Save all the blocks in the RAM. Merge blocks. + * If the aggregation is two-level - parallelize to the number of buckets. + * + * 2. In one thread, read blocks from different servers in order. + * RAM stores only one block from each server. + * If the aggregation is a two-level aggregation, we consistently merge the blocks of each next level. + * + * The second option consumes less memory (up to 256 times less) + * in the case of two-level aggregation, which is used for large results after GROUP BY, + * but it can work more slowly. + */ + + const Settings & settings = context.getSettingsRef(); + + Aggregator::Params params(header_before_merge, keys, aggregates, overflow_row, settings.max_threads); + + auto transform_params = std::make_shared(params, final); + + if (!settings.distributed_aggregation_memory_efficient) + { + /// We union several sources into one, parallelizing the work. + pipeline.resize(1); + + /// Now merge the aggregated blocks + pipeline.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header, transform_params, settings.max_threads); + }); + } + else + { + /// pipeline.resize(max_streams); - Seem we don't need it. + auto num_merge_threads = settings.aggregation_memory_efficient_merge_threads + ? static_cast(settings.aggregation_memory_efficient_merge_threads) + : static_cast(settings.max_threads); + + auto pipe = createMergingAggregatedMemoryEfficientPipe( + pipeline.getHeader(), + transform_params, + pipeline.getNumStreams(), + num_merge_threads); + + pipeline.addPipe(std::move(pipe)); + } +} + void InterpreterSelectQuery::executeHaving(Pipeline & pipeline, const ExpressionActionsPtr & expression) { @@ -1354,6 +1651,18 @@ void InterpreterSelectQuery::executeHaving(Pipeline & pipeline, const Expression }); } +void InterpreterSelectQuery::executeHaving(QueryPipeline & pipeline, const ExpressionActionsPtr & expression) +{ + pipeline.addSimpleTransform([&](const Block & header, QueryPipeline::StreamType stream_type) -> ProcessorPtr + { + if (stream_type == QueryPipeline::StreamType::Totals) + return nullptr; + + /// TODO: do we need to save filter there? + return std::make_shared(header, expression, getSelectQuery().having()->getColumnName(), false); + }); +} + void InterpreterSelectQuery::executeTotalsAndHaving(Pipeline & pipeline, bool has_having, const ExpressionActionsPtr & expression, bool overflow_row, bool final) { @@ -1371,6 +1680,19 @@ void InterpreterSelectQuery::executeTotalsAndHaving(Pipeline & pipeline, bool ha final); } +void InterpreterSelectQuery::executeTotalsAndHaving(QueryPipeline & pipeline, bool has_having, const ExpressionActionsPtr & expression, bool overflow_row, bool final) +{ + const Settings & settings = context.getSettingsRef(); + + auto totals_having = std::make_shared( + pipeline.getHeader(), overflow_row, expression, + has_having ? getSelectQuery().having()->getColumnName() : "", + settings.totals_mode, settings.totals_auto_threshold, final); + + pipeline.addTotalsHavingTransform(std::move(totals_having)); +} + + void InterpreterSelectQuery::executeRollupOrCube(Pipeline & pipeline, Modificator modificator) { executeUnion(pipeline); @@ -1401,6 +1723,44 @@ void InterpreterSelectQuery::executeRollupOrCube(Pipeline & pipeline, Modificato pipeline.firstStream() = std::make_shared(pipeline.firstStream(), params); } +void InterpreterSelectQuery::executeRollupOrCube(QueryPipeline & pipeline, Modificator modificator) +{ + pipeline.resize(1); + + Names key_names; + AggregateDescriptions aggregates; + query_analyzer->getAggregateInfo(key_names, aggregates); + + Block header_before_transform = pipeline.getHeader(); + + ColumnNumbers keys; + + for (const auto & name : key_names) + keys.push_back(header_before_transform.getPositionByName(name)); + + const Settings & settings = context.getSettingsRef(); + + Aggregator::Params params(header_before_transform, keys, aggregates, + false, settings.max_rows_to_group_by, settings.group_by_overflow_mode, + settings.compile ? &context.getCompiler() : nullptr, settings.min_count_to_compile, + SettingUInt64(0), SettingUInt64(0), + settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set, + context.getTemporaryPath(), settings.max_threads); + + auto transform_params = std::make_shared(params, true); + + pipeline.addSimpleTransform([&](const Block & header, QueryPipeline::StreamType stream_type) -> ProcessorPtr + { + if (stream_type == QueryPipeline::StreamType::Totals) + return nullptr; + + if (modificator == Modificator::ROLLUP) + return std::make_shared(header, std::move(transform_params)); + else + return std::make_shared(header, std::move(transform_params)); + }); +} + void InterpreterSelectQuery::executeExpression(Pipeline & pipeline, const ExpressionActionsPtr & expression) { @@ -1410,6 +1770,13 @@ void InterpreterSelectQuery::executeExpression(Pipeline & pipeline, const Expres }); } +void InterpreterSelectQuery::executeExpression(QueryPipeline & pipeline, const ExpressionActionsPtr & expression) +{ + pipeline.addSimpleTransform([&](const Block & header) -> ProcessorPtr + { + return std::make_shared(header, expression); + }); +} static SortDescription getSortDescription(const ASTSelectQuery & query) { @@ -1462,6 +1829,42 @@ void InterpreterSelectQuery::executeOrder(Pipeline & pipeline) settings.max_bytes_before_external_sort, context.getTemporaryPath()); } +void InterpreterSelectQuery::executeOrder(QueryPipeline & pipeline) +{ + auto & query = getSelectQuery(); + SortDescription order_descr = getSortDescription(query); + UInt64 limit = getLimitForSorting(query, context); + + const Settings & settings = context.getSettingsRef(); + + /// TODO: Limits on sorting +// IBlockInputStream::LocalLimits limits; +// limits.mode = IBlockInputStream::LIMITS_TOTAL; +// limits.size_limits = SizeLimits(settings.max_rows_to_sort, settings.max_bytes_to_sort, settings.sort_overflow_mode); + + + pipeline.addSimpleTransform([&](const Block & header, QueryPipeline::StreamType stream_type) + { + bool do_count_rows = stream_type == QueryPipeline::StreamType::Main; + return std::make_shared(header, order_descr, limit, do_count_rows); + }); + + /// If there are several streams, we merge them into one + pipeline.resize(1); + + /// Merge the sorted blocks. + pipeline.addSimpleTransform([&](const Block & header, QueryPipeline::StreamType stream_type) -> ProcessorPtr + { + if (stream_type == QueryPipeline::StreamType::Totals) + return nullptr; + + return std::make_shared( + header, order_descr, settings.max_block_size, limit, + settings.max_bytes_before_remerge_sort, + settings.max_bytes_before_external_sort, context.getTemporaryPath()); + }); +} + void InterpreterSelectQuery::executeMergeSorted(Pipeline & pipeline) { @@ -1490,6 +1893,27 @@ void InterpreterSelectQuery::executeMergeSorted(Pipeline & pipeline) } } +void InterpreterSelectQuery::executeMergeSorted(QueryPipeline & pipeline) +{ + auto & query = getSelectQuery(); + SortDescription order_descr = getSortDescription(query); + UInt64 limit = getLimitForSorting(query, context); + + const Settings & settings = context.getSettingsRef(); + + /// If there are several streams, then we merge them into one + if (pipeline.getNumStreams() > 1) + { + auto transform = std::make_shared( + pipeline.getHeader(), + pipeline.getNumStreams(), + order_descr, + settings.max_block_size, limit); + + pipeline.addPipe({ std::move(transform) }); + } +} + void InterpreterSelectQuery::executeProjection(Pipeline & pipeline, const ExpressionActionsPtr & expression) { @@ -1499,6 +1923,14 @@ void InterpreterSelectQuery::executeProjection(Pipeline & pipeline, const Expres }); } +void InterpreterSelectQuery::executeProjection(QueryPipeline & pipeline, const ExpressionActionsPtr & expression) +{ + pipeline.addSimpleTransform([&](const Block & header) -> ProcessorPtr + { + return std::make_shared(header, expression); + }); +} + void InterpreterSelectQuery::executeDistinct(Pipeline & pipeline, bool before_order, Names columns) { @@ -1522,6 +1954,32 @@ void InterpreterSelectQuery::executeDistinct(Pipeline & pipeline, bool before_or } } +void InterpreterSelectQuery::executeDistinct(QueryPipeline & pipeline, bool before_order, Names columns) +{ + auto & query = getSelectQuery(); + if (query.distinct) + { + const Settings & settings = context.getSettingsRef(); + + auto [limit_length, limit_offset] = getLimitLengthAndOffset(query, context); + UInt64 limit_for_distinct = 0; + + /// If after this stage of DISTINCT ORDER BY is not executed, then you can get no more than limit_length + limit_offset of different rows. + if (!query.orderBy() || !before_order) + limit_for_distinct = limit_length + limit_offset; + + SizeLimits limits(settings.max_rows_in_distinct, settings.max_bytes_in_distinct, settings.distinct_overflow_mode); + + pipeline.addSimpleTransform([&](const Block & header, QueryPipeline::StreamType stream_type) -> ProcessorPtr + { + if (stream_type == QueryPipeline::StreamType::Totals) + return nullptr; + + return std::make_shared(header, limits, limit_for_distinct, columns); + }); + } +} + void InterpreterSelectQuery::executeUnion(Pipeline & pipeline) { @@ -1558,6 +2016,24 @@ void InterpreterSelectQuery::executePreLimit(Pipeline & pipeline) } } +/// Preliminary LIMIT - is used in every source, if there are several sources, before they are combined. +void InterpreterSelectQuery::executePreLimit(QueryPipeline & pipeline) +{ + auto & query = getSelectQuery(); + /// If there is LIMIT + if (query.limitLength()) + { + auto [limit_length, limit_offset] = getLimitLengthAndOffset(query, context); + pipeline.addSimpleTransform([&, limit = limit_length + limit_offset](const Block & header, QueryPipeline::StreamType stream_type) -> ProcessorPtr + { + if (stream_type == QueryPipeline::StreamType::Totals) + return nullptr; + + return std::make_shared(header, limit, 0); + }); + } +} + void InterpreterSelectQuery::executeLimitBy(Pipeline & pipeline) { @@ -1577,6 +2053,28 @@ void InterpreterSelectQuery::executeLimitBy(Pipeline & pipeline) }); } +void InterpreterSelectQuery::executeLimitBy(QueryPipeline & pipeline) +{ + auto & query = getSelectQuery(); + if (!query.limitByLength() || !query.limitBy()) + return; + + Names columns; + for (const auto & elem : query.limitBy()->children) + columns.emplace_back(elem->getColumnName()); + + UInt64 length = getLimitUIntValue(query.limitByLength(), context); + UInt64 offset = (query.limitByOffset() ? getLimitUIntValue(query.limitByOffset(), context) : 0); + + pipeline.addSimpleTransform([&](const Block & header, QueryPipeline::StreamType stream_type) -> ProcessorPtr + { + if (stream_type == QueryPipeline::StreamType::Totals) + return nullptr; + + return std::make_shared(header, length, offset, columns); + }); +} + // TODO: move to anonymous namespace bool hasWithTotalsInAnySubqueryInFromClause(const ASTSelectQuery & query) @@ -1636,6 +2134,44 @@ void InterpreterSelectQuery::executeLimit(Pipeline & pipeline) } } +void InterpreterSelectQuery::executeLimit(QueryPipeline & pipeline) +{ + auto & query = getSelectQuery(); + /// If there is LIMIT + if (query.limitLength()) + { + /** Rare case: + * if there is no WITH TOTALS and there is a subquery in FROM, and there is WITH TOTALS on one of the levels, + * then when using LIMIT, you should read the data to the end, rather than cancel the query earlier, + * because if you cancel the query, we will not get `totals` data from the remote server. + * + * Another case: + * if there is WITH TOTALS and there is no ORDER BY, then read the data to the end, + * otherwise TOTALS is counted according to incomplete data. + */ + bool always_read_till_end = false; + + if (query.group_by_with_totals && !query.orderBy()) + always_read_till_end = true; + + if (!query.group_by_with_totals && hasWithTotalsInAnySubqueryInFromClause(query)) + always_read_till_end = true; + + UInt64 limit_length; + UInt64 limit_offset; + std::tie(limit_length, limit_offset) = getLimitLengthAndOffset(query, context); + + pipeline.addSimpleTransform([&](const Block & header, QueryPipeline::StreamType stream_type) -> ProcessorPtr + { + if (stream_type != QueryPipeline::StreamType::Main) + return nullptr; + + return std::make_shared( + header, limit_length, limit_offset, always_read_till_end); + }); + } +} + void InterpreterSelectQuery::executeExtremes(Pipeline & pipeline) { @@ -1648,6 +2184,15 @@ void InterpreterSelectQuery::executeExtremes(Pipeline & pipeline) }); } +void InterpreterSelectQuery::executeExtremes(QueryPipeline & pipeline) +{ + if (!context.getSettingsRef().extremes) + return; + + auto transform = std::make_shared(pipeline.getHeader()); + pipeline.addExtremesTransform(std::move(transform)); +} + void InterpreterSelectQuery::executeSubqueriesInSetsAndJoins(Pipeline & pipeline, SubqueriesForSets & subqueries_for_sets) { @@ -1656,6 +2201,19 @@ void InterpreterSelectQuery::executeSubqueriesInSetsAndJoins(Pipeline & pipeline pipeline.firstStream(), subqueries_for_sets, context); } +void InterpreterSelectQuery::executeSubqueriesInSetsAndJoins(QueryPipeline & pipeline, SubqueriesForSets & subqueries_for_sets) +{ + const Settings & settings = context.getSettingsRef(); + + auto creating_sets = std::make_shared( + pipeline.getHeader(), subqueries_for_sets, + SizeLimits(settings.max_rows_to_transfer, settings.max_bytes_to_transfer, settings.transfer_overflow_mode), + context); + + pipeline.addCreatingSetsTransform(std::move(creating_sets)); +} + + void InterpreterSelectQuery::unifyStreams(Pipeline & pipeline) { if (pipeline.hasMoreThanOneStream()) diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.h b/dbms/src/Interpreters/InterpreterSelectQuery.h index 75747694d9e..8dda8589cd9 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.h +++ b/dbms/src/Interpreters/InterpreterSelectQuery.h @@ -13,6 +13,7 @@ #include #include +#include namespace Poco { class Logger; } @@ -69,6 +70,9 @@ public: /// Execute the query and return multuple streams for parallel processing. BlockInputStreams executeWithMultipleStreams(); + QueryPipeline executeWithProcessors() override; + bool canExecuteWithProcessors() const override { return true; } + Block getSampleBlock(); void ignoreWithTotals(); @@ -125,10 +129,13 @@ private: { return hasMoreThanOneStream() || union_stream; } + + bool hasDelayedStream() const { return stream_with_non_joined_data != nullptr; } + bool initialized() const { return !streams.empty(); } }; - void executeImpl(Pipeline & pipeline, const BlockInputStreamPtr & prepared_input, bool dry_run); - + template + void executeImpl(TPipeline & pipeline, const BlockInputStreamPtr & prepared_input, bool dry_run); struct AnalysisResult { @@ -177,7 +184,8 @@ private: /// dry_run - don't read from table, use empty header block instead. void executeWithMultipleStreamsImpl(Pipeline & pipeline, const BlockInputStreamPtr & input, bool dry_run); - void executeFetchColumns(QueryProcessingStage::Enum processing_stage, Pipeline & pipeline, + template + void executeFetchColumns(QueryProcessingStage::Enum processing_stage, TPipeline & pipeline, const PrewhereInfoPtr & prewhere_info, const Names & columns_to_remove_after_prewhere); void executeWhere(Pipeline & pipeline, const ExpressionActionsPtr & expression, bool remove_filter); @@ -197,6 +205,22 @@ private: void executeExtremes(Pipeline & pipeline); void executeSubqueriesInSetsAndJoins(Pipeline & pipeline, std::unordered_map & subqueries_for_sets); + void executeWhere(QueryPipeline & pipeline, const ExpressionActionsPtr & expression, bool remove_fiter); + void executeAggregation(QueryPipeline & pipeline, const ExpressionActionsPtr & expression, bool overflow_row, bool final); + void executeMergeAggregated(QueryPipeline & pipeline, bool overflow_row, bool final); + void executeTotalsAndHaving(QueryPipeline & pipeline, bool has_having, const ExpressionActionsPtr & expression, bool overflow_row, bool final); + void executeHaving(QueryPipeline & pipeline, const ExpressionActionsPtr & expression); + void executeExpression(QueryPipeline & pipeline, const ExpressionActionsPtr & expression); + void executeOrder(QueryPipeline & pipeline); + void executeMergeSorted(QueryPipeline & pipeline); + void executePreLimit(QueryPipeline & pipeline); + void executeLimitBy(QueryPipeline & pipeline); + void executeLimit(QueryPipeline & pipeline); + void executeProjection(QueryPipeline & pipeline, const ExpressionActionsPtr & expression); + void executeDistinct(QueryPipeline & pipeline, bool before_order, Names columns); + void executeExtremes(QueryPipeline & pipeline); + void executeSubqueriesInSetsAndJoins(QueryPipeline & pipeline, std::unordered_map & subqueries_for_sets); + /// If pipeline has several streams with different headers, add ConvertingBlockInputStream to first header. void unifyStreams(Pipeline & pipeline); @@ -208,6 +232,8 @@ private: void executeRollupOrCube(Pipeline & pipeline, Modificator modificator); + void executeRollupOrCube(QueryPipeline & pipeline, Modificator modificator); + /** If there is a SETTINGS section in the SELECT query, then apply settings from it. * * Section SETTINGS - settings for a specific query. diff --git a/dbms/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/dbms/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index 6e0a2603837..0899fed9872 100644 --- a/dbms/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -12,6 +12,9 @@ #include #include +#include +#include + namespace DB { @@ -101,16 +104,25 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( for (size_t query_num = 0; query_num < num_selects; ++query_num) headers[query_num] = nested_interpreters[query_num]->getSampleBlock(); - result_header = headers.front(); - size_t num_columns = result_header.columns(); + result_header = getCommonHeaderForUnion(headers); + } +} - for (size_t query_num = 1; query_num < num_selects; ++query_num) - if (headers[query_num].columns() != num_columns) - throw Exception("Different number of columns in UNION ALL elements:\n" - + result_header.dumpNames() - + "\nand\n" - + headers[query_num].dumpNames() + "\n", - ErrorCodes::UNION_ALL_RESULT_STRUCTURES_MISMATCH); + +Block InterpreterSelectWithUnionQuery::getCommonHeaderForUnion(const Blocks & headers) +{ + size_t num_selects = headers.size(); + Block common_header = headers.front(); + size_t num_columns = common_header.columns(); + + for (size_t query_num = 1; query_num < num_selects; ++query_num) + { + if (headers[query_num].columns() != num_columns) + throw Exception("Different number of columns in UNION ALL elements:\n" + + common_header.dumpNames() + + "\nand\n" + + headers[query_num].dumpNames() + "\n", + ErrorCodes::UNION_ALL_RESULT_STRUCTURES_MISMATCH); for (size_t column_num = 0; column_num < num_columns; ++column_num) { @@ -119,10 +131,12 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( for (size_t i = 0; i < num_selects; ++i) columns.push_back(&headers[i].getByPosition(column_num)); - ColumnWithTypeAndName & result_elem = result_header.getByPosition(column_num); + ColumnWithTypeAndName & result_elem = common_header.getByPosition(column_num); result_elem = getLeastSuperColumn(columns); } } + + return common_header; } @@ -197,6 +211,43 @@ BlockIO InterpreterSelectWithUnionQuery::execute() } +QueryPipeline InterpreterSelectWithUnionQuery::executeWithProcessors() +{ + QueryPipeline main_pipeline; + std::vector pipelines; + bool has_main_pipeline = false; + + Blocks headers; + headers.reserve(nested_interpreters.size()); + + for (auto & interpreter : nested_interpreters) + { + if (!has_main_pipeline) + { + has_main_pipeline = true; + main_pipeline = interpreter->executeWithProcessors(); + headers.emplace_back(main_pipeline.getHeader()); + } + else + { + pipelines.emplace_back(interpreter->executeWithProcessors()); + headers.emplace_back(pipelines.back().getHeader()); + } + } + + if (!has_main_pipeline) + main_pipeline.init({ std::make_shared(getSampleBlock()) }); + + if (!pipelines.empty()) + { + auto common_header = getCommonHeaderForUnion(headers); + main_pipeline.unitePipelines(std::move(pipelines), common_header, context); + } + + return main_pipeline; +} + + void InterpreterSelectWithUnionQuery::ignoreWithTotals() { for (auto & interpreter : nested_interpreters) diff --git a/dbms/src/Interpreters/InterpreterSelectWithUnionQuery.h b/dbms/src/Interpreters/InterpreterSelectWithUnionQuery.h index 84d562a5308..9f2a4a96494 100644 --- a/dbms/src/Interpreters/InterpreterSelectWithUnionQuery.h +++ b/dbms/src/Interpreters/InterpreterSelectWithUnionQuery.h @@ -5,6 +5,7 @@ #include #include +#include namespace DB { @@ -30,6 +31,9 @@ public: /// Execute the query without union of streams. BlockInputStreams executeWithMultipleStreams(); + QueryPipeline executeWithProcessors() override; + bool canExecuteWithProcessors() const override { return true; } + Block getSampleBlock(); static Block getSampleBlock( @@ -48,6 +52,8 @@ private: std::vector> nested_interpreters; Block result_header; + + static Block getCommonHeaderForUnion(const Blocks & headers); }; } diff --git a/dbms/src/Interpreters/ThreadStatusExt.cpp b/dbms/src/Interpreters/ThreadStatusExt.cpp index 4af414d1eef..f9b872ae292 100644 --- a/dbms/src/Interpreters/ThreadStatusExt.cpp +++ b/dbms/src/Interpreters/ThreadStatusExt.cpp @@ -6,6 +6,13 @@ #include #include +#if defined(__linux__) +#include +#include + +#include +#endif + /// Implement some methods of ThreadStatus and CurrentThread here to avoid extra linking dependencies in clickhouse_common_io /// TODO It doesn't make sense. @@ -13,14 +20,19 @@ namespace DB { +namespace ErrorCodes +{ + extern const int CANNOT_SET_THREAD_PRIORITY; +} + + void ThreadStatus::attachQueryContext(Context & query_context_) { query_context = &query_context_; + query_id = query_context->getCurrentQueryId(); if (!global_context) global_context = &query_context->getGlobalContext(); - query_id = query_context->getCurrentQueryId(); - if (thread_group) { std::lock_guard lock(thread_group->mutex); @@ -30,7 +42,7 @@ void ThreadStatus::attachQueryContext(Context & query_context_) } } -const std::string & ThreadStatus::getQueryId() const +StringRef ThreadStatus::getQueryId() const { return query_id; } @@ -39,8 +51,7 @@ void CurrentThread::defaultThreadDeleter() { if (unlikely(!current_thread)) return; - ThreadStatus & thread = CurrentThread::get(); - thread.detachQuery(true, true); + current_thread->detachQuery(true, true); } void ThreadStatus::initializeQuery() @@ -94,6 +105,26 @@ void ThreadStatus::attachQuery(const ThreadGroupStatusPtr & thread_group_, bool thread_group->thread_numbers.emplace_back(thread_number); } + if (query_context) + query_id = query_context->getCurrentQueryId(); + +#if defined(__linux__) + /// Set "nice" value if required. + if (query_context) + { + Int32 new_os_thread_priority = query_context->getSettingsRef().os_thread_priority; + if (new_os_thread_priority && hasLinuxCapability(CAP_SYS_NICE)) + { + LOG_TRACE(log, "Setting nice to " << new_os_thread_priority); + + if (0 != setpriority(PRIO_PROCESS, os_thread_id, new_os_thread_priority)) + throwFromErrno("Cannot 'setpriority'", ErrorCodes::CANNOT_SET_THREAD_PRIORITY); + + os_thread_priority = new_os_thread_priority; + } + } +#endif + initPerformanceCounters(); thread_state = ThreadState::AttachedToQuery; } @@ -144,6 +175,18 @@ void ThreadStatus::detachQuery(bool exit_if_already_detached, bool thread_exits) thread_group.reset(); thread_state = thread_exits ? ThreadState::Died : ThreadState::DetachedFromQuery; + +#if defined(__linux__) + if (os_thread_priority) + { + LOG_TRACE(log, "Resetting nice"); + + if (0 != setpriority(PRIO_PROCESS, os_thread_id, 0)) + LOG_ERROR(log, "Cannot 'setpriority' back to zero: " << errnoToString(ErrorCodes::CANNOT_SET_THREAD_PRIORITY, errno)); + + os_thread_priority = 0; + } +#endif } void ThreadStatus::logToQueryThreadLog(QueryThreadLog & thread_log) @@ -197,62 +240,59 @@ void CurrentThread::initializeQuery() { if (unlikely(!current_thread)) return; - get().initializeQuery(); - get().deleter = CurrentThread::defaultThreadDeleter; + current_thread->initializeQuery(); + current_thread->deleter = CurrentThread::defaultThreadDeleter; } void CurrentThread::attachTo(const ThreadGroupStatusPtr & thread_group) { if (unlikely(!current_thread)) return; - get().attachQuery(thread_group, true); - get().deleter = CurrentThread::defaultThreadDeleter; + current_thread->attachQuery(thread_group, true); + current_thread->deleter = CurrentThread::defaultThreadDeleter; } void CurrentThread::attachToIfDetached(const ThreadGroupStatusPtr & thread_group) { if (unlikely(!current_thread)) return; - get().attachQuery(thread_group, false); - get().deleter = CurrentThread::defaultThreadDeleter; + current_thread->attachQuery(thread_group, false); + current_thread->deleter = CurrentThread::defaultThreadDeleter; } -const std::string & CurrentThread::getQueryId() +StringRef CurrentThread::getQueryId() { if (unlikely(!current_thread)) - { - const static std::string empty; - return empty; - } - return get().getQueryId(); + return {}; + return current_thread->getQueryId(); } void CurrentThread::attachQueryContext(Context & query_context) { if (unlikely(!current_thread)) return; - return get().attachQueryContext(query_context); + current_thread->attachQueryContext(query_context); } void CurrentThread::finalizePerformanceCounters() { if (unlikely(!current_thread)) return; - get().finalizePerformanceCounters(); + current_thread->finalizePerformanceCounters(); } void CurrentThread::detachQuery() { if (unlikely(!current_thread)) return; - get().detachQuery(false); + current_thread->detachQuery(false); } void CurrentThread::detachQueryIfNotDetached() { if (unlikely(!current_thread)) return; - get().detachQuery(true); + current_thread->detachQuery(true); } diff --git a/dbms/src/Interpreters/executeQuery.cpp b/dbms/src/Interpreters/executeQuery.cpp index 4b6a5da2d67..5c7617aa8a1 100644 --- a/dbms/src/Interpreters/executeQuery.cpp +++ b/dbms/src/Interpreters/executeQuery.cpp @@ -28,8 +28,11 @@ #include #include #include -#include "DNSCacheUpdater.h" +#include +#include +#include +#include namespace DB { @@ -151,7 +154,7 @@ static std::tuple executeQueryImpl( { time_t current_time = time(nullptr); - context.setQueryContext(context); + context.makeQueryContext(); CurrentThread::attachQueryContext(context); const Settings & settings = context.getSettingsRef(); @@ -200,6 +203,7 @@ static std::tuple executeQueryImpl( /// Copy query into string. It will be written to log and presented in processlist. If an INSERT query, string will not include data to insertion. String query(begin, query_end); BlockIO res; + QueryPipeline & pipeline = res.pipeline; try { @@ -208,11 +212,10 @@ static std::tuple executeQueryImpl( { ReplaceQueryParameterVisitor visitor(context.getQueryParameters()); visitor.visit(ast); - } - /// Get new query after substitutions. - if (context.hasQueryParameters()) + /// Get new query after substitutions. query = serializeAST(*ast); + } logQuery(query.substr(0, settings.log_queries_cut_to_length), context, internal); @@ -236,9 +239,20 @@ static std::tuple executeQueryImpl( context.initializeExternalTablesIfSet(); auto interpreter = InterpreterFactory::get(ast, context, stage); - res = interpreter->execute(); + bool use_processors = settings.experimental_use_processors && interpreter->canExecuteWithProcessors(); + + if (use_processors) + pipeline = interpreter->executeWithProcessors(); + else + res = interpreter->execute(); + if (auto * insert_interpreter = typeid_cast(&*interpreter)) - context.setInsertionTable(insert_interpreter->getDatabaseTable()); + { + /// Save insertion table (not table function). TODO: support remote() table function. + auto db_table = insert_interpreter->getDatabaseTable(); + if (!db_table.second.empty()) + context.setInsertionTable(std::move(db_table)); + } if (process_list_entry) { @@ -246,36 +260,57 @@ static std::tuple executeQueryImpl( if ((*process_list_entry)->isKilled()) throw Exception("Query '" + (*process_list_entry)->getInfo().client_info.current_query_id + "' is killed in pending state", ErrorCodes::QUERY_WAS_CANCELLED); - else + else if (!use_processors) (*process_list_entry)->setQueryStreams(res); } /// Hold element of process list till end of query execution. res.process_list_entry = process_list_entry; - if (res.in) + IBlockInputStream::LocalLimits limits; + limits.mode = IBlockInputStream::LIMITS_CURRENT; + limits.size_limits = SizeLimits(settings.max_result_rows, settings.max_result_bytes, settings.result_overflow_mode); + + if (use_processors) { - res.in->setProgressCallback(context.getProgressCallback()); - res.in->setProcessListElement(context.getProcessListElement()); + pipeline.setProgressCallback(context.getProgressCallback()); + pipeline.setProcessListElement(context.getProcessListElement()); /// Limits on the result, the quota on the result, and also callback for progress. /// Limits apply only to the final result. if (stage == QueryProcessingStage::Complete) { - IBlockInputStream::LocalLimits limits; - limits.mode = IBlockInputStream::LIMITS_CURRENT; - limits.size_limits = SizeLimits(settings.max_result_rows, settings.max_result_bytes, settings.result_overflow_mode); - - res.in->setLimits(limits); - res.in->setQuota(quota); + pipeline.resize(1); + pipeline.addSimpleTransform([&](const Block & header) + { + auto transform = std::make_shared(header, limits); + transform->setQuota(quota); + return transform; + }); } } - - if (res.out) + else { - if (auto stream = dynamic_cast(res.out.get())) + if (res.in) { - stream->setProcessListElement(context.getProcessListElement()); + res.in->setProgressCallback(context.getProgressCallback()); + res.in->setProcessListElement(context.getProcessListElement()); + + /// Limits on the result, the quota on the result, and also callback for progress. + /// Limits apply only to the final result. + if (stage == QueryProcessingStage::Complete) + { + res.in->setLimits(limits); + res.in->setQuota(quota); + } + } + + if (res.out) + { + if (auto stream = dynamic_cast(res.out.get())) + { + stream->setProcessListElement(context.getProcessListElement()); + } } } @@ -305,7 +340,7 @@ static std::tuple executeQueryImpl( } /// Also make possible for caller to log successful query finish and exception during execution. - res.finish_callback = [elem, &context, log_queries] (IBlockInputStream * stream_in, IBlockOutputStream * stream_out) mutable + auto finish_callback = [elem, &context, log_queries] (IBlockInputStream * stream_in, IBlockOutputStream * stream_out) mutable { QueryStatus * process_list_elem = context.getProcessListElement(); @@ -374,7 +409,7 @@ static std::tuple executeQueryImpl( } }; - res.exception_callback = [elem, &context, log_queries] () mutable + auto exception_callback = [elem, &context, log_queries] () mutable { context.getQuota().addError(); @@ -417,6 +452,9 @@ static std::tuple executeQueryImpl( } }; + res.finish_callback = std::move(finish_callback); + res.exception_callback = std::move(exception_callback); + if (!internal && res.in) { std::stringstream log_str; @@ -499,6 +537,8 @@ void executeQuery( std::tie(ast, streams) = executeQueryImpl(begin, end, context, false, QueryProcessingStage::Complete, may_have_tail); + auto & pipeline = streams.pipeline; + try { if (streams.out) @@ -552,6 +592,63 @@ void executeQuery( copyData(*streams.in, *out); } + + if (pipeline.initialized()) + { + const ASTQueryWithOutput * ast_query_with_output = dynamic_cast(ast.get()); + + WriteBuffer * out_buf = &ostr; + std::optional out_file_buf; + if (ast_query_with_output && ast_query_with_output->out_file) + { + if (!allow_into_outfile) + throw Exception("INTO OUTFILE is not allowed", ErrorCodes::INTO_OUTFILE_NOT_ALLOWED); + + const auto & out_file = typeid_cast(*ast_query_with_output->out_file).value.safeGet(); + out_file_buf.emplace(out_file, DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_EXCL | O_CREAT); + out_buf = &*out_file_buf; + } + + String format_name = ast_query_with_output && (ast_query_with_output->format != nullptr) + ? *getIdentifierName(ast_query_with_output->format) + : context.getDefaultFormat(); + + if (ast_query_with_output && ast_query_with_output->settings_ast) + InterpreterSetQuery(ast_query_with_output->settings_ast, context).executeForCurrentContext(); + + pipeline.addSimpleTransform([](const Block & header) + { + return std::make_shared(header); + }); + + auto out = context.getOutputFormatProcessor(format_name, *out_buf, pipeline.getHeader()); + + /// Save previous progress callback if any. TODO Do it more conveniently. + auto previous_progress_callback = context.getProgressCallback(); + + /// NOTE Progress callback takes shared ownership of 'out'. + pipeline.setProgressCallback([out, previous_progress_callback] (const Progress & progress) + { + if (previous_progress_callback) + previous_progress_callback(progress); + out->onProgress(progress); + }); + + if (set_content_type) + set_content_type(out->getContentType()); + + if (set_query_id) + set_query_id(context.getClientInfo().current_query_id); + + pipeline.setOutput(std::move(out)); + + { + auto executor = pipeline.execute(); + executor->execute(context.getSettingsRef().max_threads); + } + + pipeline.finalize(); + } } catch (...) { diff --git a/dbms/src/Interpreters/executeQuery.h b/dbms/src/Interpreters/executeQuery.h index d43ca57af10..afb1829fdf0 100644 --- a/dbms/src/Interpreters/executeQuery.h +++ b/dbms/src/Interpreters/executeQuery.h @@ -3,6 +3,7 @@ #include #include +#include namespace DB { @@ -45,4 +46,13 @@ BlockIO executeQuery( bool may_have_embedded_data = false /// If insert query may have embedded data ); + +QueryPipeline executeQueryWithProcessors( + const String & query, /// Query text without INSERT data. The latter must be written to BlockIO::out. + Context & context, /// DB, tables, data types, storage engines, functions, aggregate functions... + bool internal = false, /// If true, this query is caused by another query and thus needn't be registered in the ProcessList. + QueryProcessingStage::Enum stage = QueryProcessingStage::Complete, /// To which stage the query must be executed. + bool may_have_embedded_data = false /// If insert query may have embedded data +); + } diff --git a/dbms/src/Interpreters/loadMetadata.cpp b/dbms/src/Interpreters/loadMetadata.cpp index e0caa8f433d..84a3adffe07 100644 --- a/dbms/src/Interpreters/loadMetadata.cpp +++ b/dbms/src/Interpreters/loadMetadata.cpp @@ -33,7 +33,6 @@ static void executeCreateQuery( Context & context, const String & database, const String & file_name, - ThreadPool * pool, bool has_force_restore_data_flag) { ParserCreateQuery parser; @@ -45,8 +44,6 @@ static void executeCreateQuery( InterpreterCreateQuery interpreter(ast, context); interpreter.setInternal(true); - if (pool) - interpreter.setDatabaseLoadingThreadpool(*pool); interpreter.setForceRestoreData(has_force_restore_data_flag); interpreter.execute(); } @@ -56,7 +53,6 @@ static void loadDatabase( Context & context, const String & database, const String & database_path, - ThreadPool * thread_pool, bool force_restore_data) { /// There may exist .sql file with database creation statement. @@ -73,7 +69,8 @@ static void loadDatabase( else database_attach_query = "ATTACH DATABASE " + backQuoteIfNeed(database); - executeCreateQuery(database_attach_query, context, database, database_metadata_file, thread_pool, force_restore_data); + executeCreateQuery(database_attach_query, context, database, + database_metadata_file, force_restore_data); } @@ -92,9 +89,6 @@ void loadMetadata(Context & context) Poco::File force_restore_data_flag_file(context.getFlagsPath() + "force_restore_data"); bool has_force_restore_data_flag = force_restore_data_flag_file.exists(); - /// For parallel tables loading. - ThreadPool thread_pool(SettingMaxThreads().getAutoValue()); - /// Loop over databases. std::map databases; Poco::DirectoryIterator dir_end; @@ -113,10 +107,8 @@ void loadMetadata(Context & context) databases.emplace(unescapeForFileName(it.name()), it.path().toString()); } - for (const auto & elem : databases) - loadDatabase(context, elem.first, elem.second, &thread_pool, has_force_restore_data_flag); - - thread_pool.wait(); + for (const auto & [name, path] : databases) + loadDatabase(context, name, path, has_force_restore_data_flag); if (has_force_restore_data_flag) { @@ -138,7 +130,7 @@ void loadMetadataSystem(Context & context) if (Poco::File(path).exists()) { /// 'has_force_restore_data_flag' is true, to not fail on loading query_log table, if it is corrupted. - loadDatabase(context, SYSTEM_DATABASE, path, nullptr, true); + loadDatabase(context, SYSTEM_DATABASE, path, true); } else { diff --git a/dbms/src/Interpreters/tests/CMakeLists.txt b/dbms/src/Interpreters/tests/CMakeLists.txt index 3de6c321de2..d83c5975c08 100644 --- a/dbms/src/Interpreters/tests/CMakeLists.txt +++ b/dbms/src/Interpreters/tests/CMakeLists.txt @@ -12,11 +12,11 @@ target_link_libraries (aggregate PRIVATE dbms) add_executable (hash_map hash_map.cpp) target_include_directories (hash_map SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) -target_link_libraries (hash_map PRIVATE dbms clickhouse_compression) +target_link_libraries (hash_map PRIVATE dbms) add_executable (hash_map_lookup hash_map_lookup.cpp) target_include_directories (hash_map_lookup SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) -target_link_libraries (hash_map_lookup PRIVATE dbms clickhouse_compression) +target_link_libraries (hash_map_lookup PRIVATE dbms) add_executable (hash_map3 hash_map3.cpp) target_include_directories(hash_map3 SYSTEM BEFORE PRIVATE ${METROHASH_INCLUDE_DIR}) @@ -24,22 +24,22 @@ target_link_libraries (hash_map3 PRIVATE dbms ${FARMHASH_LIBRARIES} ${METROHASH_ add_executable (hash_map_string hash_map_string.cpp) target_include_directories (hash_map_string SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) -target_link_libraries (hash_map_string PRIVATE dbms clickhouse_compression) +target_link_libraries (hash_map_string PRIVATE dbms) add_executable (hash_map_string_2 hash_map_string_2.cpp) -target_link_libraries (hash_map_string_2 PRIVATE dbms clickhouse_compression) +target_link_libraries (hash_map_string_2 PRIVATE dbms) add_executable (hash_map_string_3 hash_map_string_3.cpp) target_include_directories(hash_map_string_3 SYSTEM BEFORE PRIVATE ${METROHASH_INCLUDE_DIR}) -target_link_libraries (hash_map_string_3 PRIVATE dbms clickhouse_compression ${FARMHASH_LIBRARIES} ${METROHASH_LIBRARIES}) +target_link_libraries (hash_map_string_3 PRIVATE dbms ${FARMHASH_LIBRARIES} ${METROHASH_LIBRARIES}) add_executable (hash_map_string_small hash_map_string_small.cpp) target_include_directories (hash_map_string_small SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) -target_link_libraries (hash_map_string_small PRIVATE dbms clickhouse_compression) +target_link_libraries (hash_map_string_small PRIVATE dbms) add_executable (two_level_hash_map two_level_hash_map.cpp) target_include_directories (two_level_hash_map SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) -target_link_libraries (two_level_hash_map PRIVATE dbms clickhouse_compression) +target_link_libraries (two_level_hash_map PRIVATE dbms) add_executable (compiler_test compiler_test.cpp) target_link_libraries (compiler_test PRIVATE dbms) diff --git a/dbms/src/Interpreters/tests/create_query.cpp b/dbms/src/Interpreters/tests/create_query.cpp index b49a87c6945..47e1f202db7 100644 --- a/dbms/src/Interpreters/tests/create_query.cpp +++ b/dbms/src/Interpreters/tests/create_query.cpp @@ -79,11 +79,12 @@ try ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0); Context context = Context::createGlobal(); + context.makeGlobalContext(); context.setPath("./"); auto database = std::make_shared("test", "./metadata/test/", context); context.addDatabase("test", database); - database->loadTables(context, nullptr, false); + database->loadTables(context, false); context.setCurrentDatabase("test"); InterpreterCreateQuery interpreter(ast, context); diff --git a/dbms/src/Interpreters/tests/expression.cpp b/dbms/src/Interpreters/tests/expression.cpp index 73502d9067e..2dbf01d148b 100644 --- a/dbms/src/Interpreters/tests/expression.cpp +++ b/dbms/src/Interpreters/tests/expression.cpp @@ -47,6 +47,7 @@ int main(int argc, char ** argv) std::cerr << std::endl; Context context = Context::createGlobal(); + context.makeGlobalContext(); NamesAndTypesList columns { {"x", std::make_shared()}, diff --git a/dbms/src/Interpreters/tests/expression_analyzer.cpp b/dbms/src/Interpreters/tests/expression_analyzer.cpp index 3732e7dce72..079a22620bc 100644 --- a/dbms/src/Interpreters/tests/expression_analyzer.cpp +++ b/dbms/src/Interpreters/tests/expression_analyzer.cpp @@ -97,6 +97,7 @@ int main() }; Context context = Context::createGlobal(); + context.makeGlobalContext(); auto system_database = std::make_shared("system"); context.addDatabase("system", system_database); diff --git a/dbms/src/Interpreters/tests/hash_map_string.cpp b/dbms/src/Interpreters/tests/hash_map_string.cpp index 9076a1e582e..cad701b49ca 100644 --- a/dbms/src/Interpreters/tests/hash_map_string.cpp +++ b/dbms/src/Interpreters/tests/hash_map_string.cpp @@ -277,7 +277,7 @@ struct Grower : public HashTableGrower<> } /// Set the buffer size by the number of elements in the hash table. Used when deserializing a hash table. - void set(size_t /*num_elems*/) + [[noreturn]] void set(size_t /*num_elems*/) { throw Poco::Exception(__PRETTY_FUNCTION__); } diff --git a/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp b/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp index ecb21c58831..7c90769b37b 100644 --- a/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp +++ b/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp @@ -38,6 +38,7 @@ public: std::string getRemoteTableName() const { return remote_table; } std::string getTableName() const override { return ""; } + std::string getDatabaseName() const override { return ""; } protected: StorageDistributedFake(const std::string & remote_database_, const std::string & remote_table_, size_t shard_count_) @@ -1158,6 +1159,7 @@ bool run() TestResult check(const TestEntry & entry) { static DB::Context context = DB::Context::createGlobal(); + context.makeGlobalContext(); try { diff --git a/dbms/src/Interpreters/tests/select_query.cpp b/dbms/src/Interpreters/tests/select_query.cpp index 951d8e0723a..1283ae6e659 100644 --- a/dbms/src/Interpreters/tests/select_query.cpp +++ b/dbms/src/Interpreters/tests/select_query.cpp @@ -31,6 +31,7 @@ try DateLUT::instance(); Context context = Context::createGlobal(); + context.makeGlobalContext(); context.setPath("./"); @@ -38,7 +39,7 @@ try DatabasePtr system = std::make_shared("system", "./metadata/system/", context); context.addDatabase("system", system); - system->loadTables(context, nullptr, false); + system->loadTables(context, false); attachSystemTablesLocal(*context.getDatabase("system")); context.setCurrentDatabase("default"); diff --git a/dbms/src/Parsers/ASTCheckQuery.h b/dbms/src/Parsers/ASTCheckQuery.h index 595b6c2ecb6..40665f6f2b6 100644 --- a/dbms/src/Parsers/ASTCheckQuery.h +++ b/dbms/src/Parsers/ASTCheckQuery.h @@ -1,12 +1,17 @@ #pragma once #include +#include + namespace DB { struct ASTCheckQuery : public ASTQueryWithTableAndOutput { + + ASTPtr partition; + /** Get the text that identifies this element. */ String getID(char delim) const override { return "CheckQuery" + (delim + database) + delim + table; } @@ -19,7 +24,7 @@ struct ASTCheckQuery : public ASTQueryWithTableAndOutput } protected: - void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked frame) const override + void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override { std::string nl_or_nothing = settings.one_line ? "" : "\n"; @@ -37,6 +42,12 @@ protected: } settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << backQuoteIfNeed(table) << (settings.hilite ? hilite_none : ""); } + + if (partition) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << " PARTITION " << (settings.hilite ? hilite_none : ""); + partition->formatImpl(settings, state, frame); + } } }; diff --git a/dbms/src/Parsers/ParserCheckQuery.cpp b/dbms/src/Parsers/ParserCheckQuery.cpp index cd25e60b887..5ba8119571d 100644 --- a/dbms/src/Parsers/ParserCheckQuery.cpp +++ b/dbms/src/Parsers/ParserCheckQuery.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB @@ -11,9 +12,11 @@ namespace DB bool ParserCheckQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { ParserKeyword s_check_table("CHECK TABLE"); + ParserKeyword s_partition("PARTITION"); ParserToken s_dot(TokenType::Dot); ParserIdentifier table_parser; + ParserPartition partition_parser; ASTPtr table; ASTPtr database; @@ -23,24 +26,28 @@ bool ParserCheckQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) if (!table_parser.parse(pos, database, expected)) return false; + auto query = std::make_shared(); if (s_dot.ignore(pos)) { if (!table_parser.parse(pos, table, expected)) return false; - auto query = std::make_shared(); getIdentifierName(database, query->database); getIdentifierName(table, query->table); - node = query; } else { table = database; - auto query = std::make_shared(); getIdentifierName(table, query->table); - node = query; } + if (s_partition.ignore(pos, expected)) + { + if (!partition_parser.parse(pos, query->partition, expected)) + return false; + } + + node = query; return true; } diff --git a/dbms/src/Processors/CMakeLists.txt b/dbms/src/Processors/CMakeLists.txt new file mode 100644 index 00000000000..99ba159eaf4 --- /dev/null +++ b/dbms/src/Processors/CMakeLists.txt @@ -0,0 +1,4 @@ +if (ENABLE_TESTS) + add_subdirectory (tests) +endif () + diff --git a/dbms/src/Processors/Chunk.cpp b/dbms/src/Processors/Chunk.cpp new file mode 100644 index 00000000000..f68502fc4b4 --- /dev/null +++ b/dbms/src/Processors/Chunk.cpp @@ -0,0 +1,155 @@ +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int POSITION_OUT_OF_BOUND; +} + +Chunk::Chunk(DB::Columns columns_, UInt64 num_rows_) : columns(std::move(columns_)), num_rows(num_rows_) +{ + checkNumRowsIsConsistent(); +} + +Chunk::Chunk(Columns columns_, UInt64 num_rows_, ChunkInfoPtr chunk_info_) + : columns(std::move(columns_)), num_rows(num_rows_), chunk_info(std::move(chunk_info_)) +{ + checkNumRowsIsConsistent(); +} + +static Columns unmuteColumns(MutableColumns && mut_columns) +{ + Columns columns; + columns.reserve(mut_columns.size()); + for (auto & col : mut_columns) + columns.emplace_back(std::move(col)); + + return columns; +} + +Chunk::Chunk(MutableColumns columns_, UInt64 num_rows_) + : columns(unmuteColumns(std::move(columns_))), num_rows(num_rows_) +{ +} + +Chunk::Chunk(MutableColumns columns_, UInt64 num_rows_, ChunkInfoPtr chunk_info_) + : columns(unmuteColumns(std::move(columns_))), num_rows(num_rows_), chunk_info(std::move(chunk_info_)) +{ +} + +Chunk Chunk::clone() const +{ + return Chunk(getColumns(), getNumRows()); +} + +void Chunk::setColumns(Columns columns_, UInt64 num_rows_) +{ + columns = std::move(columns_); + num_rows = num_rows_; + checkNumRowsIsConsistent(); +} + +void Chunk::setColumns(MutableColumns columns_, UInt64 num_rows_) +{ + columns = unmuteColumns(std::move(columns_)); + num_rows = num_rows_; + checkNumRowsIsConsistent(); +} + +void Chunk::checkNumRowsIsConsistent() +{ + for (auto & column : columns) + if (column->size() != num_rows) + throw Exception("Invalid number of rows in Chunk column " + column->getName()+ ": expected " + + toString(num_rows) + ", got " + toString(column->size()), ErrorCodes::LOGICAL_ERROR); +} + +MutableColumns Chunk::mutateColumns() +{ + size_t num_columns = columns.size(); + MutableColumns mut_columns(num_columns); + for (size_t i = 0; i < num_columns; ++i) + mut_columns[i] = (*std::move(columns[i])).mutate(); + + columns.clear(); + num_rows = 0; + + return mut_columns; +} + +MutableColumns Chunk::cloneEmptyColumns() const +{ + size_t num_columns = columns.size(); + MutableColumns mut_columns(num_columns); + for (size_t i = 0; i < num_columns; ++i) + mut_columns[i] = columns[i]->cloneEmpty(); + return mut_columns; +} + +Columns Chunk::detachColumns() +{ + num_rows = 0; + return std::move(columns); +} + +void Chunk::erase(size_t position) +{ + if (columns.empty()) + throw Exception("Chunk is empty", ErrorCodes::POSITION_OUT_OF_BOUND); + + if (position >= columns.size()) + throw Exception("Position " + toString(position) + " out of bound in Chunk::erase(), max position = " + + toString(columns.size() - 1), ErrorCodes::POSITION_OUT_OF_BOUND); + + columns.erase(columns.begin() + position); +} + +UInt64 Chunk::bytes() const +{ + UInt64 res = 0; + for (const auto & column : columns) + res += column->byteSize(); + + return res; +} + +UInt64 Chunk::allocatedBytes() const +{ + UInt64 res = 0; + for (const auto & column : columns) + res += column->allocatedBytes(); + + return res; +} + +std::string Chunk::dumpStructure() const +{ + WriteBufferFromOwnString out; + for (auto & column : columns) + out << ' ' << column->dumpStructure(); + + return out.str(); +} + + +void ChunkMissingValues::setBit(size_t column_idx, size_t row_idx) +{ + RowsBitMask & mask = rows_mask_by_column_id[column_idx]; + mask.resize(row_idx + 1); + mask[row_idx] = true; +} + +const ChunkMissingValues::RowsBitMask & ChunkMissingValues::getDefaultsBitmask(size_t column_idx) const +{ + static RowsBitMask none; + auto it = rows_mask_by_column_id.find(column_idx); + if (it != rows_mask_by_column_id.end()) + return it->second; + return none; +} + +} diff --git a/dbms/src/Processors/Chunk.h b/dbms/src/Processors/Chunk.h new file mode 100644 index 00000000000..7e33d8cf1c0 --- /dev/null +++ b/dbms/src/Processors/Chunk.h @@ -0,0 +1,117 @@ +#pragma once + +#include +#include + +namespace DB +{ + +class ChunkInfo +{ +public: + virtual ~ChunkInfo() = default; + ChunkInfo() = default; +}; + +using ChunkInfoPtr = std::shared_ptr; + +class Chunk +{ +public: + Chunk() = default; + Chunk(const Chunk & other) = delete; + Chunk(Chunk && other) noexcept + : columns(std::move(other.columns)) + , num_rows(other.num_rows) + , chunk_info(std::move(other.chunk_info)) + { + other.num_rows = 0; + } + + Chunk(Columns columns_, UInt64 num_rows_); + Chunk(Columns columns_, UInt64 num_rows_, ChunkInfoPtr chunk_info_); + Chunk(MutableColumns columns_, UInt64 num_rows_); + Chunk(MutableColumns columns_, UInt64 num_rows_, ChunkInfoPtr chunk_info_); + + Chunk & operator=(const Chunk & other) = delete; + Chunk & operator=(Chunk && other) noexcept + { + columns = std::move(other.columns); + chunk_info = std::move(other.chunk_info); + num_rows = other.num_rows; + other.num_rows = 0; + return *this; + } + + Chunk clone() const; + + void swap(Chunk & other) + { + columns.swap(other.columns); + chunk_info.swap(other.chunk_info); + std::swap(num_rows, other.num_rows); + } + + void clear() + { + num_rows = 0; + columns.clear(); + chunk_info.reset(); + } + + const Columns & getColumns() const { return columns; } + void setColumns(Columns columns_, UInt64 num_rows_); + void setColumns(MutableColumns columns_, UInt64 num_rows_); + Columns detachColumns(); + MutableColumns mutateColumns(); + /** Get empty columns with the same types as in block. */ + MutableColumns cloneEmptyColumns() const; + + const ChunkInfoPtr & getChunkInfo() const { return chunk_info; } + void setChunkInfo(ChunkInfoPtr chunk_info_) { chunk_info = std::move(chunk_info_); } + + UInt64 getNumRows() const { return num_rows; } + UInt64 getNumColumns() const { return columns.size(); } + bool hasNoRows() const { return num_rows == 0; } + bool hasNoColumns() const { return columns.empty(); } + bool empty() const { return hasNoRows() && hasNoColumns(); } + operator bool() const { return !empty(); } + + void erase(size_t position); + + UInt64 bytes() const; + UInt64 allocatedBytes() const; + + std::string dumpStructure() const; + +private: + Columns columns; + UInt64 num_rows = 0; + ChunkInfoPtr chunk_info; + + void checkNumRowsIsConsistent(); +}; + +using Chunks = std::vector; + +/// Block extension to support delayed defaults. AddingDefaultsProcessor uses it to replace missing values with column defaults. +class ChunkMissingValues : public ChunkInfo +{ +public: + using RowsBitMask = std::vector; /// a bit per row for a column + + const RowsBitMask & getDefaultsBitmask(size_t column_idx) const; + void setBit(size_t column_idx, size_t row_idx); + bool empty() const { return rows_mask_by_column_id.empty(); } + size_t size() const { return rows_mask_by_column_id.size(); } + void clear() { rows_mask_by_column_id.clear(); } + +private: + using RowsMaskByColumnId = std::unordered_map; + + /// If rows_mask_by_column_id[column_id][row_id] is true related value in Block should be replaced with column default. + /// It could contain less columns and rows then related block. + RowsMaskByColumnId rows_mask_by_column_id; +}; + +} diff --git a/dbms/src/Processors/ConcatProcessor.cpp b/dbms/src/Processors/ConcatProcessor.cpp new file mode 100644 index 00000000000..d3333c320c0 --- /dev/null +++ b/dbms/src/Processors/ConcatProcessor.cpp @@ -0,0 +1,63 @@ +#include + + +namespace DB +{ + +ConcatProcessor::Status ConcatProcessor::prepare() +{ + auto & output = outputs.front(); + + /// Check can output. + + if (output.isFinished()) + { + for (; current_input != inputs.end(); ++current_input) + current_input->close(); + + return Status::Finished; + } + + if (!output.isNeeded()) + { + if (current_input != inputs.end()) + current_input->setNotNeeded(); + + return Status::PortFull; + } + + if (!output.canPush()) + return Status::PortFull; + + /// Check can input. + + if (current_input == inputs.end()) + return Status::Finished; + + if (current_input->isFinished()) + { + ++current_input; + if (current_input == inputs.end()) + { + output.finish(); + return Status::Finished; + } + } + + auto & input = *current_input; + + input.setNeeded(); + + if (!input.hasData()) + return Status::NeedData; + + /// Move data. + output.push(input.pull()); + + /// Now, we pushed to output, and it must be full. + return Status::PortFull; +} + +} + + diff --git a/dbms/src/Processors/ConcatProcessor.h b/dbms/src/Processors/ConcatProcessor.h new file mode 100644 index 00000000000..4aa5099b38a --- /dev/null +++ b/dbms/src/Processors/ConcatProcessor.h @@ -0,0 +1,35 @@ +#pragma once + +#include + + +namespace DB +{ + +/** Has arbitary non zero number of inputs and one output. + * All of them have the same structure. + * + * Pulls all data from first input, then all data from second input, etc... + * Doesn't do any heavy calculations. + * Preserves an order of data. + */ +class ConcatProcessor : public IProcessor +{ +public: + ConcatProcessor(const Block & header, size_t num_inputs) + : IProcessor(InputPorts(num_inputs, header), OutputPorts{header}), current_input(inputs.begin()) + { + } + + String getName() const override { return "Concat"; } + + Status prepare() override; + + OutputPort & getOutputPort() { return outputs.front(); } + +private: + InputPorts::iterator current_input; +}; + +} + diff --git a/dbms/src/Processors/Executors/ParallelPipelineExecutor.cpp b/dbms/src/Processors/Executors/ParallelPipelineExecutor.cpp new file mode 100644 index 00000000000..7f0969e6451 --- /dev/null +++ b/dbms/src/Processors/Executors/ParallelPipelineExecutor.cpp @@ -0,0 +1,101 @@ +#include +#include +#include +#include + + +namespace DB +{ +// +//ParallelPipelineExecutor::ParallelPipelineExecutor(const std::vector & processors, ThreadPool & pool) +// : processors(processors), pool(pool) +//{ +//} +// +// +//ParallelPipelineExecutor::Status ParallelPipelineExecutor::prepare() +//{ +// current_processor = nullptr; +// +// bool has_someone_to_wait = false; +// +// for (auto & element : processors) +// { +// traverse(*element, +// [&] (IProcessor & processor) +// { +// { +// std::lock_guard lock(mutex); +// if (active_processors.count(&processor)) +// { +// has_someone_to_wait = true; +// return Status::Wait; +// } +// } +// +// Status status = processor.prepare(); +// +// if (status == Status::Wait) +// has_someone_to_wait = true; +// +// if (status == Status::Ready || status == Status::Async) +// { +// current_processor = &processor; +// current_status = status; +// } +// +// return status; +// }); +// +// if (current_processor) +// break; +// } +// +// if (current_processor) +// return Status::Async; +// +// if (has_someone_to_wait) +// return Status::Wait; +// +// for (auto & element : processors) +// { +// if (element->prepare() == Status::NeedData) +// throw Exception("Pipeline stuck: " + element->getName() + " processor needs input data but no one is going to generate it", ErrorCodes::LOGICAL_ERROR); +// if (element->prepare() == Status::PortFull) +// throw Exception("Pipeline stuck: " + element->getName() + " processor has data in output port but no one is going to consume it", ErrorCodes::LOGICAL_ERROR); +// } +// +// return Status::Finished; +//} +// +// +//void ParallelPipelineExecutor::schedule(EventCounter & watch) +//{ +// if (!current_processor) +// throw Exception("Bad pipeline", ErrorCodes::LOGICAL_ERROR); +// +// if (current_status == Status::Async) +// { +// current_processor->schedule(watch); +// } +// else +// { +// { +// std::lock_guard lock(mutex); +// active_processors.insert(current_processor); +// } +// +// pool.schedule([processor = current_processor, &watch, this] +// { +// processor->work(); +// { +// std::lock_guard lock(mutex); +// active_processors.erase(processor); +// } +// watch.notify(); +// }); +// } +//} + +} + diff --git a/dbms/src/Processors/Executors/ParallelPipelineExecutor.h b/dbms/src/Processors/Executors/ParallelPipelineExecutor.h new file mode 100644 index 00000000000..4997f73f699 --- /dev/null +++ b/dbms/src/Processors/Executors/ParallelPipelineExecutor.h @@ -0,0 +1,41 @@ +#pragma once + +#include +#include +#include +#include + +template +class ThreadPoolImpl; +class ThreadFromGlobalPool; +using ThreadPool = ThreadPoolImpl; + +namespace DB +{ + +/** Wraps pipeline in a single processor. + * This processor has no inputs and outputs and just executes the pipeline, + * performing all synchronous work within a threadpool. + */ +//class ParallelPipelineExecutor : public IProcessor +//{ +//private: +// Processors processors; +// ThreadPool & pool; +// +// std::set active_processors; +// std::mutex mutex; +// +// IProcessor * current_processor = nullptr; +// Status current_status; +// +//public: +// ParallelPipelineExecutor(const Processors & processors, ThreadPool & pool); +// +// String getName() const override { return "ParallelPipelineExecutor"; } +// +// Status prepare() override; +// void schedule(EventCounter & watch) override; +//}; + +} diff --git a/dbms/src/Processors/Executors/PipelineExecutor.cpp b/dbms/src/Processors/Executors/PipelineExecutor.cpp new file mode 100644 index 00000000000..9da53793e81 --- /dev/null +++ b/dbms/src/Processors/Executors/PipelineExecutor.cpp @@ -0,0 +1,653 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int TOO_MANY_ROWS_OR_BYTES; + extern const int QUOTA_EXPIRED; + extern const int QUERY_WAS_CANCELLED; +} + +static bool checkCanAddAdditionalInfoToException(const DB::Exception & exception) +{ + /// Don't add additional info to limits and quota exceptions, and in case of kill query (to pass tests). + return exception.code() != ErrorCodes::TOO_MANY_ROWS_OR_BYTES + && exception.code() != ErrorCodes::QUOTA_EXPIRED + && exception.code() != ErrorCodes::QUERY_WAS_CANCELLED; +} + +PipelineExecutor::PipelineExecutor(Processors & processors) + : processors(processors) + , cancelled(false) + , finished(false) + , num_processing_executors(0) + , expand_pipeline_task(nullptr) +{ + buildGraph(); +} + +bool PipelineExecutor::addEdges(UInt64 node) +{ + auto throwUnknownProcessor = [](const IProcessor * proc, const IProcessor * parent, bool from_input_port) + { + String msg = "Processor " + proc->getName() + " was found as " + (from_input_port ? "input" : "output") + + " for processor " + parent->getName() + ", but not found in list of processors."; + + throw Exception(msg, ErrorCodes::LOGICAL_ERROR); + }; + + const IProcessor * cur = graph[node].processor; + + auto add_edge = [&](auto & from_port, const IProcessor * to_proc, Edges & edges) + { + auto it = processors_map.find(to_proc); + if (it == processors_map.end()) + throwUnknownProcessor(to_proc, cur, true); + + UInt64 proc_num = it->second; + Edge * edge_ptr = nullptr; + + for (auto & edge : edges) + if (edge.to == proc_num) + edge_ptr = &edge; + + if (!edge_ptr) + { + edge_ptr = &edges.emplace_back(); + edge_ptr->to = proc_num; + } + + from_port.setVersion(&edge_ptr->version); + }; + + bool was_edge_added = false; + + auto & inputs = processors[node]->getInputs(); + auto from_input = graph[node].backEdges.size(); + + if (from_input < inputs.size()) + { + was_edge_added = true; + + for (auto it = std::next(inputs.begin(), from_input); it != inputs.end(); ++it) + { + const IProcessor * proc = &it->getOutputPort().getProcessor(); + add_edge(*it, proc, graph[node].backEdges); + } + } + + auto & outputs = processors[node]->getOutputs(); + auto from_output = graph[node].directEdges.size(); + + if (from_output < outputs.size()) + { + was_edge_added = true; + + for (auto it = std::next(outputs.begin(), from_output); it != outputs.end(); ++it) + { + const IProcessor * proc = &it->getInputPort().getProcessor(); + add_edge(*it, proc, graph[node].directEdges); + } + } + + return was_edge_added; +} + +void PipelineExecutor::buildGraph() +{ + UInt64 num_processors = processors.size(); + + graph.reserve(num_processors); + for (UInt64 node = 0; node < num_processors; ++node) + { + IProcessor * proc = processors[node].get(); + processors_map[proc] = node; + graph.emplace_back(proc, node); + } + + for (UInt64 node = 0; node < num_processors; ++node) + addEdges(node); +} + +void PipelineExecutor::addChildlessProcessorsToStack(Stack & stack) +{ + UInt64 num_processors = processors.size(); + for (UInt64 proc = 0; proc < num_processors; ++proc) + { + if (graph[proc].directEdges.empty()) + { + stack.push(proc); + graph[proc].status = ExecStatus::Preparing; + } + } +} + +static void executeJob(IProcessor * processor) +{ + try + { + processor->work(); + } + catch (Exception & exception) + { + if (checkCanAddAdditionalInfoToException(exception)) + exception.addMessage("While executing " + processor->getName() + " (" + + toString(reinterpret_cast(processor)) + ") "); + throw; + } +} + +void PipelineExecutor::addJob(ExecutionState * execution_state) +{ + auto job = [execution_state]() + { + try + { + Stopwatch watch; + executeJob(execution_state->processor); + execution_state->execution_time_ns += watch.elapsed(); + + ++execution_state->num_executed_jobs; + } + catch (...) + { + execution_state->exception = std::current_exception(); + } + }; + + execution_state->job = std::move(job); +} + +void PipelineExecutor::expandPipeline(Stack & stack, UInt64 pid) +{ + auto & cur_node = graph[pid]; + auto new_processors = cur_node.processor->expandPipeline(); + + for (const auto & processor : new_processors) + { + if (processors_map.count(processor.get())) + throw Exception("Processor " + processor->getName() + " was already added to pipeline.", + ErrorCodes::LOGICAL_ERROR); + + processors_map[processor.get()] = graph.size(); + graph.emplace_back(processor.get(), graph.size()); + } + + processors.insert(processors.end(), new_processors.begin(), new_processors.end()); + UInt64 num_processors = processors.size(); + + for (UInt64 node = 0; node < num_processors; ++node) + { + if (addEdges(node)) + { + if (graph[node].status == ExecStatus::Idle || graph[node].status == ExecStatus::New) + { + graph[node].status = ExecStatus::Preparing; + stack.push(node); + } + } + } +} + +bool PipelineExecutor::tryAddProcessorToStackIfUpdated(Edge & edge, Stack & stack) +{ + /// In this method we have ownership on edge, but node can be concurrently accessed. + + auto & node = graph[edge.to]; + + ExecStatus status = node.status.load(); + + /// Don't add processor if nothing was read from port. + if (status != ExecStatus::New && edge.version == edge.prev_version) + return false; + + if (status == ExecStatus::Finished) + return false; + + /// Signal that node need to be prepared. + node.need_to_be_prepared = true; + edge.prev_version = edge.version; + + /// Try to get ownership for node. + + /// Assume that current status is New or Idle. Otherwise, can't prepare node. + if (status != ExecStatus::New) + status = ExecStatus::Idle; + + /// Statuses but New and Idle are not interesting because they own node. + /// Prepare will be called in owning thread before changing status. + while (!node.status.compare_exchange_weak(status, ExecStatus::Preparing)) + if (!(status == ExecStatus::New || status == ExecStatus::Idle) || !node.need_to_be_prepared) + return false; + + stack.push(edge.to); + return true; + +} + +bool PipelineExecutor::prepareProcessor(UInt64 pid, Stack & children, Stack & parents, size_t thread_number, bool async) +{ + /// In this method we have ownership on node. + auto & node = graph[pid]; + + { + /// Stopwatch watch; + + /// Disable flag before prepare call. Otherwise, we can skip prepare request. + /// Prepare can be called more times than needed, but it's ok. + node.need_to_be_prepared = false; + + auto status = node.processor->prepare(); + + /// node.execution_state->preparation_time_ns += watch.elapsed(); + node.last_processor_status = status; + } + + auto add_neighbours_to_prepare_queue = [&] () + { + for (auto & edge : node.backEdges) + tryAddProcessorToStackIfUpdated(edge, parents); + + for (auto & edge : node.directEdges) + tryAddProcessorToStackIfUpdated(edge, children); + }; + + auto try_release_ownership = [&] () + { + /// This function can be called after expand pipeline, where node from outer scope is not longer valid. + auto & node_ = graph[pid]; + ExecStatus expected = ExecStatus::Idle; + node_.status = ExecStatus::Idle; + + if (node_.need_to_be_prepared) + { + while (!node_.status.compare_exchange_weak(expected, ExecStatus::Preparing)) + if (!(expected == ExecStatus::Idle) || !node_.need_to_be_prepared) + return; + + children.push(pid); + } + }; + + switch (node.last_processor_status) + { + case IProcessor::Status::NeedData: + { + add_neighbours_to_prepare_queue(); + try_release_ownership(); + + break; + } + case IProcessor::Status::PortFull: + { + add_neighbours_to_prepare_queue(); + try_release_ownership(); + + break; + } + case IProcessor::Status::Finished: + { + add_neighbours_to_prepare_queue(); + node.status = ExecStatus::Finished; + break; + } + case IProcessor::Status::Ready: + { + node.status = ExecStatus::Executing; + return true; + } + case IProcessor::Status::Async: + { + throw Exception("Async is temporary not supported.", ErrorCodes::LOGICAL_ERROR); + +// node.status = ExecStatus::Executing; +// addAsyncJob(pid); +// break; + } + case IProcessor::Status::Wait: + { + if (!async) + throw Exception("Processor returned status Wait before Async.", ErrorCodes::LOGICAL_ERROR); + break; + } + case IProcessor::Status::ExpandPipeline: + { + executor_contexts[thread_number]->task_list.emplace_back( + node.execution_state.get(), + &parents + ); + + ExpandPipelineTask * desired = &executor_contexts[thread_number]->task_list.back(); + ExpandPipelineTask * expected = nullptr; + + while (!expand_pipeline_task.compare_exchange_strong(expected, desired)) + { + doExpandPipeline(expected, true); + expected = nullptr; + } + + doExpandPipeline(desired, true); + + /// node is not longer valid after pipeline was expanded + graph[pid].need_to_be_prepared = true; + try_release_ownership(); + break; + } + } + + return false; +} + +void PipelineExecutor::doExpandPipeline(ExpandPipelineTask * task, bool processing) +{ + std::unique_lock lock(task->mutex); + + if (processing) + ++task->num_waiting_processing_threads; + + task->condvar.wait(lock, [&]() + { + return task->num_waiting_processing_threads >= num_processing_executors || expand_pipeline_task != task; + }); + + /// After condvar.wait() task may point to trash. Can change it only if it is still in expand_pipeline_task. + if (expand_pipeline_task == task) + { + expandPipeline(*task->stack, task->node_to_expand->processors_id); + + expand_pipeline_task = nullptr; + + lock.unlock(); + task->condvar.notify_all(); + } +} + +void PipelineExecutor::finish() +{ + { + std::lock_guard lock(task_queue_mutex); + finished = true; + } + + task_queue_condvar.notify_all(); +} + +void PipelineExecutor::execute(size_t num_threads) +{ + try + { + executeImpl(num_threads); + + /// Execution can be stopped because of exception. Check and rethrow if any. + for (auto & node : graph) + if (node.execution_state->exception) + std::rethrow_exception(node.execution_state->exception); + } + catch (Exception & exception) + { + if (checkCanAddAdditionalInfoToException(exception)) + exception.addMessage("\nCurrent state:\n" + dumpPipeline()); + + throw; + } + + if (cancelled) + return; + + bool all_processors_finished = true; + for (auto & node : graph) + if (node.status != ExecStatus::Finished) + all_processors_finished = false; + + if (!all_processors_finished) + throw Exception("Pipeline stuck. Current state:\n" + dumpPipeline(), ErrorCodes::LOGICAL_ERROR); +} + +void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads) +{ + UInt64 total_time_ns = 0; + UInt64 execution_time_ns = 0; + UInt64 processing_time_ns = 0; + UInt64 wait_time_ns = 0; + + Stopwatch total_time_watch; + ExecutionState * state = nullptr; + + auto prepare_processor = [&](UInt64 pid, Stack & children, Stack & parents) + { + try + { + return prepareProcessor(pid, children, parents, thread_num, false); + } + catch (...) + { + graph[pid].execution_state->exception = std::current_exception(); + finish(); + } + + return false; + }; + + using Queue = std::queue; + + auto prepare_all_processors = [&](Queue & queue, Stack & stack, Stack & children, Stack & parents) + { + while (!stack.empty() && !finished) + { + auto current_processor = stack.top(); + stack.pop(); + + if (prepare_processor(current_processor, children, parents)) + queue.push(graph[current_processor].execution_state.get()); + } + }; + + while (!finished) + { + + /// First, find any processor to execute. + /// Just travers graph and prepare any processor. + while (!finished) + { + std::unique_lock lock(task_queue_mutex); + + if (!task_queue.empty()) + { + state = task_queue.front(); + task_queue.pop(); + break; + } + + ++num_waiting_threads; + + if (num_waiting_threads == num_threads) + { + finished = true; + lock.unlock(); + task_queue_condvar.notify_all(); + break; + } + + task_queue_condvar.wait(lock, [&]() + { + return finished || !task_queue.empty(); + }); + + --num_waiting_threads; + } + + if (finished) + break; + + while (state) + { + if (finished) + break; + + addJob(state); + + { + Stopwatch execution_time_watch; + state->job(); + execution_time_ns += execution_time_watch.elapsed(); + } + + if (state->exception) + finish(); + + if (finished) + break; + + Stopwatch processing_time_watch; + + /// Try to execute neighbour processor. + { + Stack children; + Stack parents; + Queue queue; + + ++num_processing_executors; + while (auto task = expand_pipeline_task.load()) + doExpandPipeline(task, true); + + /// Execute again if can. + if (!prepare_processor(state->processors_id, children, parents)) + state = nullptr; + + /// Process all neighbours. Children will be on the top of stack, then parents. + prepare_all_processors(queue, children, children, parents); + + if (!state && !queue.empty()) + { + state = queue.front(); + queue.pop(); + } + + prepare_all_processors(queue, parents, parents, parents); + + if (!queue.empty()) + { + std::lock_guard lock(task_queue_mutex); + + while (!queue.empty() && !finished) + { + task_queue.push(queue.front()); + queue.pop(); + } + + task_queue_condvar.notify_all(); + } + + --num_processing_executors; + while (auto task = expand_pipeline_task.load()) + doExpandPipeline(task, false); + } + + processing_time_ns += processing_time_watch.elapsed(); + } + } + + total_time_ns = total_time_watch.elapsed(); + wait_time_ns = total_time_ns - execution_time_ns - processing_time_ns; + + LOG_TRACE(log, "Thread finished." + << " Total time: " << (total_time_ns / 1e9) << " sec." + << " Execution time: " << (execution_time_ns / 1e9) << " sec." + << " Processing time: " << (processing_time_ns / 1e9) << " sec." + << " Wait time: " << (wait_time_ns / 1e9) << "sec."); +} + +void PipelineExecutor::executeImpl(size_t num_threads) +{ + Stack stack; + + executor_contexts.reserve(num_threads); + for (size_t i = 0; i < num_threads; ++i) + executor_contexts.emplace_back(std::make_unique()); + + addChildlessProcessorsToStack(stack); + + while (!stack.empty()) + { + UInt64 proc = stack.top(); + stack.pop(); + + if (prepareProcessor(proc, stack, stack, 0, false)) + { + auto cur_state = graph[proc].execution_state.get(); + task_queue.push(cur_state); + } + } + + ThreadPool pool(num_threads); + + SCOPE_EXIT( + finish(); + pool.wait() + ); + + auto thread_group = CurrentThread::getGroup(); + + for (size_t i = 0; i < num_threads; ++i) + { + pool.schedule([this, thread_group, thread_num = i, num_threads] + { + /// ThreadStatus thread_status; + + if (thread_group) + CurrentThread::attachTo(thread_group); + + SCOPE_EXIT( + if (thread_group) + CurrentThread::detachQueryIfNotDetached(); + ); + + executeSingleThread(thread_num, num_threads); + }); + } + + pool.wait(); +} + +String PipelineExecutor::dumpPipeline() const +{ + for (auto & node : graph) + { + if (node.execution_state) + node.processor->setDescription( + "(" + std::to_string(node.execution_state->num_executed_jobs) + " jobs, execution time: " + + std::to_string(node.execution_state->execution_time_ns / 1e9) + " sec., preparation time: " + + std::to_string(node.execution_state->preparation_time_ns / 1e9) + " sec.)"); + } + + std::vector statuses; + std::vector proc_list; + statuses.reserve(graph.size()); + proc_list.reserve(graph.size()); + + for (auto & proc : graph) + { + proc_list.emplace_back(proc.processor); + statuses.emplace_back(proc.last_processor_status); + } + + WriteBufferFromOwnString out; + printPipeline(processors, statuses, out); + out.finish(); + + return out.str(); +} + +} diff --git a/dbms/src/Processors/Executors/PipelineExecutor.h b/dbms/src/Processors/Executors/PipelineExecutor.h new file mode 100644 index 00000000000..0994532f953 --- /dev/null +++ b/dbms/src/Processors/Executors/PipelineExecutor.h @@ -0,0 +1,195 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace DB +{ + + +/// Executes query pipeline. +class PipelineExecutor +{ +public: + /// Get pipeline as a set of processors. + /// Processors should represent full graph. All ports must be connected, all connected nodes are mentioned in set. + /// Executor doesn't own processors, just stores reference. + /// During pipeline execution new processors can appear. They will be added to existing set. + /// + /// Explicit graph representation is built in constructor. Throws if graph is not correct. + explicit PipelineExecutor(Processors & processors); + + /// Execute pipeline in multiple threads. Must be called once. + /// In case of exception during execution throws any occurred. + void execute(size_t num_threads); + + String getName() const { return "PipelineExecutor"; } + + const Processors & getProcessors() const { return processors; } + + /// Cancel execution. May be called from another thread. + void cancel() + { + cancelled = true; + finish(); + } + +private: + Processors & processors; + + struct Edge + { + UInt64 to = std::numeric_limits::max(); + + /// Edge version is increased when port's state is changed (e.g. when data is pushed). See Port.h for details. + /// To compare version with prev_version we can decide if neighbour processor need to be prepared. + UInt64 version = 0; + UInt64 prev_version = 0; + }; + + /// Use std::list because new ports can be added to processor during execution. + using Edges = std::list; + + /// Status for processor. + /// Can be owning or not. Owning means that executor who set this status can change node's data and nobody else can. + enum class ExecStatus + { + New, /// prepare wasn't called yet. Initial state. Non-owning. + Idle, /// prepare returned NeedData or PortFull. Non-owning. + Preparing, /// some executor is preparing processor, or processor is in task_queue. Owning. + Executing, /// prepare returned Ready and task is executing. Owning. + Finished, /// prepare returned Finished. Non-owning. + Async /// prepare returned Async. Owning. + }; + + /// Small structure with context of executing job. + struct ExecutionState + { + std::exception_ptr exception; + std::function job; + + IProcessor * processor; + UInt64 processors_id; + + /// Counters for profiling. + size_t num_executed_jobs = 0; + UInt64 execution_time_ns = 0; + UInt64 preparation_time_ns = 0; + }; + + struct Node + { + IProcessor * processor = nullptr; + Edges directEdges; + Edges backEdges; + + std::atomic status; + /// This flag can be set by any executor. + /// When enabled, any executor can try to atomically set Preparing state to status. + std::atomic_bool need_to_be_prepared; + /// Last state for profiling. + IProcessor::Status last_processor_status = IProcessor::Status::NeedData; + + std::unique_ptr execution_state; + + Node(IProcessor * processor_, UInt64 processor_id) + : processor(processor_), status(ExecStatus::New), need_to_be_prepared(false) + { + execution_state = std::make_unique(); + execution_state->processor = processor; + execution_state->processors_id = processor_id; + } + + Node(Node && other) noexcept + : processor(other.processor), status(other.status.load()) + , need_to_be_prepared(other.need_to_be_prepared.load()), execution_state(std::move(other.execution_state)) + { + } + }; + + using Nodes = std::vector; + + Nodes graph; + + using Stack = std::stack; + + using TaskQueue = std::queue; + + /// Queue with pointers to tasks. Each thread will concurrently read from it until finished flag is set. + /// Stores processors need to be prepared. Preparing status is already set for them. + TaskQueue task_queue; + std::mutex task_queue_mutex; + std::condition_variable task_queue_condvar; + + std::atomic_bool cancelled; + std::atomic_bool finished; + + Poco::Logger * log = &Poco::Logger::get("PipelineExecutor"); + + /// Num threads waiting condvar. Last thread finish execution if task_queue is empty. + size_t num_waiting_threads = 0; + + /// Things to stop execution to expand pipeline. + struct ExpandPipelineTask + { + ExecutionState * node_to_expand; + Stack * stack; + size_t num_waiting_processing_threads = 0; + std::mutex mutex; + std::condition_variable condvar; + + ExpandPipelineTask(ExecutionState * node_to_expand_, Stack * stack_) + : node_to_expand(node_to_expand_), stack(stack_) {} + }; + + std::atomic num_processing_executors; + std::atomic expand_pipeline_task; + + /// Context for each thread. + struct ExecutorContext + { + /// Will store context for all expand pipeline tasks (it's easy and we don't expect many). + /// This can be solved by using atomic shard ptr. + std::list task_list; + }; + + std::vector> executor_contexts; + + /// Processor ptr -> node number + using ProcessorsMap = std::unordered_map; + ProcessorsMap processors_map; + + /// Graph related methods. + bool addEdges(UInt64 node); + void buildGraph(); + void expandPipeline(Stack & stack, UInt64 pid); + + /// Pipeline execution related methods. + void addChildlessProcessorsToStack(Stack & stack); + bool tryAddProcessorToStackIfUpdated(Edge & edge, Stack & stack); + static void addJob(ExecutionState * execution_state); + // TODO: void addAsyncJob(UInt64 pid); + + /// Prepare processor with pid number. + /// Check parents and children of current processor and push them to stacks if they also need to be prepared. + /// If processor wants to be expanded, ExpandPipelineTask from thread_number's execution context will be used. + bool prepareProcessor(size_t pid, Stack & children, Stack & parents, size_t thread_number, bool async); + void doExpandPipeline(ExpandPipelineTask * task, bool processing); + + void executeImpl(size_t num_threads); + void executeSingleThread(size_t thread_num, size_t num_threads); + void finish(); + + String dumpPipeline() const; +}; + +using PipelineExecutorPtr = std::shared_ptr; + +} diff --git a/dbms/src/Processors/Executors/SequentialPipelineExecutor.cpp b/dbms/src/Processors/Executors/SequentialPipelineExecutor.cpp new file mode 100644 index 00000000000..d660936eff8 --- /dev/null +++ b/dbms/src/Processors/Executors/SequentialPipelineExecutor.cpp @@ -0,0 +1,79 @@ +#include +#include + + +namespace DB +{ + +//SequentialPipelineExecutor::SequentialPipelineExecutor(const Processors & processors) +// : processors(processors) +//{ +//} +// +// +//SequentialPipelineExecutor::Status SequentialPipelineExecutor::prepare() +//{ +// current_processor = nullptr; +// +// bool has_someone_to_wait = false; +// Status found_status = Status::Finished; +// +// for (auto & element : processors) +// { +// traverse(*element, +// [&] (IProcessor & processor) +// { +// Status status = processor.prepare(); +// +// if (status == Status::Wait) +// has_someone_to_wait = true; +// +// if (status == Status::Ready || status == Status::Async) +// { +// current_processor = &processor; +// found_status = status; +// } +// +// return status; +// }); +// +// if (current_processor) +// break; +// } +// +// if (current_processor) +// return found_status; +// if (has_someone_to_wait) +// return Status::Wait; +// +// for (auto & element : processors) +// { +// if (element->prepare() == Status::NeedData) +// throw Exception("Pipeline stuck: " + element->getName() + " processor needs input data but no one is going to generate it", ErrorCodes::LOGICAL_ERROR); +// if (element->prepare() == Status::PortFull) +// throw Exception("Pipeline stuck: " + element->getName() + " processor has data in output port but no one is going to consume it", ErrorCodes::LOGICAL_ERROR); +// } +// +// return Status::Finished; +//} +// +// +//void SequentialPipelineExecutor::work() +//{ +// if (!current_processor) +// throw Exception("Bad pipeline", ErrorCodes::LOGICAL_ERROR); +// +// current_processor->work(); +//} +// +// +//void SequentialPipelineExecutor::schedule(EventCounter & watch) +//{ +// if (!current_processor) +// throw Exception("Bad pipeline", ErrorCodes::LOGICAL_ERROR); +// +// current_processor->schedule(watch); +//} + +} + diff --git a/dbms/src/Processors/Executors/SequentialPipelineExecutor.h b/dbms/src/Processors/Executors/SequentialPipelineExecutor.h new file mode 100644 index 00000000000..5dbd8b73fee --- /dev/null +++ b/dbms/src/Processors/Executors/SequentialPipelineExecutor.h @@ -0,0 +1,31 @@ +#pragma once + +#include + +#include + + +namespace DB +{ + +/** Wraps pipeline in a single processor. + * This processor has no inputs and outputs and just executes the pipeline, + * performing all synchronous work from the current thread. + */ +//class SequentialPipelineExecutor : public IProcessor +//{ +//private: +// Processors processors; +// IProcessor * current_processor = nullptr; +// +//public: +// SequentialPipelineExecutor(const Processors & processors); +// +// String getName() const override { return "SequentialPipelineExecutor"; } +// +// Status prepare() override; +// void work() override; +// void schedule(EventCounter & watch) override; +//}; + +} diff --git a/dbms/src/Processors/Executors/traverse.h b/dbms/src/Processors/Executors/traverse.h new file mode 100644 index 00000000000..2fd89adcb43 --- /dev/null +++ b/dbms/src/Processors/Executors/traverse.h @@ -0,0 +1,30 @@ +#pragma once + +#include + + +namespace DB +{ + +/// Look for first Ready or Async processor by depth-first search in needed input ports and full output ports. +/// NOTE: Pipeline must not have cycles. +//template +//void traverse(IProcessor & processor, Visit && visit) +//{ +// IProcessor::Status status = visit(processor); +// +// if (status == IProcessor::Status::Ready || status == IProcessor::Status::Async) +// return; +// +// if (status == IProcessor::Status::NeedData) +// for (auto & input : processor.getInputs()) +// if (input.isNeeded() && !input.hasData()) +// traverse(input.getOutputPort().getProcessor(), std::forward(visit)); +// +// if (status == IProcessor::Status::PortFull) +// for (auto & output : processor.getOutputs()) +// if (output.hasData()) +// traverse(output.getInputPort().getProcessor(), std::forward(visit)); +//} + +} diff --git a/dbms/src/Processors/ForkProcessor.cpp b/dbms/src/Processors/ForkProcessor.cpp new file mode 100644 index 00000000000..30903f22433 --- /dev/null +++ b/dbms/src/Processors/ForkProcessor.cpp @@ -0,0 +1,81 @@ +#include + + +namespace DB +{ + +ForkProcessor::Status ForkProcessor::prepare() +{ + auto & input = inputs.front(); + + /// Check can output. + + bool all_finished = true; + bool all_can_push = true; + size_t num_active_outputs = 0; + + for (const auto & output : outputs) + { + if (!output.isFinished()) + { + all_finished = false; + ++num_active_outputs; + + /// The order is important. + if (!output.canPush()) + all_can_push = false; + } + } + + if (all_finished) + { + input.close(); + return Status::Finished; + } + + if (!all_can_push) + { + input.setNotNeeded(); + return Status::PortFull; + } + + /// Check can input. + + if (input.isFinished()) + { + for (auto & output : outputs) + output.finish(); + + return Status::Finished; + } + + input.setNeeded(); + + if (!input.hasData()) + return Status::NeedData; + + /// Move data. + + auto data = input.pull(); + size_t num_processed_outputs = 0; + + for (auto & output : outputs) + { + if (!output.isFinished()) /// Skip finished outputs. + { + ++num_processed_outputs; + if (num_processed_outputs == num_active_outputs) + output.push(std::move(data)); /// Can push because no full or unneeded outputs. + else + output.push(data.clone()); + } + } + + /// Now, we pulled from input. It must be empty. + return Status::NeedData; +} + +} + + + diff --git a/dbms/src/Processors/ForkProcessor.h b/dbms/src/Processors/ForkProcessor.h new file mode 100644 index 00000000000..8839f73584f --- /dev/null +++ b/dbms/src/Processors/ForkProcessor.h @@ -0,0 +1,35 @@ +#pragma once + +#include + + +namespace DB +{ + +/** Has one input and arbitrary non zero number of outputs. + * All of them have the same structure. + * + * Pulls data input and copies it to every output. + * You may have heard about it under the name 'tee'. + * + * Doesn't do any heavy calculations. + * Preserves an order of data. + */ +class ForkProcessor : public IProcessor +{ +public: + ForkProcessor(const Block & header, size_t num_outputs) + : IProcessor(InputPorts{header}, OutputPorts(num_outputs, header)) + { + } + + String getName() const override { return "Fork"; } + + Status prepare() override; + + InputPort & getInputPort() { return inputs.front(); } +}; + +} + + diff --git a/dbms/src/Processors/Formats/IInputFormat.h b/dbms/src/Processors/Formats/IInputFormat.h new file mode 100644 index 00000000000..7e4e00c1a33 --- /dev/null +++ b/dbms/src/Processors/Formats/IInputFormat.h @@ -0,0 +1,32 @@ +#pragma once + +#include + + +namespace DB +{ + +class ReadBuffer; + +/** Input format is a source, that reads data from ReadBuffer. + */ +class IInputFormat : public ISource +{ +protected: + + /// Skip GCC warning: ‘maybe_unused’ attribute ignored +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wattributes" + + ReadBuffer & in [[maybe_unused]]; + +#pragma GCC diagnostic pop + +public: + IInputFormat(Block header, ReadBuffer & in) + : ISource(std::move(header)), in(in) + { + } +}; + +} diff --git a/dbms/src/Processors/Formats/IOutputFormat.cpp b/dbms/src/Processors/Formats/IOutputFormat.cpp new file mode 100644 index 00000000000..63e846aa796 --- /dev/null +++ b/dbms/src/Processors/Formats/IOutputFormat.cpp @@ -0,0 +1,78 @@ +#include +#include + + +namespace DB +{ + +IOutputFormat::IOutputFormat(const Block & header, WriteBuffer & out) + : IProcessor({header, header, header}, {}), out(out) +{ +} + +IOutputFormat::Status IOutputFormat::prepare() +{ + if (has_input) + return Status::Ready; + + for (auto kind : {Main, Totals, Extremes}) + { + auto & input = getPort(kind); + + if (kind != Main && !input.isConnected()) + continue; + + if (input.isFinished()) + continue; + + input.setNeeded(); + + if (!input.hasData()) + return Status::NeedData; + + current_chunk = input.pull(); + current_block_kind = kind; + has_input = true; + return Status::Ready; + } + + finished = true; + + if (!finalized) + return Status::Ready; + + return Status::Finished; +} + +void IOutputFormat::work() +{ + if (finished && !finalized) + { + finalize(); + finalized = true; + return; + } + + switch (current_block_kind) + { + case Main: + consume(std::move(current_chunk)); + break; + case Totals: + consumeTotals(std::move(current_chunk)); + break; + case Extremes: + consumeExtremes(std::move(current_chunk)); + break; + } + + has_input = false; +} + +void IOutputFormat::flush() +{ + out.next(); +} + +} + diff --git a/dbms/src/Processors/Formats/IOutputFormat.h b/dbms/src/Processors/Formats/IOutputFormat.h new file mode 100644 index 00000000000..280f64593b7 --- /dev/null +++ b/dbms/src/Processors/Formats/IOutputFormat.h @@ -0,0 +1,63 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ + +class WriteBuffer; + +/** Output format have three inputs and no outputs. It writes data from WriteBuffer. + * + * First input is for main resultset, second is for "totals" and third is for "extremes". + * It's not necessarily to connect "totals" or "extremes" ports (they may remain dangling). + * + * Data from input ports are pulled in order: first, from main input, then totals, then extremes. + * + * By default, data for "totals" and "extremes" is ignored. + */ +class IOutputFormat : public IProcessor +{ +public: + enum PortKind { Main = 0, Totals = 1, Extremes = 2 }; + +protected: + WriteBuffer & out; + + Chunk current_chunk; + PortKind current_block_kind = PortKind::Main; + bool has_input = false; + bool finished = false; + bool finalized = false; + + virtual void consume(Chunk) = 0; + virtual void consumeTotals(Chunk) {} + virtual void consumeExtremes(Chunk) {} + virtual void finalize() {} + +public: + IOutputFormat(const Block & header, WriteBuffer & out); + + Status prepare() override; + void work() override; + + /// Flush output buffers if any. + virtual void flush(); + + /// Value for rows_before_limit_at_least field. + virtual void setRowsBeforeLimit(size_t /*rows_before_limit*/) {} + + /// Notify about progress. Method could be called from different threads. + /// Passed value are delta, that must be summarized. + virtual void onProgress(const Progress & /*progress*/) {} + + /// Content-Type to set when sending HTTP response. + virtual std::string getContentType() const { return "text/plain; charset=UTF-8"; } + + InputPort & getPort(PortKind kind) { return *std::next(inputs.begin(), kind); } +}; +} + diff --git a/dbms/src/Processors/Formats/IRowInputFormat.cpp b/dbms/src/Processors/Formats/IRowInputFormat.cpp new file mode 100644 index 00000000000..b0212d2b89f --- /dev/null +++ b/dbms/src/Processors/Formats/IRowInputFormat.cpp @@ -0,0 +1,151 @@ +#include +#include // toString + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int CANNOT_PARSE_INPUT_ASSERTION_FAILED; + extern const int CANNOT_PARSE_QUOTED_STRING; + extern const int CANNOT_PARSE_DATE; + extern const int CANNOT_PARSE_DATETIME; + extern const int CANNOT_READ_ARRAY_FROM_TEXT; + extern const int CANNOT_PARSE_NUMBER; + extern const int CANNOT_PARSE_UUID; + extern const int TOO_LARGE_STRING_SIZE; + extern const int INCORRECT_NUMBER_OF_COLUMNS; +} + + +static bool isParseError(int code) +{ + return code == ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED + || code == ErrorCodes::CANNOT_PARSE_QUOTED_STRING + || code == ErrorCodes::CANNOT_PARSE_DATE + || code == ErrorCodes::CANNOT_PARSE_DATETIME + || code == ErrorCodes::CANNOT_READ_ARRAY_FROM_TEXT + || code == ErrorCodes::CANNOT_PARSE_NUMBER + || code == ErrorCodes::CANNOT_PARSE_UUID + || code == ErrorCodes::TOO_LARGE_STRING_SIZE; +} + + +Chunk IRowInputFormat::generate() +{ + if (total_rows == 0) + readPrefix(); + + const Block & header = getPort().getHeader(); + + size_t num_columns = header.columns(); + MutableColumns columns = header.cloneEmptyColumns(); + size_t prev_rows = total_rows; + + auto chunk_missing_values = std::make_unique(); + + try + { + for (size_t rows = 0; rows < params.max_block_size; ++rows) + { + try + { + ++total_rows; + + RowReadExtension info; + if (!readRow(columns, info)) + break; + + for (size_t column_idx = 0; column_idx < info.read_columns.size(); ++column_idx) + { + if (!info.read_columns[column_idx]) + { + size_t column_size = columns[column_idx]->size(); + if (column_size == 0) + throw Exception("Unexpected empty column", ErrorCodes::INCORRECT_NUMBER_OF_COLUMNS); + chunk_missing_values->setBit(column_idx, column_size - 1); + } + } + } + catch (Exception & e) + { + /// Logic for possible skipping of errors. + + if (!isParseError(e.code())) + throw; + + if (params.allow_errors_num == 0 && params.allow_errors_ratio == 0) + throw; + + ++num_errors; + Float64 current_error_ratio = static_cast(num_errors) / total_rows; + + if (num_errors > params.allow_errors_num + && current_error_ratio > params.allow_errors_ratio) + { + e.addMessage("(Already have " + toString(num_errors) + " errors" + " out of " + toString(total_rows) + " rows" + ", which is " + toString(current_error_ratio) + " of all rows)"); + throw; + } + + if (!allowSyncAfterError()) + { + e.addMessage("(Input format doesn't allow to skip errors)"); + throw; + } + + syncAfterError(); + + /// Truncate all columns in block to minimal size (remove values, that was appended to only part of columns). + + size_t min_size = std::numeric_limits::max(); + for (size_t column_idx = 0; column_idx < num_columns; ++column_idx) + min_size = std::min(min_size, columns[column_idx]->size()); + + for (size_t column_idx = 0; column_idx < num_columns; ++column_idx) + { + auto & column = columns[column_idx]; + if (column->size() > min_size) + column->popBack(column->size() - min_size); + } + } + } + } + catch (Exception & e) + { + if (!isParseError(e.code())) + throw; + + String verbose_diagnostic; + try + { + verbose_diagnostic = getDiagnosticInfo(); + } + catch (...) + { + /// Error while trying to obtain verbose diagnostic. Ok to ignore. + } + + e.addMessage("(at row " + toString(total_rows) + ")\n" + verbose_diagnostic); + throw; + } + + if (columns.empty() || columns[0]->empty()) + { + readSuffix(); + return {}; + } + + Chunk chunk(std::move(columns), total_rows - prev_rows); + chunk.setChunkInfo(std::move(chunk_missing_values)); + return chunk; +} + +void IRowInputFormat::syncAfterError() +{ + throw Exception("Method syncAfterError is not implemented for input format", ErrorCodes::NOT_IMPLEMENTED); +} + +} diff --git a/dbms/src/Processors/Formats/IRowInputFormat.h b/dbms/src/Processors/Formats/IRowInputFormat.h new file mode 100644 index 00000000000..9d0ee7cb0c9 --- /dev/null +++ b/dbms/src/Processors/Formats/IRowInputFormat.h @@ -0,0 +1,72 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ + +/// Contains extra information about read data. +struct RowReadExtension +{ + /// IRowInputStream.read() output. It contains non zero for columns that actually read from the source and zero otherwise. + /// It's used to attach defaults for partially filled rows. + std::vector read_columns; +}; + +/// Common parameters for generating blocks. +struct RowInputFormatParams +{ + size_t max_block_size; + + UInt64 allow_errors_num; + Float64 allow_errors_ratio; +}; + +///Row oriented input format: reads data row by row. +class IRowInputFormat : public IInputFormat +{ +public: + using Params = RowInputFormatParams; + + IRowInputFormat( + Block header, + ReadBuffer & in_, + Params params) + : IInputFormat(std::move(header), in_), params(params) + { + } + + Chunk generate() override; + +protected: + /** Read next row and append it to the columns. + * If no more rows - return false. + */ + virtual bool readRow(MutableColumns & columns, RowReadExtension & extra) = 0; + + virtual void readPrefix() {} /// delimiter before begin of result + virtual void readSuffix() {} /// delimiter after end of result + + /// Skip data until next row. + /// This is intended for text streams, that allow skipping of errors. + /// By default - throws not implemented exception. + virtual bool allowSyncAfterError() const { return false; } + virtual void syncAfterError(); + + /// In case of parse error, try to roll back and parse last one or two rows very carefully + /// and collect as much as possible diagnostic information about error. + /// If not implemented, returns empty string. + virtual std::string getDiagnosticInfo() { return {}; } + +private: + Params params; + + size_t total_rows = 0; + size_t num_errors = 0; +}; + +} + diff --git a/dbms/src/Processors/Formats/IRowOutputFormat.cpp b/dbms/src/Processors/Formats/IRowOutputFormat.cpp new file mode 100644 index 00000000000..6db2802f232 --- /dev/null +++ b/dbms/src/Processors/Formats/IRowOutputFormat.cpp @@ -0,0 +1,101 @@ +#include +#include +#include + + +namespace DB +{ + +void IRowOutputFormat::consume(DB::Chunk chunk) +{ + writePrefixIfNot(); + + auto num_rows = chunk.getNumRows(); + auto & columns = chunk.getColumns(); + + for (UInt64 row = 0; row < num_rows; ++row) + { + if (!first_row) + writeRowBetweenDelimiter(); + first_row = false; + + write(columns, row); + } +} + +void IRowOutputFormat::consumeTotals(DB::Chunk chunk) +{ + writePrefixIfNot(); + writeSuffixIfNot(); + + auto num_rows = chunk.getNumRows(); + if (num_rows != 1) + throw Exception("Got " + toString(num_rows) + " in totals chunk, expected 1", ErrorCodes::LOGICAL_ERROR); + + auto & columns = chunk.getColumns(); + + writeBeforeTotals(); + writeTotals(columns, 0); + writeAfterTotals(); +} + +void IRowOutputFormat::consumeExtremes(DB::Chunk chunk) +{ + writePrefixIfNot(); + writeSuffixIfNot(); + + auto num_rows = chunk.getNumRows(); + auto & columns = chunk.getColumns(); + if (num_rows != 2) + throw Exception("Got " + toString(num_rows) + " in extremes chunk, expected 2", ErrorCodes::LOGICAL_ERROR); + + writeBeforeExtremes(); + writeMinExtreme(columns, 0); + writeRowBetweenDelimiter(); + writeMaxExtreme(columns, 1); + writeAfterExtremes(); +} + +void IRowOutputFormat::finalize() +{ + writePrefixIfNot(); + writeSuffixIfNot(); + writeLastSuffix(); +} + +void IRowOutputFormat::write(const Columns & columns, size_t row_num) +{ + size_t num_columns = columns.size(); + + writeRowStartDelimiter(); + + for (size_t i = 0; i < num_columns; ++i) + { + if (i != 0) + writeFieldDelimiter(); + + writeField(*columns[i], *types[i], row_num); + } + + writeRowEndDelimiter(); +} + +void IRowOutputFormat::writeMinExtreme(const DB::Columns & columns, size_t row_num) +{ + write(columns, row_num); +} + +void IRowOutputFormat::writeMaxExtreme(const DB::Columns & columns, size_t row_num) +{ + write(columns, row_num); +} + +void IRowOutputFormat::writeTotals(const DB::Columns & columns, size_t row_num) +{ + write(columns, row_num); +} + +} + + + diff --git a/dbms/src/Processors/Formats/IRowOutputFormat.h b/dbms/src/Processors/Formats/IRowOutputFormat.h new file mode 100644 index 00000000000..09bfe17a1b5 --- /dev/null +++ b/dbms/src/Processors/Formats/IRowOutputFormat.h @@ -0,0 +1,80 @@ +#pragma once + +#include +#include + + +namespace DB +{ + +class WriteBuffer; + +/** Output format that writes data row by row. + */ +class IRowOutputFormat : public IOutputFormat +{ +protected: + DataTypes types; + + void consume(Chunk chunk) override; + void consumeTotals(Chunk chunk) override; + void consumeExtremes(Chunk chunk) override; + void finalize() override; + +public: + IRowOutputFormat(const Block & header, WriteBuffer & out_) + : IOutputFormat(header, out_), types(header.getDataTypes()) + { + } + + /** Write a row. + * Default implementation calls methods to write single values and delimiters + * (except delimiter between rows (writeRowBetweenDelimiter())). + */ + virtual void write(const Columns & columns, size_t row_num); + virtual void writeMinExtreme(const Columns & columns, size_t row_num); + virtual void writeMaxExtreme(const Columns & columns, size_t row_num); + virtual void writeTotals(const Columns & columns, size_t row_num); + + /** Write single value. */ + virtual void writeField(const IColumn & column, const IDataType & type, size_t row_num) = 0; + + /** Write delimiter. */ + virtual void writeFieldDelimiter() {} /// delimiter between values + virtual void writeRowStartDelimiter() {} /// delimiter before each row + virtual void writeRowEndDelimiter() {} /// delimiter after each row + virtual void writeRowBetweenDelimiter() {} /// delimiter between rows + virtual void writePrefix() {} /// delimiter before resultset + virtual void writeSuffix() {} /// delimiter after resultset + virtual void writeBeforeTotals() {} + virtual void writeAfterTotals() {} + virtual void writeBeforeExtremes() {} + virtual void writeAfterExtremes() {} + virtual void writeLastSuffix() {} /// Write something after resultset, totals end extremes. + +private: + bool first_row = true; + bool prefix_written = false; + bool suffix_written = false; + + void writePrefixIfNot() + { + if (!prefix_written) + writePrefix(); + + prefix_written = true; + } + + void writeSuffixIfNot() + { + if (!suffix_written) + writeSuffix(); + + suffix_written = true; + } + +}; + +} + + diff --git a/dbms/src/Processors/Formats/Impl/BinaryRowInputFormat.cpp b/dbms/src/Processors/Formats/Impl/BinaryRowInputFormat.cpp new file mode 100644 index 00000000000..20f40fe1e41 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/BinaryRowInputFormat.cpp @@ -0,0 +1,82 @@ +#include +#include +#include +#include + + +namespace DB +{ + +BinaryRowInputFormat::BinaryRowInputFormat(ReadBuffer & in_, Block header, Params params, bool with_names_, bool with_types_) + : IRowInputFormat(std::move(header), in_, params), with_names(with_names_), with_types(with_types_) +{ +} + + +bool BinaryRowInputFormat::readRow(MutableColumns & columns, RowReadExtension &) +{ + if (in.eof()) + return false; + + size_t num_columns = columns.size(); + for (size_t i = 0; i < num_columns; ++i) + getPort().getHeader().getByPosition(i).type->deserializeBinary(*columns[i], in); + + return true; +} + + +void BinaryRowInputFormat::readPrefix() +{ + /// NOTE The header is completely ignored. This can be easily improved. + + UInt64 columns = 0; + String tmp; + + if (with_names || with_types) + { + readVarUInt(columns, in); + } + + if (with_names) + { + for (size_t i = 0; i < columns; ++i) + { + readStringBinary(tmp, in); + } + } + + if (with_types) + { + for (size_t i = 0; i < columns; ++i) + { + readStringBinary(tmp, in); + } + } +} + + +void registerInputFormatProcessorRowBinary(FormatFactory & factory) +{ + factory.registerInputFormatProcessor("RowBinary", []( + ReadBuffer & buf, + const Block & sample, + const Context &, + const IRowInputFormat::Params & params, + const FormatSettings &) + { + return std::make_shared(buf, sample, params, false, false); + }); + + factory.registerInputFormatProcessor("RowBinaryWithNamesAndTypes", []( + ReadBuffer & buf, + const Block & sample, + const Context &, + const IRowInputFormat::Params & params, + const FormatSettings &) + { + return std::make_shared(buf, sample, params, true, true); + }); +} + +} diff --git a/dbms/src/Processors/Formats/Impl/BinaryRowInputFormat.h b/dbms/src/Processors/Formats/Impl/BinaryRowInputFormat.h new file mode 100644 index 00000000000..9a5a3fe63e1 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/BinaryRowInputFormat.h @@ -0,0 +1,30 @@ +#pragma once + +#include +#include + + +namespace DB +{ + +class ReadBuffer; + + +/** A stream for inputting data in a binary line-by-line format. + */ +class BinaryRowInputFormat : public IRowInputFormat +{ +public: + BinaryRowInputFormat(ReadBuffer & in_, Block header, Params params, bool with_names_, bool with_types_); + + bool readRow(MutableColumns & columns, RowReadExtension &) override; + void readPrefix() override; + + String getName() const override { return "BinaryRowInputFormat"; } + +private: + bool with_names; + bool with_types; +}; + +} diff --git a/dbms/src/Processors/Formats/Impl/BinaryRowOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/BinaryRowOutputFormat.cpp new file mode 100644 index 00000000000..fb728d1293b --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/BinaryRowOutputFormat.cpp @@ -0,0 +1,71 @@ +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +BinaryRowOutputFormat::BinaryRowOutputFormat(WriteBuffer & out_, const Block & header, bool with_names_, bool with_types_) + : IRowOutputFormat(header, out_), with_names(with_names_), with_types(with_types_) +{ +} + +void BinaryRowOutputFormat::writePrefix() +{ + auto & header = getPort(PortKind::Main).getHeader(); + size_t columns = header.columns(); + + if (with_names || with_types) + { + writeVarUInt(columns, out); + } + + if (with_names) + { + for (size_t i = 0; i < columns; ++i) + { + writeStringBinary(header.safeGetByPosition(i).name, out); + } + } + + if (with_types) + { + for (size_t i = 0; i < columns; ++i) + { + writeStringBinary(header.safeGetByPosition(i).type->getName(), out); + } + } +} + +void BinaryRowOutputFormat::writeField(const IColumn & column, const IDataType & type, size_t row_num) +{ + type.serializeBinary(column, row_num, out); +} + + +void registerOutputFormatProcessorRowBinary(FormatFactory & factory) +{ + factory.registerOutputFormatProcessor("RowBinary", []( + WriteBuffer & buf, + const Block & sample, + const Context &, + const FormatSettings &) + { + return std::make_shared(buf, sample, false, false); + }); + + factory.registerOutputFormatProcessor("RowBinaryWithNamesAndTypes", []( + WriteBuffer & buf, + const Block & sample, + const Context &, + const FormatSettings &) + { + return std::make_shared(buf, sample, true, true); + }); +} + +} diff --git a/dbms/src/Processors/Formats/Impl/BinaryRowOutputFormat.h b/dbms/src/Processors/Formats/Impl/BinaryRowOutputFormat.h new file mode 100644 index 00000000000..23e7be0a558 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/BinaryRowOutputFormat.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include + + +namespace DB +{ + +class IColumn; +class IDataType; +class WriteBuffer; + + +/** A stream for outputting data in a binary line-by-line format. + */ +class BinaryRowOutputFormat: public IRowOutputFormat +{ +public: + BinaryRowOutputFormat(WriteBuffer & out_, const Block & header, bool with_names_, bool with_types_); + + String getName() const override { return "BinaryRowOutputFormat"; } + + void writeField(const IColumn & column, const IDataType & type, size_t row_num) override; + void writePrefix() override; + + String getContentType() const override { return "application/octet-stream"; } + +protected: + bool with_names; + bool with_types; +}; + +} + diff --git a/dbms/src/Processors/Formats/Impl/CMakeLists.txt b/dbms/src/Processors/Formats/Impl/CMakeLists.txt new file mode 100644 index 00000000000..65172356645 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/CMakeLists.txt @@ -0,0 +1,3 @@ +if (ENABLE_TESTS) + add_subdirectory (tests) +endif () diff --git a/dbms/src/Processors/Formats/Impl/CSVRowInputFormat.cpp b/dbms/src/Processors/Formats/Impl/CSVRowInputFormat.cpp new file mode 100644 index 00000000000..3038c9e02f6 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/CSVRowInputFormat.cpp @@ -0,0 +1,367 @@ +#include +#include + +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int INCORRECT_DATA; + extern const int LOGICAL_ERROR; +} + + +CSVRowInputFormat::CSVRowInputFormat( + ReadBuffer & in_, Block header, Params params, bool with_names_, const FormatSettings & format_settings) + : IRowInputFormat(std::move(header), in_, params), with_names(with_names_), format_settings(format_settings) +{ + auto & sample = getPort().getHeader(); + size_t num_columns = sample.columns(); + data_types.resize(num_columns); + for (size_t i = 0; i < num_columns; ++i) + data_types[i] = sample.safeGetByPosition(i).type; +} + + +static void skipEndOfLine(ReadBuffer & istr) +{ + /// \n (Unix) or \r\n (DOS/Windows) or \n\r (Mac OS Classic) + + if (*istr.position() == '\n') + { + ++istr.position(); + if (!istr.eof() && *istr.position() == '\r') + ++istr.position(); + } + else if (*istr.position() == '\r') + { + ++istr.position(); + if (!istr.eof() && *istr.position() == '\n') + ++istr.position(); + else + throw Exception("Cannot parse CSV format: found \\r (CR) not followed by \\n (LF)." + " Line must end by \\n (LF) or \\r\\n (CR LF) or \\n\\r.", ErrorCodes::INCORRECT_DATA); + } + else if (!istr.eof()) + throw Exception("Expected end of line", ErrorCodes::INCORRECT_DATA); +} + + +static void skipDelimiter(ReadBuffer & istr, const char delimiter, bool is_last_column) +{ + if (is_last_column) + { + if (istr.eof()) + return; + + /// we support the extra delimiter at the end of the line + if (*istr.position() == delimiter) + { + ++istr.position(); + if (istr.eof()) + return; + } + + skipEndOfLine(istr); + } + else + assertChar(delimiter, istr); +} + + +/// Skip `whitespace` symbols allowed in CSV. +static inline void skipWhitespacesAndTabs(ReadBuffer & buf) +{ + while (!buf.eof() + && (*buf.position() == ' ' + || *buf.position() == '\t')) + ++buf.position(); +} + + +static void skipRow(ReadBuffer & istr, const FormatSettings::CSV & settings, size_t num_columns) +{ + String tmp; + for (size_t i = 0; i < num_columns; ++i) + { + skipWhitespacesAndTabs(istr); + readCSVString(tmp, istr, settings); + skipWhitespacesAndTabs(istr); + + skipDelimiter(istr, settings.delimiter, i + 1 == num_columns); + } +} + + +void CSVRowInputFormat::readPrefix() +{ + /// In this format, we assume, that if first string field contain BOM as value, it will be written in quotes, + /// so BOM at beginning of stream cannot be confused with BOM in first string value, and it is safe to skip it. + skipBOMIfExists(in); + + size_t num_columns = data_types.size(); + String tmp; + + if (with_names) + skipRow(in, format_settings.csv, num_columns); +} + + +bool CSVRowInputFormat::readRow(MutableColumns & columns, RowReadExtension &) +{ + if (in.eof()) + return false; + + updateDiagnosticInfo(); + + size_t size = data_types.size(); + + for (size_t i = 0; i < size; ++i) + { + skipWhitespacesAndTabs(in); + data_types[i]->deserializeAsTextCSV(*columns[i], in, format_settings); + skipWhitespacesAndTabs(in); + + skipDelimiter(in, format_settings.csv.delimiter, i + 1 == size); + } + + return true; +} + + +String CSVRowInputFormat::getDiagnosticInfo() +{ + if (in.eof()) /// Buffer has gone, cannot extract information about what has been parsed. + return {}; + + WriteBufferFromOwnString out; + + auto & header = getPort().getHeader(); + MutableColumns columns = header.cloneEmptyColumns(); + + /// It is possible to display detailed diagnostics only if the last and next to last rows are still in the read buffer. + size_t bytes_read_at_start_of_buffer = in.count() - in.offset(); + if (bytes_read_at_start_of_buffer != bytes_read_at_start_of_buffer_on_prev_row) + { + out << "Could not print diagnostic info because two last rows aren't in buffer (rare case)\n"; + return out.str(); + } + + size_t max_length_of_column_name = 0; + for (size_t i = 0; i < header.columns(); ++i) + if (header.safeGetByPosition(i).name.size() > max_length_of_column_name) + max_length_of_column_name = header.safeGetByPosition(i).name.size(); + + size_t max_length_of_data_type_name = 0; + for (size_t i = 0; i < header.columns(); ++i) + if (header.safeGetByPosition(i).type->getName().size() > max_length_of_data_type_name) + max_length_of_data_type_name = header.safeGetByPosition(i).type->getName().size(); + + /// Roll back the cursor to the beginning of the previous or current row and parse all over again. But now we derive detailed information. + + if (pos_of_prev_row) + { + in.position() = pos_of_prev_row; + + out << "\nRow " << (row_num - 1) << ":\n"; + if (!parseRowAndPrintDiagnosticInfo(columns, out, max_length_of_column_name, max_length_of_data_type_name)) + return out.str(); + } + else + { + if (!pos_of_current_row) + { + out << "Could not print diagnostic info because parsing of data hasn't started.\n"; + return out.str(); + } + + in.position() = pos_of_current_row; + } + + out << "\nRow " << row_num << ":\n"; + parseRowAndPrintDiagnosticInfo(columns, out, max_length_of_column_name, max_length_of_data_type_name); + out << "\n"; + + return out.str(); +} + + +bool CSVRowInputFormat::parseRowAndPrintDiagnosticInfo(MutableColumns & columns, + WriteBuffer & out, size_t max_length_of_column_name, size_t max_length_of_data_type_name) +{ + const char delimiter = format_settings.csv.delimiter; + auto & header = getPort().getHeader(); + + size_t size = data_types.size(); + for (size_t i = 0; i < size; ++i) + { + if (i == 0 && in.eof()) + { + out << "\n"; + return false; + } + + out << "Column " << i << ", " << std::string((i < 10 ? 2 : i < 100 ? 1 : 0), ' ') + << "name: " << header.safeGetByPosition(i).name << ", " << std::string(max_length_of_column_name - header.safeGetByPosition(i).name.size(), ' ') + << "type: " << data_types[i]->getName() << ", " << std::string(max_length_of_data_type_name - data_types[i]->getName().size(), ' '); + + BufferBase::Position prev_position = in.position(); + BufferBase::Position curr_position = in.position(); + std::exception_ptr exception; + + try + { + skipWhitespacesAndTabs(in); + prev_position = in.position(); + data_types[i]->deserializeAsTextCSV(*columns[i], in, format_settings); + curr_position = in.position(); + skipWhitespacesAndTabs(in); + } + catch (...) + { + exception = std::current_exception(); + } + + if (curr_position < prev_position) + throw Exception("Logical error: parsing is non-deterministic.", ErrorCodes::LOGICAL_ERROR); + + if (isNumber(data_types[i]) || isDateOrDateTime(data_types[i])) + { + /// An empty string instead of a value. + if (curr_position == prev_position) + { + out << "ERROR: text "; + verbosePrintString(prev_position, std::min(prev_position + 10, in.buffer().end()), out); + out << " is not like " << data_types[i]->getName() << "\n"; + return false; + } + } + + out << "parsed text: "; + verbosePrintString(prev_position, curr_position, out); + + if (exception) + { + if (data_types[i]->getName() == "DateTime") + out << "ERROR: DateTime must be in YYYY-MM-DD hh:mm:ss or NNNNNNNNNN (unix timestamp, exactly 10 digits) format.\n"; + else if (data_types[i]->getName() == "Date") + out << "ERROR: Date must be in YYYY-MM-DD format.\n"; + else + out << "ERROR\n"; + return false; + } + + out << "\n"; + + if (data_types[i]->haveMaximumSizeOfValue()) + { + if (*curr_position != '\n' && *curr_position != '\r' && *curr_position != delimiter) + { + out << "ERROR: garbage after " << data_types[i]->getName() << ": "; + verbosePrintString(curr_position, std::min(curr_position + 10, in.buffer().end()), out); + out << "\n"; + + if (data_types[i]->getName() == "DateTime") + out << "ERROR: DateTime must be in YYYY-MM-DD hh:mm:ss or NNNNNNNNNN (unix timestamp, exactly 10 digits) format.\n"; + else if (data_types[i]->getName() == "Date") + out << "ERROR: Date must be in YYYY-MM-DD format.\n"; + + return false; + } + } + + /// Delimiters + if (i + 1 == size) + { + if (in.eof()) + return false; + + /// we support the extra delimiter at the end of the line + if (*in.position() == delimiter) + { + ++in.position(); + if (in.eof()) + break; + } + + if (!in.eof() && *in.position() != '\n' && *in.position() != '\r') + { + out << "ERROR: There is no line feed. "; + verbosePrintString(in.position(), in.position() + 1, out); + out << " found instead.\n" + " It's like your file has more columns than expected.\n" + "And if your file have right number of columns, maybe it have unquoted string value with comma.\n"; + + return false; + } + + skipEndOfLine(in); + } + else + { + try + { + assertChar(delimiter, in); + } + catch (const DB::Exception &) + { + if (*in.position() == '\n' || *in.position() == '\r') + { + out << "ERROR: Line feed found where delimiter (" << delimiter << ") is expected." + " It's like your file has less columns than expected.\n" + "And if your file have right number of columns, maybe it have unescaped quotes in values.\n"; + } + else + { + out << "ERROR: There is no delimiter (" << delimiter << "). "; + verbosePrintString(in.position(), in.position() + 1, out); + out << " found instead.\n"; + } + return false; + } + } + } + + return true; +} + + +void CSVRowInputFormat::syncAfterError() +{ + skipToNextLineOrEOF(in); +} + +void CSVRowInputFormat::updateDiagnosticInfo() +{ + ++row_num; + + bytes_read_at_start_of_buffer_on_prev_row = bytes_read_at_start_of_buffer_on_current_row; + bytes_read_at_start_of_buffer_on_current_row = in.count() - in.offset(); + + pos_of_prev_row = pos_of_current_row; + pos_of_current_row = in.position(); +} + + +void registerInputFormatProcessorCSV(FormatFactory & factory) +{ + for (bool with_names : {false, true}) + { + factory.registerInputFormatProcessor(with_names ? "CSVWithNames" : "CSV", [=]( + ReadBuffer & buf, + const Block & sample, + const Context &, + IRowInputFormat::Params params, + const FormatSettings & settings) + { + return std::make_shared(buf, sample, params, with_names, settings); + }); + } +} + +} diff --git a/dbms/src/Processors/Formats/Impl/CSVRowInputFormat.h b/dbms/src/Processors/Formats/Impl/CSVRowInputFormat.h new file mode 100644 index 00000000000..db7041bc90a --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/CSVRowInputFormat.h @@ -0,0 +1,56 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ + +class ReadBuffer; + +/** A stream for inputting data in csv format. + * Does not conform with https://tools.ietf.org/html/rfc4180 because it skips spaces and tabs between values. + */ +class CSVRowInputFormat : public IRowInputFormat +{ +public: + /** with_names - in the first line the header with column names + * with_types - on the next line header with type names + */ + CSVRowInputFormat(ReadBuffer & in_, Block header, Params params, bool with_names, const FormatSettings & format_settings); + + String getName() const override { return "CSVRowInputFormat"; } + + bool readRow(MutableColumns & columns, RowReadExtension &) override; + void readPrefix() override; + bool allowSyncAfterError() const override { return true; } + void syncAfterError() override; + + std::string getDiagnosticInfo() override; + +private: + bool with_names; + DataTypes data_types; + + const FormatSettings format_settings; + + /// For convenient diagnostics in case of an error. + + size_t row_num = 0; + + /// How many bytes were read, not counting those that are still in the buffer. + size_t bytes_read_at_start_of_buffer_on_current_row = 0; + size_t bytes_read_at_start_of_buffer_on_prev_row = 0; + + char * pos_of_current_row = nullptr; + char * pos_of_prev_row = nullptr; + + void updateDiagnosticInfo(); + + bool parseRowAndPrintDiagnosticInfo(MutableColumns & columns, + WriteBuffer & out, size_t max_length_of_column_name, size_t max_length_of_data_type_name); +}; + +} diff --git a/dbms/src/Processors/Formats/Impl/CSVRowOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/CSVRowOutputFormat.cpp new file mode 100644 index 00000000000..c5246dfb1cc --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/CSVRowOutputFormat.cpp @@ -0,0 +1,87 @@ +#include +#include + +#include + + +namespace DB +{ + + +CSVRowOutputFormat::CSVRowOutputFormat(WriteBuffer & out_, const Block & header, bool with_names_, const FormatSettings & format_settings) + : IRowOutputFormat(header, out_), with_names(with_names_), format_settings(format_settings) +{ + auto & sample = getPort(PortKind::Main).getHeader(); + size_t columns = sample.columns(); + data_types.resize(columns); + for (size_t i = 0; i < columns; ++i) + data_types[i] = sample.safeGetByPosition(i).type; +} + + +void CSVRowOutputFormat::writePrefix() +{ + auto & sample = getPort(PortKind::Main).getHeader(); + size_t columns = sample.columns(); + + if (with_names) + { + for (size_t i = 0; i < columns; ++i) + { + writeCSVString(sample.safeGetByPosition(i).name, out); + + char delimiter = format_settings.csv.delimiter; + if (i + 1 == columns) + delimiter = '\n'; + + writeChar(delimiter, out); + } + } +} + + +void CSVRowOutputFormat::writeField(const IColumn & column, const IDataType & type, size_t row_num) +{ + type.serializeAsTextCSV(column, row_num, out, format_settings); +} + + +void CSVRowOutputFormat::writeFieldDelimiter() +{ + writeChar(format_settings.csv.delimiter, out); +} + + +void CSVRowOutputFormat::writeRowEndDelimiter() +{ + writeChar('\n', out); +} + +void CSVRowOutputFormat::writeBeforeTotals() +{ + writeChar('\n', out); +} + +void CSVRowOutputFormat::writeBeforeExtremes() +{ + writeChar('\n', out); +} + + + +void registerOutputFormatProcessorCSV(FormatFactory & factory) +{ + for (bool with_names : {false, true}) + { + factory.registerOutputFormatProcessor(with_names ? "CSVWithNames" : "CSV", [=]( + WriteBuffer & buf, + const Block & sample, + const Context &, + const FormatSettings & format_settings) + { + return std::make_shared(buf, sample, with_names, format_settings); + }); + } +} + +} diff --git a/dbms/src/Processors/Formats/Impl/CSVRowOutputFormat.h b/dbms/src/Processors/Formats/Impl/CSVRowOutputFormat.h new file mode 100644 index 00000000000..5593fc98455 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/CSVRowOutputFormat.h @@ -0,0 +1,48 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ + +class WriteBuffer; + + +/** The stream for outputting data in csv format. + * Does not conform with https://tools.ietf.org/html/rfc4180 because it uses LF, not CR LF. + */ +class CSVRowOutputFormat : public IRowOutputFormat +{ +public: + /** with_names - output in the first line a header with column names + * with_types - output in the next line header with the names of the types + */ + CSVRowOutputFormat(WriteBuffer & out_, const Block & header, bool with_names_, const FormatSettings & format_settings); + + String getName() const override { return "CSVRowOutputFormat"; } + + void writeField(const IColumn & column, const IDataType & type, size_t row_num) override; + void writeFieldDelimiter() override; + void writeRowEndDelimiter() override; + void writePrefix() override; + void writeBeforeTotals() override; + void writeBeforeExtremes() override; + + /// https://www.iana.org/assignments/media-types/text/csv + String getContentType() const override + { + return String("text/csv; charset=UTF-8; header=") + (with_names ? "present" : "absent"); + } + +protected: + + bool with_names; + const FormatSettings format_settings; + DataTypes data_types; +}; + +} + diff --git a/dbms/src/Processors/Formats/Impl/CapnProtoRowInputFormat.cpp b/dbms/src/Processors/Formats/Impl/CapnProtoRowInputFormat.cpp new file mode 100644 index 00000000000..9ed84e89780 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/CapnProtoRowInputFormat.cpp @@ -0,0 +1,314 @@ +#include "config_formats.h" +#if USE_CAPNP + +#include +#include +#include // Y_IGNORE +#include +#include +#include // Y_IGNORE +#include // Y_IGNORE +#include // Y_IGNORE +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int BAD_TYPE_OF_FIELD; + extern const int BAD_ARGUMENTS; + extern const int THERE_IS_NO_COLUMN; + extern const int LOGICAL_ERROR; +} + +static CapnProtoRowInputFormat::NestedField split(const Block & header, size_t i) +{ + CapnProtoRowInputFormat::NestedField field = {{}, i}; + + // Remove leading dot in field definition, e.g. ".msg" -> "msg" + String name(header.safeGetByPosition(i).name); + if (!name.empty() && name[0] == '.') + name.erase(0, 1); + + boost::split(field.tokens, name, boost::is_any_of("._")); + return field; +} + + +static Field convertNodeToField(const capnp::DynamicValue::Reader & value) +{ + switch (value.getType()) + { + case capnp::DynamicValue::UNKNOWN: + throw Exception("Unknown field type", ErrorCodes::BAD_TYPE_OF_FIELD); + case capnp::DynamicValue::VOID: + return Field(); + case capnp::DynamicValue::BOOL: + return value.as() ? 1u : 0u; + case capnp::DynamicValue::INT: + return value.as(); + case capnp::DynamicValue::UINT: + return value.as(); + case capnp::DynamicValue::FLOAT: + return value.as(); + case capnp::DynamicValue::TEXT: + { + auto arr = value.as(); + return String(arr.begin(), arr.size()); + } + case capnp::DynamicValue::DATA: + { + auto arr = value.as().asChars(); + return String(arr.begin(), arr.size()); + } + case capnp::DynamicValue::LIST: + { + auto listValue = value.as(); + Array res(listValue.size()); + for (auto i : kj::indices(listValue)) + res[i] = convertNodeToField(listValue[i]); + + return res; + } + case capnp::DynamicValue::ENUM: + return value.as().getRaw(); + case capnp::DynamicValue::STRUCT: + { + auto structValue = value.as(); + const auto & fields = structValue.getSchema().getFields(); + + Field field = Tuple(TupleBackend(fields.size())); + TupleBackend & tuple = get(field).toUnderType(); + for (auto i : kj::indices(fields)) + tuple[i] = convertNodeToField(structValue.get(fields[i])); + + return field; + } + case capnp::DynamicValue::CAPABILITY: + throw Exception("CAPABILITY type not supported", ErrorCodes::BAD_TYPE_OF_FIELD); + case capnp::DynamicValue::ANY_POINTER: + throw Exception("ANY_POINTER type not supported", ErrorCodes::BAD_TYPE_OF_FIELD); + } + return Field(); +} + +static capnp::StructSchema::Field getFieldOrThrow(capnp::StructSchema node, const std::string & field) +{ + KJ_IF_MAYBE(child, node.findFieldByName(field)) + return *child; + else + throw Exception("Field " + field + " doesn't exist in schema " + node.getShortDisplayName().cStr(), ErrorCodes::THERE_IS_NO_COLUMN); +} + + +void CapnProtoRowInputFormat::createActions(const NestedFieldList & sorted_fields, capnp::StructSchema reader) +{ + /// Columns in a table can map to fields in Cap'n'Proto or to structs. + + /// Store common parents and their tokens in order to backtrack. + std::vector parents; + std::vector parent_tokens; + + capnp::StructSchema cur_reader = reader; + + for (const auto & field : sorted_fields) + { + if (field.tokens.empty()) + throw Exception("Logical error in CapnProtoRowInputFormat", ErrorCodes::LOGICAL_ERROR); + + // Backtrack to common parent + while (field.tokens.size() < parent_tokens.size() + 1 + || !std::equal(parent_tokens.begin(), parent_tokens.end(), field.tokens.begin())) + { + actions.push_back({Action::POP}); + parents.pop_back(); + parent_tokens.pop_back(); + + if (parents.empty()) + { + cur_reader = reader; + break; + } + else + cur_reader = parents.back().getType().asStruct(); + } + + // Go forward + while (parent_tokens.size() + 1 < field.tokens.size()) + { + const auto & token = field.tokens[parents.size()]; + auto node = getFieldOrThrow(cur_reader, token); + if (node.getType().isStruct()) + { + // Descend to field structure + parents.emplace_back(node); + parent_tokens.emplace_back(token); + cur_reader = node.getType().asStruct(); + actions.push_back({Action::PUSH, node}); + } + else if (node.getType().isList()) + { + break; // Collect list + } + else + throw Exception("Field " + token + " is neither Struct nor List", ErrorCodes::BAD_TYPE_OF_FIELD); + } + + // Read field from the structure + auto node = getFieldOrThrow(cur_reader, field.tokens[parents.size()]); + if (node.getType().isList() && !actions.empty() && actions.back().field == node) + { + // The field list here flattens Nested elements into multiple arrays + // In order to map Nested types in Cap'nProto back, they need to be collected + // Since the field names are sorted, the order of field positions must be preserved + // For example, if the fields are { b @0 :Text, a @1 :Text }, the `a` would come first + // even though it's position is second. + auto & columns = actions.back().columns; + auto it = std::upper_bound(columns.cbegin(), columns.cend(), field.pos); + columns.insert(it, field.pos); + } + else + { + actions.push_back({Action::READ, node, {field.pos}}); + } + } +} + +CapnProtoRowInputFormat::CapnProtoRowInputFormat(ReadBuffer & in_, Block header, Params params, const FormatSchemaInfo & info) + : IRowInputFormat(std::move(header), in_, params), parser(std::make_shared()) +{ + // Parse the schema and fetch the root object + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" + auto schema = parser->impl.parseDiskFile(info.schemaPath(), info.absoluteSchemaPath(), {}); +#pragma GCC diagnostic pop + + root = schema.getNested(info.messageName()).asStruct(); + + /** + * The schema typically consists of fields in various nested structures. + * Here we gather the list of fields and sort them in a way so that fields in the same structure are adjacent, + * and the nesting level doesn't decrease to make traversal easier. + */ + auto & sample = getPort().getHeader(); + NestedFieldList list; + size_t num_columns = sample.columns(); + for (size_t i = 0; i < num_columns; ++i) + list.push_back(split(sample, i)); + + // Order list first by value of strings then by length of string vector. + std::sort(list.begin(), list.end(), [](const NestedField & a, const NestedField & b) { return a.tokens < b.tokens; }); + createActions(list, root); +} + + +bool CapnProtoRowInputFormat::readRow(MutableColumns & columns, RowReadExtension &) +{ + if (in.eof()) + return false; + + // Read from underlying buffer directly + auto buf = in.buffer(); + auto base = reinterpret_cast(in.position()); + + // Check if there's enough bytes in the buffer to read the full message + kj::Array heap_array; + auto array = kj::arrayPtr(base, buf.size() - in.offset()); + auto expected_words = capnp::expectedSizeInWordsFromPrefix(array); + if (expected_words * sizeof(capnp::word) > array.size()) + { + // We'll need to reassemble the message in a contiguous buffer + heap_array = kj::heapArray(expected_words); + in.readStrict(heap_array.asChars().begin(), heap_array.asChars().size()); + array = heap_array.asPtr(); + } + + +#if CAPNP_VERSION >= 8000 + capnp::UnalignedFlatArrayMessageReader msg(array); +#else + capnp::FlatArrayMessageReader msg(array); +#endif + std::vector stack; + stack.push_back(msg.getRoot(root)); + + for (auto action : actions) + { + switch (action.type) + { + case Action::READ: + { + Field value = convertNodeToField(stack.back().get(action.field)); + if (action.columns.size() > 1) + { + // Nested columns must be flattened into several arrays + // e.g. Array(Tuple(x ..., y ...)) -> Array(x ...), Array(y ...) + const auto & collected = DB::get(value); + size_t size = collected.size(); + // The flattened array contains an array of a part of the nested tuple + Array flattened(size); + for (size_t column_index = 0; column_index < action.columns.size(); ++column_index) + { + // Populate array with a single tuple elements + for (size_t off = 0; off < size; ++off) + { + const TupleBackend & tuple = DB::get(collected[off]).toUnderType(); + flattened[off] = tuple[column_index]; + } + auto & col = columns[action.columns[column_index]]; + col->insert(flattened); + } + } + else + { + auto & col = columns[action.columns[0]]; + col->insert(value); + } + + break; + } + case Action::POP: + stack.pop_back(); + break; + case Action::PUSH: + stack.push_back(stack.back().get(action.field).as()); + break; + } + } + + // Advance buffer position if used directly + if (heap_array.size() == 0) + { + auto parsed = (msg.getEnd() - base) * sizeof(capnp::word); + in.position() += parsed; + } + + return true; +} + +void registerInputFormatProcessorCapnProto(FormatFactory & factory) +{ + factory.registerInputFormatProcessor( + "CapnProto", + [](ReadBuffer & buf, const Block & sample, const Context & context, IRowInputFormat::Params params, const FormatSettings &) + { + return std::make_shared(buf, sample, params, FormatSchemaInfo(context, "capnp")); + }); +} + +} + +#else + +namespace DB +{ + class FormatFactory; + void registerInputFormatProcessorCapnProto(FormatFactory &) {} +} + +#endif // USE_CAPNP diff --git a/dbms/src/Processors/Formats/Impl/CapnProtoRowInputFormat.h b/dbms/src/Processors/Formats/Impl/CapnProtoRowInputFormat.h new file mode 100644 index 00000000000..8d768538719 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/CapnProtoRowInputFormat.h @@ -0,0 +1,75 @@ +#pragma once +#include +#if USE_CAPNP + +#include +#include + +#include + +namespace DB +{ + +class FormatSchemaInfo; +class ReadBuffer; + +/** A stream for reading messages in Cap'n Proto format in given schema. + * Like Protocol Buffers and Thrift (but unlike JSON or MessagePack), + * Cap'n Proto messages are strongly-typed and not self-describing. + * The schema in this case cannot be compiled in, so it uses a runtime schema parser. + * See https://capnproto.org/cxx.html + */ +class CapnProtoRowInputFormat : public IRowInputFormat +{ +public: + struct NestedField + { + std::vector tokens; + size_t pos; + }; + using NestedFieldList = std::vector; + + /** schema_dir - base path for schema files + * schema_file - location of the capnproto schema, e.g. "schema.capnp" + * root_object - name to the root object, e.g. "Message" + */ + CapnProtoRowInputFormat(ReadBuffer & in_, Block header, Params params, const FormatSchemaInfo & info); + + String getName() const override { return "CapnProtoRowInputFormat"; } + + bool readRow(MutableColumns & columns, RowReadExtension &) override; + +private: + // Build a traversal plan from a sorted list of fields + void createActions(const NestedFieldList & sortedFields, capnp::StructSchema reader); + + /* Action for state machine for traversing nested structures. */ + using BlockPositionList = std::vector; + struct Action + { + enum Type { POP, PUSH, READ }; + Type type; + capnp::StructSchema::Field field = {}; + BlockPositionList columns = {}; + }; + + // Wrapper for classes that could throw in destructor + // https://github.com/capnproto/capnproto/issues/553 + template + struct DestructorCatcher + { + T impl; + template + DestructorCatcher(Arg && ... args) : impl(kj::fwd(args)...) {} + ~DestructorCatcher() noexcept try { } catch (...) { return; } + }; + using SchemaParser = DestructorCatcher; + + std::shared_ptr parser; + capnp::StructSchema root; + std::vector actions; +}; + +} + +#endif // USE_CAPNP diff --git a/dbms/src/Processors/Formats/Impl/JSONCompactRowOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/JSONCompactRowOutputFormat.cpp new file mode 100644 index 00000000000..0f63a4cf72d --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/JSONCompactRowOutputFormat.cpp @@ -0,0 +1,92 @@ +#include +#include + +#include + + +namespace DB +{ + +JSONCompactRowOutputFormat::JSONCompactRowOutputFormat( + WriteBuffer & out_, const Block & header, const FormatSettings & settings_) + : JSONRowOutputFormat(out_, header, settings_) +{ +} + + +void JSONCompactRowOutputFormat::writeField(const IColumn & column, const IDataType & type, size_t row_num) +{ + type.serializeAsTextJSON(column, row_num, *ostr, settings); + ++field_number; +} + + +void JSONCompactRowOutputFormat::writeFieldDelimiter() +{ + writeCString(", ", *ostr); +} + +void JSONCompactRowOutputFormat::writeTotalsFieldDelimiter() +{ + writeCString(",", *ostr); +} + + +void JSONCompactRowOutputFormat::writeRowStartDelimiter() +{ + if (row_count > 0) + writeCString(",\n", *ostr); + writeCString("\t\t[", *ostr); +} + + +void JSONCompactRowOutputFormat::writeRowEndDelimiter() +{ + writeChar(']', *ostr); + field_number = 0; + ++row_count; +} + +void JSONCompactRowOutputFormat::writeBeforeTotals() +{ + writeCString(",\n", *ostr); + writeChar('\n', *ostr); + writeCString("\t\"totals\": [", *ostr); +} + +void JSONCompactRowOutputFormat::writeAfterTotals() +{ + writeChar(']', *ostr); +} + +void JSONCompactRowOutputFormat::writeExtremesElement(const char * title, const Columns & columns, size_t row_num) +{ + writeCString("\t\t\"", *ostr); + writeCString(title, *ostr); + writeCString("\": [", *ostr); + + size_t extremes_columns = columns.size(); + for (size_t i = 0; i < extremes_columns; ++i) + { + if (i != 0) + writeTotalsFieldDelimiter(); + + writeField(*columns[i], *types[i], row_num); + } + + writeChar(']', *ostr); +} + +void registerOutputFormatProcessorJSONCompact(FormatFactory & factory) +{ + factory.registerOutputFormatProcessor("JSONCompact", []( + WriteBuffer & buf, + const Block & sample, + const Context &, + const FormatSettings & format_settings) + { + return std::make_shared(buf, sample, format_settings); + }); +} + +} diff --git a/dbms/src/Processors/Formats/Impl/JSONCompactRowOutputFormat.h b/dbms/src/Processors/Formats/Impl/JSONCompactRowOutputFormat.h new file mode 100644 index 00000000000..e2b5ce76f88 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/JSONCompactRowOutputFormat.h @@ -0,0 +1,43 @@ +#pragma once + +#include +#include +#include +#include + + +namespace DB +{ + +struct FormatSettings; + +/** The stream for outputting data in the JSONCompact format. + */ +class JSONCompactRowOutputFormat : public JSONRowOutputFormat +{ +public: + JSONCompactRowOutputFormat(WriteBuffer & out_, const Block & header, const FormatSettings & settings_); + + String getName() const override { return "JSONCompactRowOutputFormat"; } + + void writeField(const IColumn & column, const IDataType & type, size_t row_num) override; + void writeFieldDelimiter() override; + void writeRowStartDelimiter() override; + void writeRowEndDelimiter() override; + + void writeBeforeTotals() override; + void writeAfterTotals() override; + +protected: + void writeExtremesElement(const char * title, const Columns & columns, size_t row_num) override; + + void writeTotalsField(const IColumn & column, const IDataType & type, size_t row_num) override + { + return writeField(column, type, row_num); + } + + void writeTotalsFieldDelimiter() override; + +}; + +} diff --git a/dbms/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp b/dbms/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp new file mode 100644 index 00000000000..b1dde6d2275 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp @@ -0,0 +1,270 @@ +#include + +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int INCORRECT_DATA; + extern const int CANNOT_READ_ALL_DATA; + extern const int LOGICAL_ERROR; +} + +namespace +{ + +enum +{ + UNKNOWN_FIELD = size_t(-1), + NESTED_FIELD = size_t(-2) +}; + +} + + +JSONEachRowRowInputFormat::JSONEachRowRowInputFormat( + ReadBuffer & in_, const Block & header, Params params, const FormatSettings & format_settings) + : IRowInputFormat(header, in_, params), format_settings(format_settings), name_map(header.columns()) +{ + /// In this format, BOM at beginning of stream cannot be confused with value, so it is safe to skip it. + skipBOMIfExists(in); + + size_t num_columns = getPort().getHeader().columns(); + for (size_t i = 0; i < num_columns; ++i) + { + const String & column_name = columnName(i); + name_map[column_name] = i; /// NOTE You could place names more cache-locally. + if (format_settings.import_nested_json) + { + const auto splitted = Nested::splitName(column_name); + if (!splitted.second.empty()) + { + const StringRef table_name(column_name.data(), splitted.first.size()); + name_map[table_name] = NESTED_FIELD; + } + } + } + + prev_positions.assign(num_columns, name_map.end()); +} + +const String & JSONEachRowRowInputFormat::columnName(size_t i) const +{ + return getPort().getHeader().getByPosition(i).name; +} + +inline size_t JSONEachRowRowInputFormat::columnIndex(const StringRef & name, size_t key_index) +{ + /// Optimization by caching the order of fields (which is almost always the same) + /// and a quick check to match the next expected field, instead of searching the hash table. + + if (prev_positions.size() > key_index + && prev_positions[key_index] != name_map.end() + && name == prev_positions[key_index]->getFirst()) + { + return prev_positions[key_index]->getSecond(); + } + else + { + const auto it = name_map.find(name); + + if (name_map.end() != it) + { + if (key_index < prev_positions.size()) + prev_positions[key_index] = it; + + return it->getSecond(); + } + else + return UNKNOWN_FIELD; + } +} + +/** Read the field name and convert it to column name + * (taking into account the current nested name prefix) + * Resulting StringRef is valid only before next read from buf. + */ +StringRef JSONEachRowRowInputFormat::readColumnName(ReadBuffer & buf) +{ + // This is just an optimization: try to avoid copying the name into current_column_name + + if (nested_prefix_length == 0 && buf.position() + 1 < buf.buffer().end()) + { + char * next_pos = find_first_symbols<'\\', '"'>(buf.position() + 1, buf.buffer().end()); + + if (next_pos != buf.buffer().end() && *next_pos != '\\') + { + /// The most likely option is that there is no escape sequence in the key name, and the entire name is placed in the buffer. + assertChar('"', buf); + StringRef res(buf.position(), next_pos - buf.position()); + buf.position() = next_pos + 1; + return res; + } + } + + current_column_name.resize(nested_prefix_length); + readJSONStringInto(current_column_name, buf); + return current_column_name; +} + + +static inline void skipColonDelimeter(ReadBuffer & istr) +{ + skipWhitespaceIfAny(istr); + assertChar(':', istr); + skipWhitespaceIfAny(istr); +} + +void JSONEachRowRowInputFormat::skipUnknownField(const StringRef & name_ref) +{ + if (!format_settings.skip_unknown_fields) + throw Exception("Unknown field found while parsing JSONEachRow format: " + name_ref.toString(), ErrorCodes::INCORRECT_DATA); + + skipJSONField(in, name_ref); +} + +void JSONEachRowRowInputFormat::readField(size_t index, MutableColumns & columns) +{ + if (read_columns[index]) + throw Exception("Duplicate field found while parsing JSONEachRow format: " + columnName(index), ErrorCodes::INCORRECT_DATA); + + try + { + auto & header = getPort().getHeader(); + header.getByPosition(index).type->deserializeAsTextJSON(*columns[index], in, format_settings); + } + catch (Exception & e) + { + e.addMessage("(while read the value of key " + columnName(index) + ")"); + throw; + } + + read_columns[index] = true; +} + +inline bool JSONEachRowRowInputFormat::advanceToNextKey(size_t key_index) +{ + skipWhitespaceIfAny(in); + + if (in.eof()) + throw Exception("Unexpected end of stream while parsing JSONEachRow format", ErrorCodes::CANNOT_READ_ALL_DATA); + else if (*in.position() == '}') + { + ++in.position(); + return false; + } + + if (key_index > 0) + { + assertChar(',', in); + skipWhitespaceIfAny(in); + } + return true; +} + +void JSONEachRowRowInputFormat::readJSONObject(MutableColumns & columns) +{ + assertChar('{', in); + + for (size_t key_index = 0; advanceToNextKey(key_index); ++key_index) + { + StringRef name_ref = readColumnName(in); + const size_t column_index = columnIndex(name_ref, key_index); + + if (unlikely(ssize_t(column_index) < 0)) + { + /// name_ref may point directly to the input buffer + /// and input buffer may be filled with new data on next read + /// If we want to use name_ref after another reads from buffer, we must copy it to temporary string. + + current_column_name.assign(name_ref.data, name_ref.size); + name_ref = StringRef(current_column_name); + + skipColonDelimeter(in); + + if (column_index == UNKNOWN_FIELD) + skipUnknownField(name_ref); + else if (column_index == NESTED_FIELD) + readNestedData(name_ref.toString(), columns); + else + throw Exception("Logical error: illegal value of column_index", ErrorCodes::LOGICAL_ERROR); + } + else + { + skipColonDelimeter(in); + readField(column_index, columns); + } + } +} + +void JSONEachRowRowInputFormat::readNestedData(const String & name, MutableColumns & columns) +{ + current_column_name = name; + current_column_name.push_back('.'); + nested_prefix_length = current_column_name.size(); + readJSONObject(columns); + nested_prefix_length = 0; +} + + +bool JSONEachRowRowInputFormat::readRow(MutableColumns & columns, RowReadExtension & ext) +{ + skipWhitespaceIfAny(in); + + /// We consume ;, or \n before scanning a new row, instead scanning to next row at the end. + /// The reason is that if we want an exact number of rows read with LIMIT x + /// from a streaming table engine with text data format, like File or Kafka + /// then seeking to next ;, or \n would trigger reading of an extra row at the end. + + /// Semicolon is added for convenience as it could be used at end of INSERT query. + if (!in.eof() && (*in.position() == ',' || *in.position() == ';')) + ++in.position(); + + skipWhitespaceIfAny(in); + if (in.eof()) + return false; + + size_t num_columns = columns.size(); + + /// Set of columns for which the values were read. The rest will be filled with default values. + read_columns.assign(num_columns, false); + + nested_prefix_length = 0; + readJSONObject(columns); + + auto & header = getPort().getHeader(); + /// Fill non-visited columns with the default values. + for (size_t i = 0; i < num_columns; ++i) + if (!read_columns[i]) + header.getByPosition(i).type->insertDefaultInto(*columns[i]); + + /// return info about defaults set + ext.read_columns = read_columns; + return true; +} + + +void JSONEachRowRowInputFormat::syncAfterError() +{ + skipToUnescapedNextLineOrEOF(in); +} + + +void registerInputFormatProcessorJSONEachRow(FormatFactory & factory) +{ + factory.registerInputFormatProcessor("JSONEachRow", []( + ReadBuffer & buf, + const Block & sample, + const Context &, + IRowInputFormat::Params params, + const FormatSettings & settings) + { + return std::make_shared(buf, sample, params, settings); + }); +} + +} diff --git a/dbms/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.h b/dbms/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.h new file mode 100644 index 00000000000..901e35af6e8 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.h @@ -0,0 +1,68 @@ +#pragma once + +#include +#include +#include +#include + + +namespace DB +{ + +class ReadBuffer; + + +/** A stream for reading data in JSON format, where each row is represented by a separate JSON object. + * Objects can be separated by feed return, other whitespace characters in any number and possibly a comma. + * Fields can be listed in any order (including, in different lines there may be different order), + * and some fields may be missing. + */ +class JSONEachRowRowInputFormat : public IRowInputFormat +{ +public: + JSONEachRowRowInputFormat(ReadBuffer & in_, const Block & header, Params params, const FormatSettings & format_settings); + + String getName() const override { return "JSONEachRowRowInputFormat"; } + + bool readRow(MutableColumns & columns, RowReadExtension & ext) override; + bool allowSyncAfterError() const override { return true; } + void syncAfterError() override; + +private: + const String & columnName(size_t i) const; + size_t columnIndex(const StringRef & name, size_t key_index); + bool advanceToNextKey(size_t key_index); + void skipUnknownField(const StringRef & name_ref); + StringRef readColumnName(ReadBuffer & buf); + void readField(size_t index, MutableColumns & columns); + void readJSONObject(MutableColumns & columns); + void readNestedData(const String & name, MutableColumns & columns); + +private: + + const FormatSettings format_settings; + + /// Buffer for the read from the stream field name. Used when you have to copy it. + /// Also, if processing of Nested data is in progress, it holds the common prefix + /// of the nested column names (so that appending the field name to it produces + /// the full column name) + String current_column_name; + + /// If processing Nested data, holds the length of the common prefix + /// of the names of related nested columns. For example, for a table + /// created as follows + /// CREATE TABLE t (n Nested (i Int32, s String)) + /// the nested column names are 'n.i' and 'n.s' and the nested prefix is 'n.' + size_t nested_prefix_length = 0; + + std::vector read_columns; + + /// Hash table match `field name -> position in the block`. NOTE You can use perfect hash map. + using NameMap = HashMap; + NameMap name_map; + + /// Cached search results for previous row (keyed as index in JSON object) - used as a hint. + std::vector prev_positions; +}; + +} diff --git a/dbms/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp new file mode 100644 index 00000000000..112021dce42 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp @@ -0,0 +1,66 @@ +#include +#include +#include +#include + + +namespace DB +{ + + +JSONEachRowRowOutputFormat::JSONEachRowRowOutputFormat(WriteBuffer & out_, const Block & header, const FormatSettings & settings) + : IRowOutputFormat(header, out_), settings(settings) +{ + auto & sample = getPort(PortKind::Main).getHeader(); + size_t columns = sample.columns(); + fields.resize(columns); + + for (size_t i = 0; i < columns; ++i) + { + WriteBufferFromString buf(fields[i]); + writeJSONString(sample.getByPosition(i).name, buf, settings); + } +} + + +void JSONEachRowRowOutputFormat::writeField(const IColumn & column, const IDataType & type, size_t row_num) +{ + writeString(fields[field_number], out); + writeChar(':', out); + type.serializeAsTextJSON(column, row_num, out, settings); + ++field_number; +} + + +void JSONEachRowRowOutputFormat::writeFieldDelimiter() +{ + writeChar(',', out); +} + + +void JSONEachRowRowOutputFormat::writeRowStartDelimiter() +{ + writeChar('{', out); +} + + +void JSONEachRowRowOutputFormat::writeRowEndDelimiter() +{ + writeCString("}\n", out); + field_number = 0; +} + + +void registerOutputFormatProcessorJSONEachRow(FormatFactory & factory) +{ + factory.registerOutputFormatProcessor("JSONEachRow", []( + WriteBuffer & buf, + const Block & sample, + const Context &, + const FormatSettings & format_settings) + { + return std::make_shared(buf, sample, format_settings); + }); +} + +} diff --git a/dbms/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.h b/dbms/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.h new file mode 100644 index 00000000000..0ceeb5352c7 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include +#include +#include + + +namespace DB +{ + +/** The stream for outputting data in JSON format, by object per line. + * Does not validate UTF-8. + */ +class JSONEachRowRowOutputFormat : public IRowOutputFormat +{ +public: + JSONEachRowRowOutputFormat(WriteBuffer & out_, const Block & header, const FormatSettings & settings); + + String getName() const override { return "JSONEachRowRowOutputFormat"; } + + void writeField(const IColumn & column, const IDataType & type, size_t row_num) override; + void writeFieldDelimiter() override; + void writeRowStartDelimiter() override; + void writeRowEndDelimiter() override; + +private: + size_t field_number = 0; + Names fields; + + FormatSettings settings; +}; + +} + diff --git a/dbms/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp new file mode 100644 index 00000000000..f0d791a6554 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp @@ -0,0 +1,257 @@ +#include +#include +#include +#include +#include + + +namespace DB +{ + +JSONRowOutputFormat::JSONRowOutputFormat(WriteBuffer & out_, const Block & header, const FormatSettings & settings_) + : IRowOutputFormat(header, out_), settings(settings_) +{ + auto & sample = getPort(PortKind::Main).getHeader(); + NamesAndTypesList columns(sample.getNamesAndTypesList()); + fields.assign(columns.begin(), columns.end()); + + bool need_validate_utf8 = false; + for (size_t i = 0; i < sample.columns(); ++i) + { + if (!sample.getByPosition(i).type->textCanContainOnlyValidUTF8()) + need_validate_utf8 = true; + + WriteBufferFromOwnString buf; + writeJSONString(fields[i].name, buf, settings); + + fields[i].name = buf.str(); + } + + if (need_validate_utf8) + { + validating_ostr = std::make_unique(out); + ostr = validating_ostr.get(); + } + else + ostr = &out; +} + + +void JSONRowOutputFormat::writePrefix() +{ + writeCString("{\n", *ostr); + writeCString("\t\"meta\":\n", *ostr); + writeCString("\t[\n", *ostr); + + for (size_t i = 0; i < fields.size(); ++i) + { + writeCString("\t\t{\n", *ostr); + + writeCString("\t\t\t\"name\": ", *ostr); + writeString(fields[i].name, *ostr); + writeCString(",\n", *ostr); + writeCString("\t\t\t\"type\": ", *ostr); + writeJSONString(fields[i].type->getName(), *ostr, settings); + writeChar('\n', *ostr); + + writeCString("\t\t}", *ostr); + if (i + 1 < fields.size()) + writeChar(',', *ostr); + writeChar('\n', *ostr); + } + + writeCString("\t],\n", *ostr); + writeChar('\n', *ostr); + writeCString("\t\"data\":\n", *ostr); + writeCString("\t[\n", *ostr); +} + + +void JSONRowOutputFormat::writeField(const IColumn & column, const IDataType & type, size_t row_num) +{ + writeCString("\t\t\t", *ostr); + writeString(fields[field_number].name, *ostr); + writeCString(": ", *ostr); + type.serializeAsTextJSON(column, row_num, *ostr, settings); + ++field_number; +} + +void JSONRowOutputFormat::writeTotalsField(const IColumn & column, const IDataType & type, size_t row_num) +{ + writeCString("\t\t", *ostr); + writeString(fields[field_number].name, *ostr); + writeCString(": ", *ostr); + type.serializeAsTextJSON(column, row_num, *ostr, settings); + ++field_number; +} + +void JSONRowOutputFormat::writeFieldDelimiter() +{ + writeCString(",\n", *ostr); +} + + +void JSONRowOutputFormat::writeRowStartDelimiter() +{ + writeCString("\t\t{\n", *ostr); +} + + +void JSONRowOutputFormat::writeRowEndDelimiter() +{ + writeChar('\n', *ostr); + writeCString("\t\t}", *ostr); + field_number = 0; + ++row_count; +} + + +void JSONRowOutputFormat::writeRowBetweenDelimiter() +{ + writeCString(",\n", *ostr); +} + + +void JSONRowOutputFormat::writeSuffix() +{ + writeChar('\n', *ostr); + writeCString("\t]", *ostr); +} + +void JSONRowOutputFormat::writeBeforeTotals() +{ + writeCString(",\n", *ostr); + writeChar('\n', *ostr); + writeCString("\t\"totals\":\n", *ostr); + writeCString("\t{\n", *ostr); +} + +void JSONRowOutputFormat::writeTotals(const Columns & columns, size_t row_num) +{ + size_t num_columns = columns.size(); + + for (size_t i = 0; i < num_columns; ++i) + { + if (i != 0) + writeTotalsFieldDelimiter(); + + writeTotalsField(*columns[i], *types[i], row_num); + } +} + +void JSONRowOutputFormat::writeAfterTotals() +{ + writeChar('\n', *ostr); + writeCString("\t}", *ostr); + field_number = 0; +} + +void JSONRowOutputFormat::writeBeforeExtremes() +{ + writeCString(",\n", *ostr); + writeChar('\n', *ostr); + writeCString("\t\"extremes\":\n", *ostr); + writeCString("\t{\n", *ostr); +} + +void JSONRowOutputFormat::writeExtremesElement(const char * title, const Columns & columns, size_t row_num) +{ + writeCString("\t\t\"", *ostr); + writeCString(title, *ostr); + writeCString("\":\n", *ostr); + writeCString("\t\t{\n", *ostr); + + size_t extremes_columns = columns.size(); + for (size_t i = 0; i < extremes_columns; ++i) + { + if (i != 0) + writeFieldDelimiter(); + + writeField(*columns[i], *types[i], row_num); + } + + writeChar('\n', *ostr); + writeCString("\t\t}", *ostr); + field_number = 0; +} + +void JSONRowOutputFormat::writeMinExtreme(const Columns & columns, size_t row_num) +{ + writeExtremesElement("min", columns, row_num); +} + +void JSONRowOutputFormat::writeMaxExtreme(const Columns & columns, size_t row_num) +{ + writeExtremesElement("max", columns, row_num); +} + +void JSONRowOutputFormat::writeAfterExtremes() +{ + writeChar('\n', *ostr); + writeCString("\t}", *ostr); +} + +void JSONRowOutputFormat::writeLastSuffix() +{ + writeCString(",\n\n", *ostr); + writeCString("\t\"rows\": ", *ostr); + writeIntText(row_count, *ostr); + + writeRowsBeforeLimitAtLeast(); + + if (settings.write_statistics) + writeStatistics(); + + writeChar('\n', *ostr); + writeCString("}\n", *ostr); + ostr->next(); +} + +void JSONRowOutputFormat::writeRowsBeforeLimitAtLeast() +{ + if (applied_limit) + { + writeCString(",\n\n", *ostr); + writeCString("\t\"rows_before_limit_at_least\": ", *ostr); + writeIntText(rows_before_limit, *ostr); + } +} + +void JSONRowOutputFormat::writeStatistics() +{ + writeCString(",\n\n", *ostr); + writeCString("\t\"statistics\":\n", *ostr); + writeCString("\t{\n", *ostr); + + writeCString("\t\t\"elapsed\": ", *ostr); + writeText(watch.elapsedSeconds(), *ostr); + writeCString(",\n", *ostr); + writeCString("\t\t\"rows_read\": ", *ostr); + writeText(progress.read_rows.load(), *ostr); + writeCString(",\n", *ostr); + writeCString("\t\t\"bytes_read\": ", *ostr); + writeText(progress.read_bytes.load(), *ostr); + writeChar('\n', *ostr); + + writeCString("\t}", *ostr); +} + +void JSONRowOutputFormat::onProgress(const Progress & value) +{ + progress.incrementPiecewiseAtomically(value); +} + + +void registerOutputFormatProcessorJSON(FormatFactory & factory) +{ + factory.registerOutputFormatProcessor("JSON", []( + WriteBuffer & buf, + const Block & sample, + const Context &, + const FormatSettings & format_settings) + { + return std::make_shared(buf, sample, format_settings); + }); +} + +} diff --git a/dbms/src/Processors/Formats/Impl/JSONRowOutputFormat.h b/dbms/src/Processors/Formats/Impl/JSONRowOutputFormat.h new file mode 100644 index 00000000000..5df98475ea3 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/JSONRowOutputFormat.h @@ -0,0 +1,84 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +/** Stream for output data in JSON format. + */ +class JSONRowOutputFormat : public IRowOutputFormat +{ +public: + JSONRowOutputFormat(WriteBuffer & out_, const Block & header, const FormatSettings & settings_); + + String getName() const override { return "JSONRowOutputFormat"; } + + void writeField(const IColumn & column, const IDataType & type, size_t row_num) override; + void writeFieldDelimiter() override; + void writeRowStartDelimiter() override; + void writeRowEndDelimiter() override; + void writeRowBetweenDelimiter() override; + void writePrefix() override; + void writeSuffix() override; + + void writeMinExtreme(const Columns & columns, size_t row_num) override; + void writeMaxExtreme(const Columns & columns, size_t row_num) override; + void writeTotals(const Columns & columns, size_t row_num) override; + + void writeBeforeTotals() override; + void writeAfterTotals() override; + void writeBeforeExtremes() override; + void writeAfterExtremes() override; + + void writeLastSuffix() override; + + void flush() override + { + ostr->next(); + + if (validating_ostr) + out.next(); + } + + void setRowsBeforeLimit(size_t rows_before_limit_) override + { + applied_limit = true; + rows_before_limit = rows_before_limit_; + } + + void onProgress(const Progress & value) override; + + String getContentType() const override { return "application/json; charset=UTF-8"; } + +protected: + virtual void writeTotalsField(const IColumn & column, const IDataType & type, size_t row_num); + virtual void writeExtremesElement(const char * title, const Columns & columns, size_t row_num); + virtual void writeTotalsFieldDelimiter() { writeFieldDelimiter(); } + + void writeRowsBeforeLimitAtLeast(); + void writeStatistics(); + + + std::unique_ptr validating_ostr; /// Validates UTF-8 sequences, replaces bad sequences with replacement character. + WriteBuffer * ostr; + + size_t field_number = 0; + size_t row_count = 0; + bool applied_limit = false; + size_t rows_before_limit = 0; + NamesAndTypes fields; + + Progress progress; + Stopwatch watch; + FormatSettings settings; +}; + +} + diff --git a/dbms/src/Processors/Formats/Impl/MySQLOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/MySQLOutputFormat.cpp new file mode 100644 index 00000000000..4dc4a32c672 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/MySQLOutputFormat.cpp @@ -0,0 +1,105 @@ +#include + +#include +#include +#include +#include + +#include +#include + +namespace DB +{ + +using namespace MySQLProtocol; + + +MySQLOutputFormat::MySQLOutputFormat(WriteBuffer & out_, const Block & header, const Context & context, const FormatSettings & settings) + : IOutputFormat(header, out_) + , context(context) + , packet_sender(std::make_shared(out, const_cast(context.sequence_id))) /// TODO: fix it + , format_settings(settings) +{ +} + +void MySQLOutputFormat::consume(Chunk chunk) +{ + auto & header = getPort(PortKind::Main).getHeader(); + + if (!initialized) + { + initialized = true; + + + if (header.columns()) + { + + packet_sender->sendPacket(LengthEncodedNumber(header.columns())); + + for (const ColumnWithTypeAndName & column : header.getColumnsWithTypeAndName()) + { + ColumnDefinition column_definition(column.name, CharacterSet::binary, 0, ColumnType::MYSQL_TYPE_STRING, + 0, 0); + packet_sender->sendPacket(column_definition); + } + + if (!(context.client_capabilities & Capability::CLIENT_DEPRECATE_EOF)) + { + packet_sender->sendPacket(EOF_Packet(0, 0)); + } + } + } + + size_t rows = chunk.getNumRows(); + auto & columns = chunk.getColumns(); + + for (size_t i = 0; i < rows; i++) + { + ResultsetRow row_packet; + for (size_t col = 0; col < columns.size(); ++col) + { + String column_value; + WriteBufferFromString ostr(column_value); + header.getByPosition(col).type->serializeAsText(*columns[col], i, ostr, format_settings); + ostr.finish(); + + row_packet.appendColumn(std::move(column_value)); + } + packet_sender->sendPacket(row_packet); + } +} + +void MySQLOutputFormat::finalize() +{ + QueryStatus * process_list_elem = context.getProcessListElement(); + CurrentThread::finalizePerformanceCounters(); + QueryStatusInfo info = process_list_elem->getInfo(); + size_t affected_rows = info.written_rows; + + std::stringstream human_readable_info; + human_readable_info << std::fixed << std::setprecision(3) + << "Read " << info.read_rows << " rows, " << formatReadableSizeWithBinarySuffix(info.read_bytes) << " in " << info.elapsed_seconds << " sec., " + << static_cast(info.read_rows / info.elapsed_seconds) << " rows/sec., " + << formatReadableSizeWithBinarySuffix(info.read_bytes / info.elapsed_seconds) << "/sec."; + + auto & header = getPort(PortKind::Main).getHeader(); + + if (header.columns() == 0) + packet_sender->sendPacket(OK_Packet(0x0, context.client_capabilities, affected_rows, 0, 0, "", human_readable_info.str()), true); + else + if (context.client_capabilities & CLIENT_DEPRECATE_EOF) + packet_sender->sendPacket(OK_Packet(0xfe, context.client_capabilities, affected_rows, 0, 0, "", human_readable_info.str()), true); + else + packet_sender->sendPacket(EOF_Packet(0, 0), true); +} + +void registerOutputFormatProcessorMySQLWrite(FormatFactory & factory) +{ + factory.registerOutputFormatProcessor( + "MySQLWire", [](WriteBuffer & buf, const Block & sample, const Context & context, const FormatSettings & settings) + { + return std::make_shared(buf, sample, context, settings); + }); +} + +} diff --git a/dbms/src/Processors/Formats/Impl/MySQLOutputFormat.h b/dbms/src/Processors/Formats/Impl/MySQLOutputFormat.h new file mode 100644 index 00000000000..b959342ca4d --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/MySQLOutputFormat.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include + +#include +#include + +namespace DB +{ + +class IColumn; +class IDataType; +class WriteBuffer; +class Context; + +/** A stream for outputting data in a binary line-by-line format. + */ +class MySQLOutputFormat: public IOutputFormat +{ +public: + MySQLOutputFormat(WriteBuffer & out_, const Block & header, const Context & context, const FormatSettings & settings); + + String getName() const override { return "MySQLOutputFormat"; } + + void consume(Chunk) override; + void finalize() override; + +private: + + bool initialized = false; + + const Context & context; + std::shared_ptr packet_sender; + FormatSettings format_settings; +}; + +} + diff --git a/dbms/src/Processors/Formats/Impl/NativeFormat.cpp b/dbms/src/Processors/Formats/Impl/NativeFormat.cpp new file mode 100644 index 00000000000..0abab1e964d --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/NativeFormat.cpp @@ -0,0 +1,170 @@ +#include +#include +#include +#include +#include + + +namespace DB +{ + +class NativeInputFormatFromNativeBlockInputStream : public IInputFormat +{ +public: + NativeInputFormatFromNativeBlockInputStream(const Block & header, ReadBuffer & in_) + : IInputFormat(header, in_) + , stream(std::make_shared(in, header, 0)) + { + } + + String getName() const override { return "NativeInputFormatFromNativeBlockInputStream"; } + +protected: + Chunk generate() override + { + /// TODO: do something with totals and extremes. + + if (!read_prefix) + { + stream->readPrefix(); + read_prefix = true; + } + + auto block = stream->read(); + if (!block) + { + if (!read_suffix) + { + stream->readSuffix(); + read_suffix = true; + } + + return Chunk(); + } + + assertBlocksHaveEqualStructure(getPort().getHeader(), block, getName()); + block.checkNumberOfRows(); + + UInt64 num_rows = block.rows(); + return Chunk(block.getColumns(), num_rows); + } + +private: + std::shared_ptr stream; + bool read_prefix = false; + bool read_suffix = false; +}; + + +class NativeOutputFormatFromNativeBlockOutputStream : public IOutputFormat +{ +public: + NativeOutputFormatFromNativeBlockOutputStream(const Block & header, WriteBuffer & out_) + : IOutputFormat(header, out_) + , stream(std::make_shared(out, 0, header)) + { + } + + String getName() const override { return "NativeOutputFormatFromNativeBlockOutputStream"; } + + void setRowsBeforeLimit(size_t rows_before_limit) override + { + stream->setRowsBeforeLimit(rows_before_limit); + } + + void onProgress(const Progress & progress) override + { + stream->onProgress(progress); + } + + std::string getContentType() const override + { + return stream->getContentType(); + } + +protected: + void consume(Chunk chunk) override + { + writePrefixIfNot(); + + if (chunk) + { + + auto block = getPort(PortKind::Main).getHeader(); + block.setColumns(chunk.detachColumns()); + stream->write(block); + } + } + + void consumeTotals(Chunk chunk) override + { + writePrefixIfNot(); + + auto block = getPort(PortKind::Totals).getHeader(); + block.setColumns(chunk.detachColumns()); + stream->setTotals(block); + } + + void consumeExtremes(Chunk chunk) override + { + writePrefixIfNot(); + + auto block = getPort(PortKind::Extremes).getHeader(); + block.setColumns(chunk.detachColumns()); + stream->setExtremes(block); + } + + void finalize() override + { + writePrefixIfNot(); + writeSuffixIfNot(); + } + +private: + std::shared_ptr stream; + bool prefix_written = false; + bool suffix_written = false; + + void writePrefixIfNot() + { + if (!prefix_written) + stream->writePrefix(); + + prefix_written = true; + } + + void writeSuffixIfNot() + { + if (!suffix_written) + stream->writeSuffix(); + + suffix_written = true; + } +}; + +void registerInputFormatProcessorNative(FormatFactory & factory) +{ + factory.registerInputFormatProcessor("Native", []( + ReadBuffer & buf, + const Block & sample, + const Context &, + const RowInputFormatParams &, + const FormatSettings &) + { + return std::make_shared(sample, buf); + }); +} + +void registerOutputFormatProcessorNative(FormatFactory & factory) +{ + factory.registerOutputFormatProcessor("Native", []( + WriteBuffer & buf, + const Block & sample, + const Context &, + const FormatSettings &) + { + return std::make_shared(sample, buf); + }); +} + +} diff --git a/dbms/src/Processors/Formats/Impl/NullFormat.cpp b/dbms/src/Processors/Formats/Impl/NullFormat.cpp new file mode 100644 index 00000000000..be754294c45 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/NullFormat.cpp @@ -0,0 +1,31 @@ +#include +#include + + +namespace DB +{ + +class NullOutputFormat : public IOutputFormat +{ +public: + NullOutputFormat(const Block & header, WriteBuffer & out_) : IOutputFormat(header, out_) {} + + String getName() const override { return "NullOutputFormat"; } + +protected: + void consume(Chunk) override {} +}; + +void registerOutputFormatProcessorNull(FormatFactory & factory) +{ + factory.registerOutputFormatProcessor("Null", []( + WriteBuffer & buf, + const Block & sample, + const Context &, + const FormatSettings &) + { + return std::make_shared(sample, buf); + }); +} + +} diff --git a/dbms/src/Processors/Formats/Impl/ODBCDriver2BlockOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/ODBCDriver2BlockOutputFormat.cpp new file mode 100644 index 00000000000..543edade0e8 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/ODBCDriver2BlockOutputFormat.cpp @@ -0,0 +1,113 @@ +#include +#include +#include +#include +#include + + +#include + + +namespace DB +{ +ODBCDriver2BlockOutputFormat::ODBCDriver2BlockOutputFormat( + WriteBuffer & out_, const Block & header, const FormatSettings & format_settings) + : IOutputFormat(header, out_), format_settings(format_settings) +{ +} + +static void writeODBCString(WriteBuffer & out, const std::string & str) +{ + writeIntBinary(Int32(str.size()), out); + out.write(str.data(), str.size()); +} + +void ODBCDriver2BlockOutputFormat::writeRow(const Block & header, const Columns & columns, size_t row_idx, std::string & buffer) +{ + size_t num_columns = columns.size(); + for (size_t column_idx = 0; column_idx < num_columns; ++column_idx) + { + buffer.clear(); + auto & column = columns[column_idx]; + + if (column->isNullAt(row_idx)) + { + writeIntBinary(Int32(-1), out); + } + else + { + { + WriteBufferFromString text_out(buffer); + header.getByPosition(row_idx).type->serializeAsText(*column, row_idx, text_out, format_settings); + } + writeODBCString(out, buffer); + } + } +} + +void ODBCDriver2BlockOutputFormat::write(Chunk chunk, PortKind port_kind) +{ + String text_value; + auto & header = getPort(port_kind).getHeader(); + auto & columns = chunk.getColumns(); + const size_t rows = chunk.getNumRows(); + for (size_t i = 0; i < rows; ++i) + writeRow(header, columns, i, text_value); +} + +void ODBCDriver2BlockOutputFormat::consume(Chunk chunk) +{ + writePrefixIfNot(); + write(std::move(chunk), PortKind::Main); +} + +void ODBCDriver2BlockOutputFormat::consumeTotals(Chunk chunk) +{ + writePrefixIfNot(); + write(std::move(chunk), PortKind::Totals); +} + +void ODBCDriver2BlockOutputFormat::finalize() +{ + writePrefixIfNot(); +} + +void ODBCDriver2BlockOutputFormat::writePrefix() +{ + auto & header = getPort(PortKind::Main).getHeader(); + const size_t columns = header.columns(); + + /// Number of header rows. + writeIntBinary(Int32(2), out); + + /// Names of columns. + /// Number of columns + 1 for first name column. + writeIntBinary(Int32(columns + 1), out); + writeODBCString(out, "name"); + for (size_t i = 0; i < columns; ++i) + { + const ColumnWithTypeAndName & col = header.getByPosition(i); + writeODBCString(out, col.name); + } + + /// Types of columns. + writeIntBinary(Int32(columns + 1), out); + writeODBCString(out, "type"); + for (size_t i = 0; i < columns; ++i) + { + const ColumnWithTypeAndName & col = header.getByPosition(i); + writeODBCString(out, col.type->getName()); + } +} + + +void registerOutputFormatProcessorODBCDriver2(FormatFactory & factory) +{ + factory.registerOutputFormatProcessor( + "ODBCDriver2", [](WriteBuffer & buf, const Block & sample, const Context &, const FormatSettings & format_settings) + { + return std::make_shared(buf, sample, format_settings); + }); +} + +} diff --git a/dbms/src/Processors/Formats/Impl/ODBCDriver2BlockOutputFormat.h b/dbms/src/Processors/Formats/Impl/ODBCDriver2BlockOutputFormat.h new file mode 100644 index 00000000000..5a6ed4efc09 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/ODBCDriver2BlockOutputFormat.h @@ -0,0 +1,55 @@ +#pragma once + +#include +#include +#include +#include + + +namespace DB +{ +class WriteBuffer; + + +/** A data format designed to simplify the implementation of the ODBC driver. + * ODBC driver is designed to be build for different platforms without dependencies from the main code, + * so the format is made that way so that it can be as easy as possible to parse it. + * A header is displayed with the required information. + * The data is then output in the order of the rows. Each value is displayed as follows: length in Int32 format (-1 for NULL), then data in text form. + */ +class ODBCDriver2BlockOutputFormat final : public IOutputFormat +{ +public: + ODBCDriver2BlockOutputFormat(WriteBuffer & out_, const Block & header, const FormatSettings & format_settings); + + String getName() const override { return "ODBCDriver2BlockOutputFormat"; } + + void consume(Chunk) override; + void consumeTotals(Chunk) override; + void finalize() override; + + std::string getContentType() const override + { + return "application/octet-stream"; + } + +private: + const FormatSettings format_settings; + bool prefix_written = false; + + void writePrefixIfNot() + { + if (!prefix_written) + writePrefix(); + + prefix_written = true; + } + + void writeRow(const Block & header, const Columns & columns, size_t row_idx, std::string & buffer); + void write(Chunk chunk, PortKind port_kind); + void writePrefix(); +}; + + + +} diff --git a/dbms/src/Processors/Formats/Impl/ODBCDriverBlockOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/ODBCDriverBlockOutputFormat.cpp new file mode 100644 index 00000000000..fc8796c8799 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/ODBCDriverBlockOutputFormat.cpp @@ -0,0 +1,79 @@ +#include +#include +#include +#include +#include + + +namespace DB +{ + +ODBCDriverBlockOutputFormat::ODBCDriverBlockOutputFormat(WriteBuffer & out_, const Block & header, const FormatSettings & format_settings) + : IOutputFormat(header, out_), format_settings(format_settings) +{ +} + +void ODBCDriverBlockOutputFormat::consume(Chunk chunk) +{ + writePrefixIfNot(); + + const size_t num_rows = chunk.getNumRows(); + const size_t num_columns = chunk.getNumColumns(); + auto & columns = chunk.getColumns(); + auto & header = getPort(PortKind::Main).getHeader(); + String text_value; + + for (size_t i = 0; i < num_rows; ++i) + { + for (size_t j = 0; j < num_columns; ++j) + { + text_value.resize(0); + auto & column = columns[j]; + auto & type = header.getByPosition(j).type; + + { + WriteBufferFromString text_out(text_value); + type->serializeAsText(*column, i, text_out, format_settings); + } + + writeStringBinary(text_value, out); + } + } +} + +void ODBCDriverBlockOutputFormat::writePrefix() +{ + auto & header = getPort(PortKind::Main).getHeader(); + const size_t columns = header.columns(); + + /// Number of columns. + writeVarUInt(columns, out); + + /// Names and types of columns. + for (size_t i = 0; i < columns; ++i) + { + const ColumnWithTypeAndName & col = header.getByPosition(i); + + writeStringBinary(col.name, out); + writeStringBinary(col.type->getName(), out); + } +} + +void ODBCDriverBlockOutputFormat::finalize() +{ + writePrefixIfNot(); +} + +void registerOutputFormatProcessorODBCDriver(FormatFactory & factory) +{ + factory.registerOutputFormatProcessor("ODBCDriver", []( + WriteBuffer & buf, + const Block & sample, + const Context &, + const FormatSettings & format_settings) + { + return std::make_shared(buf, sample, format_settings); + }); +} + +} diff --git a/dbms/src/Processors/Formats/Impl/ODBCDriverBlockOutputFormat.h b/dbms/src/Processors/Formats/Impl/ODBCDriverBlockOutputFormat.h new file mode 100644 index 00000000000..3a0e6e29c40 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/ODBCDriverBlockOutputFormat.h @@ -0,0 +1,48 @@ +#pragma once + +#include +#include +#include +#include + + +namespace DB +{ + +class WriteBuffer; + + +/** A data format designed to simplify the implementation of the ODBC driver. + * ODBC driver is designed to be build for different platforms without dependencies from the main code, + * so the format is made that way so that it can be as easy as possible to parse it. + * A header is displayed with the required information. + * The data is then output in the order of the rows. Each value is displayed as follows: length in VarUInt format, then data in text form. + */ +class ODBCDriverBlockOutputFormat : public IOutputFormat +{ +public: + ODBCDriverBlockOutputFormat(WriteBuffer & out_, const Block & header, const FormatSettings & format_settings); + + String getName() const override { return "ODBCDriverBlockOutputFormat"; } + + void consume(Chunk) override; + void finalize() override; + + std::string getContentType() const override { return "application/octet-stream"; } + +private: + const FormatSettings format_settings; + bool prefix_written = false; + + void writePrefixIfNot() + { + if (!prefix_written) + writePrefix(); + + prefix_written = true; + } + + void writePrefix(); +}; + +} diff --git a/dbms/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp b/dbms/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp new file mode 100644 index 00000000000..9e19f0881ed --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp @@ -0,0 +1,497 @@ +#include "config_formats.h" + +#if USE_PARQUET +#include "ParquetBlockInputFormat.h" + +#include +#include +#include +// TODO: clear includes +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +//#include +//#include +#include +//#include +//#include +#include + +#include // REMOVE ME + +namespace DB +{ +namespace ErrorCodes +{ + extern const int UNKNOWN_TYPE; + extern const int VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE; + extern const int CANNOT_READ_ALL_DATA; + extern const int EMPTY_DATA_PASSED; + extern const int SIZES_OF_COLUMNS_DOESNT_MATCH; + extern const int CANNOT_CONVERT_TYPE; + extern const int CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN; + extern const int THERE_IS_NO_COLUMN; +} + +ParquetBlockInputFormat::ParquetBlockInputFormat(ReadBuffer & in_, Block header, const Context & context) + : IInputFormat(std::move(header), in_), context{context} +{ +} + +/// Inserts numeric data right into internal column data to reduce an overhead +template > +static void fillColumnWithNumericData(std::shared_ptr & arrow_column, MutableColumnPtr & internal_column) +{ + auto & column_data = static_cast(*internal_column).getData(); + column_data.reserve(arrow_column->length()); + + for (size_t chunk_i = 0, num_chunks = static_cast(arrow_column->data()->num_chunks()); chunk_i < num_chunks; ++chunk_i) + { + std::shared_ptr chunk = arrow_column->data()->chunk(chunk_i); + /// buffers[0] is a null bitmap and buffers[1] are actual values + std::shared_ptr buffer = chunk->data()->buffers[1]; + + const auto * raw_data = reinterpret_cast(buffer->data()); + column_data.insert_assume_reserved(raw_data, raw_data + chunk->length()); + } +} + +/// Inserts chars and offsets right into internal column data to reduce an overhead. +/// Internal offsets are shifted by one to the right in comparison with Arrow ones. So the last offset should map to the end of all chars. +/// Also internal strings are null terminated. +static void fillColumnWithStringData(std::shared_ptr & arrow_column, MutableColumnPtr & internal_column) +{ + PaddedPODArray & column_chars_t = static_cast(*internal_column).getChars(); + PaddedPODArray & column_offsets = static_cast(*internal_column).getOffsets(); + + size_t chars_t_size = 0; + for (size_t chunk_i = 0, num_chunks = static_cast(arrow_column->data()->num_chunks()); chunk_i < num_chunks; ++chunk_i) + { + arrow::BinaryArray & chunk = static_cast(*(arrow_column->data()->chunk(chunk_i))); + const size_t chunk_length = chunk.length(); + + chars_t_size += chunk.value_offset(chunk_length - 1) + chunk.value_length(chunk_length - 1); + chars_t_size += chunk_length; /// additional space for null bytes + } + + column_chars_t.reserve(chars_t_size); + column_offsets.reserve(arrow_column->length()); + + for (size_t chunk_i = 0, num_chunks = static_cast(arrow_column->data()->num_chunks()); chunk_i < num_chunks; ++chunk_i) + { + arrow::BinaryArray & chunk = static_cast(*(arrow_column->data()->chunk(chunk_i))); + std::shared_ptr buffer = chunk.value_data(); + const size_t chunk_length = chunk.length(); + + for (size_t offset_i = 0; offset_i != chunk_length; ++offset_i) + { + if (!chunk.IsNull(offset_i) && buffer) + { + const UInt8 * raw_data = buffer->data() + chunk.value_offset(offset_i); + column_chars_t.insert_assume_reserved(raw_data, raw_data + chunk.value_length(offset_i)); + } + column_chars_t.emplace_back('\0'); + + column_offsets.emplace_back(column_chars_t.size()); + } + } +} + +static void fillColumnWithBooleanData(std::shared_ptr & arrow_column, MutableColumnPtr & internal_column) +{ + auto & column_data = static_cast &>(*internal_column).getData(); + column_data.resize(arrow_column->length()); + + for (size_t chunk_i = 0, num_chunks = static_cast(arrow_column->data()->num_chunks()); chunk_i < num_chunks; ++chunk_i) + { + arrow::BooleanArray & chunk = static_cast(*(arrow_column->data()->chunk(chunk_i))); + /// buffers[0] is a null bitmap and buffers[1] are actual values + std::shared_ptr buffer = chunk.data()->buffers[1]; + + for (size_t bool_i = 0; bool_i != static_cast(chunk.length()); ++bool_i) + column_data[bool_i] = chunk.Value(bool_i); + } +} + +/// Arrow stores Parquet::DATE in Int32, while ClickHouse stores Date in UInt16. Therefore, it should be checked before saving +static void fillColumnWithDate32Data(std::shared_ptr & arrow_column, MutableColumnPtr & internal_column) +{ + PaddedPODArray & column_data = static_cast &>(*internal_column).getData(); + column_data.reserve(arrow_column->length()); + + for (size_t chunk_i = 0, num_chunks = static_cast(arrow_column->data()->num_chunks()); chunk_i < num_chunks; ++chunk_i) + { + arrow::Date32Array & chunk = static_cast(*(arrow_column->data()->chunk(chunk_i))); + + for (size_t value_i = 0, length = static_cast(chunk.length()); value_i < length; ++value_i) + { + UInt32 days_num = static_cast(chunk.Value(value_i)); + if (days_num > DATE_LUT_MAX_DAY_NUM) + { + // TODO: will it rollback correctly? + throw Exception{"Input value " + std::to_string(days_num) + " of a column \"" + arrow_column->name() + + "\" is greater than " + "max allowed Date value, which is " + + std::to_string(DATE_LUT_MAX_DAY_NUM), + ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE}; + } + + column_data.emplace_back(days_num); + } + } +} + +/// Arrow stores Parquet::DATETIME in Int64, while ClickHouse stores DateTime in UInt32. Therefore, it should be checked before saving +static void fillColumnWithDate64Data(std::shared_ptr & arrow_column, MutableColumnPtr & internal_column) +{ + auto & column_data = static_cast &>(*internal_column).getData(); + column_data.reserve(arrow_column->length()); + + for (size_t chunk_i = 0, num_chunks = static_cast(arrow_column->data()->num_chunks()); chunk_i < num_chunks; ++chunk_i) + { + auto & chunk = static_cast(*(arrow_column->data()->chunk(chunk_i))); + for (size_t value_i = 0, length = static_cast(chunk.length()); value_i < length; ++value_i) + { + auto timestamp = static_cast(chunk.Value(value_i) / 1000); // Always? in ms + column_data.emplace_back(timestamp); + } + } +} + +static void fillColumnWithTimestampData(std::shared_ptr & arrow_column, MutableColumnPtr & internal_column) +{ + auto & column_data = static_cast &>(*internal_column).getData(); + column_data.reserve(arrow_column->length()); + + for (size_t chunk_i = 0, num_chunks = static_cast(arrow_column->data()->num_chunks()); chunk_i < num_chunks; ++chunk_i) + { + auto & chunk = static_cast(*(arrow_column->data()->chunk(chunk_i))); + const auto & type = static_cast(*chunk.type()); + + UInt32 divide = 1; + const auto unit = type.unit(); + switch (unit) + { + case arrow::TimeUnit::SECOND: + divide = 1; + break; + case arrow::TimeUnit::MILLI: + divide = 1000; + break; + case arrow::TimeUnit::MICRO: + divide = 1000000; + break; + case arrow::TimeUnit::NANO: + divide = 1000000000; + break; + } + + for (size_t value_i = 0, length = static_cast(chunk.length()); value_i < length; ++value_i) + { + auto timestamp = static_cast(chunk.Value(value_i) / divide); // ms! TODO: check other 's' 'ns' ... + column_data.emplace_back(timestamp); + } + } +} + +static void fillColumnWithDecimalData(std::shared_ptr & arrow_column, MutableColumnPtr & internal_column) +{ + auto & column = static_cast &>(*internal_column); + auto & column_data = column.getData(); + column_data.reserve(arrow_column->length()); + + for (size_t chunk_i = 0, num_chunks = static_cast(arrow_column->data()->num_chunks()); chunk_i < num_chunks; ++chunk_i) + { + auto & chunk = static_cast(*(arrow_column->data()->chunk(chunk_i))); + for (size_t value_i = 0, length = static_cast(chunk.length()); value_i < length; ++value_i) + { + column_data.emplace_back(chunk.IsNull(value_i) ? Decimal128(0) : *reinterpret_cast(chunk.Value(value_i))); // TODO: copy column + } + } +} + +/// Creates a null bytemap from arrow's null bitmap +static void fillByteMapFromArrowColumn(std::shared_ptr & arrow_column, MutableColumnPtr & bytemap) +{ + PaddedPODArray & bytemap_data = static_cast &>(*bytemap).getData(); + bytemap_data.reserve(arrow_column->length()); + + for (size_t chunk_i = 0; chunk_i != static_cast(arrow_column->data()->num_chunks()); ++chunk_i) + { + std::shared_ptr chunk = arrow_column->data()->chunk(chunk_i); + + for (size_t value_i = 0; value_i != static_cast(chunk->length()); ++value_i) + bytemap_data.emplace_back(chunk->IsNull(value_i)); + } +} + +# define FOR_ARROW_NUMERIC_TYPES(M) \ + M(arrow::Type::UINT8, UInt8) \ + M(arrow::Type::INT8, Int8) \ + M(arrow::Type::UINT16, UInt16) \ + M(arrow::Type::INT16, Int16) \ + M(arrow::Type::UINT32, UInt32) \ + M(arrow::Type::INT32, Int32) \ + M(arrow::Type::UINT64, UInt64) \ + M(arrow::Type::INT64, Int64) \ + M(arrow::Type::FLOAT, Float32) \ + M(arrow::Type::DOUBLE, Float64) +//M(arrow::Type::HALF_FLOAT, Float32) // TODO + + +using NameToColumnPtr = std::unordered_map>; + +const std::unordered_map> arrow_type_to_internal_type = { + //{arrow::Type::DECIMAL, std::make_shared()}, + {arrow::Type::UINT8, std::make_shared()}, + {arrow::Type::INT8, std::make_shared()}, + {arrow::Type::UINT16, std::make_shared()}, + {arrow::Type::INT16, std::make_shared()}, + {arrow::Type::UINT32, std::make_shared()}, + {arrow::Type::INT32, std::make_shared()}, + {arrow::Type::UINT64, std::make_shared()}, + {arrow::Type::INT64, std::make_shared()}, + {arrow::Type::HALF_FLOAT, std::make_shared()}, + {arrow::Type::FLOAT, std::make_shared()}, + {arrow::Type::DOUBLE, std::make_shared()}, + + {arrow::Type::BOOL, std::make_shared()}, + //{arrow::Type::DATE32, std::make_shared()}, + {arrow::Type::DATE32, std::make_shared()}, + //{arrow::Type::DATE32, std::make_shared()}, + {arrow::Type::DATE64, std::make_shared()}, + {arrow::Type::TIMESTAMP, std::make_shared()}, + //{arrow::Type::TIME32, std::make_shared()}, + + + {arrow::Type::STRING, std::make_shared()}, + {arrow::Type::BINARY, std::make_shared()}, + //{arrow::Type::FIXED_SIZE_BINARY, std::make_shared()}, + //{arrow::Type::UUID, std::make_shared()}, + + + // TODO: add other types that are convertable to internal ones: + // 0. ENUM? + // 1. UUID -> String + // 2. JSON -> String + // Full list of types: contrib/arrow/cpp/src/arrow/type.h +}; + + +Chunk ParquetBlockInputFormat::generate() +{ + Chunk res; + Columns columns_list; + UInt64 num_rows = 0; + auto & header = getPort().getHeader(); + columns_list.reserve(header.rows()); + + if (!in.eof()) + { + /* + First we load whole stream into string (its very bad and limiting .parquet file size to half? of RAM) + Then producing blocks for every row_group (dont load big .parquet files with one row_group - it can eat x10+ RAM from .parquet file size) + */ + + if (row_group_current < row_group_total) + throw Exception{"Got new data, but data from previous chunks not readed " + std::to_string(row_group_current) + "/" + std::to_string(row_group_total), ErrorCodes::CANNOT_READ_ALL_DATA}; + + file_data.clear(); + { + WriteBufferFromString file_buffer(file_data); + copyData(in, file_buffer); + } + + buffer = std::make_unique(file_data); + // TODO: maybe use parquet::RandomAccessSource? + auto reader = parquet::ParquetFileReader::Open(std::make_shared<::arrow::io::BufferReader>(*buffer)); + file_reader = std::make_unique(::arrow::default_memory_pool(), std::move(reader)); + row_group_total = file_reader->num_row_groups(); + row_group_current = 0; + } + //DUMP(row_group_current, row_group_total); + if (row_group_current >= row_group_total) + return res; + + // TODO: also catch a ParquetException thrown by filereader? + //arrow::Status read_status = filereader.ReadTable(&table); + std::shared_ptr table; + arrow::Status read_status = file_reader->ReadRowGroup(row_group_current, &table); + + if (!read_status.ok()) + throw Exception{"Error while reading parquet data: " + read_status.ToString(), ErrorCodes::CANNOT_READ_ALL_DATA}; + + if (0 == table->num_rows()) + throw Exception{"Empty table in input data", ErrorCodes::EMPTY_DATA_PASSED}; + + if (header.columns() > static_cast(table->num_columns())) + // TODO: What if some columns were not presented? Insert NULLs? What if a column is not nullable? + throw Exception{"Number of columns is less than the table has", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH}; + + ++row_group_current; + + NameToColumnPtr name_to_column_ptr; + for (size_t i = 0, num_columns = static_cast(table->num_columns()); i < num_columns; ++i) + { + std::shared_ptr arrow_column = table->column(i); + name_to_column_ptr[arrow_column->name()] = arrow_column; + } + + for (size_t column_i = 0, columns = header.columns(); column_i < columns; ++column_i) + { + ColumnWithTypeAndName header_column = header.getByPosition(column_i); + + if (name_to_column_ptr.find(header_column.name) == name_to_column_ptr.end()) + // TODO: What if some columns were not presented? Insert NULLs? What if a column is not nullable? + throw Exception{"Column \"" + header_column.name + "\" is not presented in input data", ErrorCodes::THERE_IS_NO_COLUMN}; + + std::shared_ptr arrow_column = name_to_column_ptr[header_column.name]; + arrow::Type::type arrow_type = arrow_column->type()->id(); + + // TODO: check if a column is const? + if (!header_column.type->isNullable() && arrow_column->null_count()) + { + throw Exception{"Can not insert NULL data into non-nullable column \"" + header_column.name + "\"", + ErrorCodes::CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN}; + } + + const bool target_column_is_nullable = header_column.type->isNullable() || arrow_column->null_count(); + + DataTypePtr internal_nested_type; + + if (arrow_type == arrow::Type::DECIMAL) + { + const auto decimal_type = static_cast(arrow_column->type().get()); + internal_nested_type = std::make_shared>(decimal_type->precision(), decimal_type->scale()); + } + else if (arrow_type_to_internal_type.find(arrow_type) != arrow_type_to_internal_type.end()) + { + internal_nested_type = arrow_type_to_internal_type.at(arrow_type); + } + else + { + throw Exception{"The type \"" + arrow_column->type()->name() + "\" of an input column \"" + arrow_column->name() + + "\" is not supported for conversion from a Parquet data format", + ErrorCodes::CANNOT_CONVERT_TYPE}; + } + + const DataTypePtr internal_type = target_column_is_nullable ? makeNullable(internal_nested_type) : internal_nested_type; + const std::string internal_nested_type_name = internal_nested_type->getName(); + + const DataTypePtr column_nested_type = header_column.type->isNullable() + ? static_cast(header_column.type.get())->getNestedType() + : header_column.type; + + const DataTypePtr column_type = header_column.type; + + const std::string column_nested_type_name = column_nested_type->getName(); + + ColumnWithTypeAndName column; + column.name = header_column.name; + column.type = internal_type; + + /// Data + MutableColumnPtr read_column = internal_nested_type->createColumn(); + switch (arrow_type) + { + case arrow::Type::STRING: + case arrow::Type::BINARY: + //case arrow::Type::FIXED_SIZE_BINARY: + fillColumnWithStringData(arrow_column, read_column); + break; + case arrow::Type::BOOL: + fillColumnWithBooleanData(arrow_column, read_column); + break; + case arrow::Type::DATE32: + fillColumnWithDate32Data(arrow_column, read_column); + break; + case arrow::Type::DATE64: + fillColumnWithDate64Data(arrow_column, read_column); + break; + case arrow::Type::TIMESTAMP: + fillColumnWithTimestampData(arrow_column, read_column); + break; + case arrow::Type::DECIMAL: + //fillColumnWithNumericData>(arrow_column, read_column); // Have problems with trash values under NULL, but faster + fillColumnWithDecimalData(arrow_column, read_column /*, internal_nested_type*/); + break; +# define DISPATCH(ARROW_NUMERIC_TYPE, CPP_NUMERIC_TYPE) \ + case ARROW_NUMERIC_TYPE: \ + fillColumnWithNumericData(arrow_column, read_column); \ + break; + + FOR_ARROW_NUMERIC_TYPES(DISPATCH) +# undef DISPATCH + // TODO: support TIMESTAMP_MICROS and TIMESTAMP_MILLIS with truncated micro- and milliseconds? + // TODO: read JSON as a string? + // TODO: read UUID as a string? + default: + throw Exception{"Unsupported parquet type \"" + arrow_column->type()->name() + "\" of an input column \"" + + arrow_column->name() + "\"", + ErrorCodes::UNKNOWN_TYPE}; + } + + if (column.type->isNullable()) + { + MutableColumnPtr null_bytemap = DataTypeUInt8().createColumn(); + fillByteMapFromArrowColumn(arrow_column, null_bytemap); + column.column = ColumnNullable::create(std::move(read_column), std::move(null_bytemap)); + } + else + { + column.column = std::move(read_column); + } + + column.column = castColumn(column, column_type, context); + column.type = column_type; + num_rows = column.column->size(); + columns_list.push_back(std::move(column.column)); + } + + res.setColumns(columns_list, num_rows); + return res; +} + +void registerInputFormatProcessorParquet(FormatFactory & factory) +{ + factory.registerInputFormatProcessor( + "Parquet", + [](ReadBuffer & buf, + const Block & sample, + const Context & context, + const RowInputFormatParams &, + const FormatSettings & /* settings */){ return std::make_shared(buf, sample, context); }); +} + +} + +#else + +namespace DB +{ +class FormatFactory; +void registerInputFormatProcessorParquet(FormatFactory &) +{ +} +} + +#endif diff --git a/dbms/src/Processors/Formats/Impl/ParquetBlockInputFormat.h b/dbms/src/Processors/Formats/Impl/ParquetBlockInputFormat.h new file mode 100644 index 00000000000..bbb7abdd5d4 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/ParquetBlockInputFormat.h @@ -0,0 +1,43 @@ +#pragma once + +#include +#if USE_PARQUET +# include +//# include +//# include +//# include + + +namespace parquet { namespace arrow { class FileReader; } } +namespace arrow { class Buffer; } + +namespace DB +{ +class Context; + +class ParquetBlockInputFormat: public IInputFormat +{ +public: + ParquetBlockInputFormat(ReadBuffer & in_, Block header, const Context & context); + + String getName() const override { return "ParquetBlockInputFormat"; } + +protected: + Chunk generate() override; + +private: + + // TODO: check that this class implements every part of its parent + + const Context & context; + + std::unique_ptr file_reader; + std::string file_data; + std::unique_ptr buffer; + int row_group_total = 0; + int row_group_current = 0; +}; + +} + +#endif diff --git a/dbms/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp new file mode 100644 index 00000000000..26b3dd84f14 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp @@ -0,0 +1,451 @@ +#include "config_formats.h" + +#if USE_PARQUET +# include "ParquetBlockOutputFormat.h" + +// TODO: clean includes +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include + +# include // REMOVE ME + +namespace DB +{ +namespace ErrorCodes +{ + extern const int UNKNOWN_EXCEPTION; + extern const int UNKNOWN_TYPE; +} + +ParquetBlockOutputFormat::ParquetBlockOutputFormat(WriteBuffer & out_, const Block & header, const FormatSettings & format_settings) + : IOutputFormat(header, out_), format_settings{format_settings} +{ +} + +static void checkStatus(arrow::Status & status, const std::string & column_name) +{ + if (!status.ok()) + throw Exception{"Error with a parquet column \"" + column_name + "\": " + status.ToString(), ErrorCodes::UNKNOWN_EXCEPTION}; +} + +template +static void fillArrowArrayWithNumericColumnData( + ColumnPtr write_column, std::shared_ptr & arrow_array, const PaddedPODArray * null_bytemap) +{ + const PaddedPODArray & internal_data = static_cast &>(*write_column).getData(); + ArrowBuilderType builder; + arrow::Status status; + + const UInt8 * arrow_null_bytemap_raw_ptr = nullptr; + PaddedPODArray arrow_null_bytemap; + if (null_bytemap) + { + /// Invert values since Arrow interprets 1 as a non-null value, while CH as a null + arrow_null_bytemap.reserve(null_bytemap->size()); + for (size_t i = 0, size = null_bytemap->size(); i < size; ++i) + arrow_null_bytemap.emplace_back(1 ^ (*null_bytemap)[i]); + + arrow_null_bytemap_raw_ptr = arrow_null_bytemap.data(); + } + + status = builder.AppendValues(internal_data.data(), internal_data.size(), arrow_null_bytemap_raw_ptr); + checkStatus(status, write_column->getName()); + + status = builder.Finish(&arrow_array); + checkStatus(status, write_column->getName()); +} + +template +static void fillArrowArrayWithStringColumnData( + ColumnPtr write_column, std::shared_ptr & arrow_array, const PaddedPODArray * null_bytemap) +{ + const auto & internal_column = static_cast(*write_column); + arrow::StringBuilder builder; + arrow::Status status; + + for (size_t string_i = 0, size = internal_column.size(); string_i < size; ++string_i) + { + if (null_bytemap && (*null_bytemap)[string_i]) + { + status = builder.AppendNull(); + } + else + { + StringRef string_ref = internal_column.getDataAt(string_i); + status = builder.Append(string_ref.data, string_ref.size); + } + + checkStatus(status, write_column->getName()); + } + + status = builder.Finish(&arrow_array); + checkStatus(status, write_column->getName()); +} + +static void fillArrowArrayWithDateColumnData( + ColumnPtr write_column, std::shared_ptr & arrow_array, const PaddedPODArray * null_bytemap) +{ + const PaddedPODArray & internal_data = static_cast &>(*write_column).getData(); + //arrow::Date32Builder date_builder; + arrow::UInt16Builder builder; + arrow::Status status; + + for (size_t value_i = 0, size = internal_data.size(); value_i < size; ++value_i) + { + if (null_bytemap && (*null_bytemap)[value_i]) + status = builder.AppendNull(); + else + /// Implicitly converts UInt16 to Int32 + status = builder.Append(internal_data[value_i]); + checkStatus(status, write_column->getName()); + } + + status = builder.Finish(&arrow_array); + checkStatus(status, write_column->getName()); +} + +static void fillArrowArrayWithDateTimeColumnData( + ColumnPtr write_column, std::shared_ptr & arrow_array, const PaddedPODArray * null_bytemap) +{ + auto & internal_data = static_cast &>(*write_column).getData(); + //arrow::Date64Builder builder; + arrow::UInt32Builder builder; + arrow::Status status; + + for (size_t value_i = 0, size = internal_data.size(); value_i < size; ++value_i) + { + if (null_bytemap && (*null_bytemap)[value_i]) + status = builder.AppendNull(); + else + /// Implicitly converts UInt16 to Int32 + //status = date_builder.Append(static_cast(internal_data[value_i]) * 1000); // now ms. TODO check other units + status = builder.Append(internal_data[value_i]); + + checkStatus(status, write_column->getName()); + } + + status = builder.Finish(&arrow_array); + checkStatus(status, write_column->getName()); +} + +template +static void fillArrowArrayWithDecimalColumnData( + ColumnPtr write_column, + std::shared_ptr & arrow_array, + const PaddedPODArray * null_bytemap, + const DataType * decimal_type) +{ + const auto & column = static_cast(*write_column); + arrow::DecimalBuilder builder(arrow::decimal(decimal_type->getPrecision(), decimal_type->getScale())); + arrow::Status status; + + for (size_t value_i = 0, size = column.size(); value_i < size; ++value_i) + { + if (null_bytemap && (*null_bytemap)[value_i]) + status = builder.AppendNull(); + else + status = builder.Append( + arrow::Decimal128(reinterpret_cast(&column.getElement(value_i).value))); // TODO: try copy column + + checkStatus(status, write_column->getName()); + } + status = builder.Finish(&arrow_array); + checkStatus(status, write_column->getName()); + +/* TODO column copy + const auto & internal_data = static_cast(*write_column).getData(); + //ArrowBuilderType numeric_builder; + arrow::DecimalBuilder builder(arrow::decimal(decimal_type->getPrecision(), decimal_type->getScale())); + arrow::Status status; + + const uint8_t * arrow_null_bytemap_raw_ptr = nullptr; + PaddedPODArray arrow_null_bytemap; + if (null_bytemap) + { + /// Invert values since Arrow interprets 1 as a non-null value, while CH as a null + arrow_null_bytemap.reserve(null_bytemap->size()); + for (size_t i = 0, size = null_bytemap->size(); i < size; ++i) + arrow_null_bytemap.emplace_back(1 ^ (*null_bytemap)[i]); + + arrow_null_bytemap_raw_ptr = arrow_null_bytemap.data(); + } + + status = builder.AppendValues(reinterpret_cast(internal_data.data()), internal_data.size(), arrow_null_bytemap_raw_ptr); + checkStatus(status, write_column->getName()); + + status = builder.Finish(&arrow_array); + checkStatus(status, write_column->getName()); +*/ +} + +# define FOR_INTERNAL_NUMERIC_TYPES(M) \ + M(UInt8, arrow::UInt8Builder) \ + M(Int8, arrow::Int8Builder) \ + M(UInt16, arrow::UInt16Builder) \ + M(Int16, arrow::Int16Builder) \ + M(UInt32, arrow::UInt32Builder) \ + M(Int32, arrow::Int32Builder) \ + M(UInt64, arrow::UInt64Builder) \ + M(Int64, arrow::Int64Builder) \ + M(Float32, arrow::FloatBuilder) \ + M(Float64, arrow::DoubleBuilder) + +const std::unordered_map> internal_type_to_arrow_type = { + {"UInt8", arrow::uint8()}, + {"Int8", arrow::int8()}, + {"UInt16", arrow::uint16()}, + {"Int16", arrow::int16()}, + {"UInt32", arrow::uint32()}, + {"Int32", arrow::int32()}, + {"UInt64", arrow::uint64()}, + {"Int64", arrow::int64()}, + {"Float32", arrow::float32()}, + {"Float64", arrow::float64()}, + + //{"Date", arrow::date64()}, + //{"Date", arrow::date32()}, + {"Date", arrow::uint16()}, // CHECK + //{"DateTime", arrow::date64()}, // BUG! saves as date32 + {"DateTime", arrow::uint32()}, + + // TODO: ClickHouse can actually store non-utf8 strings! + {"String", arrow::utf8()}, + {"FixedString", arrow::utf8()}, +}; + +static const PaddedPODArray * extractNullBytemapPtr(ColumnPtr column) +{ + ColumnPtr null_column = static_cast(*column).getNullMapColumnPtr(); + const PaddedPODArray & null_bytemap = static_cast &>(*null_column).getData(); + return &null_bytemap; +} + + +class OstreamOutputStream : public parquet::OutputStream +{ +public: + explicit OstreamOutputStream(WriteBuffer & ostr_) : ostr(ostr_) {} + virtual ~OstreamOutputStream() {} + virtual void Close() {} + virtual int64_t Tell() { return total_length; } + virtual void Write(const uint8_t * data, int64_t length) + { + ostr.write(reinterpret_cast(data), length); + total_length += length; + } + +private: + WriteBuffer & ostr; + int64_t total_length = 0; + + PARQUET_DISALLOW_COPY_AND_ASSIGN(OstreamOutputStream); +}; + + +void ParquetBlockOutputFormat::consume(Chunk chunk) +{ + auto & header = getPort(PortKind::Main).getHeader(); + const size_t columns_num = chunk.getNumColumns(); + + /// For arrow::Schema and arrow::Table creation + std::vector> arrow_fields; + std::vector> arrow_arrays; + arrow_fields.reserve(columns_num); + arrow_arrays.reserve(columns_num); + + for (size_t column_i = 0; column_i < columns_num; ++column_i) + { + // TODO: constructed every iteration + ColumnWithTypeAndName column = header.safeGetByPosition(column_i); + column.column = chunk.getColumns()[column_i]; + + const bool is_column_nullable = column.type->isNullable(); + const auto & column_nested_type + = is_column_nullable ? static_cast(column.type.get())->getNestedType() : column.type; + const std::string column_nested_type_name = column_nested_type->getFamilyName(); + + if (isDecimal(column_nested_type)) + { + const auto add_decimal_field = [&](const auto & types) -> bool { + using Types = std::decay_t; + using ToDataType = typename Types::LeftType; + + if constexpr ( + std::is_same_v< + ToDataType, + DataTypeDecimal< + Decimal32>> || std::is_same_v> || std::is_same_v>) + { + const auto & decimal_type = static_cast(column_nested_type.get()); + arrow_fields.emplace_back(std::make_shared( + column.name, arrow::decimal(decimal_type->getPrecision(), decimal_type->getScale()), is_column_nullable)); + } + + return false; + }; + callOnIndexAndDataType(column_nested_type->getTypeId(), add_decimal_field); + } + else + { + if (internal_type_to_arrow_type.find(column_nested_type_name) == internal_type_to_arrow_type.end()) + { + throw Exception{"The type \"" + column_nested_type_name + "\" of a column \"" + column.name + + "\"" + " is not supported for conversion into a Parquet data format", + ErrorCodes::UNKNOWN_TYPE}; + } + + arrow_fields.emplace_back(std::make_shared(column.name, internal_type_to_arrow_type.at(column_nested_type_name), is_column_nullable)); + } + + std::shared_ptr arrow_array; + + ColumnPtr nested_column + = is_column_nullable ? static_cast(*column.column).getNestedColumnPtr() : column.column; + const PaddedPODArray * null_bytemap = is_column_nullable ? extractNullBytemapPtr(column.column) : nullptr; + + if ("String" == column_nested_type_name) + { + fillArrowArrayWithStringColumnData(nested_column, arrow_array, null_bytemap); + } + else if ("FixedString" == column_nested_type_name) + { + fillArrowArrayWithStringColumnData(nested_column, arrow_array, null_bytemap); + } + else if ("Date" == column_nested_type_name) + { + fillArrowArrayWithDateColumnData(nested_column, arrow_array, null_bytemap); + } + else if ("DateTime" == column_nested_type_name) + { + fillArrowArrayWithDateTimeColumnData(nested_column, arrow_array, null_bytemap); + } + + else if (isDecimal(column_nested_type)) + { + auto fill_decimal = [&](const auto & types) -> bool + { + using Types = std::decay_t; + using ToDataType = typename Types::LeftType; + if constexpr ( + std::is_same_v< + ToDataType, + DataTypeDecimal< + Decimal32>> || std::is_same_v> || std::is_same_v>) + { + const auto & decimal_type = static_cast(column_nested_type.get()); + fillArrowArrayWithDecimalColumnData(nested_column, arrow_array, null_bytemap, decimal_type); + } + return false; + }; + callOnIndexAndDataType(column_nested_type->getTypeId(), fill_decimal); + } +# define DISPATCH(CPP_NUMERIC_TYPE, ARROW_BUILDER_TYPE) \ + else if (#CPP_NUMERIC_TYPE == column_nested_type_name) \ + { \ + fillArrowArrayWithNumericColumnData(nested_column, arrow_array, null_bytemap); \ + } + + FOR_INTERNAL_NUMERIC_TYPES(DISPATCH) +# undef DISPATCH + else + { + throw Exception{"Internal type \"" + column_nested_type_name + "\" of a column \"" + column.name + + "\"" + " is not supported for conversion into a Parquet data format", + ErrorCodes::UNKNOWN_TYPE}; + } + + + arrow_arrays.emplace_back(std::move(arrow_array)); + } + + std::shared_ptr arrow_schema = std::make_shared(std::move(arrow_fields)); + + std::shared_ptr arrow_table = arrow::Table::Make(arrow_schema, arrow_arrays); + + auto sink = std::make_shared(out); + + if (!file_writer) + { + + parquet::WriterProperties::Builder builder; +#if USE_SNAPPY + builder.compression(parquet::Compression::SNAPPY); +#endif + auto props = builder.build(); + auto status = parquet::arrow::FileWriter::Open( + *arrow_table->schema(), + arrow::default_memory_pool(), + sink, + props, /*parquet::default_writer_properties(),*/ + parquet::arrow::default_arrow_writer_properties(), + &file_writer); + if (!status.ok()) + throw Exception{"Error while opening a table: " + status.ToString(), ErrorCodes::UNKNOWN_EXCEPTION}; + } + + // TODO: calculate row_group_size depending on a number of rows and table size + auto status = file_writer->WriteTable(*arrow_table, format_settings.parquet.row_group_size); + + if (!status.ok()) + throw Exception{"Error while writing a table: " + status.ToString(), ErrorCodes::UNKNOWN_EXCEPTION}; +} + +void ParquetBlockOutputFormat::finalize() +{ + if (file_writer) + { + auto status = file_writer->Close(); + if (!status.ok()) + throw Exception{"Error while closing a table: " + status.ToString(), ErrorCodes::UNKNOWN_EXCEPTION}; + } +} + + +void registerOutputFormatProcessorParquet(FormatFactory & factory) +{ + factory.registerOutputFormatProcessor( + "Parquet", [](WriteBuffer & buf, const Block & sample, const Context & /*context*/, const FormatSettings & format_settings) + { + auto impl = std::make_shared(buf, sample, format_settings); + /// TODO + // auto res = std::make_shared(impl, impl->getHeader(), format_settings.parquet.row_group_size, 0); + // res->disableFlush(); + return impl; + }); +} + +} + + +#else + +namespace DB +{ +class FormatFactory; +void registerOutputFormatProcessorParquet(FormatFactory &) +{ +} +} + + +#endif diff --git a/dbms/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h b/dbms/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h new file mode 100644 index 00000000000..de2deba50b3 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h @@ -0,0 +1,43 @@ +#pragma once + +#include +#if USE_PARQUET +# include +# include + +namespace arrow +{ +class Array; +class DataType; +} + +namespace parquet +{ +namespace arrow +{ + class FileWriter; +} +} + +namespace DB +{ +class ParquetBlockOutputFormat : public IOutputFormat +{ +public: + ParquetBlockOutputFormat(WriteBuffer & out_, const Block & header, const FormatSettings & format_settings); + + String getName() const override { return "ParquetBlockOutputFormat"; } + void consume(Chunk) override; + void finalize() override; + + String getContentType() const override { return "application/octet-stream"; } + +private: + const FormatSettings format_settings; + + std::unique_ptr file_writer; +}; + +} + +#endif diff --git a/dbms/src/Processors/Formats/Impl/PrettyBlockOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/PrettyBlockOutputFormat.cpp new file mode 100644 index 00000000000..6868a3b2987 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/PrettyBlockOutputFormat.cpp @@ -0,0 +1,281 @@ +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; +} + + +PrettyBlockOutputFormat::PrettyBlockOutputFormat( + WriteBuffer & out_, const Block & header, const FormatSettings & format_settings_) + : IOutputFormat(header, out_), format_settings(format_settings_) +{ + struct winsize w; + if (0 == ioctl(STDOUT_FILENO, TIOCGWINSZ, &w)) + terminal_width = w.ws_col; +} + + +/// Evaluate the visible width of the values and column names. +/// Note that number of code points is just a rough approximation of visible string width. +void PrettyBlockOutputFormat::calculateWidths( + const Block & header, const Chunk & chunk, + WidthsPerColumn & widths, Widths & max_widths, Widths & name_widths) +{ + size_t num_rows = chunk.getNumRows(); + size_t num_columns = chunk.getNumColumns(); + auto & columns = chunk.getColumns(); + + widths.resize(num_columns); + max_widths.resize_fill(num_columns); + name_widths.resize(num_columns); + + /// Calculate widths of all values. + String serialized_value; + size_t prefix = 2; // Tab character adjustment + for (size_t i = 0; i < num_columns; ++i) + { + auto & elem = header.getByPosition(i); + auto & column = columns[i]; + + widths[i].resize(num_rows); + + for (size_t j = 0; j < num_rows; ++j) + { + { + WriteBufferFromString out(serialized_value); + elem.type->serializeAsText(*column, j, out, format_settings); + } + + widths[i][j] = std::min(format_settings.pretty.max_column_pad_width, + UTF8::computeWidth(reinterpret_cast(serialized_value.data()), serialized_value.size(), prefix)); + max_widths[i] = std::max(max_widths[i], widths[i][j]); + } + + /// And also calculate widths for names of columns. + { + // name string doesn't contain Tab, no need to pass `prefix` + name_widths[i] = std::min(format_settings.pretty.max_column_pad_width, + UTF8::computeWidth(reinterpret_cast(elem.name.data()), elem.name.size())); + max_widths[i] = std::max(max_widths[i], name_widths[i]); + } + prefix += max_widths[i] + 3; + } +} + + +void PrettyBlockOutputFormat::write(const Chunk & chunk, PortKind port_kind) +{ + UInt64 max_rows = format_settings.pretty.max_rows; + + if (total_rows >= max_rows) + { + total_rows += chunk.getNumRows(); + return; + } + + auto num_rows = chunk.getNumRows(); + auto num_columns = chunk.getNumColumns(); + auto & columns = chunk.getColumns(); + auto & header = getPort(port_kind).getHeader(); + + WidthsPerColumn widths; + Widths max_widths; + Widths name_widths; + calculateWidths(header, chunk, widths, max_widths, name_widths); + + /// Create separators + std::stringstream top_separator; + std::stringstream middle_names_separator; + std::stringstream middle_values_separator; + std::stringstream bottom_separator; + + top_separator << "┏"; + middle_names_separator << "┡"; + middle_values_separator << "├"; + bottom_separator << "└"; + for (size_t i = 0; i < num_columns; ++i) + { + if (i != 0) + { + top_separator << "┳"; + middle_names_separator << "╇"; + middle_values_separator << "┼"; + bottom_separator << "┴"; + } + + for (size_t j = 0; j < max_widths[i] + 2; ++j) + { + top_separator << "━"; + middle_names_separator << "━"; + middle_values_separator << "─"; + bottom_separator << "─"; + } + } + top_separator << "┓\n"; + middle_names_separator << "┩\n"; + middle_values_separator << "┤\n"; + bottom_separator << "┘\n"; + + std::string top_separator_s = top_separator.str(); + std::string middle_names_separator_s = middle_names_separator.str(); + std::string middle_values_separator_s = middle_values_separator.str(); + std::string bottom_separator_s = bottom_separator.str(); + + /// Output the block + writeString(top_separator_s, out); + + /// Names + writeCString("┃ ", out); + for (size_t i = 0; i < num_columns; ++i) + { + if (i != 0) + writeCString(" ┃ ", out); + + auto & col = header.getByPosition(i); + + if (format_settings.pretty.color) + writeCString("\033[1m", out); + + if (col.type->shouldAlignRightInPrettyFormats()) + { + for (size_t k = 0; k < max_widths[i] - name_widths[i]; ++k) + writeChar(' ', out); + + writeString(col.name, out); + } + else + { + writeString(col.name, out); + + for (size_t k = 0; k < max_widths[i] - name_widths[i]; ++k) + writeChar(' ', out); + } + + if (format_settings.pretty.color) + writeCString("\033[0m", out); + } + writeCString(" ┃\n", out); + + writeString(middle_names_separator_s, out); + + for (size_t i = 0; i < num_rows && total_rows + i < max_rows; ++i) + { + if (i != 0) + writeString(middle_values_separator_s, out); + + writeCString("│ ", out); + + for (size_t j = 0; j < num_columns; ++j) + { + if (j != 0) + writeCString(" │ ", out); + + auto & type = *header.getByPosition(j).type; + writeValueWithPadding(*columns[j], type, i, widths[j].empty() ? max_widths[j] : widths[j][i], max_widths[j]); + } + + writeCString(" │\n", out); + } + + writeString(bottom_separator_s, out); + + total_rows += num_rows; +} + + +void PrettyBlockOutputFormat::writeValueWithPadding( + const IColumn & column, const IDataType & type, size_t row_num, size_t value_width, size_t pad_to_width) +{ + auto writePadding = [&]() + { + for (size_t k = 0; k < pad_to_width - value_width; ++k) + writeChar(' ', out); + }; + + if (type.shouldAlignRightInPrettyFormats()) + { + writePadding(); + type.serializeAsText(column, row_num, out, format_settings); + } + else + { + type.serializeAsText(column, row_num, out, format_settings); + writePadding(); + } +} + + +void PrettyBlockOutputFormat::consume(Chunk chunk) +{ + write(chunk, PortKind::Main); +} + +void PrettyBlockOutputFormat::consumeTotals(Chunk chunk) +{ + total_rows = 0; + writeSuffixIfNot(); + writeCString("\nExtremes:\n", out); + write(chunk, PortKind::Totals); +} + +void PrettyBlockOutputFormat::consumeExtremes(Chunk chunk) +{ + total_rows = 0; + writeSuffixIfNot(); + writeCString("\nTotals:\n", out); + write(chunk, PortKind::Extremes); +} + + +void PrettyBlockOutputFormat::writeSuffix() +{ + if (total_rows >= format_settings.pretty.max_rows) + { + writeCString(" Showed first ", out); + writeIntText(format_settings.pretty.max_rows, out); + writeCString(".\n", out); + } +} + +void PrettyBlockOutputFormat::finalize() +{ + writeSuffixIfNot(); +} + + +void registerOutputFormatProcessorPretty(FormatFactory & factory) +{ + factory.registerOutputFormatProcessor("Pretty", []( + WriteBuffer & buf, + const Block & sample, + const Context &, + const FormatSettings & format_settings) + { + return std::make_shared(buf, sample, format_settings); + }); + + factory.registerOutputFormatProcessor("PrettyNoEscapes", []( + WriteBuffer & buf, + const Block & sample, + const Context &, + const FormatSettings & format_settings) + { + FormatSettings changed_settings = format_settings; + changed_settings.pretty.color = false; + return std::make_shared(buf, sample, changed_settings); + }); +} + +} diff --git a/dbms/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h b/dbms/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h new file mode 100644 index 00000000000..34bbbc3000c --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h @@ -0,0 +1,61 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ + +class WriteBuffer; +class Context; + + +/** Prints the result in the form of beautiful tables. + */ +class PrettyBlockOutputFormat : public IOutputFormat +{ +public: + /// no_escapes - do not use ANSI escape sequences - to display in the browser, not in the console. + PrettyBlockOutputFormat(WriteBuffer & out_, const Block & header, const FormatSettings & format_settings_); + + String getName() const override { return "PrettyBlockOutputFormat"; } + + void consume(Chunk) override; + void consumeTotals(Chunk) override; + void consumeExtremes(Chunk) override; + + void finalize() override; + +protected: + size_t total_rows = 0; + size_t terminal_width = 0; + bool suffix_written = false; + + const FormatSettings format_settings; + + using Widths = PODArray; + using WidthsPerColumn = std::vector; + + virtual void write(const Chunk & chunk, PortKind port_kind); + virtual void writeSuffix(); + + + void writeSuffixIfNot() + { + if (!suffix_written) + writeSuffix(); + + suffix_written = true; + } + + void calculateWidths( + const Block & header, const Chunk & chunk, + WidthsPerColumn & widths, Widths & max_widths, Widths & name_widths); + + void writeValueWithPadding( + const IColumn & column, const IDataType & type, size_t row_num, size_t value_width, size_t pad_to_width); +}; + +} diff --git a/dbms/src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.cpp new file mode 100644 index 00000000000..53fef75a97c --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.cpp @@ -0,0 +1,167 @@ +#include +#include +///#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + +extern const int ILLEGAL_COLUMN; + +} + +void PrettyCompactBlockOutputFormat::writeHeader( + const Block & block, + const Widths & max_widths, + const Widths & name_widths) +{ + /// Names + writeCString("┌─", out); + for (size_t i = 0; i < max_widths.size(); ++i) + { + if (i != 0) + writeCString("─┬─", out); + + const ColumnWithTypeAndName & col = block.getByPosition(i); + + if (col.type->shouldAlignRightInPrettyFormats()) + { + for (size_t k = 0; k < max_widths[i] - name_widths[i]; ++k) + writeCString("─", out); + + if (format_settings.pretty.color) + writeCString("\033[1m", out); + writeString(col.name, out); + if (format_settings.pretty.color) + writeCString("\033[0m", out); + } + else + { + if (format_settings.pretty.color) + writeCString("\033[1m", out); + writeString(col.name, out); + if (format_settings.pretty.color) + writeCString("\033[0m", out); + + for (size_t k = 0; k < max_widths[i] - name_widths[i]; ++k) + writeCString("─", out); + } + } + writeCString("─┐\n", out); +} + +void PrettyCompactBlockOutputFormat::writeBottom(const Widths & max_widths) +{ + /// Create delimiters + std::stringstream bottom_separator; + + bottom_separator << "└"; + for (size_t i = 0; i < max_widths.size(); ++i) + { + if (i != 0) + bottom_separator << "┴"; + + for (size_t j = 0; j < max_widths[i] + 2; ++j) + bottom_separator << "─"; + } + bottom_separator << "┘\n"; + + writeString(bottom_separator.str(), out); +} + +void PrettyCompactBlockOutputFormat::writeRow( + size_t row_num, + const Block & header, + const Columns & columns, + const WidthsPerColumn & widths, + const Widths & max_widths) +{ + size_t num_columns = max_widths.size(); + + writeCString("│ ", out); + + for (size_t j = 0; j < num_columns; ++j) + { + if (j != 0) + writeCString(" │ ", out); + + auto & type = *header.getByPosition(j).type; + auto & cur_widths = widths[j].empty() ? max_widths[j] : widths[j][row_num]; + writeValueWithPadding(*columns[j], type, row_num, cur_widths, max_widths[j]); + } + + writeCString(" │\n", out); +} + +void PrettyCompactBlockOutputFormat::write(const Chunk & chunk, PortKind port_kind) +{ + UInt64 max_rows = format_settings.pretty.max_rows; + + if (total_rows >= max_rows) + { + total_rows += chunk.getNumRows(); + return; + } + + size_t num_rows = chunk.getNumRows(); + auto & header = getPort(port_kind).getHeader(); + auto & columns = chunk.getColumns(); + + WidthsPerColumn widths; + Widths max_widths; + Widths name_widths; + calculateWidths(header, chunk, widths, max_widths, name_widths); + + writeHeader(header, max_widths, name_widths); + + for (size_t i = 0; i < num_rows && total_rows + i < max_rows; ++i) + writeRow(i, header, columns, widths, max_widths); + + writeBottom(max_widths); + + total_rows += num_rows; +} + + +void registerOutputFormatProcessorPrettyCompact(FormatFactory & factory) +{ + factory.registerOutputFormatProcessor("PrettyCompact", []( + WriteBuffer & buf, + const Block & sample, + const Context &, + const FormatSettings & format_settings) + { + return std::make_shared(buf, sample, format_settings); + }); + + factory.registerOutputFormatProcessor("PrettyCompactNoEscapes", []( + WriteBuffer & buf, + const Block & sample, + const Context &, + const FormatSettings & format_settings) + { + FormatSettings changed_settings = format_settings; + changed_settings.pretty.color = false; + return std::make_shared(buf, sample, changed_settings); + }); + +/// TODO +// factory.registerOutputFormat("PrettyCompactMonoBlock", []( +// WriteBuffer & buf, +// const Block & sample, +// const Context &, +// const FormatSettings & format_settings) +// { +// BlockOutputStreamPtr impl = std::make_shared(buf, sample, format_settings); +// auto res = std::make_shared(impl, impl->getHeader(), format_settings.pretty.max_rows, 0); +// res->disableFlush(); +// return res; +// }); +} + +} diff --git a/dbms/src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.h b/dbms/src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.h new file mode 100644 index 00000000000..8130577995a --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.h @@ -0,0 +1,31 @@ +#pragma once + +#include + + +namespace DB +{ + +/** Prints the result in the form of beautiful tables, but with fewer delimiter lines. + */ +class PrettyCompactBlockOutputFormat : public PrettyBlockOutputFormat +{ +public: + PrettyCompactBlockOutputFormat(WriteBuffer & out_, const Block & header, const FormatSettings & format_settings_) + : PrettyBlockOutputFormat(out_, header, format_settings_) {} + + String getName() const override { return "PrettyCompactBlockOutputFormat"; } + +protected: + void write(const Chunk & chunk, PortKind port_kind) override; + void writeHeader(const Block & block, const Widths & max_widths, const Widths & name_widths); + void writeBottom(const Widths & max_widths); + void writeRow( + size_t row_num, + const Block & header, + const Columns & columns, + const WidthsPerColumn & widths, + const Widths & max_widths); +}; + +} diff --git a/dbms/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp new file mode 100644 index 00000000000..2b75d867327 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp @@ -0,0 +1,123 @@ +#include +#include +#include +#include + + +namespace DB +{ + + +void PrettySpaceBlockOutputFormat::write(const Chunk & chunk, PortKind port_kind) +{ + UInt64 max_rows = format_settings.pretty.max_rows; + + if (total_rows >= max_rows) + { + total_rows += chunk.getNumRows(); + return; + } + + size_t num_rows = chunk.getNumRows(); + size_t num_columns = chunk.getNumColumns(); + auto & header = getPort(port_kind).getHeader(); + auto & columns = chunk.getColumns(); + + WidthsPerColumn widths; + Widths max_widths; + Widths name_widths; + calculateWidths(header, chunk, widths, max_widths, name_widths); + + /// Do not align on too long values. + if (terminal_width > 80) + for (size_t i = 0; i < num_columns; ++i) + if (max_widths[i] > terminal_width / 2) + max_widths[i] = terminal_width / 2; + + /// Names + for (size_t i = 0; i < num_columns; ++i) + { + if (i != 0) + writeCString(" ", out); + + const ColumnWithTypeAndName & col = header.getByPosition(i); + + if (col.type->shouldAlignRightInPrettyFormats()) + { + for (ssize_t k = 0; k < std::max(static_cast(0), static_cast(max_widths[i] - name_widths[i])); ++k) + writeChar(' ', out); + + if (format_settings.pretty.color) + writeCString("\033[1m", out); + writeString(col.name, out); + if (format_settings.pretty.color) + writeCString("\033[0m", out); + } + else + { + if (format_settings.pretty.color) + writeCString("\033[1m", out); + writeString(col.name, out); + if (format_settings.pretty.color) + writeCString("\033[0m", out); + + for (ssize_t k = 0; k < std::max(static_cast(0), static_cast(max_widths[i] - name_widths[i])); ++k) + writeChar(' ', out); + } + } + writeCString("\n\n", out); + + for (size_t row = 0; row < num_rows && total_rows + row < max_rows; ++row) + { + for (size_t column = 0; column < num_columns; ++column) + { + if (column != 0) + writeCString(" ", out); + + auto & type = *header.getByPosition(column).type; + auto & cur_width = widths[column].empty() ? max_widths[column] : widths[column][row]; + writeValueWithPadding(*columns[column], type, row, cur_width, max_widths[column]); + } + + writeChar('\n', out); + } + + total_rows += num_rows; +} + + +void PrettySpaceBlockOutputFormat::writeSuffix() +{ + if (total_rows >= format_settings.pretty.max_rows) + { + writeCString("\nShowed first ", out); + writeIntText(format_settings.pretty.max_rows, out); + writeCString(".\n", out); + } +} + + +void registerOutputFormatProcessorPrettySpace(FormatFactory & factory) +{ + factory.registerOutputFormatProcessor("PrettySpace", []( + WriteBuffer & buf, + const Block & sample, + const Context &, + const FormatSettings & format_settings) + { + return std::make_shared(buf, sample, format_settings); + }); + + factory.registerOutputFormatProcessor("PrettySpaceNoEscapes", []( + WriteBuffer & buf, + const Block & sample, + const Context &, + const FormatSettings & format_settings) + { + FormatSettings changed_settings = format_settings; + changed_settings.pretty.color = false; + return std::make_shared(buf, sample, changed_settings); + }); +} + +} diff --git a/dbms/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.h b/dbms/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.h new file mode 100644 index 00000000000..a041d324fd3 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.h @@ -0,0 +1,24 @@ +#pragma once + +#include + + +namespace DB +{ + +/** Prints the result, aligned with spaces. + */ +class PrettySpaceBlockOutputFormat : public PrettyBlockOutputFormat +{ +public: + PrettySpaceBlockOutputFormat(WriteBuffer & out_, const Block & header, const FormatSettings & format_settings_) + : PrettyBlockOutputFormat(out_, header, format_settings_) {} + + String getName() const override { return "PrettySpaceBlockOutputFormat"; } + +protected: + void write(const Chunk & chunk, PortKind port_kind) override; + void writeSuffix() override; +}; + +} diff --git a/dbms/src/Processors/Formats/Impl/ProtobufBlockOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/ProtobufBlockOutputFormat.cpp new file mode 100644 index 00000000000..03c8ef1b2ac --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/ProtobufBlockOutputFormat.cpp @@ -0,0 +1,73 @@ +#include + +#include "config_formats.h" +#if USE_PROTOBUF + +#include + +#include +#include +#include +#include +#include + + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; + extern const int NO_DATA_FOR_REQUIRED_PROTOBUF_FIELD; +} + + +ProtobufBlockOutputFormat::ProtobufBlockOutputFormat( + WriteBuffer & out_, + const Block & header, + const FormatSchemaInfo & format_schema) + : IOutputFormat(header, out_) + , data_types(header.getDataTypes()) + , writer(out, ProtobufSchemas::instance().getMessageTypeForFormatSchema(format_schema), header.getNames()) +{ + value_indices.resize(header.columns()); +} + +void ProtobufBlockOutputFormat::consume(Chunk chunk) +{ + auto & columns = chunk.getColumns(); + auto num_rows = chunk.getNumRows(); + + for (UInt64 row_num = 0; row_num < num_rows; ++row_num) + { + writer.startMessage(); + std::fill(value_indices.begin(), value_indices.end(), 0); + size_t column_index; + while (writer.writeField(column_index)) + data_types[column_index]->serializeProtobuf( + *columns[column_index], row_num, writer, value_indices[column_index]); + writer.endMessage(); + } +} + + +void registerOutputFormatProcessorProtobuf(FormatFactory & factory) +{ + factory.registerOutputFormatProcessor( + "Protobuf", [](WriteBuffer & buf, const Block & header, const Context & context, const FormatSettings &) + { + return std::make_shared(buf, header, FormatSchemaInfo(context, "Protobuf")); + }); +} + +} + +#else + +namespace DB +{ + class FormatFactory; + void registerOutputFormatProcessorProtobuf(FormatFactory &) {} +} + +#endif diff --git a/dbms/src/Processors/Formats/Impl/ProtobufBlockOutputFormat.h b/dbms/src/Processors/Formats/Impl/ProtobufBlockOutputFormat.h new file mode 100644 index 00000000000..9cc5c7af68e --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/ProtobufBlockOutputFormat.h @@ -0,0 +1,53 @@ +#pragma once + +#include +#if USE_PROTOBUF + +#include +#include +#include +#include +#include + + +namespace google +{ +namespace protobuf +{ + class Message; +} +} + + +namespace DB +{ +/** Stream designed to serialize data in the google protobuf format. + * Each row is written as a separated message. + * These messages are delimited according to documentation + * https://github.com/protocolbuffers/protobuf/blob/master/src/google/protobuf/util/delimited_message_util.h + * Serializing in the protobuf format requires the 'format_schema' setting to be set, e.g. + * SELECT * from table FORMAT Protobuf SETTINGS format_schema = 'schema:Message' + * where schema is the name of "schema.proto" file specifying protobuf schema. + */ +class ProtobufBlockOutputFormat : public IOutputFormat +{ +public: + ProtobufBlockOutputFormat( + WriteBuffer & out_, + const Block & header, + const FormatSchemaInfo & format_schema); + + String getName() const override { return "ProtobufBlockOutputFormat"; } + + void consume(Chunk) override; + + std::string getContentType() const override { return "application/octet-stream"; } + +private: + DataTypes data_types; + ProtobufWriter writer; + std::vector value_indices; +}; + +} +#endif diff --git a/dbms/src/Processors/Formats/Impl/ProtobufRowInputFormat.cpp b/dbms/src/Processors/Formats/Impl/ProtobufRowInputFormat.cpp new file mode 100644 index 00000000000..a2f3a47def3 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/ProtobufRowInputFormat.cpp @@ -0,0 +1,92 @@ +#include "config_formats.h" +#if USE_PROTOBUF + +#include +#include +#include +#include +#include + + +namespace DB +{ + +ProtobufRowInputFormat::ProtobufRowInputFormat(ReadBuffer & in_, const Block & header, Params params, const FormatSchemaInfo & info) + : IRowInputFormat(header, in_, params) + , data_types(header.getDataTypes()) + , reader(in, ProtobufSchemas::instance().getMessageTypeForFormatSchema(info), header.getNames()) +{ +} + +ProtobufRowInputFormat::~ProtobufRowInputFormat() = default; + +bool ProtobufRowInputFormat::readRow(MutableColumns & columns, RowReadExtension & extra) +{ + if (!reader.startMessage()) + return false; // EOF reached, no more messages. + + // Set of columns for which the values were read. The rest will be filled with default values. + auto & read_columns = extra.read_columns; + read_columns.assign(columns.size(), false); + + // Read values from this message and put them to the columns while it's possible. + size_t column_index; + while (reader.readColumnIndex(column_index)) + { + bool allow_add_row = !static_cast(read_columns[column_index]); + do + { + bool row_added; + data_types[column_index]->deserializeProtobuf(*columns[column_index], reader, allow_add_row, row_added); + if (row_added) + { + read_columns[column_index] = true; + allow_add_row = false; + } + } while (reader.maybeCanReadValue()); + } + + // Fill non-visited columns with the default values. + for (column_index = 0; column_index < read_columns.size(); ++column_index) + if (!read_columns[column_index]) + data_types[column_index]->insertDefaultInto(*columns[column_index]); + + reader.endMessage(); + return true; +} + +bool ProtobufRowInputFormat::allowSyncAfterError() const +{ + return true; +} + +void ProtobufRowInputFormat::syncAfterError() +{ + reader.endMessage(); +} + + +void registerInputFormatProcessorProtobuf(FormatFactory & factory) +{ + factory.registerInputFormatProcessor("Protobuf", []( + ReadBuffer & buf, + const Block & sample, + const Context & context, + IRowInputFormat::Params params, + const FormatSettings &) + { + return std::make_shared(buf, sample, params, FormatSchemaInfo(context, "proto")); + }); +} + +} + +#else + +namespace DB +{ +class FormatFactory; +void registerInputFormatProcessorProtobuf(FormatFactory &) {} +} + +#endif diff --git a/dbms/src/Processors/Formats/Impl/ProtobufRowInputFormat.h b/dbms/src/Processors/Formats/Impl/ProtobufRowInputFormat.h new file mode 100644 index 00000000000..77c6350d371 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/ProtobufRowInputFormat.h @@ -0,0 +1,36 @@ +#pragma once + +#include +#if USE_PROTOBUF + +#include +#include +#include + +namespace DB +{ +class Block; +class FormatSchemaInfo; + + +/** Interface of stream, that allows to read data by rows. + */ +class ProtobufRowInputFormat : public IRowInputFormat +{ +public: + ProtobufRowInputFormat(ReadBuffer & in_, const Block & header, Params params, const FormatSchemaInfo & info); + ~ProtobufRowInputFormat() override; + + String getName() const override { return "ProtobufRowInputFormat"; } + + bool readRow(MutableColumns & columns, RowReadExtension & extra) override; + bool allowSyncAfterError() const override; + void syncAfterError() override; + +private: + DataTypes data_types; + ProtobufReader reader; +}; + +} +#endif diff --git a/dbms/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp b/dbms/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp new file mode 100644 index 00000000000..7750962beec --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp @@ -0,0 +1,207 @@ +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int INCORRECT_DATA; + extern const int CANNOT_PARSE_ESCAPE_SEQUENCE; + extern const int CANNOT_READ_ALL_DATA; + extern const int CANNOT_PARSE_INPUT_ASSERTION_FAILED; +} + + +TSKVRowInputFormat::TSKVRowInputFormat(ReadBuffer & in_, Block header, Params params, const FormatSettings & format_settings) + : IRowInputFormat(std::move(header), in_, params), format_settings(format_settings), name_map(header.columns()) +{ + /// In this format, we assume that column name cannot contain BOM, + /// so BOM at beginning of stream cannot be confused with name of field, and it is safe to skip it. + skipBOMIfExists(in); + + size_t num_columns = header.columns(); + for (size_t i = 0; i < num_columns; ++i) + name_map[header.safeGetByPosition(i).name] = i; /// NOTE You could place names more cache-locally. +} + + +/** Read the field name in the `tskv` format. + * Return true if the field is followed by an equal sign, + * otherwise (field with no value) return false. + * The reference to the field name will be written to `ref`. + * A temporary `tmp` buffer can also be used to copy the field name to it. + * When reading, skips the name and the equal sign after it. + */ +static bool readName(ReadBuffer & buf, StringRef & ref, String & tmp) +{ + tmp.clear(); + + while (!buf.eof()) + { + const char * next_pos = find_first_symbols<'\t', '\n', '\\', '='>(buf.position(), buf.buffer().end()); + + if (next_pos == buf.buffer().end()) + { + tmp.append(buf.position(), next_pos - buf.position()); + buf.next(); + continue; + } + + /// Came to the end of the name. + if (*next_pos != '\\') + { + bool have_value = *next_pos == '='; + if (tmp.empty()) + { + /// No need to copy data, you can refer directly to the `buf`. + ref = StringRef(buf.position(), next_pos - buf.position()); + buf.position() += next_pos + have_value - buf.position(); + } + else + { + /// Copy the data to a temporary string and return a reference to it. + tmp.append(buf.position(), next_pos - buf.position()); + buf.position() += next_pos + have_value - buf.position(); + ref = StringRef(tmp); + } + return have_value; + } + /// The name has an escape sequence. + else + { + tmp.append(buf.position(), next_pos - buf.position()); + buf.position() += next_pos + 1 - buf.position(); + if (buf.eof()) + throw Exception("Cannot parse escape sequence", ErrorCodes::CANNOT_PARSE_ESCAPE_SEQUENCE); + + tmp.push_back(parseEscapeSequence(*buf.position())); + ++buf.position(); + continue; + } + } + + throw Exception("Unexpected end of stream while reading key name from TSKV format", ErrorCodes::CANNOT_READ_ALL_DATA); +} + + +bool TSKVRowInputFormat::readRow(MutableColumns & columns, RowReadExtension & ext) +{ + if (in.eof()) + return false; + + auto & header = getPort().getHeader(); + size_t num_columns = columns.size(); + + /// Set of columns for which the values were read. The rest will be filled with default values. + read_columns.assign(num_columns, false); + + if (unlikely(*in.position() == '\n')) + { + /// An empty string. It is permissible, but it is unclear why. + ++in.position(); + } + else + { + while (true) + { + StringRef name_ref; + bool has_value = readName(in, name_ref, name_buf); + ssize_t index = -1; + + if (has_value) + { + /// NOTE Optimization is possible by caching the order of fields (which is almost always the same) + /// and quickly checking for the next expected field, instead of searching the hash table. + + auto it = name_map.find(name_ref); + if (name_map.end() == it) + { + if (!format_settings.skip_unknown_fields) + throw Exception("Unknown field found while parsing TSKV format: " + name_ref.toString(), ErrorCodes::INCORRECT_DATA); + + /// If the key is not found, skip the value. + NullSink sink; + readEscapedStringInto(sink, in); + } + else + { + index = it->getSecond(); + + if (read_columns[index]) + throw Exception("Duplicate field found while parsing TSKV format: " + name_ref.toString(), ErrorCodes::INCORRECT_DATA); + + read_columns[index] = true; + + header.getByPosition(index).type->deserializeAsTextEscaped(*columns[index], in, format_settings); + } + } + else + { + /// The only thing that can go without value is `tskv` fragment that is ignored. + if (!(name_ref.size == 4 && 0 == memcmp(name_ref.data, "tskv", 4))) + throw Exception("Found field without value while parsing TSKV format: " + name_ref.toString(), ErrorCodes::INCORRECT_DATA); + } + + if (in.eof()) + { + throw Exception("Unexpected end of stream after field in TSKV format: " + name_ref.toString(), ErrorCodes::CANNOT_READ_ALL_DATA); + } + else if (*in.position() == '\t') + { + ++in.position(); + continue; + } + else if (*in.position() == '\n') + { + ++in.position(); + break; + } + else + { + /// Possibly a garbage was written into column, remove it + if (index >= 0) + { + columns[index]->popBack(1); + read_columns[index] = false; + } + + throw Exception("Found garbage after field in TSKV format: " + name_ref.toString(), ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED); + } + } + } + + /// Fill in the not met columns with default values. + for (size_t i = 0; i < num_columns; ++i) + if (!read_columns[i]) + header.getByPosition(i).type->insertDefaultInto(*columns[i]); + + /// return info about defaults set + ext.read_columns = read_columns; + + return true; +} + + +void TSKVRowInputFormat::syncAfterError() +{ + skipToUnescapedNextLineOrEOF(in); +} + + +void registerInputFormatProcessorTSKV(FormatFactory & factory) +{ + factory.registerInputFormatProcessor("TSKV", []( + ReadBuffer & buf, + const Block & sample, + const Context &, + IRowInputFormat::Params params, + const FormatSettings & settings) + { + return std::make_shared(buf, sample, params, settings); + }); +} + +} diff --git a/dbms/src/Processors/Formats/Impl/TSKVRowInputFormat.h b/dbms/src/Processors/Formats/Impl/TSKVRowInputFormat.h new file mode 100644 index 00000000000..4d9c55f6efc --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/TSKVRowInputFormat.h @@ -0,0 +1,47 @@ +#pragma once + +#include +#include +#include +#include + + +namespace DB +{ + +class ReadBuffer; + + +/** Stream for reading data in TSKV format. + * TSKV is a very inefficient data format. + * Similar to TSV, but each field is written as key=value. + * Fields can be listed in any order (including, in different lines there may be different order), + * and some fields may be missing. + * An equal sign can be escaped in the field name. + * Also, as an additional element there may be a useless tskv fragment - it needs to be ignored. + */ +class TSKVRowInputFormat : public IRowInputFormat +{ +public: + TSKVRowInputFormat(ReadBuffer & in_, Block header, Params params, const FormatSettings & format_settings); + + String getName() const override { return "TSKVRowInputFormat"; } + + bool readRow(MutableColumns & columns, RowReadExtension &) override; + bool allowSyncAfterError() const override { return true; } + void syncAfterError() override; + +private: + const FormatSettings format_settings; + + /// Buffer for the read from the stream the field name. Used when you have to copy it. + String name_buf; + + /// Hash table matching `field name -> position in the block`. NOTE You can use perfect hash map. + using NameMap = HashMap; + NameMap name_map; + + std::vector read_columns; +}; + +} diff --git a/dbms/src/Processors/Formats/Impl/TSKVRowOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/TSKVRowOutputFormat.cpp new file mode 100644 index 00000000000..f1c13af9c5d --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/TSKVRowOutputFormat.cpp @@ -0,0 +1,55 @@ +#include +#include +#include +#include + + + +namespace DB +{ + +TSKVRowOutputFormat::TSKVRowOutputFormat(WriteBuffer & out_, const Block & header, const FormatSettings & format_settings_) + : TabSeparatedRowOutputFormat(out_, header, false, false, format_settings_) +{ + auto & sample = getPort(PortKind::Main).getHeader(); + NamesAndTypesList columns(sample.getNamesAndTypesList()); + fields.assign(columns.begin(), columns.end()); + + for (auto & field : fields) + { + WriteBufferFromOwnString wb; + writeAnyEscapedString<'='>(field.name.data(), field.name.data() + field.name.size(), wb); + writeCString("=", wb); + field.name = wb.str(); + } +} + + +void TSKVRowOutputFormat::writeField(const IColumn & column, const IDataType & type, size_t row_num) +{ + writeString(fields[field_number].name, out); + type.serializeAsTextEscaped(column, row_num, out, format_settings); + ++field_number; +} + + +void TSKVRowOutputFormat::writeRowEndDelimiter() +{ + writeChar('\n', out); + field_number = 0; +} + + +void registerOutputFormatProcessorTSKV(FormatFactory & factory) +{ + factory.registerOutputFormatProcessor("TSKV", []( + WriteBuffer & buf, + const Block & sample, + const Context &, + const FormatSettings & settings) + { + return std::make_shared(buf, sample, settings); + }); +} + +} diff --git a/dbms/src/Processors/Formats/Impl/TSKVRowOutputFormat.h b/dbms/src/Processors/Formats/Impl/TSKVRowOutputFormat.h new file mode 100644 index 00000000000..607fa5fdc9b --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/TSKVRowOutputFormat.h @@ -0,0 +1,30 @@ +#pragma once + +#include +#include + + +namespace DB +{ + +/** The stream for outputting data in the TSKV format. + * TSKV is similar to TabSeparated, but before every value, its name and equal sign are specified: name=value. + * This format is very inefficient. + */ +class TSKVRowOutputFormat: public TabSeparatedRowOutputFormat +{ +public: + TSKVRowOutputFormat(WriteBuffer & out_, const Block & header, const FormatSettings & format_settings); + + String getName() const override { return "TSKVRowOutputFormat"; } + + void writeField(const IColumn & column, const IDataType & type, size_t row_num) override; + void writeRowEndDelimiter() override; + +protected: + NamesAndTypes fields; + size_t field_number = 0; +}; + +} + diff --git a/dbms/src/Processors/Formats/Impl/TabSeparatedRawRowOutputFormat.h b/dbms/src/Processors/Formats/Impl/TabSeparatedRawRowOutputFormat.h new file mode 100644 index 00000000000..2d2d7cf4ad4 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/TabSeparatedRawRowOutputFormat.h @@ -0,0 +1,28 @@ +#pragma once + +#include +#include + + +namespace DB +{ + +/** A stream for outputting data in tsv format, but without escaping individual values. + * (That is, the output is irreversible.) + */ +class TabSeparatedRawRowOutputFormat : public TabSeparatedRowOutputFormat +{ +public: + TabSeparatedRawRowOutputFormat(WriteBuffer & out_, const Block & header, bool with_names_, bool with_types_, const FormatSettings & format_settings_) + : TabSeparatedRowOutputFormat(out_, header, with_names_, with_types_, format_settings_) {} + + String getName() const override { return "TabSeparatedRawRowOutputFormat"; } + + void writeField(const IColumn & column, const IDataType & type, size_t row_num) override + { + type.serializeAsText(column, row_num, out, format_settings); + } +}; + +} + diff --git a/dbms/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp b/dbms/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp new file mode 100644 index 00000000000..ab7562a0036 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp @@ -0,0 +1,370 @@ +#include +#include +#include + +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int INCORRECT_DATA; + extern const int LOGICAL_ERROR; +} + + +TabSeparatedRowInputFormat::TabSeparatedRowInputFormat( + ReadBuffer & in_, Block header, bool with_names, bool with_types, Params params, const FormatSettings & format_settings) + : IRowInputFormat(std::move(header), in_, params), with_names(with_names), with_types(with_types), format_settings(format_settings) +{ + auto & sample = getPort().getHeader(); + size_t num_columns = sample.columns(); + data_types.resize(num_columns); + for (size_t i = 0; i < num_columns; ++i) + data_types[i] = sample.safeGetByPosition(i).type; +} + + +void TabSeparatedRowInputFormat::readPrefix() +{ + auto & header = getPort().getHeader(); + size_t num_columns = header.columns(); + String tmp; + + if (with_names || with_types) + { + /// In this format, we assume that column name or type cannot contain BOM, + /// so, if format has header, + /// then BOM at beginning of stream cannot be confused with name or type of field, and it is safe to skip it. + skipBOMIfExists(in); + } + + if (with_names) + { + for (size_t i = 0; i < num_columns; ++i) + { + readEscapedString(tmp, in); + assertChar(i == num_columns - 1 ? '\n' : '\t', in); + } + } + + if (with_types) + { + for (size_t i = 0; i < num_columns; ++i) + { + readEscapedString(tmp, in); + assertChar(i == num_columns - 1 ? '\n' : '\t', in); + } + } +} + + +/** Check for a common error case - usage of Windows line feed. + */ +static void checkForCarriageReturn(ReadBuffer & in) +{ + if (in.position()[0] == '\r' || (in.position() != in.buffer().begin() && in.position()[-1] == '\r')) + throw Exception("\nYou have carriage return (\\r, 0x0D, ASCII 13) at end of first row." + "\nIt's like your input data has DOS/Windows style line separators, that are illegal in TabSeparated format." + " You must transform your file to Unix format." + "\nBut if you really need carriage return at end of string value of last column, you need to escape it as \\r.", + ErrorCodes::INCORRECT_DATA); +} + + +bool TabSeparatedRowInputFormat::readRow(MutableColumns & columns, RowReadExtension &) +{ + if (in.eof()) + return false; + + updateDiagnosticInfo(); + + size_t size = data_types.size(); + + for (size_t i = 0; i < size; ++i) + { + data_types[i]->deserializeAsTextEscaped(*columns[i], in, format_settings); + + /// skip separators + if (i + 1 == size) + { + if (!in.eof()) + { + if (unlikely(row_num == 1)) + checkForCarriageReturn(in); + + assertChar('\n', in); + } + } + else + assertChar('\t', in); + } + + return true; +} + + +String TabSeparatedRowInputFormat::getDiagnosticInfo() +{ + if (in.eof()) /// Buffer has gone, cannot extract information about what has been parsed. + return {}; + + auto & header = getPort().getHeader(); + WriteBufferFromOwnString out; + MutableColumns columns = header.cloneEmptyColumns(); + + /// It is possible to display detailed diagnostics only if the last and next to last lines are still in the read buffer. + size_t bytes_read_at_start_of_buffer = in.count() - in.offset(); + if (bytes_read_at_start_of_buffer != bytes_read_at_start_of_buffer_on_prev_row) + { + out << "Could not print diagnostic info because two last rows aren't in buffer (rare case)\n"; + return out.str(); + } + + size_t max_length_of_column_name = 0; + for (size_t i = 0; i < header.columns(); ++i) + if (header.safeGetByPosition(i).name.size() > max_length_of_column_name) + max_length_of_column_name = header.safeGetByPosition(i).name.size(); + + size_t max_length_of_data_type_name = 0; + for (size_t i = 0; i < header.columns(); ++i) + if (header.safeGetByPosition(i).type->getName().size() > max_length_of_data_type_name) + max_length_of_data_type_name = header.safeGetByPosition(i).type->getName().size(); + + /// Roll back the cursor to the beginning of the previous or current line and pars all over again. But now we derive detailed information. + + if (pos_of_prev_row) + { + in.position() = pos_of_prev_row; + + out << "\nRow " << (row_num - 1) << ":\n"; + if (!parseRowAndPrintDiagnosticInfo(columns, out, max_length_of_column_name, max_length_of_data_type_name)) + return out.str(); + } + else + { + if (!pos_of_current_row) + { + out << "Could not print diagnostic info because parsing of data hasn't started.\n"; + return out.str(); + } + + in.position() = pos_of_current_row; + } + + out << "\nRow " << row_num << ":\n"; + parseRowAndPrintDiagnosticInfo(columns, out, max_length_of_column_name, max_length_of_data_type_name); + out << "\n"; + + return out.str(); +} + + +bool TabSeparatedRowInputFormat::parseRowAndPrintDiagnosticInfo(MutableColumns & columns, + WriteBuffer & out, size_t max_length_of_column_name, size_t max_length_of_data_type_name) +{ + auto & header = getPort().getHeader(); + size_t size = data_types.size(); + for (size_t i = 0; i < size; ++i) + { + if (i == 0 && in.eof()) + { + out << "\n"; + return false; + } + + out << "Column " << i << ", " << std::string((i < 10 ? 2 : i < 100 ? 1 : 0), ' ') + << "name: " << header.safeGetByPosition(i).name << ", " << std::string(max_length_of_column_name - header.safeGetByPosition(i).name.size(), ' ') + << "type: " << data_types[i]->getName() << ", " << std::string(max_length_of_data_type_name - data_types[i]->getName().size(), ' '); + + auto prev_position = in.position(); + std::exception_ptr exception; + + try + { + data_types[i]->deserializeAsTextEscaped(*columns[i], in, format_settings); + } + catch (...) + { + exception = std::current_exception(); + } + + auto curr_position = in.position(); + + if (curr_position < prev_position) + throw Exception("Logical error: parsing is non-deterministic.", ErrorCodes::LOGICAL_ERROR); + + if (isNumber(data_types[i]) || isDateOrDateTime(data_types[i])) + { + /// An empty string instead of a value. + if (curr_position == prev_position) + { + out << "ERROR: text "; + verbosePrintString(prev_position, std::min(prev_position + 10, in.buffer().end()), out); + out << " is not like " << data_types[i]->getName() << "\n"; + return false; + } + } + + out << "parsed text: "; + verbosePrintString(prev_position, curr_position, out); + + if (exception) + { + if (data_types[i]->getName() == "DateTime") + out << "ERROR: DateTime must be in YYYY-MM-DD hh:mm:ss or NNNNNNNNNN (unix timestamp, exactly 10 digits) format.\n"; + else if (data_types[i]->getName() == "Date") + out << "ERROR: Date must be in YYYY-MM-DD format.\n"; + else + out << "ERROR\n"; + return false; + } + + out << "\n"; + + if (data_types[i]->haveMaximumSizeOfValue()) + { + if (*curr_position != '\n' && *curr_position != '\t') + { + out << "ERROR: garbage after " << data_types[i]->getName() << ": "; + verbosePrintString(curr_position, std::min(curr_position + 10, in.buffer().end()), out); + out << "\n"; + + if (data_types[i]->getName() == "DateTime") + out << "ERROR: DateTime must be in YYYY-MM-DD hh:mm:ss or NNNNNNNNNN (unix timestamp, exactly 10 digits) format.\n"; + else if (data_types[i]->getName() == "Date") + out << "ERROR: Date must be in YYYY-MM-DD format.\n"; + + return false; + } + } + + /// Delimiters + if (i + 1 == size) + { + if (!in.eof()) + { + try + { + assertChar('\n', in); + } + catch (const DB::Exception &) + { + if (*in.position() == '\t') + { + out << "ERROR: Tab found where line feed is expected." + " It's like your file has more columns than expected.\n" + "And if your file have right number of columns, maybe it have unescaped tab in value.\n"; + } + else if (*in.position() == '\r') + { + out << "ERROR: Carriage return found where line feed is expected." + " It's like your file has DOS/Windows style line separators, that is illegal in TabSeparated format.\n"; + } + else + { + out << "ERROR: There is no line feed. "; + verbosePrintString(in.position(), in.position() + 1, out); + out << " found instead.\n"; + } + return false; + } + } + } + else + { + try + { + assertChar('\t', in); + } + catch (const DB::Exception &) + { + if (*in.position() == '\n') + { + out << "ERROR: Line feed found where tab is expected." + " It's like your file has less columns than expected.\n" + "And if your file have right number of columns, maybe it have unescaped backslash in value before tab, which cause tab has escaped.\n"; + } + else if (*in.position() == '\r') + { + out << "ERROR: Carriage return found where tab is expected.\n"; + } + else + { + out << "ERROR: There is no tab. "; + verbosePrintString(in.position(), in.position() + 1, out); + out << " found instead.\n"; + } + return false; + } + } + } + + return true; +} + + +void TabSeparatedRowInputFormat::syncAfterError() +{ + skipToUnescapedNextLineOrEOF(in); +} + + +void TabSeparatedRowInputFormat::updateDiagnosticInfo() +{ + ++row_num; + + bytes_read_at_start_of_buffer_on_prev_row = bytes_read_at_start_of_buffer_on_current_row; + bytes_read_at_start_of_buffer_on_current_row = in.count() - in.offset(); + + pos_of_prev_row = pos_of_current_row; + pos_of_current_row = in.position(); +} + + +void registerInputFormatProcessorTabSeparated(FormatFactory & factory) +{ + for (auto name : {"TabSeparated", "TSV"}) + { + factory.registerInputFormatProcessor(name, []( + ReadBuffer & buf, + const Block & sample, + const Context &, + IRowInputFormat::Params params, + const FormatSettings & settings) + { + return std::make_shared(buf, sample, false, false, params, settings); + }); + } + + for (auto name : {"TabSeparatedWithNames", "TSVWithNames"}) + { + factory.registerInputFormatProcessor(name, []( + ReadBuffer & buf, + const Block & sample, + const Context &, + IRowInputFormat::Params params, + const FormatSettings & settings) + { + return std::make_shared(buf, sample, true, false, params, settings); + }); + } + + for (auto name : {"TabSeparatedWithNamesAndTypes", "TSVWithNamesAndTypes"}) + { + factory.registerInputFormatProcessor(name, []( + ReadBuffer & buf, + const Block & sample, + const Context &, + IRowInputFormat::Params params, + const FormatSettings & settings) + { + return std::make_shared(buf, sample, true, true, params, settings); + }); + } +} + +} diff --git a/dbms/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.h b/dbms/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.h new file mode 100644 index 00000000000..b91884d9db5 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.h @@ -0,0 +1,57 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ + +class ReadBuffer; + + +/** A stream to input data in tsv format. + */ +class TabSeparatedRowInputFormat : public IRowInputFormat +{ +public: + /** with_names - the first line is the header with the names of the columns + * with_types - on the next line header with type names + */ + TabSeparatedRowInputFormat( + ReadBuffer & in_, Block header, bool with_names, bool with_types, Params params, const FormatSettings & format_settings); + + String getName() const override { return "TabSeparatedRowInputFormat"; } + + bool readRow(MutableColumns & columns, RowReadExtension &) override; + void readPrefix() override; + bool allowSyncAfterError() const override { return true; } + void syncAfterError() override; + + std::string getDiagnosticInfo() override; + +private: + bool with_names; + bool with_types; + const FormatSettings format_settings; + DataTypes data_types; + + /// For convenient diagnostics in case of an error. + + size_t row_num = 0; + + /// How many bytes were read, not counting those still in the buffer. + size_t bytes_read_at_start_of_buffer_on_current_row = 0; + size_t bytes_read_at_start_of_buffer_on_prev_row = 0; + + char * pos_of_current_row = nullptr; + char * pos_of_prev_row = nullptr; + + void updateDiagnosticInfo(); + + bool parseRowAndPrintDiagnosticInfo(MutableColumns & columns, + WriteBuffer & out, size_t max_length_of_column_name, size_t max_length_of_data_type_name); +}; + +} diff --git a/dbms/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.cpp new file mode 100644 index 00000000000..608f2e8b5d0 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.cpp @@ -0,0 +1,121 @@ +#include +#include +#include +#include + + +namespace DB +{ + +TabSeparatedRowOutputFormat::TabSeparatedRowOutputFormat( + WriteBuffer & out_, const Block & header, bool with_names, bool with_types, const FormatSettings & format_settings) + : IRowOutputFormat(header, out_), with_names(with_names), with_types(with_types), format_settings(format_settings) +{ +} + + +void TabSeparatedRowOutputFormat::writePrefix() +{ + auto & header = getPort(PortKind::Main).getHeader(); + size_t columns = header.columns(); + + if (with_names) + { + for (size_t i = 0; i < columns; ++i) + { + writeEscapedString(header.safeGetByPosition(i).name, out); + writeChar(i == columns - 1 ? '\n' : '\t', out); + } + } + + if (with_types) + { + for (size_t i = 0; i < columns; ++i) + { + writeEscapedString(header.safeGetByPosition(i).type->getName(), out); + writeChar(i == columns - 1 ? '\n' : '\t', out); + } + } +} + + +void TabSeparatedRowOutputFormat::writeField(const IColumn & column, const IDataType & type, size_t row_num) +{ + type.serializeAsTextEscaped(column, row_num, out, format_settings); +} + + +void TabSeparatedRowOutputFormat::writeFieldDelimiter() +{ + writeChar('\t', out); +} + + +void TabSeparatedRowOutputFormat::writeRowEndDelimiter() +{ + writeChar('\n', out); +} + +void TabSeparatedRowOutputFormat::writeBeforeTotals() +{ + writeChar('\n', out); +} + +void TabSeparatedRowOutputFormat::writeBeforeExtremes() +{ + writeChar('\n', out); +} + + +void registerOutputFormatProcessorTabSeparated(FormatFactory & factory) +{ + for (auto name : {"TabSeparated", "TSV"}) + { + factory.registerOutputFormatProcessor(name, []( + WriteBuffer & buf, + const Block & sample, + const Context &, + const FormatSettings & settings) + { + return std::make_shared(buf, sample, false, false, settings); + }); + } + + for (auto name : {"TabSeparatedRaw", "TSVRaw"}) + { + factory.registerOutputFormatProcessor(name, []( + WriteBuffer & buf, + const Block & sample, + const Context &, + const FormatSettings & settings) + { + return std::make_shared(buf, sample, false, false, settings); + }); + } + + for (auto name : {"TabSeparatedWithNames", "TSVWithNames"}) + { + factory.registerOutputFormatProcessor(name, []( + WriteBuffer & buf, + const Block & sample, + const Context &, + const FormatSettings & settings) + { + return std::make_shared(buf, sample, true, false, settings); + }); + } + + for (auto name : {"TabSeparatedWithNamesAndTypes", "TSVWithNamesAndTypes"}) + { + factory.registerOutputFormatProcessor(name, []( + WriteBuffer & buf, + const Block & sample, + const Context &, + const FormatSettings & settings) + { + return std::make_shared(buf, sample, true, true, settings); + }); + } +} + +} diff --git a/dbms/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.h b/dbms/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.h new file mode 100644 index 00000000000..7ebe12bc30d --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.h @@ -0,0 +1,43 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ + +class WriteBuffer; + +/** A stream for outputting data in tsv format. + */ +class TabSeparatedRowOutputFormat : public IRowOutputFormat +{ +public: + /** with_names - output in the first line a header with column names + * with_types - output the next line header with the names of the types + */ + TabSeparatedRowOutputFormat(WriteBuffer & out_, const Block & header, bool with_names, bool with_types, const FormatSettings & format_settings); + + String getName() const override { return "TabSeparatedRowOutputFormat"; } + + void writeField(const IColumn & column, const IDataType & type, size_t row_num) override; + void writeFieldDelimiter() override; + void writeRowEndDelimiter() override; + void writePrefix() override; + void writeBeforeTotals() override; + void writeBeforeExtremes() override; + + /// https://www.iana.org/assignments/media-types/text/tab-separated-values + String getContentType() const override { return "text/tab-separated-values; charset=UTF-8"; } + +protected: + + bool with_names; + bool with_types; + const FormatSettings format_settings; +}; + +} + diff --git a/dbms/src/Processors/Formats/Impl/ValuesRowInputFormat.cpp b/dbms/src/Processors/Formats/Impl/ValuesRowInputFormat.cpp new file mode 100644 index 00000000000..7b0287596ad --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/ValuesRowInputFormat.cpp @@ -0,0 +1,166 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int CANNOT_PARSE_INPUT_ASSERTION_FAILED; + extern const int CANNOT_PARSE_QUOTED_STRING; + extern const int CANNOT_PARSE_NUMBER; + extern const int CANNOT_PARSE_DATE; + extern const int CANNOT_PARSE_DATETIME; + extern const int CANNOT_READ_ARRAY_FROM_TEXT; + extern const int CANNOT_PARSE_DATE; + extern const int SYNTAX_ERROR; + extern const int VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE; +} + + +ValuesRowInputFormat::ValuesRowInputFormat( + ReadBuffer & in_, Block header, Params params, const Context & context_, const FormatSettings & format_settings) + : IRowInputFormat(std::move(header), in_, params) + , context(std::make_unique(context_)), format_settings(format_settings) +{ + /// In this format, BOM at beginning of stream cannot be confused with value, so it is safe to skip it. + skipBOMIfExists(in); +} + + +bool ValuesRowInputFormat::readRow(MutableColumns & columns, RowReadExtension &) +{ + size_t num_columns = columns.size(); + auto & header = getPort().getHeader(); + + skipWhitespaceIfAny(in); + + if (in.eof() || *in.position() == ';') + return false; + + /** Typically, this is the usual format for streaming parsing. + * But as an exception, it also supports processing arbitrary expressions instead of values. + * This is very inefficient. But if there are no expressions, then there is no overhead. + */ + ParserExpression parser; + + assertChar('(', in); + + for (size_t i = 0; i < num_columns; ++i) + { + skipWhitespaceIfAny(in); + + char * prev_in_position = in.position(); + size_t prev_in_bytes = in.count() - in.offset(); + + bool rollback_on_exception = false; + try + { + header.getByPosition(i).type->deserializeAsTextQuoted(*columns[i], in, format_settings); + rollback_on_exception = true; + skipWhitespaceIfAny(in); + + if (i != num_columns - 1) + assertChar(',', in); + else + assertChar(')', in); + } + catch (const Exception & e) + { + if (!format_settings.values.interpret_expressions) + throw; + + /** The normal streaming parser could not parse the value. + * Let's try to parse it with a SQL parser as a constant expression. + * This is an exceptional case. + */ + if (e.code() == ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED + || e.code() == ErrorCodes::CANNOT_PARSE_QUOTED_STRING + || e.code() == ErrorCodes::CANNOT_PARSE_NUMBER + || e.code() == ErrorCodes::CANNOT_PARSE_DATE + || e.code() == ErrorCodes::CANNOT_PARSE_DATETIME + || e.code() == ErrorCodes::CANNOT_READ_ARRAY_FROM_TEXT) + { + /// TODO Case when the expression does not fit entirely in the buffer. + + /// If the beginning of the value is no longer in the buffer. + if (in.count() - in.offset() != prev_in_bytes) + throw; + + if (rollback_on_exception) + columns[i]->popBack(1); + + const IDataType & type = *header.getByPosition(i).type; + + Expected expected; + + Tokens tokens(prev_in_position, in.buffer().end()); + TokenIterator token_iterator(tokens); + + ASTPtr ast; + if (!parser.parse(token_iterator, ast, expected)) + throw Exception("Cannot parse expression of type " + type.getName() + " here: " + + String(prev_in_position, std::min(SHOW_CHARS_ON_SYNTAX_ERROR, in.buffer().end() - prev_in_position)), + ErrorCodes::SYNTAX_ERROR); + + in.position() = const_cast(token_iterator->begin); + + std::pair value_raw = evaluateConstantExpression(ast, *context); + Field value = convertFieldToType(value_raw.first, type, value_raw.second.get()); + + /// Check that we are indeed allowed to insert a NULL. + if (value.isNull()) + { + if (!type.isNullable()) + throw Exception{"Expression returns value " + applyVisitor(FieldVisitorToString(), value) + + ", that is out of range of type " + type.getName() + + ", at: " + String(prev_in_position, std::min(SHOW_CHARS_ON_SYNTAX_ERROR, in.buffer().end() - prev_in_position)), + ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE}; + } + + columns[i]->insert(value); + + skipWhitespaceIfAny(in); + + if (i != num_columns - 1) + assertChar(',', in); + else + assertChar(')', in); + } + else + throw; + } + } + + skipWhitespaceIfAny(in); + if (!in.eof() && *in.position() == ',') + ++in.position(); + + return true; +} + + +void registerInputFormatProcessorValues(FormatFactory & factory) +{ + factory.registerInputFormatProcessor("Values", []( + ReadBuffer & buf, + const Block & sample, + const Context & context, + IRowInputFormat::Params params, + const FormatSettings & settings) + { + return std::make_shared(buf, sample, params, context, settings); + }); +} + +} diff --git a/dbms/src/Processors/Formats/Impl/ValuesRowInputFormat.h b/dbms/src/Processors/Formats/Impl/ValuesRowInputFormat.h new file mode 100644 index 00000000000..f7ad3b470e6 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/ValuesRowInputFormat.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ + +class Context; +class ReadBuffer; + + +/** Stream to read data in VALUES format (as in INSERT query). + */ +class ValuesRowInputFormat : public IRowInputFormat +{ +public: + /** Data is parsed using fast, streaming parser. + * If interpret_expressions is true, it will, in addition, try to use SQL parser and interpreter + * in case when streaming parser could not parse field (this is very slow). + */ + ValuesRowInputFormat(ReadBuffer & in_, Block header, Params params, const Context & context_, const FormatSettings & format_settings); + + String getName() const override { return "ValuesRowInputFormat"; } + + bool readRow(MutableColumns & columns, RowReadExtension &) override; + +private: + std::unique_ptr context; /// pimpl + const FormatSettings format_settings; +}; + +} diff --git a/dbms/src/Processors/Formats/Impl/ValuesRowOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/ValuesRowOutputFormat.cpp new file mode 100644 index 00000000000..234a9da5c67 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/ValuesRowOutputFormat.cpp @@ -0,0 +1,56 @@ +#include +#include + +#include +#include +#include + + +namespace DB +{ + + +ValuesRowOutputFormat::ValuesRowOutputFormat(WriteBuffer & out_, const Block & header, const FormatSettings & format_settings) + : IRowOutputFormat(header, out_), format_settings(format_settings) +{ +} + +void ValuesRowOutputFormat::writeField(const IColumn & column, const IDataType & type, size_t row_num) +{ + type.serializeAsTextQuoted(column, row_num, out, format_settings); +} + +void ValuesRowOutputFormat::writeFieldDelimiter() +{ + writeChar(',', out); +} + +void ValuesRowOutputFormat::writeRowStartDelimiter() +{ + writeChar('(', out); +} + +void ValuesRowOutputFormat::writeRowEndDelimiter() +{ + writeChar(')', out); +} + +void ValuesRowOutputFormat::writeRowBetweenDelimiter() +{ + writeCString(",", out); +} + + +void registerOutputFormatProcessorValues(FormatFactory & factory) +{ + factory.registerOutputFormatProcessor("Values", []( + WriteBuffer & buf, + const Block & sample, + const Context &, + const FormatSettings & settings) + { + return std::make_shared(buf, sample, settings); + }); +} + +} diff --git a/dbms/src/Processors/Formats/Impl/ValuesRowOutputFormat.h b/dbms/src/Processors/Formats/Impl/ValuesRowOutputFormat.h new file mode 100644 index 00000000000..5f82e78d3c0 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/ValuesRowOutputFormat.h @@ -0,0 +1,33 @@ +#pragma once + +#include +#include + + +namespace DB +{ + +class WriteBuffer; + + +/** A stream for outputting data in the VALUES format (as in the INSERT request). + */ +class ValuesRowOutputFormat : public IRowOutputFormat +{ +public: + ValuesRowOutputFormat(WriteBuffer & out_, const Block & header, const FormatSettings & format_settings); + + String getName() const override { return "ValuesRowOutputFormat"; } + + void writeField(const IColumn & column, const IDataType & type, size_t row_num) override; + void writeFieldDelimiter() override; + void writeRowStartDelimiter() override; + void writeRowEndDelimiter() override; + void writeRowBetweenDelimiter() override; + +private: + const FormatSettings format_settings; +}; + +} + diff --git a/dbms/src/Processors/Formats/Impl/VerticalRowOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/VerticalRowOutputFormat.cpp new file mode 100644 index 00000000000..8e9d0cd37c5 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/VerticalRowOutputFormat.cpp @@ -0,0 +1,177 @@ +#include + +#include +#include +#include +#include +#include + + +namespace DB +{ + +VerticalRowOutputFormat::VerticalRowOutputFormat( + WriteBuffer & out_, const Block & header, const FormatSettings & format_settings) + : IRowOutputFormat(header, out_), format_settings(format_settings) +{ + auto & sample = getPort(PortKind::Main).getHeader(); + size_t columns = sample.columns(); + + using Widths = std::vector; + Widths name_widths(columns); + size_t max_name_width = 0; + + String serialized_value; + + for (size_t i = 0; i < columns; ++i) + { + /// Note that number of code points is just a rough approximation of visible string width. + const String & name = sample.getByPosition(i).name; + + name_widths[i] = UTF8::computeWidth(reinterpret_cast(name.data()), name.size()); + + if (name_widths[i] > max_name_width) + max_name_width = name_widths[i]; + } + + names_and_paddings.resize(columns); + for (size_t i = 0; i < columns; ++i) + { + WriteBufferFromString buf(names_and_paddings[i]); + writeString(sample.getByPosition(i).name, buf); + writeCString(": ", buf); + } + + for (size_t i = 0; i < columns; ++i) + { + size_t new_size = max_name_width - name_widths[i] + names_and_paddings[i].size(); + names_and_paddings[i].resize(new_size, ' '); + } +} + + +void VerticalRowOutputFormat::writeField(const IColumn & column, const IDataType & type, size_t row_num) +{ + if (row_number > format_settings.pretty.max_rows) + return; + + writeString(names_and_paddings[field_number], out); + writeValue(column, type, row_num); + writeChar('\n', out); + + ++field_number; +} + + +void VerticalRowOutputFormat::writeValue(const IColumn & column, const IDataType & type, size_t row_num) const +{ + type.serializeAsText(column, row_num, out, format_settings); +} + + +void VerticalRowOutputFormat::writeRowStartDelimiter() +{ + ++row_number; + + if (row_number > format_settings.pretty.max_rows) + return; + + writeCString("Row ", out); + writeIntText(row_number, out); + writeCString(":\n", out); + + size_t width = log10(row_number + 1) + 1 + strlen("Row :"); + for (size_t i = 0; i < width; ++i) + writeCString("─", out); + writeChar('\n', out); +} + + +void VerticalRowOutputFormat::writeRowBetweenDelimiter() +{ + if (row_number > format_settings.pretty.max_rows) + return; + + writeCString("\n", out); + field_number = 0; +} + + +void VerticalRowOutputFormat::writeSuffix() +{ + if (row_number > format_settings.pretty.max_rows) + { + writeCString("Showed first ", out); + writeIntText(format_settings.pretty.max_rows, out); + writeCString(".\n", out); + } +} + +void VerticalRowOutputFormat::writeBeforeTotals() +{ + writeCString("\n", out); +} + +void VerticalRowOutputFormat::writeBeforeExtremes() +{ + if (!was_totals_written) + writeCString("\n", out); +} + +void VerticalRowOutputFormat::writeMinExtreme(const Columns & columns, size_t row_num) +{ + writeSpecialRow(columns, row_num, PortKind::Totals, "Min"); +} + +void VerticalRowOutputFormat::writeMaxExtreme(const Columns & columns, size_t row_num) +{ + writeSpecialRow(columns, row_num, PortKind::Totals, "Max"); +} + +void VerticalRowOutputFormat::writeTotals(const Columns & columns, size_t row_num) +{ + writeSpecialRow(columns, row_num, PortKind::Totals, "Totals"); + was_totals_written = true; +} + +void VerticalRowOutputFormat::writeSpecialRow(const Columns & columns, size_t row_num, PortKind port_kind, const char * title) +{ + writeCString("\n", out); + + row_number = 0; + field_number = 0; + + auto & header = getPort(port_kind).getHeader(); + size_t num_columns = columns.size(); + + writeCString(title, out); + writeCString(":\n", out); + + size_t width = strlen(title) + 1; + for (size_t i = 0; i < width; ++i) + writeCString("─", out); + writeChar('\n', out); + + for (size_t i = 0; i < num_columns; ++i) + { + if (i != 0) + writeFieldDelimiter(); + + auto & col = header.getByPosition(i); + writeField(*columns[i], *col.type, row_num); + } +} + +void registerOutputFormatProcessorVertical(FormatFactory & factory) +{ + factory.registerOutputFormatProcessor("Vertical", []( + WriteBuffer & buf, + const Block & sample, + const Context &, + const FormatSettings & settings) + { + return std::make_shared(buf, sample, settings); + }); +} + +} diff --git a/dbms/src/Processors/Formats/Impl/VerticalRowOutputFormat.h b/dbms/src/Processors/Formats/Impl/VerticalRowOutputFormat.h new file mode 100644 index 00000000000..a535d1e9c5b --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/VerticalRowOutputFormat.h @@ -0,0 +1,53 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ + +class WriteBuffer; +class Context; + + +/** Stream to output data in format "each value in separate row". + * Usable to show few rows with many columns. + */ +class VerticalRowOutputFormat : public IRowOutputFormat +{ +public: + VerticalRowOutputFormat(WriteBuffer & out_, const Block & header, const FormatSettings & format_settings); + + String getName() const override { return "VerticalRowOutputFormat"; } + + void writeField(const IColumn & column, const IDataType & type, size_t row_num) override; + void writeRowStartDelimiter() override; + void writeRowBetweenDelimiter() override; + void writeSuffix() override; + + void writeMinExtreme(const Columns & columns, size_t row_num) override; + void writeMaxExtreme(const Columns & columns, size_t row_num) override; + void writeTotals(const Columns & columns, size_t row_num) override; + + void writeBeforeTotals() override; + void writeBeforeExtremes() override; + +protected: + virtual void writeValue(const IColumn & column, const IDataType & type, size_t row_num) const; + + /// For totals and extremes. + void writeSpecialRow(const Columns & columns, size_t row_num, PortKind port_kind, const char * title); + + const FormatSettings format_settings; + size_t field_number = 0; + size_t row_number = 0; + bool was_totals_written = false; + + using NamesAndPaddings = std::vector; + NamesAndPaddings names_and_paddings; +}; + +} + diff --git a/dbms/src/Processors/Formats/Impl/XMLRowOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/XMLRowOutputFormat.cpp new file mode 100644 index 00000000000..5df58a5c733 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/XMLRowOutputFormat.cpp @@ -0,0 +1,255 @@ +#include +#include +#include +#include + + +namespace DB +{ + +XMLRowOutputFormat::XMLRowOutputFormat(WriteBuffer & out_, const Block & header, const FormatSettings & format_settings) + : IRowOutputFormat(header, out_), format_settings(format_settings) +{ + auto & sample = getPort(PortKind::Main).getHeader(); + NamesAndTypesList columns(sample.getNamesAndTypesList()); + fields.assign(columns.begin(), columns.end()); + field_tag_names.resize(sample.columns()); + + bool need_validate_utf8 = false; + for (size_t i = 0; i < sample.columns(); ++i) + { + if (!sample.getByPosition(i).type->textCanContainOnlyValidUTF8()) + need_validate_utf8 = true; + + /// As element names, we will use the column name if it has a valid form, or "field", otherwise. + /// The condition below is more strict than the XML standard requires. + bool is_column_name_suitable = true; + const char * begin = fields[i].name.data(); + const char * end = begin + fields[i].name.size(); + for (const char * pos = begin; pos != end; ++pos) + { + char c = *pos; + if (!(isAlphaASCII(c) + || (pos != begin && isNumericASCII(c)) + || c == '_' + || c == '-' + || c == '.')) + { + is_column_name_suitable = false; + break; + } + } + + field_tag_names[i] = is_column_name_suitable + ? fields[i].name + : "field"; + } + + if (need_validate_utf8) + { + validating_ostr = std::make_unique(out); + ostr = validating_ostr.get(); + } + else + ostr = &out; +} + + +void XMLRowOutputFormat::writePrefix() +{ + writeCString("\n", *ostr); + writeCString("\n", *ostr); + writeCString("\t\n", *ostr); + writeCString("\t\t\n", *ostr); + + for (const auto & field : fields) + { + writeCString("\t\t\t\n", *ostr); + + writeCString("\t\t\t\t", *ostr); + writeXMLString(field.name, *ostr); + writeCString("\n", *ostr); + writeCString("\t\t\t\t", *ostr); + writeXMLString(field.type->getName(), *ostr); + writeCString("\n", *ostr); + + writeCString("\t\t\t\n", *ostr); + } + + writeCString("\t\t\n", *ostr); + writeCString("\t\n", *ostr); + writeCString("\t\n", *ostr); +} + + +void XMLRowOutputFormat::writeField(const IColumn & column, const IDataType & type, size_t row_num) +{ + writeCString("\t\t\t<", *ostr); + writeString(field_tag_names[field_number], *ostr); + writeCString(">", *ostr); + type.serializeAsTextXML(column, row_num, *ostr, format_settings); + writeCString("\n", *ostr); + ++field_number; +} + + +void XMLRowOutputFormat::writeRowStartDelimiter() +{ + writeCString("\t\t\n", *ostr); +} + + +void XMLRowOutputFormat::writeRowEndDelimiter() +{ + writeCString("\t\t\n", *ostr); + field_number = 0; + ++row_count; +} + + +void XMLRowOutputFormat::writeSuffix() +{ + writeCString("\t\n", *ostr); + +} + + +void XMLRowOutputFormat::writeBeforeTotals() +{ + writeCString("\t\n", *ostr); +} + +void XMLRowOutputFormat::writeTotals(const Columns & columns, size_t row_num) +{ + size_t totals_columns = columns.size(); + auto & header = getPort(PortKind::Totals).getHeader(); + for (size_t i = 0; i < totals_columns; ++i) + { + const ColumnWithTypeAndName & column = header.safeGetByPosition(i); + + writeCString("\t\t<", *ostr); + writeString(field_tag_names[i], *ostr); + writeCString(">", *ostr); + column.type->serializeAsTextXML(*columns[i], row_num, *ostr, format_settings); + writeCString("\n", *ostr); + } +} + +void XMLRowOutputFormat::writeAfterTotals() +{ + writeCString("\t\n", *ostr); +} + + +void XMLRowOutputFormat::writeBeforeExtremes() +{ + writeCString("\t\n", *ostr); +} + +void XMLRowOutputFormat::writeMinExtreme(const Columns & columns, size_t row_num) +{ + writeExtremesElement("min", columns, row_num); +} + +void XMLRowOutputFormat::writeMaxExtreme(const Columns & columns, size_t row_num) +{ + writeExtremesElement("max", columns, row_num); +} + +void XMLRowOutputFormat::writeAfterExtremes() +{ + writeCString("\t\n", *ostr); +} + +void XMLRowOutputFormat::writeExtremesElement(const char * title, const Columns & columns, size_t row_num) +{ + auto & header = getPort(PortKind::Extremes).getHeader(); + + writeCString("\t\t<", *ostr); + writeCString(title, *ostr); + writeCString(">\n", *ostr); + + size_t extremes_columns = columns.size(); + for (size_t i = 0; i < extremes_columns; ++i) + { + const ColumnWithTypeAndName & column = header.safeGetByPosition(i); + + writeCString("\t\t\t<", *ostr); + writeString(field_tag_names[i], *ostr); + writeCString(">", *ostr); + column.type->serializeAsTextXML(*columns[i], row_num, *ostr, format_settings); + writeCString("\n", *ostr); + } + + writeCString("\t\t\n", *ostr); +} + + +void XMLRowOutputFormat::onProgress(const Progress & value) +{ + progress.incrementPiecewiseAtomically(value); +} + +void XMLRowOutputFormat::writeLastSuffix() +{ + + writeCString("\t", *ostr); + writeIntText(row_count, *ostr); + writeCString("\n", *ostr); + + writeRowsBeforeLimitAtLeast(); + + if (format_settings.write_statistics) + writeStatistics(); + + writeCString("\n", *ostr); + ostr->next(); +} + +void XMLRowOutputFormat::writeRowsBeforeLimitAtLeast() +{ + if (applied_limit) + { + writeCString("\t", *ostr); + writeIntText(rows_before_limit, *ostr); + writeCString("\n", *ostr); + } +} + +void XMLRowOutputFormat::writeStatistics() +{ + writeCString("\t\n", *ostr); + writeCString("\t\t", *ostr); + writeText(watch.elapsedSeconds(), *ostr); + writeCString("\n", *ostr); + writeCString("\t\t", *ostr); + writeText(progress.read_rows.load(), *ostr); + writeCString("\n", *ostr); + writeCString("\t\t", *ostr); + writeText(progress.read_bytes.load(), *ostr); + writeCString("\n", *ostr); + writeCString("\t\n", *ostr); +} + + +void registerOutputFormatProcessorXML(FormatFactory & factory) +{ + factory.registerOutputFormatProcessor("XML", []( + WriteBuffer & buf, + const Block & sample, + const Context &, + const FormatSettings & settings) + { + return std::make_shared(buf, sample, settings); + }); +} + +} diff --git a/dbms/src/Processors/Formats/Impl/XMLRowOutputFormat.h b/dbms/src/Processors/Formats/Impl/XMLRowOutputFormat.h new file mode 100644 index 00000000000..102b11490fe --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/XMLRowOutputFormat.h @@ -0,0 +1,78 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +/** A stream for outputting data in XML format. + */ +class XMLRowOutputFormat : public IRowOutputFormat +{ +public: + XMLRowOutputFormat(WriteBuffer & out_, const Block & header, const FormatSettings & format_settings); + + String getName() const override { return "XMLRowOutputFormat"; } + + void writeField(const IColumn & column, const IDataType & type, size_t row_num) override; + void writeRowStartDelimiter() override; + void writeRowEndDelimiter() override; + void writePrefix() override; + void writeSuffix() override; + void writeLastSuffix() override; + + void writeMinExtreme(const Columns & columns, size_t row_num) override; + void writeMaxExtreme(const Columns & columns, size_t row_num) override; + void writeTotals(const Columns & columns, size_t row_num) override; + + void writeBeforeTotals() override; + void writeAfterTotals() override; + void writeBeforeExtremes() override; + void writeAfterExtremes() override; + + void flush() override + { + ostr->next(); + + if (validating_ostr) + out.next(); + } + + void setRowsBeforeLimit(size_t rows_before_limit_) override + { + applied_limit = true; + rows_before_limit = rows_before_limit_; + } + + void onProgress(const Progress & value) override; + + String getContentType() const override { return "application/xml; charset=UTF-8"; } + +protected: + void writeExtremesElement(const char * title, const Columns & columns, size_t row_num); + void writeRowsBeforeLimitAtLeast(); + void writeStatistics(); + + std::unique_ptr validating_ostr; /// Validates UTF-8 sequences, replaces bad sequences with replacement character. + WriteBuffer * ostr; + + size_t field_number = 0; + size_t row_count = 0; + bool applied_limit = false; + size_t rows_before_limit = 0; + NamesAndTypes fields; + Names field_tag_names; + + Progress progress; + Stopwatch watch; + const FormatSettings format_settings; +}; + +} + diff --git a/dbms/src/Processors/Formats/LazyOutputFormat.cpp b/dbms/src/Processors/Formats/LazyOutputFormat.cpp new file mode 100644 index 00000000000..129d1a06b9d --- /dev/null +++ b/dbms/src/Processors/Formats/LazyOutputFormat.cpp @@ -0,0 +1,61 @@ +#include +#include + + +namespace DB +{ + +WriteBuffer LazyOutputFormat::out(nullptr, 0); + +Block LazyOutputFormat::getBlock(UInt64 milliseconds) +{ + if (finished_processing) + { + if (queue.size() == 0) + return {}; + } + + Chunk chunk; + if (!queue.tryPop(chunk, milliseconds)) + return {}; + + if (!chunk) + return {}; + + auto block = getPort(PortKind::Main).getHeader().cloneWithColumns(chunk.detachColumns()); + info.update(block); + + if (auto chunk_info = chunk.getChunkInfo()) + { + if (auto * agg_info = typeid_cast(chunk_info.get())) + { + block.info.bucket_num = agg_info->bucket_num; + block.info.is_overflows = agg_info->is_overflows; + } + } + + return block; +} + +Block LazyOutputFormat::getTotals() +{ + if (!totals) + return {}; + + return getPort(PortKind::Totals).getHeader().cloneWithColumns(totals.detachColumns()); +} + +Block LazyOutputFormat::getExtremes() +{ + if (!extremes) + return {}; + + return getPort(PortKind::Extremes).getHeader().cloneWithColumns(extremes.detachColumns()); +} + +void LazyOutputFormat::setRowsBeforeLimit(size_t rows_before_limit) +{ + info.setRowsBeforeLimit(rows_before_limit); +} + +} diff --git a/dbms/src/Processors/Formats/LazyOutputFormat.h b/dbms/src/Processors/Formats/LazyOutputFormat.h new file mode 100644 index 00000000000..56aaf249480 --- /dev/null +++ b/dbms/src/Processors/Formats/LazyOutputFormat.h @@ -0,0 +1,64 @@ +#pragma once +#include +#include +#include +#include + +namespace DB +{ + +class LazyOutputFormat : public IOutputFormat +{ + +public: + explicit LazyOutputFormat(const Block & header) + : IOutputFormat(header, out), queue(2), finished_processing(false) {} + + String getName() const override { return "LazyOutputFormat"; } + + Block getBlock(UInt64 milliseconds = 0); + Block getTotals(); + Block getExtremes(); + + bool isFinished() { return finished_processing; } + + BlockStreamProfileInfo & getProfileInfo() { return info; } + + void setRowsBeforeLimit(size_t rows_before_limit) override; + + void finish() { finished_processing = true; } + void clearQueue() { queue.clear(); } + +protected: + void consume(Chunk chunk) override + { + if (!finished_processing) + queue.emplace(std::move(chunk)); + } + + void consumeTotals(Chunk chunk) override { totals = std::move(chunk); } + void consumeExtremes(Chunk chunk) override { extremes = std::move(chunk); } + + void finalize() override + { + finished_processing = true; + + /// In case we are waiting for result. + queue.emplace(Chunk()); + } + +private: + + ConcurrentBoundedQueue queue; + Chunk totals; + Chunk extremes; + + /// Is not used. + static WriteBuffer out; + + BlockStreamProfileInfo info; + + std::atomic finished_processing; +}; + +} diff --git a/dbms/src/Processors/IAccumulatingTransform.cpp b/dbms/src/Processors/IAccumulatingTransform.cpp new file mode 100644 index 00000000000..0abb923584d --- /dev/null +++ b/dbms/src/Processors/IAccumulatingTransform.cpp @@ -0,0 +1,91 @@ +#include + + +namespace DB +{ + +IAccumulatingTransform::IAccumulatingTransform(Block input_header, Block output_header) + : IProcessor({std::move(input_header)}, {std::move(output_header)}), + input(inputs.front()), output(outputs.front()) +{ +} + +IAccumulatingTransform::Status IAccumulatingTransform::prepare() +{ + /// Check can output. + if (output.isFinished()) + { + input.close(); + return Status::Finished; + } + + if (!output.canPush()) + { + input.setNotNeeded(); + return Status::PortFull; + } + + /// Output if has data. + if (current_output_chunk) + output.push(std::move(current_output_chunk)); + + if (finished_generate) + { + output.finish(); + return Status::Finished; + } + + /// Generate output block. + if (input.isFinished()) + { + finished_input = true; + return Status::Ready; + } + + /// Close input if flag was set manually. + if (finished_input) + { + input.close(); + return Status::Ready; + } + + /// Check can input. + if (!has_input) + { + input.setNeeded(); + if (!input.hasData()) + return Status::NeedData; + + current_input_chunk = input.pull(); + has_input = true; + } + + return Status::Ready; +} + +void IAccumulatingTransform::work() +{ + if (!finished_input) + { + consume(std::move(current_input_chunk)); + has_input = false; + } + else + { + current_output_chunk = generate(); + if (!current_output_chunk) + finished_generate = true; + } +} + +void IAccumulatingTransform::setReadyChunk(Chunk chunk) +{ + if (current_output_chunk) + throw Exception("IAccumulatingTransform already has input. Cannot set another chunk. " + "Probably, setReadyChunk method was called twice per consume().", ErrorCodes::LOGICAL_ERROR); + + current_output_chunk = std::move(chunk); +} + +} + diff --git a/dbms/src/Processors/IAccumulatingTransform.h b/dbms/src/Processors/IAccumulatingTransform.h new file mode 100644 index 00000000000..2d3a51f7b2e --- /dev/null +++ b/dbms/src/Processors/IAccumulatingTransform.h @@ -0,0 +1,42 @@ +#pragma once + +#include + + +namespace DB +{ + +/** Has one input and one output. + * Pulls all blocks from input, and only then produce output. + * Examples: ORDER BY, GROUP BY. + */ +class IAccumulatingTransform : public IProcessor +{ +protected: + InputPort & input; + OutputPort & output; + + Chunk current_input_chunk; + Chunk current_output_chunk; + bool has_input = false; + bool finished_input = false; + bool finished_generate = false; + + virtual void consume(Chunk chunk) = 0; + virtual Chunk generate() = 0; + + /// This method can be called once per consume call. In case if some chunks are ready. + void setReadyChunk(Chunk chunk); + void finishConsume() { finished_input = true; } + +public: + IAccumulatingTransform(Block input_header, Block output_header); + + Status prepare() override; + void work() override; + + InputPort & getInputPort() { return input; } + OutputPort & getOutputPort() { return output; } +}; + +} diff --git a/dbms/src/Processors/IInflatingTransform.cpp b/dbms/src/Processors/IInflatingTransform.cpp new file mode 100644 index 00000000000..a1206d52427 --- /dev/null +++ b/dbms/src/Processors/IInflatingTransform.cpp @@ -0,0 +1,85 @@ +#include + +namespace DB +{ + +IInflatingTransform::IInflatingTransform(Block input_header, Block output_header) + : IProcessor({std::move(input_header)}, {std::move(output_header)}) + , input(inputs.front()), output(outputs.front()) +{ + +} + +IInflatingTransform::Status IInflatingTransform::prepare() +{ + /// Check can output. + + if (output.isFinished()) + { + input.close(); + return Status::Finished; + } + + if (!output.canPush()) + { + input.setNotNeeded(); + return Status::PortFull; + } + + /// Output if has data. + if (generated) + { + output.push(std::move(current_chunk)); + generated = false; + } + + if (can_generate) + return Status::Ready; + + /// Check can input. + if (!has_input) + { + if (input.isFinished()) + { + output.finish(); + return Status::Finished; + } + + input.setNeeded(); + + if (!input.hasData()) + return Status::NeedData; + + current_chunk = input.pull(); + has_input = true; + } + + /// Now transform. + return Status::Ready; +} + +void IInflatingTransform::work() +{ + if (can_generate) + { + if (generated) + throw Exception("IInflatingTransform cannot consume chunk because it already was generated", + ErrorCodes::LOGICAL_ERROR); + + current_chunk = generate(); + generated = true; + can_generate = canGenerate(); + } + else + { + if (!has_input) + throw Exception("IInflatingTransform cannot consume chunk because it wasn't read", + ErrorCodes::LOGICAL_ERROR); + + consume(std::move(current_chunk)); + has_input = false; + can_generate = canGenerate(); + } +} + +} diff --git a/dbms/src/Processors/IInflatingTransform.h b/dbms/src/Processors/IInflatingTransform.h new file mode 100644 index 00000000000..45edf5302e5 --- /dev/null +++ b/dbms/src/Processors/IInflatingTransform.h @@ -0,0 +1,32 @@ +#pragma once +#include + +namespace DB +{ + +class IInflatingTransform : public IProcessor +{ +protected: + InputPort & input; + OutputPort & output; + + Chunk current_chunk; + bool has_input = false; + bool generated = false; + bool can_generate = false; + + virtual void consume(Chunk chunk) = 0; + virtual bool canGenerate() = 0; + virtual Chunk generate() = 0; + +public: + IInflatingTransform(Block input_header, Block output_header); + + Status prepare() override; + void work() override; + + InputPort & getInputPort() { return input; } + OutputPort & getOutputPort() { return output; } +}; + +} diff --git a/dbms/src/Processors/IProcessor.cpp b/dbms/src/Processors/IProcessor.cpp new file mode 100644 index 00000000000..36beeea8476 --- /dev/null +++ b/dbms/src/Processors/IProcessor.cpp @@ -0,0 +1,46 @@ +#include +#include + + +namespace DB +{ + +void IProcessor::dump() const +{ + std::cerr << getName() << "\n"; + + std::cerr << "inputs:\n"; + for (const auto & port : inputs) + std::cerr << "\t" << port.hasData() << " " << port.isFinished() << "\n"; + + std::cerr << "outputs:\n"; + for (const auto & port : outputs) + std::cerr << "\t" << port.hasData() << " " << port.isNeeded() << "\n"; +} + + +std::string IProcessor::statusToName(Status status) +{ + switch (status) + { + case Status::NeedData: + return "NeedData"; + case Status::PortFull: + return "PortFull"; + case Status::Finished: + return "Finished"; + case Status::Ready: + return "Ready"; + case Status::Async: + return "Async"; + case Status::Wait: + return "Wait"; + case Status::ExpandPipeline: + return "ExpandPipeline"; + } + + __builtin_unreachable(); +} + +} + diff --git a/dbms/src/Processors/IProcessor.h b/dbms/src/Processors/IProcessor.h new file mode 100644 index 00000000000..c5f9ef64b4a --- /dev/null +++ b/dbms/src/Processors/IProcessor.h @@ -0,0 +1,231 @@ +#pragma once + +#include +#include + + +class EventCounter; + + +namespace DB +{ + +class IProcessor; +using ProcessorPtr = std::shared_ptr; +using Processors = std::vector; + +/** Processor is an element (low level building block) of a query execution pipeline. + * It has zero or more input ports and zero or more output ports. + * + * Blocks of data are transferred over ports. + * Each port has fixed structure: names and types of columns and values of constants. + * + * Processors may pull data from input ports, do some processing and push data to output ports. + * Processor may indicate that it requires input data to proceed and indicate that it needs data from some ports. + * + * Synchronous work must only use CPU - don't do any sleep, IO wait, network wait. + * + * Processor may want to do work asynchronously (example: fetch data from remote server) + * - in this case it will initiate background job and allow to subscribe to it. + * + * Processor may throw an exception to indicate some runtime error. + * + * Different ports may have different structure. For example, ports may correspond to different resultsets + * or semantically different parts of result. + * + * Processor may modify its ports (create another processors and connect to them) on the fly. + * Example: first execute the subquery; on basis of subquery result + * determine how to execute the rest of query and build the corresponding pipeline. + * + * Processor may simply wait for another processor to execute without transferring any data from it. + * For this purpose it should connect its input port to another processor, and indicate need of data. + * + * Examples: + * + * Source. Has no input ports and single output port. Generates data itself and pushes it to its output port. + * + * Sink. Has single input port and no output ports. Consumes data that was passed to its input port. + * + * Empty source. Immediately says that data on its output port is finished. + * + * Null sink. Consumes data and does nothing. + * + * Simple transformation. Has single input and single output port. Pulls data, transforms it and pushes to output port. + * Example: expression calculator. + * TODO Better to make each function a separate processor. It's better for pipeline analysis. Also keep in mind 'sleep' and 'rand' functions. + * + * Squashing or filtering transformation. Pulls data, possibly accumulates it, and sometimes pushes it to output port. + * Examples: DISTINCT, WHERE, squashing of blocks for INSERT SELECT. + * + * Accumulating transformation. Pulls and accumulates all data from input until it it exhausted, then pushes data to output port. + * Examples: ORDER BY, GROUP BY. + * + * Limiting transformation. Pulls data from input and passes to output. + * When there was enough data, says that it doesn't need data on its input and that data on its output port is finished. + * + * Resize. Has arbitary number of inputs and arbitary number of outputs. + * Pulls data from whatever ready input and pushes it to randomly choosed free output. + * Examples: + * Union - merge data from number of inputs to one output in arbitary order. + * Split - read data from one input and pass it to arbitary output. + * + * Concat. Has many inputs and only one output. Pulls all data from first input until it is exhausted, + * then all data from second input, etc. and pushes all data to output. + * + * Ordered merge. Has many inputs but only one output. Pulls data from selected input in specific order, merges and pushes it to output. + * + * Fork. Has one input and many outputs. Pulls data from input and copies it to all outputs. + * Used to process multiple queries with common source of data. + * + * Select. Has one or multiple inputs and one output. + * Read blocks from inputs and check that blocks on inputs are "parallel": correspond to each other in number of rows. + * Construct a new block by selecting some subset (or all) of columns from inputs. + * Example: collect columns - function arguments before function execution. + * + * + * TODO Processors may carry algebraic properties about transformations they do. + * For example, that processor doesn't change number of rows; doesn't change order of rows, doesn't change the set of rows, etc. + * + * TODO Ports may carry algebraic properties about streams of data. + * For example, that data comes ordered by specific key; or grouped by specific key; or have unique values of specific key. + * And also simple properties, including lower and upper bound on number of rows. + * + * TODO Processor should have declarative representation, that is able to be serialized and parsed. + * Example: read_from_merge_tree(database, table, Columns(a, b, c), Piece(0, 10), Parts(Part('name', MarkRanges(MarkRange(0, 100), ...)), ...)) + * It's reasonable to have an intermediate language for declaration of pipelines. + * + * TODO Processor with all its parameters should represent "pure" function on streams of data from its input ports. + * It's in question, what kind of "pure" function do we mean. + * For example, data streams are considered equal up to order unless ordering properties are stated explicitly. + * Another example: we should support the notion of "arbitary N-th of M substream" of full stream of data. + */ + +class IProcessor +{ +protected: + InputPorts inputs; + OutputPorts outputs; + +public: + IProcessor() = default; + + IProcessor(InputPorts inputs_, OutputPorts outputs_) + : inputs(std::move(inputs_)), outputs(std::move(outputs_)) + { + for (auto & port : inputs) + port.processor = this; + for (auto & port : outputs) + port.processor = this; + } + + virtual String getName() const = 0; + + enum class Status + { + /// Processor needs some data at its inputs to proceed. + /// You need to run another processor to generate required input and then call 'prepare' again. + NeedData, + + /// Processor cannot proceed because output port is full or not isNeeded(). + /// You need to transfer data from output port to the input port of another processor and then call 'prepare' again. + PortFull, + + /// All work is done (all data is processed or all output are closed), nothing more to do. + Finished, + + /// No one needs data on output ports. + /// Unneeded, + + /// You may call 'work' method and processor will do some work synchronously. + Ready, + + /// You may call 'schedule' method and processor will initiate some background work. + Async, + + /// Processor is doing some work in background. + /// You may wait for next event or do something else and then you should call 'prepare' again. + Wait, + + /// Processor wants to add other processors to pipeline. + /// New processors must be obtained by expandPipeline() call. + ExpandPipeline, + }; + + static std::string statusToName(Status status); + + /** Method 'prepare' is responsible for all cheap ("instantenous": O(1) of data volume, no wait) calculations. + * + * It may access input and output ports, + * indicate the need for work by another processor by returning NeedData or PortFull, + * or indicate the absense of work by returning Finished or Unneeded, + * it may pull data from input ports and push data to output ports. + * + * The method is not thread-safe and must be called from a single thread in one moment of time, + * even for different connected processors. + * + * Instead of all long work (CPU calculations or waiting) it should just prepare all required data and return Ready or Async. + * + * Thread safety and parallel execution: + * - no methods (prepare, work, schedule) of single object can be executed in parallel; + * - method 'work' can be executed in parallel for different objects, even for connected processors; + * - method 'prepare' cannot be executed in parallel even for different objects, + * if they are connected (including indirectly) to each other by their ports; + */ + virtual Status prepare() = 0; + + /** You may call this method if 'prepare' returned Ready. + * This method cannot access any ports. It should use only data that was prepared by 'prepare' method. + * + * Method work can be executed in parallel for different processors. + */ + virtual void work() + { + throw Exception("Method 'work' is not implemented for " + getName() + " processor", ErrorCodes::NOT_IMPLEMENTED); + } + + /** You may call this method if 'prepare' returned Async. + * This method cannot access any ports. It should use only data that was prepared by 'prepare' method. + * + * This method should return instantly and fire an event (or many events) when asynchronous job will be done. + * When the job is not done, method 'prepare' will return Wait and the user may block and wait for next event before checking again. + * + * Note that it can fire many events in EventCounter while doing its job, + * and you have to wait for next event (or do something else) every time when 'prepare' returned Wait. + */ + virtual void schedule(EventCounter & /*watch*/) + { + throw Exception("Method 'schedule' is not implemented for " + getName() + " processor", ErrorCodes::NOT_IMPLEMENTED); + } + + /** You must call this method if 'prepare' returned ExpandPipeline. + * This method cannot access any port, but it can create new ports for current processor. + * + * Method should return set of new already connected processors. + * All added processors must be connected only to each other or current processor. + * + * Method can't remove or reconnect existing ports, move data from/to port or perform calculations. + * 'prepare' should be called again after expanding pipeline. + */ + virtual Processors expandPipeline() + { + throw Exception("Method 'expandPipeline' is not implemented for " + getName() + " processor", ErrorCodes::NOT_IMPLEMENTED); + } + + virtual ~IProcessor() = default; + + auto & getInputs() { return inputs; } + auto & getOutputs() { return outputs; } + + /// Debug output. + void dump() const; + + std::string processor_description; + + void setDescription(const std::string & description_) { processor_description = description_; } + const std::string & getDescription() const { return processor_description; } +}; + + + + +} diff --git a/dbms/src/Processors/ISimpleTransform.cpp b/dbms/src/Processors/ISimpleTransform.cpp new file mode 100644 index 00000000000..39736973a16 --- /dev/null +++ b/dbms/src/Processors/ISimpleTransform.cpp @@ -0,0 +1,109 @@ +#include + + +namespace DB +{ + +ISimpleTransform::ISimpleTransform(Block input_header, Block output_header, bool skip_empty_chunks) + : IProcessor({std::move(input_header)}, {std::move(output_header)}) + , input(inputs.front()) + , output(outputs.front()) + , skip_empty_chunks(skip_empty_chunks) +{ +} + +ISimpleTransform::Status ISimpleTransform::prepare() +{ + /// Check can output. + + if (output.isFinished()) + { + input.close(); + return Status::Finished; + } + + if (!output.canPush()) + { + input.setNotNeeded(); + return Status::PortFull; + } + + /// Output if has data. + if (transformed) + { + output.pushData(std::move(current_data)); + transformed = false; + } + + /// Stop if don't need more data. + if (no_more_data_needed) + { + input.close(); + output.finish(); + return Status::Finished; + } + + /// Check can input. + if (!has_input) + { + if (input.isFinished()) + { + output.finish(); + return Status::Finished; + } + + input.setNeeded(); + + if (!input.hasData()) + return Status::NeedData; + + current_data = input.pullData(); + has_input = true; + + if (current_data.exception) + { + /// Skip transform in case of exception. + has_input = false; + transformed = true; + + /// No more data needed. Exception will be thrown (or swallowed) later. + input.setNotNeeded(); + } + + if (set_input_not_needed_after_read) + input.setNotNeeded(); + } + + /// Now transform. + return Status::Ready; +} + +void ISimpleTransform::work() +{ + if (current_data.exception) + return; + + try + { + transform(current_data.chunk); + } + catch (DB::Exception &) + { + current_data.exception = std::current_exception(); + transformed = true; + has_input = false; + return; + } + + has_input = false; + + if (!skip_empty_chunks || current_data.chunk) + transformed = true; + + if (transformed && !current_data.chunk) + /// Support invariant that chunks must have the same number of columns as header. + current_data.chunk = Chunk(getOutputPort().getHeader().cloneEmpty().getColumns(), 0); +} + +} + diff --git a/dbms/src/Processors/ISimpleTransform.h b/dbms/src/Processors/ISimpleTransform.h new file mode 100644 index 00000000000..82e383ceeb0 --- /dev/null +++ b/dbms/src/Processors/ISimpleTransform.h @@ -0,0 +1,44 @@ +#pragma once + +#include + + +namespace DB +{ + +/** Has one input and one output. + * Simply pull a block from input, transform it, and push it to output. + */ +class ISimpleTransform : public IProcessor +{ +protected: + InputPort & input; + OutputPort & output; + + Port::Data current_data; + bool has_input = false; + bool transformed = false; + bool no_more_data_needed = false; + const bool skip_empty_chunks; + + /// Set input port NotNeeded after chunk was pulled. + /// Input port will become needed again only after data was transformed. + /// This allows to escape caching chunks in input port, which can lead to uneven data distribution. + bool set_input_not_needed_after_read = false; + + virtual void transform(Chunk & chunk) = 0; + void stopReading() { no_more_data_needed = true; } + +public: + ISimpleTransform(Block input_header, Block output_header, bool skip_empty_chunks); + + Status prepare() override; + void work() override; + + InputPort & getInputPort() { return input; } + OutputPort & getOutputPort() { return output; } + + void setInputNotNeededAfterRead(bool value) { set_input_not_needed_after_read = value; } +}; + +} diff --git a/dbms/src/Processors/ISink.cpp b/dbms/src/Processors/ISink.cpp new file mode 100644 index 00000000000..5c5f98cb131 --- /dev/null +++ b/dbms/src/Processors/ISink.cpp @@ -0,0 +1,36 @@ +#include + + +namespace DB +{ + +ISink::ISink(Block header) + : IProcessor({std::move(header)}, {}), input(inputs.front()) +{ +} + +ISink::Status ISink::prepare() +{ + if (has_input) + return Status::Ready; + + if (input.isFinished()) + return Status::Finished; + + input.setNeeded(); + if (!input.hasData()) + return Status::NeedData; + + current_chunk = input.pull(); + has_input = true; + return Status::Ready; +} + +void ISink::work() +{ + consume(std::move(current_chunk)); + has_input = false; +} + +} + diff --git a/dbms/src/Processors/ISink.h b/dbms/src/Processors/ISink.h new file mode 100644 index 00000000000..4e25e528098 --- /dev/null +++ b/dbms/src/Processors/ISink.h @@ -0,0 +1,27 @@ +#pragma once + +#include + + +namespace DB +{ + +class ISink : public IProcessor +{ +protected: + InputPort & input; + Chunk current_chunk; + bool has_input = false; + + virtual void consume(Chunk block) = 0; + +public: + explicit ISink(Block header); + + Status prepare() override; + void work() override; + + InputPort & getPort() { return input; } +}; + +} diff --git a/dbms/src/Processors/ISource.cpp b/dbms/src/Processors/ISource.cpp new file mode 100644 index 00000000000..d40f0e32fb7 --- /dev/null +++ b/dbms/src/Processors/ISource.cpp @@ -0,0 +1,67 @@ +#include + + +namespace DB +{ + +ISource::ISource(Block header) + : IProcessor({}, {std::move(header)}), output(outputs.front()) +{ +} + +ISource::Status ISource::prepare() +{ + if (finished) + { + output.finish(); + return Status::Finished; + } + + /// Check can output. + if (output.isFinished()) + return Status::Finished; + + if (!output.canPush()) + return Status::PortFull; + + if (!has_input) + return Status::Ready; + + output.pushData(std::move(current_chunk)); + has_input = false; + + if (got_exception) + { + finished = true; + output.finish(); + return Status::Finished; + } + + /// Now, we pushed to output, and it must be full. + return Status::PortFull; +} + +void ISource::work() +{ + try + { + current_chunk.chunk = generate(); + if (!current_chunk.chunk) + finished = true; + else + has_input = true; + } + catch (...) + { + finished = true; + throw; + } +// { +// current_chunk = std::current_exception(); +// has_input = true; +// got_exception = true; +// } +} + +} + diff --git a/dbms/src/Processors/ISource.h b/dbms/src/Processors/ISource.h new file mode 100644 index 00000000000..b1669860192 --- /dev/null +++ b/dbms/src/Processors/ISource.h @@ -0,0 +1,30 @@ +#pragma once + +#include + + +namespace DB +{ + +class ISource : public IProcessor +{ +protected: + OutputPort & output; + bool has_input = false; + bool finished = false; + bool got_exception = false; + Port::Data current_chunk; + + virtual Chunk generate() = 0; + +public: + ISource(Block header); + + Status prepare() override; + void work() override; + + OutputPort & getPort() { return output; } + const OutputPort & getPort() const { return output; } +}; + +} diff --git a/dbms/src/Processors/LimitTransform.cpp b/dbms/src/Processors/LimitTransform.cpp new file mode 100644 index 00000000000..f591ecfb046 --- /dev/null +++ b/dbms/src/Processors/LimitTransform.cpp @@ -0,0 +1,159 @@ +#include + + +namespace DB +{ + +LimitTransform::LimitTransform( + const Block & header, size_t limit, size_t offset, + bool always_read_till_end) + : IProcessor({header}, {header}) + , input(inputs.front()), output(outputs.front()) + , limit(limit), offset(offset) + , always_read_till_end(always_read_till_end) +{ +} + + +LimitTransform::Status LimitTransform::prepare() +{ + + + /// Check can output. + bool output_finished = false; + if (output.isFinished()) + { + output_finished = true; + if (!always_read_till_end) + { + input.close(); + return Status::Finished; + } + } + + if (!output_finished && !output.canPush()) + { + input.setNotNeeded(); + return Status::PortFull; + } + + /// Push block if can. + if (!output_finished && has_block && block_processed) + { + output.push(std::move(current_chunk)); + has_block = false; + block_processed = false; + } + + /// Check if we are done with pushing. + bool pushing_is_finished = rows_read >= offset + limit; + if (pushing_is_finished) + { + if (!always_read_till_end) + { + output.finish(); + input.close(); + return Status::Finished; + } + } + + /// Check can input. + + if (input.isFinished()) + { + output.finish(); + return Status::Finished; + } + + input.setNeeded(); + if (!input.hasData()) + return Status::NeedData; + + current_chunk = input.pull(); + has_block = true; + + auto rows = current_chunk.getNumRows(); + rows_before_limit_at_least += rows; + + /// Skip block (for 'always_read_till_end' case). + if (pushing_is_finished) + { + current_chunk.clear(); + has_block = false; + + if (input.isFinished()) + { + output.finish(); + return Status::Finished; + } + + /// Now, we pulled from input, and it must be empty. + return Status::NeedData; + } + + /// Process block. + + rows_read += rows; + + if (rows_read <= offset) + { + current_chunk.clear(); + has_block = false; + + if (input.isFinished()) + { + output.finish(); + return Status::Finished; + } + + /// Now, we pulled from input, and it must be empty. + return Status::NeedData; + } + + /// Return the whole block. + if (rows_read >= offset + rows && rows_read <= offset + limit) + { + if (output.hasData()) + return Status::PortFull; + + output.push(std::move(current_chunk)); + has_block = false; + + return Status::PortFull; + } + + /// No more data is needed. + if (!always_read_till_end && rows_read >= offset + limit) + input.close(); + + return Status::Ready; +} + + +void LimitTransform::work() +{ + size_t num_rows = current_chunk.getNumRows(); + size_t num_columns = current_chunk.getNumColumns(); + + /// return a piece of the block + size_t start = std::max( + static_cast(0), + static_cast(offset) - static_cast(rows_read) + static_cast(num_rows)); + + size_t length = std::min( + static_cast(limit), std::min( + static_cast(rows_read) - static_cast(offset), + static_cast(limit) + static_cast(offset) - static_cast(rows_read) + static_cast(num_rows))); + + auto columns = current_chunk.detachColumns(); + + for (size_t i = 0; i < num_columns; ++i) + columns[i] = columns[i]->cut(start, length); + + current_chunk.setColumns(std::move(columns), length); + + block_processed = true; +} + +} + diff --git a/dbms/src/Processors/LimitTransform.h b/dbms/src/Processors/LimitTransform.h new file mode 100644 index 00000000000..eb5a8fe8d5a --- /dev/null +++ b/dbms/src/Processors/LimitTransform.h @@ -0,0 +1,42 @@ +#pragma once + +#include + + +namespace DB +{ + +class LimitTransform : public IProcessor +{ +private: + InputPort & input; + OutputPort & output; + + size_t limit; + size_t offset; + size_t rows_read = 0; /// including the last read block + bool always_read_till_end; + + bool has_block = false; + bool block_processed = false; + Chunk current_chunk; + + UInt64 rows_before_limit_at_least = 0; + +public: + LimitTransform( + const Block & header, size_t limit, size_t offset, + bool always_read_till_end = false); + + String getName() const override { return "Limit"; } + + Status prepare() override; + void work() override; + + InputPort & getInputPort() { return input; } + OutputPort & getOutputPort() { return output; } + + UInt64 getRowsBeforeLimitAtLeast() const { return rows_before_limit_at_least; } +}; + +} diff --git a/dbms/src/Processors/NullSink.h b/dbms/src/Processors/NullSink.h new file mode 100644 index 00000000000..e4968daee29 --- /dev/null +++ b/dbms/src/Processors/NullSink.h @@ -0,0 +1,22 @@ +#pragma once +#include + +namespace DB +{ + +class NullSink : public IProcessor +{ +public: + explicit NullSink(Block header) : IProcessor({std::move(header)}, {}) {} + String getName() const override { return "NullSink"; } + + Status prepare() override + { + inputs.front().close(); + return Status::Finished; + } + + InputPort & getPort() { return inputs.front(); } +}; + +} diff --git a/dbms/src/Processors/Port.cpp b/dbms/src/Processors/Port.cpp new file mode 100644 index 00000000000..43f901aa4b9 --- /dev/null +++ b/dbms/src/Processors/Port.cpp @@ -0,0 +1,23 @@ +#include +#include + +namespace DB +{ + +void connect(OutputPort & output, InputPort & input) +{ + if (input.state || output.state) + throw Exception("Port is already connected", ErrorCodes::LOGICAL_ERROR); + + auto out_name = output.getProcessor().getName(); + auto in_name = input.getProcessor().getName(); + + assertBlocksHaveEqualStructure(input.getHeader(), output.getHeader(), " function connect between " + out_name + " and " + in_name); + + input.output_port = &output; + output.input_port = &input; + input.state = std::make_shared(); + output.state = input.state; +} + +} diff --git a/dbms/src/Processors/Port.h b/dbms/src/Processors/Port.h new file mode 100644 index 00000000000..99ad7df4b50 --- /dev/null +++ b/dbms/src/Processors/Port.h @@ -0,0 +1,430 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include + +namespace DB +{ + +class InputPort; +class OutputPort; +class IProcessor; + + +class Port +{ + friend void connect(OutputPort &, InputPort &); + friend class IProcessor; + +protected: + /// Shared state of two connected ports. + class State + { + public: + + struct Data + { + /// Note: std::variant can be used. But move constructor for it can't be inlined. + Chunk chunk; + std::exception_ptr exception; + }; + + private: + static std::uintptr_t getUInt(Data * data) { return reinterpret_cast(data); } + static Data * getPtr(std::uintptr_t data) { return reinterpret_cast(data); } + + public: + + /// Flags for Port state. + /// Will store them in least pointer bits. + + /// Port was set finished or closed. + static constexpr std::uintptr_t IS_FINISHED = 1; + /// Block is not needed right now, but may be will be needed later. + /// This allows to pause calculations if we are not sure that we need more data. + static constexpr std::uintptr_t IS_NEEDED = 2; + /// Check if port has data. + static constexpr std::uintptr_t HAS_DATA = 4; + + static constexpr std::uintptr_t FLAGS_MASK = IS_FINISHED | IS_NEEDED | HAS_DATA; + static constexpr std::uintptr_t PTR_MASK = ~FLAGS_MASK; + + /// Tiny smart ptr class for Data. Takes into account that ptr can have flags in least bits. + class DataPtr + { + public: + DataPtr() : data(new Data()) + { + if (unlikely((getUInt(data) & FLAGS_MASK) != 0)) + throw Exception("Not alignment memory for Port.", ErrorCodes::LOGICAL_ERROR); + } + /// Pointer can store flags in case of exception in swap. + ~DataPtr() { delete getPtr(getUInt(data) & PTR_MASK); } + + DataPtr(DataPtr const &) : data(new Data()) {} + DataPtr& operator=(DataPtr const &) = delete; + + Data * operator->() const { return data; } + Data & operator*() const { return *data; } + + Data * get() const { return data; } + explicit operator bool() const { return data; } + + Data * release() + { + Data * result = nullptr; + std::swap(result, data); + return result; + } + + uintptr_t ALWAYS_INLINE swap(std::atomic & value, std::uintptr_t flags, std::uintptr_t mask) + { + Data * expected = nullptr; + Data * desired = getPtr(flags | getUInt(data)); + + while (!value.compare_exchange_weak(expected, desired)) + desired = getPtr((getUInt(expected) & FLAGS_MASK & (~mask)) | flags | getUInt(data)); + + /// It's not very safe. In case of exception after exchange and before assigment we will get leak. + /// Don't know how to make it better. + data = getPtr(getUInt(expected) & PTR_MASK); + + return getUInt(expected) & FLAGS_MASK; + } + + private: + Data * data = nullptr; + }; + + /// Not finished, not needed, has not data. + State() : data(new Data()) + { + if (unlikely((getUInt(data) & FLAGS_MASK) != 0)) + throw Exception("Not alignment memory for Port.", ErrorCodes::LOGICAL_ERROR); + } + + ~State() + { + Data * desired = nullptr; + Data * expected = nullptr; + + while (!data.compare_exchange_weak(expected, desired)); + + expected = getPtr(getUInt(expected) & PTR_MASK); + delete expected; + } + + void ALWAYS_INLINE push(DataPtr & data_, std::uintptr_t & flags) + { + flags = data_.swap(data, HAS_DATA, HAS_DATA); + + /// It's possible to push data into finished port. Will just ignore it. + /// if (flags & IS_FINISHED) + /// throw Exception("Cannot push block to finished port.", ErrorCodes::LOGICAL_ERROR); + + /// It's possible to push data into port which is not needed now. + /// if ((flags & IS_NEEDED) == 0) + /// throw Exception("Cannot push block to port which is not needed.", ErrorCodes::LOGICAL_ERROR); + + if (unlikely(flags & HAS_DATA)) + throw Exception("Cannot push block to port which already has data.", ErrorCodes::LOGICAL_ERROR); + } + + void ALWAYS_INLINE pull(DataPtr & data_, std::uintptr_t & flags) + { + flags = data_.swap(data, 0, HAS_DATA); + + /// It's ok to check because this flag can be changed only by pulling thread. + if (unlikely((flags & IS_NEEDED) == 0)) + throw Exception("Cannot pull block from port which is not needed.", ErrorCodes::LOGICAL_ERROR); + + if (unlikely((flags & HAS_DATA) == 0)) + throw Exception("Cannot pull block from port which has no data.", ErrorCodes::LOGICAL_ERROR); + } + + std::uintptr_t ALWAYS_INLINE setFlags(std::uintptr_t flags, std::uintptr_t mask) + { + Data * expected = nullptr; + Data * desired = getPtr(flags); + + while (!data.compare_exchange_weak(expected, desired)) + desired = getPtr((getUInt(expected) & FLAGS_MASK & (~mask)) | flags | (getUInt(expected) & PTR_MASK)); + + return getUInt(expected) & FLAGS_MASK; + } + + std::uintptr_t ALWAYS_INLINE getFlags() const + { + return getUInt(data.load()) & FLAGS_MASK; + } + + private: + std::atomic data; + }; + + Block header; + std::shared_ptr state; + + /// This object is only used for data exchange between port and shared state. + State::DataPtr data; + + IProcessor * processor = nullptr; + +public: + using Data = State::Data; + + Port(Block header) : header(std::move(header)) {} + Port(Block header, IProcessor * processor) : header(std::move(header)), processor(processor) {} + + const Block & getHeader() const { return header; } + bool ALWAYS_INLINE isConnected() const { return state != nullptr; } + + void ALWAYS_INLINE assumeConnected() const + { + if (unlikely(!isConnected())) + throw Exception("Port is not connected", ErrorCodes::LOGICAL_ERROR); + } + + bool ALWAYS_INLINE hasData() const + { + assumeConnected(); + return state->getFlags() & State::HAS_DATA; + } + + IProcessor & getProcessor() + { + if (!processor) + throw Exception("Port does not belong to Processor", ErrorCodes::LOGICAL_ERROR); + return *processor; + } + + const IProcessor & getProcessor() const + { + if (!processor) + throw Exception("Port does not belong to Processor", ErrorCodes::LOGICAL_ERROR); + return *processor; + } +}; + +/// Invariants: +/// * If you close port, it isFinished(). +/// * If port isFinished(), you can do nothing with it. +/// * If port is not needed, you can only setNeeded() or close() it. +/// * You can pull only if port hasData(). +class InputPort : public Port +{ + friend void connect(OutputPort &, InputPort &); + +private: + OutputPort * output_port = nullptr; + + /// If version was set, it will be increased on each pull. + UInt64 * version = nullptr; + + mutable bool is_finished = false; + +public: + using Port::Port; + + void setVersion(UInt64 * value) { version = value; } + + Data ALWAYS_INLINE pullData() + { + if (version) + ++(*version); + + assumeConnected(); + + std::uintptr_t flags = 0; + state->pull(data, flags); + + is_finished = flags & State::IS_FINISHED; + + if (unlikely(!data->exception && data->chunk.getNumColumns() != header.columns())) + { + auto & chunk = data->chunk; + + String msg = "Invalid number of columns in chunk pulled from OutputPort. Expected " + + std::to_string(header.columns()) + ", found " + std::to_string(chunk.getNumColumns()) + '\n'; + + msg += "Header: " + header.dumpStructure() + '\n'; + msg += "Chunk: " + chunk.dumpStructure() + '\n'; + + throw Exception(msg, ErrorCodes::LOGICAL_ERROR); + } + + return std::move(*data); + } + + Chunk ALWAYS_INLINE pull() + { + auto data_ = pullData(); + + if (data_.exception) + std::rethrow_exception(data_.exception); + + return std::move(data_.chunk); + } + + bool ALWAYS_INLINE isFinished() const + { + assumeConnected(); + + if (is_finished) + return true; + + auto flags = state->getFlags(); + + is_finished = (flags & State::IS_FINISHED) && ((flags & State::HAS_DATA) == 0); + + return is_finished; + } + + void ALWAYS_INLINE setNeeded() + { + assumeConnected(); + + if ((state->setFlags(State::IS_NEEDED, State::IS_NEEDED) & State::IS_NEEDED) == 0 && version) + ++(*version); + } + + void ALWAYS_INLINE setNotNeeded() + { + assumeConnected(); + state->setFlags(0, State::IS_NEEDED); + } + + void ALWAYS_INLINE close() + { + assumeConnected(); + + if ((state->setFlags(State::IS_FINISHED, State::IS_FINISHED) & State::IS_FINISHED) == 0 && version) + ++(*version); + + is_finished = true; + } + + OutputPort & getOutputPort() + { + assumeConnected(); + return *output_port; + } + + const OutputPort & getOutputPort() const + { + assumeConnected(); + return *output_port; + } +}; + + +/// Invariants: +/// * If you finish port, it isFinished(). +/// * If port isFinished(), you can do nothing with it. +/// * If port not isNeeded(), you can only finish() it. +/// * You can hush only if port doesn't hasData(). +class OutputPort : public Port +{ + friend void connect(OutputPort &, InputPort &); + +private: + InputPort * input_port = nullptr; + + /// If version was set, it will be increased on each push. + UInt64 * version = nullptr; + +public: + using Port::Port; + + void setVersion(UInt64 * value) { version = value; } + + void ALWAYS_INLINE push(Chunk chunk) + { + pushData({.chunk = std::move(chunk), .exception = {}}); + } + + void ALWAYS_INLINE push(std::exception_ptr exception) + { + pushData({.chunk = {}, .exception = std::move(exception)}); + } + + void ALWAYS_INLINE pushData(Data data_) + { + if (unlikely(!data_.exception && data_.chunk.getNumColumns() != header.columns())) + { + String msg = "Invalid number of columns in chunk pushed to OutputPort. Expected " + + std::to_string(header.columns()) + + ", found " + std::to_string(data_.chunk.getNumColumns()) + '\n'; + + msg += "Header: " + header.dumpStructure() + '\n'; + msg += "Chunk: " + data_.chunk.dumpStructure() + '\n'; + + throw Exception(msg, ErrorCodes::LOGICAL_ERROR); + } + + if (version) + ++(*version); + + assumeConnected(); + + std::uintptr_t flags = 0; + *data = std::move(data_); + state->push(data, flags); + } + + void ALWAYS_INLINE finish() + { + assumeConnected(); + + auto flags = state->setFlags(State::IS_FINISHED, State::IS_FINISHED); + + if (version && (flags & State::IS_FINISHED) == 0) + ++(*version); + } + + bool ALWAYS_INLINE isNeeded() const + { + assumeConnected(); + return state->getFlags() & State::IS_NEEDED; + } + + bool ALWAYS_INLINE isFinished() const + { + assumeConnected(); + return state->getFlags() & State::IS_FINISHED; + } + + bool ALWAYS_INLINE canPush() const + { + assumeConnected(); + auto flags = state->getFlags(); + return (flags & State::IS_NEEDED) && ((flags & State::HAS_DATA) == 0); + } + + InputPort & getInputPort() + { + assumeConnected(); + return *input_port; + } + + const InputPort & getInputPort() const + { + assumeConnected(); + return *input_port; + } +}; + + +using InputPorts = std::list; +using OutputPorts = std::list; + + +void connect(OutputPort & output, InputPort & input); + +} diff --git a/dbms/src/Processors/QueryPipeline.cpp b/dbms/src/Processors/QueryPipeline.cpp new file mode 100644 index 00000000000..cb9be19088c --- /dev/null +++ b/dbms/src/Processors/QueryPipeline.cpp @@ -0,0 +1,634 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace DB +{ + +void QueryPipeline::checkInitialized() +{ + if (!initialized()) + throw Exception("QueryPipeline wasn't initialized.", ErrorCodes::LOGICAL_ERROR); +} + +void QueryPipeline::checkSource(const ProcessorPtr & source, bool can_have_totals) +{ + if (!source->getInputs().empty()) + throw Exception("Source for query pipeline shouldn't have any input, but " + source->getName() + " has " + + toString(source->getInputs().size()) + " inputs.", ErrorCodes::LOGICAL_ERROR); + + if (source->getOutputs().empty()) + throw Exception("Source for query pipeline should have single output, but it doesn't have any", + ErrorCodes::LOGICAL_ERROR); + + if (!can_have_totals && source->getOutputs().size() != 1) + throw Exception("Source for query pipeline should have single output, but " + source->getName() + " has " + + toString(source->getOutputs().size()) + " outputs.", ErrorCodes::LOGICAL_ERROR); + + if (source->getOutputs().size() > 2) + throw Exception("Source for query pipeline should have 1 or 2 outputs, but " + source->getName() + " has " + + toString(source->getOutputs().size()) + " outputs.", ErrorCodes::LOGICAL_ERROR); +} + +void QueryPipeline::init(Processors sources) +{ + if (initialized()) + throw Exception("Pipeline has already been initialized.", ErrorCodes::LOGICAL_ERROR); + + if (sources.empty()) + throw Exception("Can't initialize pipeline with empty source list.", ErrorCodes::LOGICAL_ERROR); + + std::vector totals; + + for (auto & source : sources) + { + checkSource(source, true); + + auto & header = source->getOutputs().front().getHeader(); + + if (current_header) + assertBlocksHaveEqualStructure(current_header, header, "QueryPipeline"); + else + current_header = header; + + if (source->getOutputs().size() > 1) + { + assertBlocksHaveEqualStructure(current_header, source->getOutputs().back().getHeader(), "QueryPipeline"); + totals.emplace_back(&source->getOutputs().back()); + } + + streams.emplace_back(&source->getOutputs().front()); + processors.emplace_back(std::move(source)); + } + + if (!totals.empty()) + { + if (totals.size() == 1) + totals_having_port = totals.back(); + else + { + auto resize = std::make_shared(current_header, totals.size(), 1); + auto in = resize->getInputs().begin(); + for (auto & total : totals) + connect(*total, *(in++)); + + totals_having_port = &resize->getOutputs().front(); + processors.emplace_back(std::move(resize)); + } + } +} + +static ProcessorPtr callProcessorGetter( + const Block & header, const QueryPipeline::ProcessorGetter & getter, QueryPipeline::StreamType) +{ + return getter(header); +} + +static ProcessorPtr callProcessorGetter( + const Block & header, const QueryPipeline::ProcessorGetterWithStreamKind & getter, QueryPipeline::StreamType kind) +{ + return getter(header, kind); +} + +template +void QueryPipeline::addSimpleTransformImpl(const TProcessorGetter & getter) +{ + checkInitialized(); + + Block header; + + auto add_transform = [&](OutputPort *& stream, StreamType stream_type) + { + if (!stream) + return; + + auto transform = callProcessorGetter(stream->getHeader(), getter, stream_type); + + if (transform) + { + if (transform->getInputs().size() != 1) + throw Exception("Processor for query pipeline transform should have single input, " + "but " + transform->getName() + " has " + + toString(transform->getInputs().size()) + " inputs.", ErrorCodes::LOGICAL_ERROR); + + if (transform->getOutputs().size() != 1) + throw Exception("Processor for query pipeline transform should have single output, " + "but " + transform->getName() + " has " + + toString(transform->getOutputs().size()) + " outputs.", ErrorCodes::LOGICAL_ERROR); + } + + auto & out_header = transform ? transform->getOutputs().front().getHeader() + : stream->getHeader(); + + if (stream_type != StreamType::Totals) + { + if (header) + assertBlocksHaveEqualStructure(header, out_header, "QueryPipeline"); + else + header = out_header; + } + + if (transform) + { + connect(*stream, transform->getInputs().front()); + stream = &transform->getOutputs().front(); + processors.emplace_back(std::move(transform)); + } + }; + + for (auto & stream : streams) + add_transform(stream, StreamType::Main); + + add_transform(delayed_stream_port, StreamType::Main); + add_transform(totals_having_port, StreamType::Totals); + add_transform(extremes_port, StreamType::Extremes); + + current_header = std::move(header); +} + +void QueryPipeline::addSimpleTransform(const ProcessorGetter & getter) +{ + addSimpleTransformImpl(getter); +} + +void QueryPipeline::addSimpleTransform(const ProcessorGetterWithStreamKind & getter) +{ + addSimpleTransformImpl(getter); +} + +void QueryPipeline::addPipe(Processors pipe) +{ + checkInitialized(); + concatDelayedStream(); + + if (pipe.empty()) + throw Exception("Can't add empty processors list to QueryPipeline.", ErrorCodes::LOGICAL_ERROR); + + auto & first = pipe.front(); + auto & last = pipe.back(); + + auto num_inputs = first->getInputs().size(); + + if (num_inputs != streams.size()) + throw Exception("Can't add processors to QueryPipeline because first processor has " + toString(num_inputs) + + " input ports, but QueryPipeline has " + toString(streams.size()) + " streams.", + ErrorCodes::LOGICAL_ERROR); + + auto stream = streams.begin(); + for (auto & input : first->getInputs()) + connect(**(stream++), input); + + Block header; + streams.clear(); + streams.reserve(last->getOutputs().size()); + for (auto & output : last->getOutputs()) + { + streams.emplace_back(&output); + if (header) + assertBlocksHaveEqualStructure(header, output.getHeader(), "QueryPipeline"); + else + header = output.getHeader(); + } + + processors.insert(processors.end(), pipe.begin(), pipe.end()); + current_header = std::move(header); +} + +void QueryPipeline::addDelayedStream(ProcessorPtr source) +{ + checkInitialized(); + + if (delayed_stream_port) + throw Exception("QueryPipeline already has stream with non joined data.", ErrorCodes::LOGICAL_ERROR); + + checkSource(source, false); + assertBlocksHaveEqualStructure(current_header, source->getOutputs().front().getHeader(), "QueryPipeline"); + + delayed_stream_port = &source->getOutputs().front(); + processors.emplace_back(std::move(source)); +} + +void QueryPipeline::concatDelayedStream() +{ + if (!delayed_stream_port) + return; + + auto resize = std::make_shared(current_header, getNumMainStreams(), 1); + auto stream = streams.begin(); + for (auto & input : resize->getInputs()) + connect(**(stream++), input); + + auto concat = std::make_shared(current_header, 2); + connect(resize->getOutputs().front(), concat->getInputs().front()); + connect(*delayed_stream_port, concat->getInputs().back()); + + streams = { &concat->getOutputs().front() }; + processors.emplace_back(std::move(resize)); + processors.emplace_back(std::move(concat)); + + delayed_stream_port = nullptr; +} + +void QueryPipeline::resize(size_t num_streams) +{ + checkInitialized(); + concatDelayedStream(); + + if (num_streams == getNumStreams()) + return; + + has_resize = true; + + auto resize = std::make_shared(current_header, getNumStreams(), num_streams); + auto stream = streams.begin(); + for (auto & input : resize->getInputs()) + connect(**(stream++), input); + + streams.clear(); + streams.reserve(num_streams); + for (auto & output : resize->getOutputs()) + streams.emplace_back(&output); + + processors.emplace_back(std::move(resize)); +} + +void QueryPipeline::addTotalsHavingTransform(ProcessorPtr transform) +{ + checkInitialized(); + + if (!typeid_cast(transform.get())) + throw Exception("TotalsHavingTransform expected for QueryPipeline::addTotalsHavingTransform.", + ErrorCodes::LOGICAL_ERROR); + + if (totals_having_port) + throw Exception("Totals having transform was already added to pipeline.", ErrorCodes::LOGICAL_ERROR); + + resize(1); + + connect(*streams.front(), transform->getInputs().front()); + + auto & outputs = transform->getOutputs(); + + streams = { &outputs.front() }; + totals_having_port = &outputs.back(); + current_header = outputs.front().getHeader(); + processors.emplace_back(std::move(transform)); +} + +void QueryPipeline::addDefaultTotals() +{ + checkInitialized(); + + if (totals_having_port) + throw Exception("Totals having transform was already added to pipeline.", ErrorCodes::LOGICAL_ERROR); + + Columns columns; + columns.reserve(current_header.columns()); + + for (size_t i = 0; i < current_header.columns(); ++i) + { + auto column = current_header.getByPosition(i).type->createColumn(); + column->insertDefault(); + columns.emplace_back(std::move(column)); + } + + auto source = std::make_shared(current_header, Chunk(std::move(columns), 1)); + totals_having_port = &source->getPort(); + processors.emplace_back(source); +} + +void QueryPipeline::addTotals(ProcessorPtr source) +{ + checkInitialized(); + + if (totals_having_port) + throw Exception("Totals having transform was already added to pipeline.", ErrorCodes::LOGICAL_ERROR); + + checkSource(source, false); + assertBlocksHaveEqualStructure(current_header, source->getOutputs().front().getHeader(), "QueryPipeline"); + + totals_having_port = &source->getOutputs().front(); + processors.emplace_back(source); +} + +void QueryPipeline::dropTotalsIfHas() +{ + if (totals_having_port) + { + auto null_sink = std::make_shared(totals_having_port->getHeader()); + connect(*totals_having_port, null_sink->getPort()); + processors.emplace_back(std::move(null_sink)); + totals_having_port = nullptr; + } +} + +void QueryPipeline::addExtremesTransform(ProcessorPtr transform) +{ + checkInitialized(); + + if (!typeid_cast(transform.get())) + throw Exception("ExtremesTransform expected for QueryPipeline::addExtremesTransform.", + ErrorCodes::LOGICAL_ERROR); + + if (extremes_port) + throw Exception("Extremes transform was already added to pipeline.", ErrorCodes::LOGICAL_ERROR); + + if (getNumStreams() != 1) + throw Exception("Cant't add Extremes transform because pipeline is expected to have single stream, " + "but it has " + toString(getNumStreams()) + " streams.", ErrorCodes::LOGICAL_ERROR); + + connect(*streams.front(), transform->getInputs().front()); + + auto & outputs = transform->getOutputs(); + + streams = { &outputs.front() }; + extremes_port = &outputs.back(); + current_header = outputs.front().getHeader(); + processors.emplace_back(std::move(transform)); +} + +void QueryPipeline::addCreatingSetsTransform(ProcessorPtr transform) +{ + checkInitialized(); + + if (!typeid_cast(transform.get())) + throw Exception("CreatingSetsTransform expected for QueryPipeline::addExtremesTransform.", + ErrorCodes::LOGICAL_ERROR); + + resize(1); + + auto concat = std::make_shared(current_header, 2); + connect(transform->getOutputs().front(), concat->getInputs().front()); + connect(*streams.back(), concat->getInputs().back()); + + streams = { &concat->getOutputs().front() }; + processors.emplace_back(std::move(transform)); + processors.emplace_back(std::move(concat)); +} + +void QueryPipeline::setOutput(ProcessorPtr output) +{ + checkInitialized(); + + auto * format = dynamic_cast(output.get()); + + if (!format) + throw Exception("IOutputFormat processor expected for QueryPipeline::setOutput.", ErrorCodes::LOGICAL_ERROR); + + if (output_format) + throw Exception("QueryPipeline already has output.", ErrorCodes::LOGICAL_ERROR); + + output_format = format; + + resize(1); + + auto & main = format->getPort(IOutputFormat::PortKind::Main); + auto & totals = format->getPort(IOutputFormat::PortKind::Totals); + auto & extremes = format->getPort(IOutputFormat::PortKind::Extremes); + + if (!totals_having_port) + { + auto null_source = std::make_shared(totals.getHeader()); + totals_having_port = &null_source->getPort(); + processors.emplace_back(std::move(null_source)); + } + + if (!extremes_port) + { + auto null_source = std::make_shared(extremes.getHeader()); + extremes_port = &null_source->getPort(); + processors.emplace_back(std::move(null_source)); + } + + processors.emplace_back(std::move(output)); + + connect(*streams.front(), main); + connect(*totals_having_port, totals); + connect(*extremes_port, extremes); +} + +void QueryPipeline::unitePipelines( + std::vector && pipelines, const Block & common_header, const Context & context) +{ + checkInitialized(); + concatDelayedStream(); + + addSimpleTransform([&](const Block & header) + { + return std::make_shared( + header, common_header, ConvertingTransform::MatchColumnsMode::Position, context); + }); + + std::vector extremes; + + for (auto & pipeline : pipelines) + { + pipeline.checkInitialized(); + pipeline.concatDelayedStream(); + + pipeline.addSimpleTransform([&](const Block & header) + { + return std::make_shared( + header, common_header, ConvertingTransform::MatchColumnsMode::Position, context); + }); + + if (pipeline.extremes_port) + { + auto converting = std::make_shared( + pipeline.current_header, common_header, ConvertingTransform::MatchColumnsMode::Position, context); + + connect(*pipeline.extremes_port, converting->getInputPort()); + extremes.push_back(&converting->getOutputPort()); + processors.push_back(std::move(converting)); + } + + /// Take totals only from first port. + if (pipeline.totals_having_port) + { + if (!totals_having_port) + { + auto converting = std::make_shared( + pipeline.current_header, common_header, ConvertingTransform::MatchColumnsMode::Position, context); + + connect(*pipeline.totals_having_port, converting->getInputPort()); + totals_having_port = &converting->getOutputPort(); + processors.push_back(std::move(converting)); + } + else + pipeline.dropTotalsIfHas(); + } + + processors.insert(processors.end(), pipeline.processors.begin(), pipeline.processors.end()); + streams.insert(streams.end(), pipeline.streams.begin(), pipeline.streams.end()); + } + + if (!extremes.empty()) + { + size_t num_inputs = extremes.size() + (extremes_port ? 1u : 0u); + + if (num_inputs == 1) + extremes_port = extremes.front(); + else + { + /// Add extra processor for extremes. + auto resize = std::make_shared(current_header, num_inputs, 1); + auto input = resize->getInputs().begin(); + + if (extremes_port) + connect(*extremes_port, *(input++)); + + for (auto & output : extremes) + connect(*output, *(input++)); + + auto transform = std::make_shared(current_header); + extremes_port = &transform->getOutputPort(); + + connect(resize->getOutputs().front(), transform->getInputPort()); + processors.emplace_back(std::move(transform)); + } + } +} + +void QueryPipeline::setProgressCallback(const ProgressCallback & callback) +{ + for (auto & processor : processors) + { + if (auto * source = typeid_cast(processor.get())) + source->getStream().setProgressCallback(callback); + + if (auto * source = typeid_cast(processor.get())) + source->setProgressCallback(callback); + } +} + +void QueryPipeline::setProcessListElement(QueryStatus * elem) +{ + for (auto & processor : processors) + { + if (auto * source = typeid_cast(processor.get())) + source->getStream().setProcessListElement(elem); + + if (auto * source = typeid_cast(processor.get())) + source->setProcessListElement(elem); + } +} + +void QueryPipeline::finalize() +{ + checkInitialized(); + + if (!output_format) + throw Exception("Cannot finalize pipeline because it doesn't have output.", ErrorCodes::LOGICAL_ERROR); + + calcRowsBeforeLimit(); +} + +void QueryPipeline::calcRowsBeforeLimit() +{ + /// TODO get from Remote + + UInt64 rows_before_limit_at_least = 0; + UInt64 rows_before_limit = 0; + + bool has_limit = false; + bool has_partial_sorting = false; + + std::unordered_set visited; + + struct QueuedEntry + { + IProcessor * processor; + bool visited_limit; + }; + + std::queue queue; + + queue.push({ output_format, false }); + visited.emplace(output_format); + + while (!queue.empty()) + { + auto processor = queue.front().processor; + auto visited_limit = queue.front().visited_limit; + queue.pop(); + + if (!visited_limit) + { + if (auto * limit = typeid_cast(processor)) + { + has_limit = visited_limit = true; + rows_before_limit_at_least += limit->getRowsBeforeLimitAtLeast(); + } + + if (auto * source = typeid_cast(processor)) + { + auto & info = source->getStream().getProfileInfo(); + if (info.hasAppliedLimit()) + { + has_limit = visited_limit = true; + rows_before_limit_at_least += info.getRowsBeforeLimit(); + } + } + } + + if (auto * sorting = typeid_cast(processor)) + { + has_partial_sorting = true; + rows_before_limit += sorting->getNumReadRows(); + + /// Don't go to children. Take rows_before_limit from last PartialSortingTransform. + /// continue; + } + + /// Skip totals and extremes port for output format. + if (auto * format = dynamic_cast(processor)) + { + auto * child_processor = &format->getPort(IOutputFormat::PortKind::Main).getOutputPort().getProcessor(); + if (visited.emplace(child_processor).second) + queue.push({ child_processor, visited_limit }); + + continue; + } + + for (auto & child_port : processor->getInputs()) + { + auto * child_processor = &child_port.getOutputPort().getProcessor(); + if (visited.emplace(child_processor).second) + queue.push({ child_processor, visited_limit }); + } + } + + /// Get num read rows from PartialSortingTransform if have it. + if (has_limit) + output_format->setRowsBeforeLimit(has_partial_sorting ? rows_before_limit : rows_before_limit_at_least); +} + +PipelineExecutorPtr QueryPipeline::execute() +{ + checkInitialized(); + + if (!output_format) + throw Exception("Cannot execute pipeline because it doesn't have output.", ErrorCodes::LOGICAL_ERROR); + + return std::make_shared(processors); +} + +} diff --git a/dbms/src/Processors/QueryPipeline.h b/dbms/src/Processors/QueryPipeline.h new file mode 100644 index 00000000000..e932360971a --- /dev/null +++ b/dbms/src/Processors/QueryPipeline.h @@ -0,0 +1,119 @@ +#pragma once +#include +#include + +#include +#include + + +namespace DB +{ + +class TableStructureReadLock; +using TableStructureReadLockPtr = std::shared_ptr; +using TableStructureReadLocks = std::vector; + +class Context; + +class IOutputFormat; + +class QueryPipeline +{ +public: + QueryPipeline() = default; + + /// Each source must have single output port and no inputs. All outputs must have same header. + void init(Processors sources); + bool initialized() { return !processors.empty(); } + + enum class StreamType + { + Main = 0, + Totals, + Extremes, + }; + + using ProcessorGetter = std::function; + using ProcessorGetterWithStreamKind = std::function; + + void addSimpleTransform(const ProcessorGetter & getter); + void addSimpleTransform(const ProcessorGetterWithStreamKind & getter); + void addPipe(Processors pipe); + void addTotalsHavingTransform(ProcessorPtr transform); + void addExtremesTransform(ProcessorPtr transform); + void addCreatingSetsTransform(ProcessorPtr transform); + void setOutput(ProcessorPtr output); + + /// Add totals which returns one chunk with single row with defaults. + void addDefaultTotals(); + + /// Add already calculated totals. + void addTotals(ProcessorPtr source); + + void dropTotalsIfHas(); + + /// Will read from this stream after all data was read from other streams. + void addDelayedStream(ProcessorPtr source); + bool hasDelayedStream() const { return delayed_stream_port; } + /// Check if resize transform was used. (In that case another distinct transform will be added). + bool hasMixedStreams() const { return has_resize || hasMoreThanOneStream(); } + + void resize(size_t num_streams); + + void unitePipelines(std::vector && pipelines, const Block & common_header, const Context & context); + + PipelineExecutorPtr execute(); + + size_t getNumStreams() const { return streams.size() + (hasDelayedStream() ? 1 : 0); } + size_t getNumMainStreams() const { return streams.size(); } + + bool hasMoreThanOneStream() const { return getNumStreams() > 1; } + bool hasTotals() const { return totals_having_port != nullptr; } + + const Block & getHeader() const { return current_header; } + + void addTableLock(const TableStructureReadLockPtr & lock) { table_locks.push_back(lock); } + + /// For compatibility with IBlockInputStream. + void setProgressCallback(const ProgressCallback & callback); + void setProcessListElement(QueryStatus * elem); + + /// Call after execution. + void finalize(); + +private: + + /// All added processors. + Processors processors; + + /// Port for each independent "stream". + std::vector streams; + + /// Special ports for extremes and totals having. + OutputPort * totals_having_port = nullptr; + OutputPort * extremes_port = nullptr; + + /// Special port for delayed stream. + OutputPort * delayed_stream_port = nullptr; + + /// If resize processor was added to pipeline. + bool has_resize = false; + + /// Common header for each stream. + Block current_header; + + TableStructureReadLocks table_locks; + + IOutputFormat * output_format = nullptr; + + void checkInitialized(); + void checkSource(const ProcessorPtr & source, bool can_have_totals); + void concatDelayedStream(); + + template + void addSimpleTransformImpl(const TProcessorGetter & getter); + + void calcRowsBeforeLimit(); +}; + +} diff --git a/dbms/src/Processors/QueueBuffer.h b/dbms/src/Processors/QueueBuffer.h new file mode 100644 index 00000000000..d81f7e779a3 --- /dev/null +++ b/dbms/src/Processors/QueueBuffer.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include + + +namespace DB +{ + +class QueueBuffer : public IAccumulatingTransform +{ +private: + std::queue chunks; +public: + String getName() const override { return "QueueBuffer"; } + + QueueBuffer(Block header) + : IAccumulatingTransform(header, header) + { + } + + void consume(Chunk block) override + { + chunks.push(std::move(block)); + } + + Chunk generate() override + { + if (chunks.empty()) + return {}; + + auto res = std::move(chunks.front()); + chunks.pop(); + return res; + } +}; + +} diff --git a/dbms/src/Processors/ResizeProcessor.cpp b/dbms/src/Processors/ResizeProcessor.cpp new file mode 100644 index 00000000000..b3cb3a1735d --- /dev/null +++ b/dbms/src/Processors/ResizeProcessor.cpp @@ -0,0 +1,157 @@ +#include + + +namespace DB +{ + +ResizeProcessor::Status ResizeProcessor::prepare() +{ + bool is_first_output = true; + auto output_end = current_output; + + bool all_outs_full_or_unneeded = true; + bool all_outs_finished = true; + + bool is_first_input = true; + auto input_end = current_input; + + bool all_inputs_finished = true; + + auto is_end_input = [&]() { return !is_first_input && current_input == input_end; }; + auto is_end_output = [&]() { return !is_first_output && current_output == output_end; }; + + auto inc_current_input = [&]() + { + is_first_input = false; + ++current_input; + + if (current_input == inputs.end()) + current_input = inputs.begin(); + }; + + auto inc_current_output = [&]() + { + is_first_output = false; + ++current_output; + + if (current_output == outputs.end()) + current_output = outputs.begin(); + }; + + /// Find next output where can push. + auto get_next_out = [&, this]() -> OutputPorts::iterator + { + while (!is_end_output()) + { + if (!current_output->isFinished()) + { + all_outs_finished = false; + + if (current_output->canPush()) + { + all_outs_full_or_unneeded = false; + auto res_output = current_output; + inc_current_output(); + return res_output; + } + } + + inc_current_output(); + } + + return outputs.end(); + }; + + /// Find next input from where can pull. + auto get_next_input = [&, this]() -> InputPorts::iterator + { + while (!is_end_input()) + { + if (!current_input->isFinished()) + { + all_inputs_finished = false; + + current_input->setNeeded(); + if (current_input->hasData()) + { + auto res_input = current_input; + inc_current_input(); + return res_input; + } + } + + inc_current_input(); + } + + return inputs.end(); + }; + + auto get_status_if_no_outputs = [&]() -> Status + { + if (all_outs_finished) + { + for (auto & in : inputs) + in.close(); + + return Status::Finished; + } + + if (all_outs_full_or_unneeded) + { + for (auto & in : inputs) + in.setNotNeeded(); + + return Status::PortFull; + } + + /// Now, we pushed to output, and it must be full. + return Status::PortFull; + }; + + auto get_status_if_no_inputs = [&]() -> Status + { + if (all_inputs_finished) + { + for (auto & out : outputs) + out.finish(); + + return Status::Finished; + } + + return Status::NeedData; + }; + + /// Set all inputs needed in order to evenly process them. + /// Otherwise, in case num_outputs < num_inputs and chunks are consumed faster than produced, + /// some inputs can be skipped. +// auto set_all_unprocessed_inputs_needed = [&]() +// { +// for (; cur_input != inputs.end(); ++cur_input) +// if (!cur_input->isFinished()) +// cur_input->setNeeded(); +// }; + + while (!is_end_input() && !is_end_output()) + { + auto output = get_next_out(); + auto input = get_next_input(); + + if (output == outputs.end()) + return get_status_if_no_outputs(); + + + if (input == inputs.end()) + return get_status_if_no_inputs(); + + output->push(input->pull()); + } + + if (is_end_input()) + return get_status_if_no_outputs(); + + /// cur_input == inputs_end() + return get_status_if_no_inputs(); +} + +} + diff --git a/dbms/src/Processors/ResizeProcessor.h b/dbms/src/Processors/ResizeProcessor.h new file mode 100644 index 00000000000..67574c384a1 --- /dev/null +++ b/dbms/src/Processors/ResizeProcessor.h @@ -0,0 +1,40 @@ +#pragma once + +#include + + +namespace DB +{ + +/** Has arbitary non zero number of inputs and arbitary non zero number of outputs. + * All of them have the same structure. + * + * Pulls data from arbitary input (whenever it is ready) and pushes it to arbitary output (whenever is is not full). + * Doesn't do any heavy calculations. + * Doesn't preserve an order of data. + * + * Examples: + * - union data from multiple inputs to single output - to serialize data that was processed in parallel. + * - split data from single input to multiple outputs - to allow further parallel processing. + */ +class ResizeProcessor : public IProcessor +{ +public: + /// TODO Check that there is non zero number of inputs and outputs. + ResizeProcessor(const Block & header, size_t num_inputs, size_t num_outputs) + : IProcessor(InputPorts(num_inputs, header), OutputPorts(num_outputs, header)) + , current_input(inputs.begin()) + , current_output(outputs.begin()) + { + } + + String getName() const override { return "Resize"; } + + Status prepare() override; + +private: + InputPorts::iterator current_input; + OutputPorts::iterator current_output; +}; + +} diff --git a/dbms/src/Processors/Sources/NullSource.h b/dbms/src/Processors/Sources/NullSource.h new file mode 100644 index 00000000000..d1f0ec5e6ca --- /dev/null +++ b/dbms/src/Processors/Sources/NullSource.h @@ -0,0 +1,18 @@ +#pragma once +#include + + +namespace DB +{ + +class NullSource : public ISource +{ +public: + explicit NullSource(Block header) : ISource(std::move(header)) {} + String getName() const override { return "NullSource"; } + +protected: + Chunk generate() override { return Chunk(); } +}; + +} diff --git a/dbms/src/Processors/Sources/SourceFromInputStream.cpp b/dbms/src/Processors/Sources/SourceFromInputStream.cpp new file mode 100644 index 00000000000..d9d74a5cde6 --- /dev/null +++ b/dbms/src/Processors/Sources/SourceFromInputStream.cpp @@ -0,0 +1,142 @@ +#include +#include +#include +#include + +namespace DB +{ + +SourceFromInputStream::SourceFromInputStream(BlockInputStreamPtr stream_, bool force_add_aggregating_info) + : ISource(stream_->getHeader()) + , force_add_aggregating_info(force_add_aggregating_info) + , stream(std::move(stream_)) +{ + auto & sample = getPort().getHeader(); + for (auto & type : sample.getDataTypes()) + if (typeid_cast(type.get())) + has_aggregate_functions = true; +} + +void SourceFromInputStream::addTotalsPort() +{ + if (has_totals_port) + throw Exception("Totals port was already added for SourceFromInputStream.", ErrorCodes::LOGICAL_ERROR); + + outputs.emplace_back(outputs.front().getHeader(), this); + has_totals_port = true; +} + +IProcessor::Status SourceFromInputStream::prepare() +{ + auto status = ISource::prepare(); + + if (status == Status::Finished) + { + is_generating_finished = true; + + /// Read postfix and get totals if needed. + if (!is_stream_finished) + return Status::Ready; + + if (has_totals_port) + { + auto & totals_out = outputs.back(); + + if (totals_out.isFinished()) + return Status::Finished; + + if (has_totals) + { + if (!totals_out.canPush()) + return Status::PortFull; + + totals_out.push(std::move(totals)); + has_totals = false; + } + + totals_out.finish(); + } + } + + return status; +} + +void SourceFromInputStream::work() +{ + if (!is_generating_finished) + { + try + { + ISource::work(); + } + catch (...) + { + /// Won't read suffix in case of exception. + is_stream_finished = true; + throw; + } + + return; + } + + if (is_stream_finished) + return; + + /// Don't cancel for RemoteBlockInputStream (otherwise readSuffix can stack) + if (!typeid_cast(stream.get())) + stream->cancel(false); + + stream->readSuffix(); + + if (auto totals_block = stream->getTotals()) + { + totals.setColumns(totals_block.getColumns(), 1); + has_totals = true; + } + + is_stream_finished = true; +} + +Chunk SourceFromInputStream::generate() +{ + if (is_stream_finished) + return {}; + + if (!is_stream_started) + { + stream->readPrefix(); + is_stream_started = true; + } + + auto block = stream->read(); + if (!block) + { + stream->readSuffix(); + + if (auto totals_block = stream->getTotals()) + { + totals.setColumns(totals_block.getColumns(), 1); + has_totals = true; + } + + is_stream_finished = true; + return {}; + } + + assertBlocksHaveEqualStructure(getPort().getHeader(), block, "SourceFromInputStream"); + + UInt64 num_rows = block.rows(); + Chunk chunk(block.getColumns(), num_rows); + + if (force_add_aggregating_info || has_aggregate_functions) + { + auto info = std::make_shared(); + info->bucket_num = block.info.bucket_num; + info->is_overflows = block.info.is_overflows; + chunk.setChunkInfo(std::move(info)); + } + + return chunk; +} + +} diff --git a/dbms/src/Processors/Sources/SourceFromInputStream.h b/dbms/src/Processors/Sources/SourceFromInputStream.h new file mode 100644 index 00000000000..46e0b3fb04b --- /dev/null +++ b/dbms/src/Processors/Sources/SourceFromInputStream.h @@ -0,0 +1,39 @@ +#pragma once +#include + +namespace DB +{ + +class IBlockInputStream; +using BlockInputStreamPtr = std::shared_ptr; + +class SourceFromInputStream : public ISource +{ +public: + explicit SourceFromInputStream(BlockInputStreamPtr stream_, bool force_add_aggregating_info = false); + String getName() const override { return "SourceFromInputStream"; } + + Status prepare() override; + void work() override; + + Chunk generate() override; + + IBlockInputStream & getStream() { return *stream; } + + void addTotalsPort(); + +private: + bool has_aggregate_functions = false; + bool force_add_aggregating_info; + BlockInputStreamPtr stream; + + Chunk totals; + bool has_totals_port = false; + bool has_totals = false; + + bool is_generating_finished = false; + bool is_stream_finished = false; + bool is_stream_started = false; +}; + +} diff --git a/dbms/src/Processors/Sources/SourceFromSingleChunk.h b/dbms/src/Processors/Sources/SourceFromSingleChunk.h new file mode 100644 index 00000000000..4384597e28b --- /dev/null +++ b/dbms/src/Processors/Sources/SourceFromSingleChunk.h @@ -0,0 +1,21 @@ +#pragma once +#include + + +namespace DB +{ + +class SourceFromSingleChunk : public ISource +{ +public: + explicit SourceFromSingleChunk(Block header, Chunk chunk_) : ISource(std::move(header)), chunk(std::move(chunk_)) {} + String getName() const override { return "SourceFromSingleChunk"; } + +protected: + Chunk generate() override { return std::move(chunk); } + +private: + Chunk chunk; +}; + +} diff --git a/dbms/src/Processors/Transforms/AggregatingTransform.cpp b/dbms/src/Processors/Transforms/AggregatingTransform.cpp new file mode 100644 index 00000000000..5993584f5c9 --- /dev/null +++ b/dbms/src/Processors/Transforms/AggregatingTransform.cpp @@ -0,0 +1,299 @@ +#include + +#include +#include +#include +#include +#include + +namespace ProfileEvents +{ + extern const Event ExternalAggregationMerge; +} + +namespace DB +{ + +namespace +{ + class SourceFromNativeStream : public ISource + { + public: + SourceFromNativeStream(const Block & header, const std::string & path) + : ISource(header), file_in(path), compressed_in(file_in) + , block_in(std::make_shared(compressed_in, ClickHouseRevision::get())) + { + block_in->readPrefix(); + } + + String getName() const override { return "SourceFromNativeStream"; } + + Chunk generate() override + { + if (!block_in) + return {}; + + auto block = block_in->read(); + if (!block) + { + block_in->readSuffix(); + block_in.reset(); + return {}; + } + + auto info = std::make_shared(); + info->bucket_num = block.info.bucket_num; + info->is_overflows = block.info.is_overflows; + + UInt64 num_rows = block.rows(); + Chunk chunk(block.getColumns(), num_rows); + chunk.setChunkInfo(std::move(info)); + + return chunk; + } + + private: + ReadBufferFromFile file_in; + CompressedReadBuffer compressed_in; + BlockInputStreamPtr block_in; + }; + + class ConvertingAggregatedToBlocksTransform : public ISource + { + public: + ConvertingAggregatedToBlocksTransform(Block header, AggregatingTransformParamsPtr params_, BlockInputStreamPtr stream) + : ISource(std::move(header)), params(std::move(params_)), stream(std::move(stream)) {} + + String getName() const override { return "ConvertingAggregatedToBlocksTransform"; } + + protected: + Chunk generate() override + { + auto block = stream->read(); + if (!block) + return {}; + + auto info = std::make_shared(); + info->bucket_num = block.info.bucket_num; + info->is_overflows = block.info.is_overflows; + + UInt64 num_rows = block.rows(); + Chunk chunk(block.getColumns(), num_rows); + chunk.setChunkInfo(std::move(info)); + + return chunk; + } + + private: + /// Store params because aggregator must be destroyed after stream. Order is important. + AggregatingTransformParamsPtr params; + BlockInputStreamPtr stream; + }; +} + +AggregatingTransform::AggregatingTransform(Block header, AggregatingTransformParamsPtr params_) + : AggregatingTransform(std::move(header), std::move(params_) + , std::make_unique(1), 0, 1, 1) +{ +} + +AggregatingTransform::AggregatingTransform( + Block header, AggregatingTransformParamsPtr params_, ManyAggregatedDataPtr many_data_, + size_t current_variant, size_t temporary_data_merge_threads, size_t max_threads) + : IProcessor({std::move(header)}, {params_->getHeader()}), params(std::move(params_)) + , key(params->params.keys_size) + , key_columns(params->params.keys_size) + , aggregate_columns(params->params.aggregates_size) + , many_data(std::move(many_data_)) + , variants(*many_data->variants[current_variant]) + , max_threads(std::min(many_data->variants.size(), max_threads)) + , temporary_data_merge_threads(temporary_data_merge_threads) +{ +} + +AggregatingTransform::~AggregatingTransform() = default; + +IProcessor::Status AggregatingTransform::prepare() +{ + auto & output = outputs.front(); + /// Last output is current. All other outputs should already be closed. + auto & input = inputs.back(); + + /// Check can output. + if (output.isFinished()) + { + input.close(); + return Status::Finished; + } + + if (!output.canPush()) + { + input.setNotNeeded(); + return Status::PortFull; + } + + /// Finish data processing, prepare to generating. + if (is_consume_finished && !is_generate_initialized) + return Status::Ready; + + if (is_generate_initialized && !is_pipeline_created && !processors.empty()) + return Status::ExpandPipeline; + + /// Only possible while consuming. + if (read_current_chunk) + return Status::Ready; + + /// Get chunk from input. + if (input.isFinished()) + { + if (is_consume_finished) + { + output.finish(); + return Status::Finished; + } + else + { + /// Finish data processing and create another pipe. + is_consume_finished = true; + return Status::Ready; + } + } + + input.setNeeded(); + if (!input.hasData()) + return Status::NeedData; + + current_chunk = input.pull(); + read_current_chunk = true; + + if (is_consume_finished) + { + output.push(std::move(current_chunk)); + read_current_chunk = false; + return Status::PortFull; + } + + return Status::Ready; +} + +void AggregatingTransform::work() +{ + if (is_consume_finished) + initGenerate(); + else + { + consume(std::move(current_chunk)); + read_current_chunk = false; + } +} + +Processors AggregatingTransform::expandPipeline() +{ + auto & out = processors.back()->getOutputs().front(); + inputs.emplace_back(out.getHeader(), this); + connect(out, inputs.back()); + is_pipeline_created = true; + return std::move(processors); +} + +void AggregatingTransform::consume(Chunk chunk) +{ + if (chunk.getNumRows() == 0 && params->params.empty_result_for_aggregation_by_empty_set) + return; + + if (!is_consume_started) + { + LOG_TRACE(log, "Aggregating"); + is_consume_started = true; + } + + src_rows += chunk.getNumRows(); + src_bytes += chunk.bytes(); + + auto block = getInputs().front().getHeader().cloneWithColumns(chunk.detachColumns()); + + if (!params->aggregator.executeOnBlock(block, variants, key_columns, aggregate_columns, key, no_more_keys)) + is_consume_finished = true; +} + +void AggregatingTransform::initGenerate() +{ + if (is_generate_initialized) + return; + + is_generate_initialized = true; + + /// If there was no data, and we aggregate without keys, and we must return single row with the result of empty aggregation. + /// To do this, we pass a block with zero rows to aggregate. + if (variants.empty() && params->params.keys_size == 0 && !params->params.empty_result_for_aggregation_by_empty_set) + params->aggregator.executeOnBlock(getInputs().front().getHeader(), variants, key_columns, aggregate_columns, key, no_more_keys); + + double elapsed_seconds = watch.elapsedSeconds(); + size_t rows = variants.sizeWithoutOverflowRow(); + LOG_TRACE(log, std::fixed << std::setprecision(3) + << "Aggregated. " << src_rows << " to " << rows << " rows (from " << src_bytes / 1048576.0 << " MiB)" + << " in " << elapsed_seconds << " sec." + << " (" << src_rows / elapsed_seconds << " rows/sec., " << src_bytes / elapsed_seconds / 1048576.0 << " MiB/sec.)"); + + if (params->aggregator.hasTemporaryFiles()) + { + if (variants.isConvertibleToTwoLevel()) + variants.convertToTwoLevel(); + + /// Flush data in the RAM to disk also. It's easier than merging on-disk and RAM data. + if (variants.size()) + params->aggregator.writeToTemporaryFile(variants); + } + + if (many_data->num_finished.fetch_add(1) + 1 < many_data->variants.size()) + return; + + if (!params->aggregator.hasTemporaryFiles()) + { + auto stream = params->aggregator.mergeAndConvertToBlocks(many_data->variants, params->final, max_threads); + processors.emplace_back(std::make_shared(stream->getHeader(), params, std::move(stream))); + } + else + { + /// If there are temporary files with partially-aggregated data on the disk, + /// then read and merge them, spending the minimum amount of memory. + + ProfileEvents::increment(ProfileEvents::ExternalAggregationMerge); + + if (many_data->variants.size() > 1) + { + /// It may happen that some data has not yet been flushed, + /// because at the time thread has finished, no data has been flushed to disk, and then some were. + for (auto & cur_variants : many_data->variants) + { + if (cur_variants->isConvertibleToTwoLevel()) + cur_variants->convertToTwoLevel(); + + if (cur_variants->size()) + params->aggregator.writeToTemporaryFile(*cur_variants); + } + } + + auto header = params->aggregator.getHeader(false); + + const auto & files = params->aggregator.getTemporaryFiles(); + BlockInputStreams input_streams; + for (const auto & file : files.files) + processors.emplace_back(std::make_unique(header, file->path())); + + LOG_TRACE(log, "Will merge " << files.files.size() << " temporary files of size " + << (files.sum_size_compressed / 1048576.0) << " MiB compressed, " + << (files.sum_size_uncompressed / 1048576.0) << " MiB uncompressed."); + + auto pipe = createMergingAggregatedMemoryEfficientPipe( + header, params, files.files.size(), temporary_data_merge_threads); + + auto input = pipe.front()->getInputs().begin(); + for (auto & processor : processors) + connect(processor->getOutputs().front(), *(input++)); + + processors.insert(processors.end(), pipe.begin(), pipe.end()); + } +} + +} diff --git a/dbms/src/Processors/Transforms/AggregatingTransform.h b/dbms/src/Processors/Transforms/AggregatingTransform.h new file mode 100644 index 00000000000..64ba10e1801 --- /dev/null +++ b/dbms/src/Processors/Transforms/AggregatingTransform.h @@ -0,0 +1,102 @@ +#pragma once +#include +#include +#include +#include +#include + +namespace DB +{ + +class AggregatedChunkInfo : public ChunkInfo +{ +public: + bool is_overflows = false; + Int32 bucket_num = -1; +}; + +class IBlockInputStream; +using BlockInputStreamPtr = std::shared_ptr; + +struct AggregatingTransformParams +{ + Aggregator::Params params; + Aggregator aggregator; + bool final; + + AggregatingTransformParams(const Aggregator::Params & params, bool final) + : params(params), aggregator(params), final(final) {} + + Block getHeader() const { return aggregator.getHeader(final); } +}; + +struct ManyAggregatedData +{ + ManyAggregatedDataVariants variants; + std::atomic num_finished = 0; + + explicit ManyAggregatedData(size_t num_threads = 0) : variants(num_threads) + { + for (auto & elem : variants) + elem = std::make_shared(); + } +}; + +using AggregatingTransformParamsPtr = std::shared_ptr; +using ManyAggregatedDataPtr = std::shared_ptr; + +class AggregatingTransform : public IProcessor +{ +public: + AggregatingTransform(Block header, AggregatingTransformParamsPtr params_); + + /// For Parallel aggregating. + AggregatingTransform(Block header, AggregatingTransformParamsPtr params_, + ManyAggregatedDataPtr many_data, size_t current_variant, + size_t temporary_data_merge_threads, size_t max_threads); + ~AggregatingTransform() override; + + String getName() const override { return "AggregatingTransform"; } + Status prepare() override; + void work() override; + Processors expandPipeline() override; + +protected: + void consume(Chunk chunk); + +private: + /// To read the data that was flushed into the temporary data file. + Processors processors; + + AggregatingTransformParamsPtr params; + Logger * log = &Logger::get("AggregatingTransform"); + + StringRefs key; + ColumnRawPtrs key_columns; + Aggregator::AggregateColumns aggregate_columns; + bool no_more_keys = false; + + ManyAggregatedDataPtr many_data; + AggregatedDataVariants & variants; + size_t max_threads = 1; + size_t temporary_data_merge_threads = 1; + + /// TODO: calculate time only for aggregation. + Stopwatch watch; + + UInt64 src_rows = 0; + UInt64 src_bytes = 0; + + bool is_generate_initialized = false; + bool is_consume_finished = false; + bool is_pipeline_created = false; + + Chunk current_chunk; + bool read_current_chunk = false; + + bool is_consume_started = false; + + void initGenerate(); +}; + +} diff --git a/dbms/src/Processors/Transforms/ConvertingTransform.cpp b/dbms/src/Processors/Transforms/ConvertingTransform.cpp new file mode 100644 index 00000000000..49dbb748591 --- /dev/null +++ b/dbms/src/Processors/Transforms/ConvertingTransform.cpp @@ -0,0 +1,127 @@ +#include + +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int THERE_IS_NO_COLUMN; + extern const int BLOCKS_HAVE_DIFFERENT_STRUCTURE; + extern const int NUMBER_OF_COLUMNS_DOESNT_MATCH; +} + +static ColumnPtr castColumnWithDiagnostic( + const ColumnWithTypeAndName & src_elem, + const ColumnWithTypeAndName & res_elem, + const Context & context) +{ + try + { + return castColumn(src_elem, res_elem.type, context); + } + catch (Exception & e) + { + e.addMessage("while converting source column " + backQuoteIfNeed(src_elem.name) + + " to destination column " + backQuoteIfNeed(res_elem.name)); + throw; + } +} + +ConvertingTransform::ConvertingTransform( + Block source_header, + Block result_header, + MatchColumnsMode mode, + const Context & context) + : ISimpleTransform(std::move(source_header), std::move(result_header), false) + , context(context) + , conversion(getOutputPort().getHeader().columns()) +{ + auto & source = getInputPort().getHeader(); + auto & result = getOutputPort().getHeader(); + + size_t num_input_columns = source.columns(); + size_t num_result_columns = result.columns(); + + if (mode == MatchColumnsMode::Position && num_input_columns != num_result_columns) + throw Exception("Number of columns doesn't match", ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH); + + for (size_t result_col_num = 0; result_col_num < num_result_columns; ++result_col_num) + { + const auto & res_elem = result.getByPosition(result_col_num); + + switch (mode) + { + case MatchColumnsMode::Position: + conversion[result_col_num] = result_col_num; + break; + + case MatchColumnsMode::Name: + if (source.has(res_elem.name)) + conversion[result_col_num] = source.getPositionByName(res_elem.name); + else + throw Exception("Cannot find column " + backQuoteIfNeed(res_elem.name) + " in source stream", + ErrorCodes::THERE_IS_NO_COLUMN); + break; + } + + const auto & src_elem = source.getByPosition(conversion[result_col_num]); + + /// Check constants. + + if (auto * res_const = typeid_cast(res_elem.column.get())) + { + if (auto * src_const = typeid_cast(src_elem.column.get())) + { + if (res_const->getField() != src_const->getField()) + throw Exception("Cannot convert column " + backQuoteIfNeed(res_elem.name) + " because " + "it is constant but values of constants are different in source and result", + ErrorCodes::BLOCKS_HAVE_DIFFERENT_STRUCTURE); + } + else + throw Exception("Cannot convert column " + backQuoteIfNeed(res_elem.name) + " because " + "it is non constant in source stream but must be constant in result", + ErrorCodes::BLOCKS_HAVE_DIFFERENT_STRUCTURE); + } + + /// Check conversion by dry run CAST function. + + castColumnWithDiagnostic(src_elem, res_elem, context); + } +} + +void ConvertingTransform::transform(Chunk & chunk) +{ + auto & source = getInputPort().getHeader(); + auto & result = getOutputPort().getHeader(); + + auto num_rows = chunk.getNumRows(); + auto src_columns = chunk.detachColumns(); + + size_t num_res_columns = conversion.size(); + + Columns res_columns; + res_columns.reserve(num_res_columns); + + for (size_t res_pos = 0; res_pos < num_res_columns; ++res_pos) + { + auto src_elem = source.getByPosition(conversion[res_pos]); + src_elem.column = src_columns[conversion[res_pos]]; + auto res_elem = result.getByPosition(res_pos); + + ColumnPtr converted = castColumnWithDiagnostic(src_elem, res_elem, context); + + if (!isColumnConst(*res_elem.column)) + converted = converted->convertToFullColumnIfConst(); + + res_columns.emplace_back(std::move(converted)); + } + + chunk.setColumns(std::move(res_columns), num_rows); +} + +} diff --git a/dbms/src/Processors/Transforms/ConvertingTransform.h b/dbms/src/Processors/Transforms/ConvertingTransform.h new file mode 100644 index 00000000000..d6e6219316a --- /dev/null +++ b/dbms/src/Processors/Transforms/ConvertingTransform.h @@ -0,0 +1,49 @@ +#include +#include + +namespace DB +{ + +/** Convert one block structure to another: + * + * Leaves only necessary columns; + * + * Columns are searched in source first by name; + * and if there is no column with same name, then by position. + * + * Converting types of matching columns (with CAST function). + * + * Materializing columns which are const in source and non-const in result, + * throw if they are const in result and non const in source, + * or if they are const and have different values. + */ +class ConvertingTransform : public ISimpleTransform +{ +public: + enum class MatchColumnsMode + { + /// Require same number of columns in source and result. Match columns by corresponding positions, regardless to names. + Position, + /// Find columns in source by their names. Allow excessive columns in source. + Name, + }; + + ConvertingTransform( + Block source_header, + Block result_header, + MatchColumnsMode mode, + const Context & context); + + String getName() const override { return "Converting"; } + +protected: + void transform(Chunk & chunk) override; + +private: + const Context & context; + + /// How to construct result block. Position in source block, where to get each column. + ColumnNumbers conversion; +}; + +} diff --git a/dbms/src/Processors/Transforms/CreatingSetsTransform.cpp b/dbms/src/Processors/Transforms/CreatingSetsTransform.cpp new file mode 100644 index 00000000000..29bc4030b81 --- /dev/null +++ b/dbms/src/Processors/Transforms/CreatingSetsTransform.cpp @@ -0,0 +1,223 @@ +#include + +#include +#include +#include + +#include +#include +#include + +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int SET_SIZE_LIMIT_EXCEEDED; +} + + +CreatingSetsTransform::CreatingSetsTransform( + Block out_header, + const SubqueriesForSets & subqueries_for_sets_, + const SizeLimits & network_transfer_limits, + const Context & context) + : IProcessor({}, {std::move(out_header)}) + , subqueries_for_sets(subqueries_for_sets_) + , cur_subquery(subqueries_for_sets.begin()) + , network_transfer_limits(network_transfer_limits) + , context(context) +{ +} + +IProcessor::Status CreatingSetsTransform::prepare() +{ + auto & output = outputs.front(); + + if (finished) + { + output.finish(); + return Status::Finished; + } + + /// Check can output. + if (output.isFinished()) + return Status::Finished; + + if (!output.canPush()) + return Status::PortFull; + + return Status::Ready; +} + +void CreatingSetsTransform::startSubquery(SubqueryForSet & subquery) +{ + LOG_TRACE(log, (subquery.set ? "Creating set. " : "") + << (subquery.join ? "Creating join. " : "") + << (subquery.table ? "Filling temporary table. " : "")); + + elapsed_nanoseconds = 0; + + if (subquery.table) + table_out = subquery.table->write({}, context); + + done_with_set = !subquery.set; + done_with_join = !subquery.join; + done_with_table = !subquery.table; + + if (done_with_set && done_with_join && done_with_table) + throw Exception("Logical error: nothing to do with subquery", ErrorCodes::LOGICAL_ERROR); + + if (table_out) + table_out->writePrefix(); +} + +void CreatingSetsTransform::finishSubquery(SubqueryForSet & subquery) +{ + size_t head_rows = 0; + const BlockStreamProfileInfo & profile_info = subquery.source->getProfileInfo(); + + head_rows = profile_info.rows; + + if (subquery.join) + subquery.join->setTotals(subquery.source->getTotals()); + + if (head_rows != 0) + { + std::stringstream msg; + msg << std::fixed << std::setprecision(3); + msg << "Created. "; + + if (subquery.set) + msg << "Set with " << subquery.set->getTotalRowCount() << " entries from " << head_rows << " rows. "; + if (subquery.join) + msg << "Join with " << subquery.join->getTotalRowCount() << " entries from " << head_rows << " rows. "; + if (subquery.table) + msg << "Table with " << head_rows << " rows. "; + + msg << "In " << (static_cast(elapsed_nanoseconds) / 1000000000ULL) << " sec."; + LOG_DEBUG(log, msg.rdbuf()); + } + else + { + LOG_DEBUG(log, "Subquery has empty result."); + } +} + +void CreatingSetsTransform::init() +{ + is_initialized = true; + + for (auto & elem : subqueries_for_sets) + if (elem.second.source && elem.second.set) + elem.second.set->setHeader(elem.second.source->getHeader()); +} + +void CreatingSetsTransform::work() +{ + if (!is_initialized) + init(); + + Stopwatch watch; + + while (cur_subquery != subqueries_for_sets.end() && cur_subquery->second.source == nullptr) + ++cur_subquery; + + if (cur_subquery == subqueries_for_sets.end()) + { + finished = true; + return; + } + + SubqueryForSet & subquery = cur_subquery->second; + + if (!started_cur_subquery) + { + startSubquery(subquery); + started_cur_subquery = true; + } + + auto finishCurrentSubquery = [&]() + { + if (table_out) + table_out->writeSuffix(); + + watch.stop(); + elapsed_nanoseconds += watch.elapsedNanoseconds(); + + finishSubquery(subquery); + + ++cur_subquery; + started_cur_subquery = false; + + while (cur_subquery != subqueries_for_sets.end() && cur_subquery->second.source == nullptr) + ++cur_subquery; + + if (cur_subquery == subqueries_for_sets.end()) + finished = true; + }; + + auto block = subquery.source->read(); + if (!block) + { + finishCurrentSubquery(); + return; + } + + if (!done_with_set) + { + if (!subquery.set->insertFromBlock(block)) + done_with_set = true; + } + + if (!done_with_join) + { + subquery.renameColumns(block); + + if (subquery.joined_block_actions) + subquery.joined_block_actions->execute(block); + + if (!subquery.join->insertFromBlock(block)) + done_with_join = true; + } + + if (!done_with_table) + { + block = materializeBlock(block); + table_out->write(block); + + rows_to_transfer += block.rows(); + bytes_to_transfer += block.bytes(); + + if (!network_transfer_limits.check(rows_to_transfer, bytes_to_transfer, "IN/JOIN external table", + ErrorCodes::SET_SIZE_LIMIT_EXCEEDED)) + done_with_table = true; + } + + if (done_with_set && done_with_join && done_with_table) + { + subquery.source->cancel(false); + finishCurrentSubquery(); + } + else + elapsed_nanoseconds += watch.elapsedNanoseconds(); +} + +void CreatingSetsTransform::setProgressCallback(const ProgressCallback & callback) +{ + for (auto & elem : subqueries_for_sets) + if (elem.second.source) + elem.second.source->setProgressCallback(callback); +} + +void CreatingSetsTransform::setProcessListElement(QueryStatus * status) +{ + for (auto & elem : subqueries_for_sets) + if (elem.second.source) + elem.second.source->setProcessListElement(status); +} + +} diff --git a/dbms/src/Processors/Transforms/CreatingSetsTransform.h b/dbms/src/Processors/Transforms/CreatingSetsTransform.h new file mode 100644 index 00000000000..b5f7ea63748 --- /dev/null +++ b/dbms/src/Processors/Transforms/CreatingSetsTransform.h @@ -0,0 +1,63 @@ +#pragma once +#include +#include +#include + +namespace DB +{ + +struct Progress; +using ProgressCallback = std::function; + +/// This processor creates sets during execution. +/// Don't return any data. Sets are created when Finish status is returned. +/// In general, several work() methods need to be called to finish. +/// TODO: several independent processors can be created for each subquery. Make subquery a piece of pipeline. +class CreatingSetsTransform : public IProcessor +{ +public: + CreatingSetsTransform( + Block out_header, + const SubqueriesForSets & subqueries_for_sets_, + const SizeLimits & network_transfer_limits, + const Context & context); + + String getName() const override { return "CreatingSetsTransform"; } + Status prepare() override; + void work() override; + + void setProgressCallback(const ProgressCallback & callback); + void setProcessListElement(QueryStatus * status); + +protected: + bool finished = false; + +private: + SubqueriesForSets subqueries_for_sets; + SubqueriesForSets::iterator cur_subquery; + + bool started_cur_subquery = false; + BlockOutputStreamPtr table_out; + UInt64 elapsed_nanoseconds = 0; + + bool done_with_set = true; + bool done_with_join = true; + bool done_with_table = true; + + SizeLimits network_transfer_limits; + const Context & context; + + size_t rows_to_transfer = 0; + size_t bytes_to_transfer = 0; + + using Logger = Poco::Logger; + Logger * log = &Logger::get("CreatingSetsBlockInputStream"); + + bool is_initialized = false; + + void init(); + void startSubquery(SubqueryForSet & subquery); + void finishSubquery(SubqueryForSet & subquery); +}; + +} diff --git a/dbms/src/Processors/Transforms/CubeTransform.cpp b/dbms/src/Processors/Transforms/CubeTransform.cpp new file mode 100644 index 00000000000..5809a480d09 --- /dev/null +++ b/dbms/src/Processors/Transforms/CubeTransform.cpp @@ -0,0 +1,61 @@ +#include +#include + +namespace DB +{ + +CubeTransform::CubeTransform(Block header, AggregatingTransformParamsPtr params_) + : IInflatingTransform(std::move(header), params_->getHeader()) + , params(std::move(params_)) + , keys(params->params.keys) +{ + if (keys.size() >= 8 * sizeof(mask)) + throw Exception("Too many keys are used for CubeTransform.", ErrorCodes::LOGICAL_ERROR); +} + +void CubeTransform::consume(Chunk chunk) +{ + consumed_chunk = std::move(chunk); + auto num_rows = consumed_chunk.getNumRows(); + mask = (UInt64(1) << keys.size()) - 1; + + current_columns = consumed_chunk.getColumns(); + current_zero_columns.clear(); + current_zero_columns.reserve(keys.size()); + + for (auto key : keys) + current_zero_columns.emplace_back(current_columns[key]->cloneEmpty()->cloneResized(num_rows)); +} + +bool CubeTransform::canGenerate() +{ + return consumed_chunk; +} + +Chunk CubeTransform::generate() +{ + auto gen_chunk = std::move(consumed_chunk); + + if (mask) + { + --mask; + + auto columns = current_columns; + auto size = keys.size(); + for (size_t i = 0; i < size; ++i) + /// Reverse bit order to support previous behaviour. + if ((mask & (UInt64(1) << (size - i - 1))) == 0) + columns[keys[i]] = current_zero_columns[i]; + + BlocksList cube_blocks = { getInputPort().getHeader().cloneWithColumns(columns) }; + auto cube_block = params->aggregator.mergeBlocks(cube_blocks, false); + + auto num_rows = cube_block.rows(); + consumed_chunk = Chunk(cube_block.getColumns(), num_rows); + } + + finalizeChunk(gen_chunk); + return gen_chunk; +} + +} diff --git a/dbms/src/Processors/Transforms/CubeTransform.h b/dbms/src/Processors/Transforms/CubeTransform.h new file mode 100644 index 00000000000..60259832e40 --- /dev/null +++ b/dbms/src/Processors/Transforms/CubeTransform.h @@ -0,0 +1,35 @@ +#pragma once +#include +#include + + +namespace DB +{ + +/// Takes blocks after grouping, with non-finalized aggregate functions. +/// Calculates all subsets of columns and aggregates over them. +class CubeTransform : public IInflatingTransform +{ +public: + CubeTransform(Block header, AggregatingTransformParamsPtr params); + String getName() const override { return "CubeTransform"; } + +protected: + void consume(Chunk chunk) override; + + bool canGenerate() override; + + Chunk generate() override; + +private: + AggregatingTransformParamsPtr params; + ColumnNumbers keys; + + Chunk consumed_chunk; + Columns current_columns; + Columns current_zero_columns; + + UInt64 mask = 0; +}; + +} diff --git a/dbms/src/Processors/Transforms/DistinctTransform.cpp b/dbms/src/Processors/Transforms/DistinctTransform.cpp new file mode 100644 index 00000000000..7cd9a54e055 --- /dev/null +++ b/dbms/src/Processors/Transforms/DistinctTransform.cpp @@ -0,0 +1,114 @@ +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int SET_SIZE_LIMIT_EXCEEDED; +} + +DistinctTransform::DistinctTransform( + const Block & header, + const SizeLimits & set_size_limits, + UInt64 limit_hint, + const Names & columns) + : ISimpleTransform(header, header, true) + , limit_hint(limit_hint) + , set_size_limits(set_size_limits) +{ + size_t num_columns = columns.empty() ? header.columns() : columns.size(); + + key_columns_pos.reserve(columns.size()); + for (size_t i = 0; i < num_columns; ++i) + { + auto pos = columns.empty() ? i + : header.getPositionByName(columns[i]); + + auto & col = header.getByPosition(pos).column; + + if (!(col && isColumnConst(*col))) + key_columns_pos.emplace_back(pos); + } +} + +template +void DistinctTransform::buildFilter( + Method & method, + const ColumnRawPtrs & columns, + IColumn::Filter & filter, + size_t rows, + SetVariants & variants) const +{ + typename Method::State state(columns, key_sizes, nullptr); + + for (size_t i = 0; i < rows; ++i) + { + auto emplace_result = state.emplaceKey(method.data, i, variants.string_pool); + + /// Emit the record if there is no such key in the current set yet. + /// Skip it otherwise. + filter[i] = emplace_result.isInserted(); + } +} + +void DistinctTransform::transform(Chunk & chunk) +{ + auto num_rows = chunk.getNumRows(); + auto columns = chunk.detachColumns(); + + /// Stop reading if we already reach the limit. + if (no_more_rows || (limit_hint && data.getTotalRowCount() >= limit_hint)) + { + stopReading(); + return; + } + + ColumnRawPtrs column_ptrs; + column_ptrs.reserve(key_columns_pos.size()); + for (auto pos : key_columns_pos) + column_ptrs.emplace_back(columns[pos].get()); + + if (column_ptrs.empty()) + { + /// Only constants. We need to return single row. + no_more_rows = true; + for (auto & column : columns) + column = column->cut(0, 1); + + chunk.setColumns(std::move(columns), 1); + return; + } + + if (data.empty()) + data.init(SetVariants::chooseMethod(column_ptrs, key_sizes)); + + const auto old_set_size = data.getTotalRowCount(); + IColumn::Filter filter(num_rows); + + switch (data.type) + { + case SetVariants::Type::EMPTY: + break; +#define M(NAME) \ + case SetVariants::Type::NAME: \ + buildFilter(*data.NAME, column_ptrs, filter, num_rows, data); \ + break; + APPLY_FOR_SET_VARIANTS(M) +#undef M + } + + /// Just go to the next chunk if there isn't any new record in the current one. + if (data.getTotalRowCount() == old_set_size) + return; + + if (!set_size_limits.check(data.getTotalRowCount(), data.getTotalByteCount(), "DISTINCT", ErrorCodes::SET_SIZE_LIMIT_EXCEEDED)) + return; + + for (auto & column : columns) + column = column->filter(filter, -1); + + chunk.setColumns(std::move(columns), data.getTotalRowCount() - old_set_size); +} + +} diff --git a/dbms/src/Processors/Transforms/DistinctTransform.h b/dbms/src/Processors/Transforms/DistinctTransform.h new file mode 100644 index 00000000000..05ef9e9c334 --- /dev/null +++ b/dbms/src/Processors/Transforms/DistinctTransform.h @@ -0,0 +1,44 @@ +#pragma once +#include +#include +#include +#include + +namespace DB +{ + +class DistinctTransform : public ISimpleTransform +{ +public: + DistinctTransform( + const Block & header, + const SizeLimits & set_size_limits, + UInt64 limit_hint, + const Names & columns); + + String getName() const override { return "DistinctTransform"; } + +protected: + void transform(Chunk & chunk) override; + +private: + ColumnNumbers key_columns_pos; + SetVariants data; + Sizes key_sizes; + UInt64 limit_hint; + + bool no_more_rows = false; + + /// Restrictions on the maximum size of the output data. + SizeLimits set_size_limits; + + template + void buildFilter( + Method & method, + const ColumnRawPtrs & key_columns, + IColumn::Filter & filter, + size_t rows, + SetVariants & variants) const; +}; + +} diff --git a/dbms/src/Processors/Transforms/ExpressionTransform.cpp b/dbms/src/Processors/Transforms/ExpressionTransform.cpp new file mode 100644 index 00000000000..c42ef92b085 --- /dev/null +++ b/dbms/src/Processors/Transforms/ExpressionTransform.cpp @@ -0,0 +1,40 @@ +#include +#include + +namespace DB +{ + +static Block transformHeader(Block header, const ExpressionActionsPtr & expression) +{ + expression->execute(header, true); + return header; +} + + +ExpressionTransform::ExpressionTransform(const Block & header, ExpressionActionsPtr expression, bool on_totals, bool default_totals) + : ISimpleTransform(header, transformHeader(header, expression), on_totals) + , expression(std::move(expression)) + , on_totals(on_totals) + , default_totals(default_totals) +{ +} + +void ExpressionTransform::transform(Chunk & chunk) +{ + auto block = getInputPort().getHeader().cloneWithColumns(chunk.detachColumns()); + + if (on_totals) + { + if (default_totals && !expression->hasTotalsInJoin()) + return; + + expression->executeOnTotals(block); + } + else + expression->execute(block); + + auto num_rows = block.rows(); + chunk.setColumns(block.getColumns(), num_rows); +} + +} diff --git a/dbms/src/Processors/Transforms/ExpressionTransform.h b/dbms/src/Processors/Transforms/ExpressionTransform.h new file mode 100644 index 00000000000..8face634f96 --- /dev/null +++ b/dbms/src/Processors/Transforms/ExpressionTransform.h @@ -0,0 +1,26 @@ +#pragma once +#include + +namespace DB +{ + +class ExpressionActions; +using ExpressionActionsPtr = std::shared_ptr; + +class ExpressionTransform : public ISimpleTransform +{ +public: + ExpressionTransform(const Block & header, ExpressionActionsPtr expression, bool on_totals = false, bool default_totals = false); + + String getName() const override { return "ExpressionTransform"; } + +protected: + void transform(Chunk & chunk) override; + +private: + ExpressionActionsPtr expression; + bool on_totals; + bool default_totals; +}; + +} diff --git a/dbms/src/Processors/Transforms/ExtremesTransform.cpp b/dbms/src/Processors/Transforms/ExtremesTransform.cpp new file mode 100644 index 00000000000..82a7605fa11 --- /dev/null +++ b/dbms/src/Processors/Transforms/ExtremesTransform.cpp @@ -0,0 +1,121 @@ +#include + +namespace DB +{ + +ExtremesTransform::ExtremesTransform(const Block & header) + : ISimpleTransform(header, header, true) +{ + /// Port for Extremes. + outputs.emplace_back(outputs.front().getHeader(), this); +} + +IProcessor::Status ExtremesTransform::prepare() +{ + if (!finished_transform) + { + auto status = ISimpleTransform::prepare(); + + if (status != Status::Finished) + return status; + + finished_transform = true; + } + + auto & totals_output = getExtremesPort(); + + /// Check can output. + if (totals_output.isFinished()) + return Status::Finished; + + if (!totals_output.canPush()) + return Status::PortFull; + + if (!extremes && !extremes_columns.empty()) + return Status::Ready; + + if (extremes) + totals_output.push(std::move(extremes)); + + totals_output.finish(); + return Status::Finished; +} + +void ExtremesTransform::work() +{ + if (finished_transform) + { + if (!extremes && !extremes_columns.empty()) + extremes.setColumns(std::move(extremes_columns), 2); + } + else + ISimpleTransform::work(); +} + +void ExtremesTransform::transform(DB::Chunk & chunk) +{ + + if (chunk.getNumRows() == 0) + return; + + size_t num_columns = chunk.getNumColumns(); + auto & columns = chunk.getColumns(); + + if (extremes_columns.empty()) + { + extremes_columns.resize(num_columns); + + for (size_t i = 0; i < num_columns; ++i) + { + const ColumnPtr & src = columns[i]; + + if (isColumnConst(*src)) + { + /// Equal min and max. + extremes_columns[i] = src->cloneResized(2); + } + else + { + Field min_value; + Field max_value; + + src->getExtremes(min_value, max_value); + + extremes_columns[i] = src->cloneEmpty(); + + extremes_columns[i]->insert(min_value); + extremes_columns[i]->insert(max_value); + } + } + } + else + { + for (size_t i = 0; i < num_columns; ++i) + { + if (isColumnConst(*extremes_columns[i])) + continue; + + Field min_value = (*extremes_columns[i])[0]; + Field max_value = (*extremes_columns[i])[1]; + + Field cur_min_value; + Field cur_max_value; + + columns[i]->getExtremes(cur_min_value, cur_max_value); + + if (cur_min_value < min_value) + min_value = cur_min_value; + if (cur_max_value > max_value) + max_value = cur_max_value; + + MutableColumnPtr new_extremes = extremes_columns[i]->cloneEmpty(); + + new_extremes->insert(min_value); + new_extremes->insert(max_value); + + extremes_columns[i] = std::move(new_extremes); + } + } +} + +} diff --git a/dbms/src/Processors/Transforms/ExtremesTransform.h b/dbms/src/Processors/Transforms/ExtremesTransform.h new file mode 100644 index 00000000000..d447e8c9cfa --- /dev/null +++ b/dbms/src/Processors/Transforms/ExtremesTransform.h @@ -0,0 +1,30 @@ +#include + +namespace DB +{ + +class ExtremesTransform : public ISimpleTransform +{ + +public: + explicit ExtremesTransform(const Block & header); + + String getName() const override { return "ExtremesTransform"; } + + OutputPort & getExtremesPort() { return outputs.back(); } + + Status prepare() override; + void work() override; + +protected: + void transform(Chunk & chunk) override; + + bool finished_transform = false; + Chunk extremes; + +private: + MutableColumns extremes_columns; +}; + +} + diff --git a/dbms/src/Processors/Transforms/FilterTransform.cpp b/dbms/src/Processors/Transforms/FilterTransform.cpp new file mode 100644 index 00000000000..725b5ceb01b --- /dev/null +++ b/dbms/src/Processors/Transforms/FilterTransform.cpp @@ -0,0 +1,199 @@ +#include +#include +#include + +namespace DB +{ + +static void replaceFilterToConstant(Block & block, const String & filter_column_name) +{ + ConstantFilterDescription constant_filter_description; + + auto filter_column = block.getPositionByName(filter_column_name); + auto & column_elem = block.safeGetByPosition(filter_column); + + /// Isn't the filter already constant? + if (column_elem.column) + constant_filter_description = ConstantFilterDescription(*column_elem.column); + + if (!constant_filter_description.always_false + && !constant_filter_description.always_true) + { + /// Replace the filter column to a constant with value 1. + FilterDescription filter_description_check(*column_elem.column); + column_elem.column = column_elem.type->createColumnConst(block.rows(), 1u); + } +} + +static Block transformHeader( + Block header, + const ExpressionActionsPtr & expression, + const String & filter_column_name, + bool remove_filter_column) +{ + expression->execute(header); + + if (remove_filter_column) + header.erase(filter_column_name); + else + replaceFilterToConstant(header, filter_column_name); + + return header; +} + +FilterTransform::FilterTransform( + const Block & header, + ExpressionActionsPtr expression_, + String filter_column_name_, + bool remove_filter_column) + : ISimpleTransform(header, transformHeader(header, expression_, filter_column_name_, remove_filter_column), true) + , expression(std::move(expression_)) + , filter_column_name(std::move(filter_column_name_)) + , remove_filter_column(remove_filter_column) +{ + transformed_header = getInputPort().getHeader(); + expression->execute(transformed_header); + filter_column_position = transformed_header.getPositionByName(filter_column_name); + + auto & column = transformed_header.getByPosition(filter_column_position).column; + if (column) + constant_filter_description = ConstantFilterDescription(*column); +} + +IProcessor::Status FilterTransform::prepare() +{ + if (constant_filter_description.always_false) + { + input.close(); + output.finish(); + return Status::Finished; + } + + return ISimpleTransform::prepare(); +} + + +void FilterTransform::removeFilterIfNeed(Chunk & chunk) +{ + if (chunk && remove_filter_column) + chunk.erase(filter_column_position); +} + +void FilterTransform::transform(Chunk & chunk) +{ + size_t num_rows_before_filtration = chunk.getNumRows(); + auto columns = chunk.detachColumns(); + + { + Block block = getInputPort().getHeader().cloneWithColumns(columns); + columns.clear(); + expression->execute(block); + num_rows_before_filtration = block.rows(); + columns = block.getColumns(); + } + + if (constant_filter_description.always_true) + { + chunk.setColumns(std::move(columns), num_rows_before_filtration); + removeFilterIfNeed(chunk); + return; + } + + size_t num_columns = columns.size(); + ColumnPtr filter_column = columns[filter_column_position]; + + /** It happens that at the stage of analysis of expressions (in sample_block) the columns-constants have not been calculated yet, + * and now - are calculated. That is, not all cases are covered by the code above. + * This happens if the function returns a constant for a non-constant argument. + * For example, `ignore` function. + */ + constant_filter_description = ConstantFilterDescription(*filter_column); + + if (constant_filter_description.always_false) + return; /// Will finish at next prepare call + + if (constant_filter_description.always_true) + { + chunk.setColumns(std::move(columns), num_rows_before_filtration); + removeFilterIfNeed(chunk); + return; + } + + FilterDescription filter_and_holder(*filter_column); + + /** Let's find out how many rows will be in result. + * To do this, we filter out the first non-constant column + * or calculate number of set bytes in the filter. + */ + size_t first_non_constant_column = num_columns; + for (size_t i = 0; i < num_columns; ++i) + { + if (!isColumnConst(*columns[i])) + { + first_non_constant_column = i; + break; + } + } + + size_t num_filtered_rows = 0; + if (first_non_constant_column != num_columns) + { + columns[first_non_constant_column] = columns[first_non_constant_column]->filter(*filter_and_holder.data, -1); + num_filtered_rows = columns[first_non_constant_column]->size(); + } + else + num_filtered_rows = countBytesInFilter(*filter_and_holder.data); + + /// If the current block is completely filtered out, let's move on to the next one. + if (num_filtered_rows == 0) + /// SimpleTransform will skip it. + return; + + /// If all the rows pass through the filter. + if (num_filtered_rows == num_rows_before_filtration) + { + if (!remove_filter_column) + { + /// Replace the column with the filter by a constant. + auto & type = transformed_header.getByPosition(filter_column_position).type; + columns[filter_column_position] = type->createColumnConst(num_filtered_rows, 1u); + } + + /// No need to touch the rest of the columns. + chunk.setColumns(std::move(columns), num_rows_before_filtration); + removeFilterIfNeed(chunk); + return; + } + + /// Filter the rest of the columns. + for (size_t i = 0; i < num_columns; ++i) + { + const auto & current_type = transformed_header.safeGetByPosition(i).type; + auto & current_column = columns[i]; + + if (i == filter_column_position) + { + /// The column with filter itself is replaced with a column with a constant `1`, since after filtering, nothing else will remain. + /// NOTE User could pass column with something different than 0 and 1 for filter. + /// Example: + /// SELECT materialize(100) AS x WHERE x + /// will work incorrectly. + current_column = current_type->createColumnConst(num_filtered_rows, 1u); + continue; + } + + if (i == first_non_constant_column) + continue; + + if (isColumnConst(*current_column)) + current_column = current_column->cut(0, num_filtered_rows); + else + current_column = current_column->filter(*filter_and_holder.data, num_filtered_rows); + } + + chunk.setColumns(std::move(columns), num_filtered_rows); + removeFilterIfNeed(chunk); +} + + +} diff --git a/dbms/src/Processors/Transforms/FilterTransform.h b/dbms/src/Processors/Transforms/FilterTransform.h new file mode 100644 index 00000000000..32cdbb79d50 --- /dev/null +++ b/dbms/src/Processors/Transforms/FilterTransform.h @@ -0,0 +1,42 @@ +#pragma once +#include +#include + +namespace DB +{ + +class ExpressionActions; +using ExpressionActionsPtr = std::shared_ptr; + +/** Has one input and one output. + * Simply pull a block from input, transform it, and push it to output. + * If remove_filter_column is true, remove filter column from block. + */ +class FilterTransform : public ISimpleTransform +{ +public: + FilterTransform( + const Block & header, ExpressionActionsPtr expression, String filter_column_name, bool remove_filter_column); + + String getName() const override { return "FilterTransform"; } + + Status prepare() override; + +protected: + void transform(Chunk & chunk) override; + +private: + ExpressionActionsPtr expression; + String filter_column_name; + bool remove_filter_column; + + ConstantFilterDescription constant_filter_description; + size_t filter_column_position = 0; + + /// Header after expression, but before removing filter column. + Block transformed_header; + + void removeFilterIfNeed(Chunk & chunk); +}; + +} diff --git a/dbms/src/Processors/Transforms/LimitByTransform.cpp b/dbms/src/Processors/Transforms/LimitByTransform.cpp new file mode 100644 index 00000000000..83268d178bd --- /dev/null +++ b/dbms/src/Processors/Transforms/LimitByTransform.cpp @@ -0,0 +1,70 @@ +#include +#include + +namespace DB +{ + +LimitByTransform::LimitByTransform(const Block & header, size_t group_length_, size_t group_offset_, const Names & columns) + : ISimpleTransform(header, header, true) + , group_length(group_length_) + , group_offset(group_offset_) +{ + key_positions.reserve(columns.size()); + + for (const auto & name : columns) + { + auto position = header.getPositionByName(name); + auto & column = header.getByPosition(position).column; + + /// Ignore all constant columns. + if (!(column && isColumnConst(*column))) + key_positions.emplace_back(position); + } +} + +void LimitByTransform::transform(Chunk & chunk) +{ + size_t num_rows = chunk.getNumRows(); + auto columns = chunk.detachColumns(); + + IColumn::Filter filter(num_rows); + size_t inserted_count = 0; + + for (size_t row = 0; row < num_rows; ++row) + { + UInt128 key(0, 0); + SipHash hash; + + for (auto position : key_positions) + columns[position]->updateHashWithValue(row, hash); + + hash.get128(key.low, key.high); + + auto count = keys_counts[key]++; + if (count >= group_offset && count < group_length + group_offset) + { + inserted_count++; + filter[row] = 1; + } + else + filter[row] = 0; + } + + /// Just go to the next block if there isn't any new records in the current one. + if (!inserted_count) + /// SimpleTransform will skip it. + return; + + if (inserted_count < num_rows) + { + for (auto & column : columns) + if (isColumnConst(*column)) + column = column->cut(0, inserted_count); + else + column = column->filter(filter, inserted_count); + } + + chunk.setColumns(std::move(columns), inserted_count); +} + +} diff --git a/dbms/src/Processors/Transforms/LimitByTransform.h b/dbms/src/Processors/Transforms/LimitByTransform.h new file mode 100644 index 00000000000..344cf6feea4 --- /dev/null +++ b/dbms/src/Processors/Transforms/LimitByTransform.h @@ -0,0 +1,28 @@ +#pragma once +#include +#include +#include + +namespace DB +{ + +class LimitByTransform : public ISimpleTransform +{ +public: + LimitByTransform(const Block & header, size_t group_length_, size_t group_offset_, const Names & columns); + + String getName() const override { return "LimitByTransform"; } + +protected: + void transform(Chunk & chunk) override; + +private: + using MapHashed = HashMap; + + MapHashed keys_counts; + std::vector key_positions; + const size_t group_length; + const size_t group_offset; +}; + +} diff --git a/dbms/src/Processors/Transforms/LimitsCheckingTransform.cpp b/dbms/src/Processors/Transforms/LimitsCheckingTransform.cpp new file mode 100644 index 00000000000..092327a0d8e --- /dev/null +++ b/dbms/src/Processors/Transforms/LimitsCheckingTransform.cpp @@ -0,0 +1,115 @@ +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int TOO_MANY_ROWS; + extern const int TOO_MANY_BYTES; + extern const int TOO_MANY_ROWS_OR_BYTES; + extern const int TIMEOUT_EXCEEDED; + extern const int TOO_SLOW; + extern const int LOGICAL_ERROR; + extern const int BLOCKS_HAVE_DIFFERENT_STRUCTURE; + extern const int TOO_DEEP_PIPELINE; +} + + +static bool handleOverflowMode(OverflowMode mode, const String & message, int code) +{ + switch (mode) + { + case OverflowMode::THROW: + throw Exception(message, code); + case OverflowMode::BREAK: + return false; + default: + throw Exception("Logical error: unknown overflow mode", ErrorCodes::LOGICAL_ERROR); + } +} + + +void ProcessorProfileInfo::update(const Chunk & block) +{ + ++blocks; + rows += block.getNumRows(); + bytes += block.bytes(); +} + +LimitsCheckingTransform::LimitsCheckingTransform(const Block & header, LocalLimits limits) + : ISimpleTransform(header, header, false) + , limits(std::move(limits)) +{ +} + +//LimitsCheckingTransform::LimitsCheckingTransform(const Block & header, LocalLimits limits, QueryStatus * process_list_elem) +// : ISimpleTransform(header, header, false) +// , limits(std::move(limits)) +// , process_list_elem(process_list_elem) +//{ +//} + +void LimitsCheckingTransform::transform(Chunk & chunk) +{ + if (!info.started) + { + info.total_stopwatch.start(); + info.started = true; + } + + if (!checkTimeLimit()) + { + stopReading(); + return; + } + + if (chunk) + { + info.update(chunk); + + if (limits.mode == LimitsMode::LIMITS_CURRENT && + !limits.size_limits.check(info.rows, info.bytes, "result", ErrorCodes::TOO_MANY_ROWS_OR_BYTES)) + stopReading(); + + if (quota != nullptr) + checkQuota(chunk); + } +} + +bool LimitsCheckingTransform::checkTimeLimit() +{ + if (limits.max_execution_time != 0 + && info.total_stopwatch.elapsed() > static_cast(limits.max_execution_time.totalMicroseconds()) * 1000) + return handleOverflowMode(limits.timeout_overflow_mode, + "Timeout exceeded: elapsed " + toString(info.total_stopwatch.elapsedSeconds()) + + " seconds, maximum: " + toString(limits.max_execution_time.totalMicroseconds() / 1000000.0), + ErrorCodes::TIMEOUT_EXCEEDED); + + return true; +} + +void LimitsCheckingTransform::checkQuota(Chunk & chunk) +{ + switch (limits.mode) + { + case LimitsMode::LIMITS_TOTAL: + /// Checked in `progress` method. + break; + + case LimitsMode::LIMITS_CURRENT: + { + time_t current_time = time(nullptr); + double total_elapsed = info.total_stopwatch.elapsedSeconds(); + + quota->checkAndAddResultRowsBytes(current_time, chunk.getNumRows(), chunk.bytes()); + quota->checkAndAddExecutionTime(current_time, Poco::Timespan((total_elapsed - prev_elapsed) * 1000000.0)); + + prev_elapsed = total_elapsed; + break; + } + } +} + +} diff --git a/dbms/src/Processors/Transforms/LimitsCheckingTransform.h b/dbms/src/Processors/Transforms/LimitsCheckingTransform.h new file mode 100644 index 00000000000..a08e9ea9c67 --- /dev/null +++ b/dbms/src/Processors/Transforms/LimitsCheckingTransform.h @@ -0,0 +1,56 @@ +#pragma once +#include +#include +#include +#include + +#include + +namespace DB +{ + +/// Information for profiling. +struct ProcessorProfileInfo +{ + bool started = false; + Stopwatch total_stopwatch {CLOCK_MONOTONIC_COARSE}; /// Time with waiting time + + size_t rows = 0; + size_t blocks = 0; + size_t bytes = 0; + + void update(const Chunk & block); +}; + +class LimitsCheckingTransform : public ISimpleTransform +{ +public: + + using LocalLimits = IBlockInputStream::LocalLimits; + using LimitsMode = IBlockInputStream::LimitsMode; + + /// LIMITS_CURRENT + LimitsCheckingTransform(const Block & header, LocalLimits limits); + /// LIMITS_TOTAL + /// LimitsCheckingTransform(const Block & header, LocalLimits limits, QueryStatus * process_list_elem); + + String getName() const override { return "LimitsCheckingTransform"; } + + void setQuota(QuotaForIntervals & quota_) { quota = "a_; } + +protected: + void transform(Chunk & chunk) override; + +private: + LocalLimits limits; + + QuotaForIntervals * quota = nullptr; + double prev_elapsed = 0; + + ProcessorProfileInfo info; + + bool checkTimeLimit(); + void checkQuota(Chunk & chunk); +}; + +} diff --git a/dbms/src/Processors/Transforms/MaterializingTransform.cpp b/dbms/src/Processors/Transforms/MaterializingTransform.cpp new file mode 100644 index 00000000000..f13d5376ebe --- /dev/null +++ b/dbms/src/Processors/Transforms/MaterializingTransform.cpp @@ -0,0 +1,21 @@ +#include +#include + +namespace DB +{ + +MaterializingTransform::MaterializingTransform(const Block & header) + : ISimpleTransform(header, materializeBlock(header), false) {} + +void MaterializingTransform::transform(Chunk & chunk) +{ + auto num_rows = chunk.getNumRows(); + auto columns = chunk.detachColumns(); + + for (auto & col : columns) + col = col->convertToFullColumnIfConst(); + + chunk.setColumns(std::move(columns), num_rows); +} + +} diff --git a/dbms/src/Processors/Transforms/MaterializingTransform.h b/dbms/src/Processors/Transforms/MaterializingTransform.h new file mode 100644 index 00000000000..914dc268021 --- /dev/null +++ b/dbms/src/Processors/Transforms/MaterializingTransform.h @@ -0,0 +1,18 @@ +#include + +namespace DB +{ + +/// Converts columns-constants to full columns ("materializes" them). +class MaterializingTransform : public ISimpleTransform +{ +public: + explicit MaterializingTransform(const Block & header); + + String getName() const override { return "MaterializingTransform"; } + +protected: + void transform(Chunk & chunk) override; +}; + +} diff --git a/dbms/src/Processors/Transforms/MergeSortingTransform.cpp b/dbms/src/Processors/Transforms/MergeSortingTransform.cpp new file mode 100644 index 00000000000..8591f5447f7 --- /dev/null +++ b/dbms/src/Processors/Transforms/MergeSortingTransform.cpp @@ -0,0 +1,659 @@ +#include + +#include +#include + +#include +#include + +#include +#include + +#include +#include + +#include +#include +#include + + +namespace ProfileEvents +{ + extern const Event ExternalSortWritePart; + extern const Event ExternalSortMerge; +} + + +namespace DB +{ + +class BufferingToFileTransform : public IAccumulatingTransform +{ +public: + BufferingToFileTransform(const Block & header, Logger * log_, std::string path_) + : IAccumulatingTransform(header, header), log(log_) + , path(std::move(path_)), file_buf_out(path), compressed_buf_out(file_buf_out) + , out_stream(std::make_shared(compressed_buf_out, 0, header)) + { + LOG_INFO(log, "Sorting and writing part of data into temporary file " + path); + ProfileEvents::increment(ProfileEvents::ExternalSortWritePart); + out_stream->writePrefix(); + } + + String getName() const override { return "BufferingToFileTransform"; } + + void consume(Chunk chunk) override + { + out_stream->write(getInputPort().getHeader().cloneWithColumns(chunk.detachColumns())); + } + + Chunk generate() override + { + if (out_stream) + { + out_stream->writeSuffix(); + compressed_buf_out.next(); + file_buf_out.next(); + LOG_INFO(log, "Done writing part of data into temporary file " + path); + + out_stream.reset(); + + file_in = std::make_unique(path); + compressed_in = std::make_unique(*file_in); + block_in = std::make_shared(*compressed_in, getOutputPort().getHeader(), 0); + } + + if (!block_in) + return {}; + + auto block = block_in->read(); + if (!block) + { + block_in->readSuffix(); + block_in.reset(); + return {}; + } + + UInt64 num_rows = block.rows(); + return Chunk(block.getColumns(), num_rows); + } + +private: + Logger * log; + std::string path; + WriteBufferFromFile file_buf_out; + CompressedWriteBuffer compressed_buf_out; + BlockOutputStreamPtr out_stream; + + std::unique_ptr file_in; + std::unique_ptr compressed_in; + BlockInputStreamPtr block_in; +}; + +/** Part of implementation. Merging array of ready (already read from somewhere) chunks. + * Returns result of merge as stream of chunks, not more than 'max_merged_block_size' rows in each. + */ +class MergeSorter +{ +public: + MergeSorter(Chunks chunks_, SortDescription & description_, size_t max_merged_block_size_, UInt64 limit_); + + Chunk read(); + +private: + Chunks chunks; + SortDescription description; + size_t max_merged_block_size; + UInt64 limit; + size_t total_merged_rows = 0; + + using CursorImpls = std::vector; + CursorImpls cursors; + + bool has_collation = false; + + std::priority_queue queue_without_collation; + std::priority_queue queue_with_collation; + + /** Two different cursors are supported - with and without Collation. + * Templates are used (instead of virtual functions in SortCursor) for zero-overhead. + */ + template + Chunk mergeImpl(std::priority_queue & queue); +}; + +class MergeSorterSource : public ISource +{ +public: + MergeSorterSource(Block header, Chunks chunks, SortDescription & description, size_t max_merged_block_size, UInt64 limit) + : ISource(std::move(header)), merge_sorter(std::move(chunks), description, max_merged_block_size, limit) {} + + String getName() const override { return "MergeSorterSource"; } + +protected: + Chunk generate() override { return merge_sorter.read(); } + +private: + MergeSorter merge_sorter; +}; + +MergeSorter::MergeSorter(Chunks chunks_, SortDescription & description_, size_t max_merged_block_size_, UInt64 limit_) + : chunks(std::move(chunks_)), description(description_), max_merged_block_size(max_merged_block_size_), limit(limit_) +{ + Chunks nonempty_chunks; + for (auto & chunk : chunks) + { + if (chunk.getNumRows() == 0) + continue; + + cursors.emplace_back(chunk.getColumns(), description); + has_collation |= cursors.back().has_collation; + + nonempty_chunks.emplace_back(std::move(chunk)); + } + + chunks.swap(nonempty_chunks); + + if (!has_collation) + { + for (auto & cursor : cursors) + queue_without_collation.push(SortCursor(&cursor)); + } + else + { + for (auto & cursor : cursors) + queue_with_collation.push(SortCursorWithCollation(&cursor)); + } +} + + +Chunk MergeSorter::read() +{ + if (chunks.empty()) + return Chunk(); + + if (chunks.size() == 1) + { + auto res = std::move(chunks[0]); + chunks.clear(); + return res; + } + + return !has_collation + ? mergeImpl(queue_without_collation) + : mergeImpl(queue_with_collation); +} + + +template +Chunk MergeSorter::mergeImpl(std::priority_queue & queue) +{ + size_t num_columns = chunks[0].getNumColumns(); + + MutableColumns merged_columns = chunks[0].cloneEmptyColumns(); + /// TODO: reserve (in each column) + + /// Take rows from queue in right order and push to 'merged'. + size_t merged_rows = 0; + while (!queue.empty()) + { + TSortCursor current = queue.top(); + queue.pop(); + + for (size_t i = 0; i < num_columns; ++i) + merged_columns[i]->insertFrom(*current->all_columns[i], current->pos); + + ++total_merged_rows; + ++merged_rows; + + if (!current->isLast()) + { + current->next(); + queue.push(current); + } + + if (limit && total_merged_rows == limit) + { + chunks.clear(); + return Chunk(std::move(merged_columns), merged_rows); + } + + if (merged_rows == max_merged_block_size) + return Chunk(std::move(merged_columns), merged_rows); + } + + chunks.clear(); + + if (merged_rows == 0) + return {}; + + return Chunk(std::move(merged_columns), merged_rows); +} + + +MergeSortingTransform::MergeSortingTransform( + const Block & header, + SortDescription & description_, + size_t max_merged_block_size_, UInt64 limit_, + size_t max_bytes_before_remerge_, + size_t max_bytes_before_external_sort_, const std::string & tmp_path_) + : IProcessor({header}, {header}) + , description(description_), max_merged_block_size(max_merged_block_size_), limit(limit_) + , max_bytes_before_remerge(max_bytes_before_remerge_) + , max_bytes_before_external_sort(max_bytes_before_external_sort_), tmp_path(tmp_path_) +{ + auto & sample = inputs.front().getHeader(); + + /// Replace column names to column position in sort_description. + for (auto & column_description : description) + { + if (!column_description.column_name.empty()) + { + column_description.column_number = sample.getPositionByName(column_description.column_name); + column_description.column_name.clear(); + } + } + + /// Remove constants from header and map old indexes to new. + size_t num_columns = sample.columns(); + ColumnNumbers map(num_columns, num_columns); + const_columns_to_remove.assign(num_columns, true); + for (size_t pos = 0; pos < num_columns; ++pos) + { + const auto & column = sample.getByPosition(pos); + if (!(column.column && isColumnConst(*column.column))) + { + map[pos] = header_without_constants.columns(); + header_without_constants.insert(column); + const_columns_to_remove[pos] = false; + } + } + + /// Remove constants from column_description and remap positions. + SortDescription description_without_constants; + description_without_constants.reserve(description.size()); + for (const auto & column_description : description) + { + auto old_pos = column_description.column_number; + auto new_pos = map[old_pos]; + if (new_pos < num_columns) + { + description_without_constants.push_back(column_description); + description_without_constants.back().column_number = new_pos; + } + } + + description.swap(description_without_constants); +} + +MergeSortingTransform::~MergeSortingTransform() = default; + +IProcessor::Status MergeSortingTransform::prepare() +{ + if (stage == Stage::Serialize) + { + if (!processors.empty()) + return Status::ExpandPipeline; + + auto status = prepareSerialize(); + if (status != Status::Finished) + return status; + + stage = Stage::Consume; + } + + if (stage == Stage::Consume) + { + auto status = prepareConsume(); + if (status != Status::Finished) + return status; + + stage = Stage::Generate; + } + + /// stage == Stage::Generate + + if (!generated_prefix) + return Status::Ready; + + if (!processors.empty()) + return Status::ExpandPipeline; + + return prepareGenerate(); +} + +IProcessor::Status MergeSortingTransform::prepareConsume() +{ + auto & input = inputs.front(); + auto & output = outputs.front(); + + /// Check can output. + + if (output.isFinished()) + { + input.close(); + return Status::Finished; + } + + if (!output.canPush()) + { + input.setNotNeeded(); + return Status::PortFull; + } + + if (generated_chunk) + output.push(std::move(generated_chunk)); + + /// Check can input. + if (!current_chunk) + { + if (input.isFinished()) + return Status::Finished; + + input.setNeeded(); + + if (!input.hasData()) + return Status::NeedData; + + current_chunk = input.pull(); + } + + /// Now consume. + return Status::Ready; +} + +IProcessor::Status MergeSortingTransform::prepareSerialize() +{ + auto & output = outputs.back(); + + if (output.isFinished()) + return Status::Finished; + + if (!output.canPush()) + return Status::PortFull; + + if (current_chunk) + output.push(std::move(current_chunk)); + + if (merge_sorter) + return Status::Ready; + + output.finish(); + return Status::Finished; +} + +IProcessor::Status MergeSortingTransform::prepareGenerate() +{ + auto & output = outputs.front(); + + if (output.isFinished()) + return Status::Finished; + + if (!output.canPush()) + return Status::PortFull; + + if (merge_sorter) + { + if (!generated_chunk) + return Status::Ready; + + output.push(std::move(generated_chunk)); + return Status::PortFull; + } + else + { + auto & input = inputs.back(); + + if (input.isFinished()) + { + output.finish(); + return Status::Finished; + } + + input.setNeeded(); + + if (!input.hasData()) + return Status::NeedData; + + auto chunk = input.pull(); + enrichChunkWithConstants(chunk); + output.push(std::move(chunk)); + return Status::PortFull; + } +} + +void MergeSortingTransform::work() +{ + if (stage == Stage::Consume) + consume(std::move(current_chunk)); + + if (stage == Stage::Serialize) + serialize(); + + if (stage == Stage::Generate) + generate(); +} + +Processors MergeSortingTransform::expandPipeline() +{ + if (processors.size() > 1) + { + /// Add external_merging_sorted. + inputs.emplace_back(header_without_constants, this); + connect(external_merging_sorted->getOutputs().front(), inputs.back()); + } + + auto & buffer = processors.front(); + + static_cast(*external_merging_sorted).addInput(); + connect(buffer->getOutputs().back(), external_merging_sorted->getInputs().back()); + + if (!buffer->getInputs().empty()) + { + /// Serialize + outputs.emplace_back(header_without_constants, this); + connect(getOutputs().back(), buffer->getInputs().back()); + /// Hack. Say buffer that we need data from port (otherwise it will return PortFull). + external_merging_sorted->getInputs().back().setNeeded(); + } + else + /// Generate + static_cast(*external_merging_sorted).setHaveAllInputs(); + + return std::move(processors); +} + +void MergeSortingTransform::consume(Chunk chunk) +{ + /** Algorithm: + * - read to memory blocks from source stream; + * - if too many of them and if external sorting is enabled, + * - merge all blocks to sorted stream and write it to temporary file; + * - at the end, merge all sorted streams from temporary files and also from rest of blocks in memory. + */ + + /// If there were only const columns in sort description, then there is no need to sort. + /// Return the chunk as is. + if (description.empty()) + { + generated_chunk = std::move(chunk); + return; + } + + removeConstColumns(chunk); + + sum_rows_in_blocks += chunk.getNumRows(); + sum_bytes_in_blocks += chunk.allocatedBytes(); + chunks.push_back(std::move(chunk)); + + /** If significant amount of data was accumulated, perform preliminary merging step. + */ + if (chunks.size() > 1 + && limit + && limit * 2 < sum_rows_in_blocks /// 2 is just a guess. + && remerge_is_useful + && max_bytes_before_remerge + && sum_bytes_in_blocks > max_bytes_before_remerge) + { + remerge(); + } + + /** If too many of them and if external sorting is enabled, + * will merge blocks that we have in memory at this moment and write merged stream to temporary (compressed) file. + * NOTE. It's possible to check free space in filesystem. + */ + if (max_bytes_before_external_sort && sum_bytes_in_blocks > max_bytes_before_external_sort) + { + Poco::File(tmp_path).createDirectories(); + temporary_files.emplace_back(std::make_unique(tmp_path)); + const std::string & path = temporary_files.back()->path(); + merge_sorter = std::make_unique(std::move(chunks), description, max_merged_block_size, limit); + auto current_processor = std::make_shared(header_without_constants, log, path); + + processors.emplace_back(current_processor); + + if (!external_merging_sorted) + { + bool quiet = false; + bool have_all_inputs = false; + + external_merging_sorted = std::make_shared( + header_without_constants, + 0, + description, + max_merged_block_size, + limit, + quiet, + have_all_inputs); + + processors.emplace_back(external_merging_sorted); + } + + stage = Stage::Serialize; + sum_bytes_in_blocks = 0; + sum_rows_in_blocks = 0; + } +} + +void MergeSortingTransform::serialize() +{ + current_chunk = merge_sorter->read(); + if (!current_chunk) + merge_sorter.reset(); +} + +void MergeSortingTransform::generate() +{ + if (!generated_prefix) + { + if (temporary_files.empty()) + merge_sorter = std::make_unique(std::move(chunks), description, max_merged_block_size, limit); + else + { + ProfileEvents::increment(ProfileEvents::ExternalSortMerge); + LOG_INFO(log, "There are " << temporary_files.size() << " temporary sorted parts to merge."); + + if (!chunks.empty()) + processors.emplace_back(std::make_shared( + header_without_constants, std::move(chunks), description, max_merged_block_size, limit)); + } + + generated_prefix = true; + } + + if (merge_sorter) + { + generated_chunk = merge_sorter->read(); + if (!generated_chunk) + merge_sorter.reset(); + else + enrichChunkWithConstants(generated_chunk); + } +} + +void MergeSortingTransform::remerge() +{ + LOG_DEBUG(log, "Re-merging intermediate ORDER BY data (" << chunks.size() + << " blocks with " << sum_rows_in_blocks << " rows) to save memory consumption"); + + /// NOTE Maybe concat all blocks and partial sort will be faster than merge? + MergeSorter remerge_sorter(std::move(chunks), description, max_merged_block_size, limit); + + Chunks new_chunks; + size_t new_sum_rows_in_blocks = 0; + size_t new_sum_bytes_in_blocks = 0; + + while (auto chunk = remerge_sorter.read()) + { + new_sum_rows_in_blocks += chunk.getNumRows(); + new_sum_bytes_in_blocks += chunk.allocatedBytes(); + new_chunks.emplace_back(std::move(chunk)); + } + + LOG_DEBUG(log, "Memory usage is lowered from " + << formatReadableSizeWithBinarySuffix(sum_bytes_in_blocks) << " to " + << formatReadableSizeWithBinarySuffix(new_sum_bytes_in_blocks)); + + /// If the memory consumption was not lowered enough - we will not perform remerge anymore. 2 is a guess. + if (new_sum_bytes_in_blocks * 2 > sum_bytes_in_blocks) + remerge_is_useful = false; + + chunks = std::move(new_chunks); + sum_rows_in_blocks = new_sum_rows_in_blocks; + sum_bytes_in_blocks = new_sum_bytes_in_blocks; +} + + +void MergeSortingTransform::removeConstColumns(Chunk & chunk) +{ + size_t num_columns = chunk.getNumColumns(); + size_t num_rows = chunk.getNumRows(); + + if (num_columns != const_columns_to_remove.size()) + throw Exception("Block has different number of columns with header: " + toString(num_columns) + + " vs " + toString(const_columns_to_remove.size()), ErrorCodes::LOGICAL_ERROR); + + auto columns = chunk.detachColumns(); + Columns column_without_constants; + column_without_constants.reserve(header_without_constants.columns()); + + for (size_t position = 0; position < num_columns; ++position) + { + if (!const_columns_to_remove[position]) + column_without_constants.push_back(std::move(columns[position])); + } + + chunk.setColumns(std::move(column_without_constants), num_rows); +} + +void MergeSortingTransform::enrichChunkWithConstants(Chunk & chunk) +{ + size_t num_rows = chunk.getNumRows(); + size_t num_result_columns = const_columns_to_remove.size(); + + auto columns = chunk.detachColumns(); + Columns column_with_constants; + column_with_constants.reserve(num_result_columns); + + auto & header = inputs.front().getHeader(); + + size_t next_non_const_column = 0; + for (size_t i = 0; i < num_result_columns; ++i) + { + if (const_columns_to_remove[i]) + column_with_constants.emplace_back(header.getByPosition(i).column->cloneResized(num_rows)); + else + { + if (next_non_const_column >= columns.size()) + throw Exception("Can't enrich chunk with constants because run out of non-constant columns.", + ErrorCodes::LOGICAL_ERROR); + + column_with_constants.emplace_back(std::move(columns[next_non_const_column])); + ++next_non_const_column; + } + } + + chunk.setColumns(std::move(column_with_constants), num_rows); +} + +} diff --git a/dbms/src/Processors/Transforms/MergeSortingTransform.h b/dbms/src/Processors/Transforms/MergeSortingTransform.h new file mode 100644 index 00000000000..0ab517fc5d4 --- /dev/null +++ b/dbms/src/Processors/Transforms/MergeSortingTransform.h @@ -0,0 +1,98 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +namespace DB +{ + +class MergeSorter; + +class MergeSortingTransform : public IProcessor +{ +public: + /// limit - if not 0, allowed to return just first 'limit' rows in sorted order. + MergeSortingTransform(const Block & header, + SortDescription & description_, + size_t max_merged_block_size_, UInt64 limit_, + size_t max_bytes_before_remerge_, + size_t max_bytes_before_external_sort_, const std::string & tmp_path_); + + ~MergeSortingTransform() override; + + String getName() const override { return "MergeSortingTransform"; } + +protected: + + Status prepare() override; + void work() override; + Processors expandPipeline() override; + +private: + SortDescription description; + size_t max_merged_block_size; + UInt64 limit; + + size_t max_bytes_before_remerge; + size_t max_bytes_before_external_sort; + const std::string tmp_path; + + Logger * log = &Logger::get("MergeSortingBlockInputStream"); + + Chunks chunks; + size_t sum_rows_in_blocks = 0; + size_t sum_bytes_in_blocks = 0; + + /// If remerge doesn't save memory at least several times, mark it as useless and don't do it anymore. + bool remerge_is_useful = true; + + /// Before operation, will remove constant columns from blocks. And after, place constant columns back. + /// (to avoid excessive virtual function calls and because constants cannot be serialized in Native format for temporary files) + /// Save original block structure here. + Block header_without_constants; + /// Columns which were constant in header and we need to remove from chunks. + std::vector const_columns_to_remove; + + /// Everything below is for external sorting. + std::vector> temporary_files; + + /// Merge all accumulated blocks to keep no more than limit rows. + void remerge(); + + void removeConstColumns(Chunk & chunk); + void enrichChunkWithConstants(Chunk & chunk); + + enum class Stage + { + Consume = 0, + Generate, + Serialize, + }; + + Stage stage = Stage::Consume; + bool generated_prefix = false; + Chunk current_chunk; + Chunk generated_chunk; + + std::unique_ptr merge_sorter; + ProcessorPtr external_merging_sorted; + Processors processors; + + Status prepareConsume(); + Status prepareSerialize(); + Status prepareGenerate(); + + void consume(Chunk chunk); + void serialize(); + void generate(); +}; + +} diff --git a/dbms/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp b/dbms/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp new file mode 100644 index 00000000000..a573a6270e9 --- /dev/null +++ b/dbms/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp @@ -0,0 +1,519 @@ +#include + +#include +#include +#include + +namespace DB +{ + +struct ChunksToMerge : public ChunkInfo +{ + std::unique_ptr chunks; + Int32 bucket_num = -1; + bool is_overflows = false; +}; + +GroupingAggregatedTransform::GroupingAggregatedTransform( + const Block & header, size_t num_inputs, AggregatingTransformParamsPtr params) + : IProcessor(InputPorts(num_inputs, header), { Block() }) + , num_inputs(num_inputs) + , params(std::move(params)) + , last_bucket_number(num_inputs, -1) + , read_from_input(num_inputs, false) +{ +} + +void GroupingAggregatedTransform::readFromAllInputs() +{ + auto in = inputs.begin(); + for (size_t i = 0; i < num_inputs; ++i, ++in) + { + if (in->isFinished()) + continue; + + if (read_from_input[i]) + continue; + + in->setNeeded(); + + if (!in->hasData()) + return; + + auto chunk = in->pull(); + read_from_input[i] = true; + addChunk(std::move(chunk), i); + } + + read_from_all_inputs = true; +} + +void GroupingAggregatedTransform::pushData(Chunks chunks, Int32 bucket, bool is_overflows) +{ + auto & output = outputs.front(); + + auto info = std::make_shared(); + info->bucket_num = bucket; + info->is_overflows = is_overflows; + info->chunks = std::make_unique(std::move(chunks)); + + Chunk chunk; + chunk.setChunkInfo(std::move(info)); + output.push(std::move(chunk)); +} + +bool GroupingAggregatedTransform::tryPushTwoLevelData() +{ + auto try_push_by_iter = [&](auto batch_it) + { + if (batch_it == chunks_map.end()) + return false; + + Chunks & cur_chunks = batch_it->second; + if (cur_chunks.empty()) + { + chunks_map.erase(batch_it); + return false; + } + + pushData(std::move(cur_chunks), batch_it->first, false); + chunks_map.erase(batch_it); + return true; + }; + + if (all_inputs_finished) + { + /// Chunks are sorted by bucket. + while (!chunks_map.empty()) + if (try_push_by_iter(chunks_map.begin())) + return true; + } + else + { + for (; next_bucket_to_push < current_bucket; ++next_bucket_to_push) + if (try_push_by_iter(chunks_map.find(next_bucket_to_push))) + return true; + } + + return false; +} + +bool GroupingAggregatedTransform::tryPushSingleLevelData() +{ + if (single_level_chunks.empty()) + return false; + + pushData(std::move(single_level_chunks), -1, false); + return true; +} + +bool GroupingAggregatedTransform::tryPushOverflowData() +{ + if (overflow_chunks.empty()) + return false; + + pushData(std::move(overflow_chunks), -1, true); + return true; +} + +IProcessor::Status GroupingAggregatedTransform::prepare() +{ + /// Check can output. + auto & output = outputs.front(); + + if (output.isFinished()) + { + for (auto & input : inputs) + input.close(); + + chunks_map.clear(); + last_bucket_number.clear(); + return Status::Finished; + } + + /// Read first time from each input to understand if we have two-level aggregation. + if (!read_from_all_inputs) + { + readFromAllInputs(); + if (!read_from_all_inputs) + return Status::NeedData; + } + + /// Convert single level to two levels if have two-level input. + if (has_two_level && !single_level_chunks.empty()) + return Status::Ready; + + /// Check can push (to avoid data caching). + if (!output.canPush()) + { + for (auto & input : inputs) + input.setNotNeeded(); + + return Status::PortFull; + } + + bool pushed_to_output = false; + + /// Output if has data. + if (has_two_level) + pushed_to_output = tryPushTwoLevelData(); + + auto need_input = [this](size_t input_num) + { + if (last_bucket_number[input_num] < current_bucket) + return true; + + return expect_several_chunks_for_single_bucket_per_source && last_bucket_number[input_num] == current_bucket; + }; + + /// Read next bucket if can. + for (; ; ++current_bucket) + { + bool finished = true; + bool need_data = false; + + auto in = inputs.begin(); + for (size_t input_num = 0; input_num < num_inputs; ++input_num, ++in) + { + if (in->isFinished()) + continue; + + finished = false; + + if (!need_input(input_num)) + continue; + + in->setNeeded(); + + if (!in->hasData()) + { + need_data = true; + continue; + } + + auto chunk = in->pull(); + addChunk(std::move(chunk), input_num); + + if (has_two_level && !single_level_chunks.empty()) + return Status::Ready; + + if (!in->isFinished() && need_input(input_num)) + need_data = true; + } + + if (finished) + { + all_inputs_finished = true; + break; + } + + if (need_data) + return Status::NeedData; + } + + if (pushed_to_output) + return Status::PortFull; + + if (has_two_level) + { + if (tryPushTwoLevelData()) + return Status::PortFull; + + /// Sanity check. If new bucket was read, we should be able to push it. + if (!all_inputs_finished) + throw Exception("GroupingAggregatedTransform has read new two-level bucket, but couldn't push it.", + ErrorCodes::LOGICAL_ERROR); + } + else + { + if (!all_inputs_finished) + throw Exception("GroupingAggregatedTransform should have read all chunks for single level aggregation, " + "but not all of the inputs are finished.", ErrorCodes::LOGICAL_ERROR); + + if (tryPushSingleLevelData()) + return Status::PortFull; + } + + /// If we haven't pushed to output, then all data was read. Push overflows if have. + if (tryPushOverflowData()) + return Status::PortFull; + + output.finish(); + return Status::Finished; +} + +void GroupingAggregatedTransform::addChunk(Chunk chunk, size_t input) +{ + auto & info = chunk.getChunkInfo(); + if (!info) + throw Exception("Chunk info was not set for chunk in GroupingAggregatedTransform.", ErrorCodes::LOGICAL_ERROR); + + auto * agg_info = typeid_cast(info.get()); + if (!agg_info) + throw Exception("Chunk should have AggregatedChunkInfo in GroupingAggregatedTransform.", ErrorCodes::LOGICAL_ERROR); + + Int32 bucket = agg_info->bucket_num; + bool is_overflows = agg_info->is_overflows; + + if (is_overflows) + overflow_chunks.emplace_back(std::move(chunk)); + else if (bucket < 0) + single_level_chunks.emplace_back(std::move(chunk)); + else + { + chunks_map[bucket].emplace_back(std::move(chunk)); + has_two_level = true; + last_bucket_number[input] = bucket; + } +} + +void GroupingAggregatedTransform::work() +{ + if (!single_level_chunks.empty()) + { + auto & header = getOutputs().front().getHeader(); + auto block = header.cloneWithColumns(single_level_chunks.back().detachColumns()); + single_level_chunks.pop_back(); + auto blocks = params->aggregator.convertBlockToTwoLevel(block); + + for (auto & cur_block : blocks) + { + Int32 bucket = cur_block.info.bucket_num; + chunks_map[bucket].emplace_back(Chunk(cur_block.getColumns(), cur_block.rows())); + } + } +} + + +MergingAggregatedBucketTransform::MergingAggregatedBucketTransform(AggregatingTransformParamsPtr params) + : ISimpleTransform({}, params->getHeader(), false), params(std::move(params)) +{ + setInputNotNeededAfterRead(true); +} + +void MergingAggregatedBucketTransform::transform(Chunk & chunk) +{ + auto & info = chunk.getChunkInfo(); + auto * chunks_to_merge = typeid_cast(info.get()); + + if (!chunks_to_merge) + throw Exception("MergingAggregatedSimpleTransform chunk must have ChunkInfo with type ChunksToMerge.", + ErrorCodes::LOGICAL_ERROR); + + auto header = params->aggregator.getHeader(false); + + BlocksList blocks_list; + for (auto & cur_chunk : *chunks_to_merge->chunks) + { + auto & cur_info = cur_chunk.getChunkInfo(); + if (!cur_info) + throw Exception("Chunk info was not set for chunk in MergingAggregatedBucketTransform.", + ErrorCodes::LOGICAL_ERROR); + + auto * agg_info = typeid_cast(cur_info.get()); + if (!agg_info) + throw Exception("Chunk should have AggregatedChunkInfo in MergingAggregatedBucketTransform.", + ErrorCodes::LOGICAL_ERROR); + + Block block = header.cloneWithColumns(cur_chunk.detachColumns()); + block.info.is_overflows = agg_info->is_overflows; + block.info.bucket_num = agg_info->bucket_num; + + blocks_list.emplace_back(std::move(block)); + } + + auto res_info = std::make_shared(); + res_info->is_overflows = chunks_to_merge->is_overflows; + res_info->bucket_num = chunks_to_merge->bucket_num; + chunk.setChunkInfo(std::move(res_info)); + + auto block = params->aggregator.mergeBlocks(blocks_list, params->final); + size_t num_rows = block.rows(); + chunk.setColumns(block.getColumns(), num_rows); +} + + +SortingAggregatedTransform::SortingAggregatedTransform(size_t num_inputs, AggregatingTransformParamsPtr params) + : IProcessor(InputPorts(num_inputs, params->getHeader()), {params->getHeader()}) + , num_inputs(num_inputs) + , params(std::move(params)) + , last_bucket_number(num_inputs, -1) + , is_input_finished(num_inputs, false) +{ +} + +bool SortingAggregatedTransform::tryPushChunk() +{ + auto & output = outputs.front(); + + if (chunks.empty()) + return false; + + /// Chunk with min current bucket. + auto it = chunks.begin(); + auto cur_bucket = it->first; + + /// Check that can push it + for (size_t input = 0; input < num_inputs; ++input) + if (!is_input_finished[input] && last_bucket_number[input] < cur_bucket) + return false; + + output.push(std::move(it->second)); + chunks.erase(it); + return true; +} + +void SortingAggregatedTransform::addChunk(Chunk chunk, size_t from_input) +{ + auto & info = chunk.getChunkInfo(); + if (!info) + throw Exception("Chunk info was not set for chunk in SortingAggregatedTransform.", ErrorCodes::LOGICAL_ERROR); + + auto * agg_info = typeid_cast(info.get()); + if (!agg_info) + throw Exception("Chunk should have AggregatedChunkInfo in SortingAggregatedTransform.", ErrorCodes::LOGICAL_ERROR); + + Int32 bucket = agg_info->bucket_num; + bool is_overflows = agg_info->is_overflows; + + if (is_overflows) + overflow_chunk = std::move(chunk); + else + { + if (chunks[bucket]) + throw Exception("SortingAggregatedTransform already got bucket with number " + toString(bucket), + ErrorCodes::LOGICAL_ERROR); + + chunks[bucket] = std::move(chunk); + last_bucket_number[from_input] = bucket; + } +} + +IProcessor::Status SortingAggregatedTransform::prepare() +{ + /// Check can output. + auto & output = outputs.front(); + + if (output.isFinished()) + { + for (auto & input : inputs) + input.close(); + + chunks.clear(); + last_bucket_number.clear(); + return Status::Finished; + } + + /// Check can push (to avoid data caching). + if (!output.canPush()) + { + for (auto & input : inputs) + input.setNotNeeded(); + + return Status::PortFull; + } + + /// Push if have min version. + bool pushed_to_output = tryPushChunk(); + + bool need_data = false; + bool all_finished = true; + + /// Try read anything. + auto in = inputs.begin(); + for (size_t input_num = 0; input_num < num_inputs; ++input_num, ++in) + { + if (in->isFinished()) + { + is_input_finished[input_num] = true; + continue; + } + + all_finished = false; + + in->setNeeded(); + + if (!in->hasData()) + { + need_data = true; + continue; + } + + auto chunk = in->pull(); + /// If chunk was pulled, then we need data from this port. + need_data = true; + + addChunk(std::move(chunk), input_num); + } + + if (pushed_to_output) + return Status::PortFull; + + if (tryPushChunk()) + return Status::PortFull; + + if (need_data) + return Status::NeedData; + + if (!all_finished) + throw Exception("SortingAggregatedTransform has read bucket, but couldn't push it.", + ErrorCodes::LOGICAL_ERROR); + + if (overflow_chunk) + { + output.push(std::move(overflow_chunk)); + return Status::PortFull; + } + + output.finish(); + return Status::Finished; +} + + +Processors createMergingAggregatedMemoryEfficientPipe( + Block header, + AggregatingTransformParamsPtr params, + size_t num_inputs, + size_t num_merging_processors) +{ + Processors processors; + processors.reserve(num_merging_processors + 2); + + auto grouping = std::make_shared(header, num_inputs, params); + processors.emplace_back(std::move(grouping)); + + if (num_merging_processors <= 1) + { + /// --> GroupingAggregated --> MergingAggregatedBucket --> + auto transform = std::make_shared(params); + connect(processors.back()->getOutputs().front(), transform->getInputPort()); + + processors.emplace_back(std::move(transform)); + return processors; + } + + /// --> --> MergingAggregatedBucket --> + /// --> GroupingAggregated --> ResizeProcessor --> MergingAggregatedBucket --> SortingAggregated --> + /// --> --> MergingAggregatedBucket --> + + auto resize = std::make_shared(Block(), 1, num_merging_processors); + connect(processors.back()->getOutputs().front(), resize->getInputs().front()); + processors.emplace_back(std::move(resize)); + + auto sorting = std::make_shared(num_merging_processors, params); + auto out = processors.back()->getOutputs().begin(); + auto in = sorting->getInputs().begin(); + + for (size_t i = 0; i < num_merging_processors; ++i, ++in, ++out) + { + auto transform = std::make_shared(params); + connect(*out, transform->getInputPort()); + connect(transform->getOutputPort(), *in); + processors.emplace_back(std::move(transform)); + } + + processors.emplace_back(std::move(sorting)); + return processors; +} + +} diff --git a/dbms/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.h b/dbms/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.h new file mode 100644 index 00000000000..eff71e954a9 --- /dev/null +++ b/dbms/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.h @@ -0,0 +1,98 @@ +#include +#include +#include +#include +#include + + +namespace DB +{ + +/// Has several inputs and single output. +/// Read from inputs chunks with partially aggregated data, group them by bucket number +/// and write data from single bucket as single chunk. +class GroupingAggregatedTransform : public IProcessor +{ +public: + GroupingAggregatedTransform(const Block & header, size_t num_inputs, AggregatingTransformParamsPtr params); + String getName() const override { return "GroupingAggregatedTransform"; } + + /// Special setting: in case if single source can return several chunks with same bucket. + void allowSeveralChunksForSingleBucketPerSource() { expect_several_chunks_for_single_bucket_per_source = true; } + +protected: + Status prepare() override; + void work() override; + +private: + size_t num_inputs; + AggregatingTransformParamsPtr params; + + std::vector last_bucket_number; + std::map chunks_map; + Chunks overflow_chunks; + Chunks single_level_chunks; + Int32 current_bucket = 0; + Int32 next_bucket_to_push = 0; /// Always <= current_bucket. + bool has_two_level = false; + + bool all_inputs_finished = false; + bool read_from_all_inputs = false; + std::vector read_from_input; + + bool expect_several_chunks_for_single_bucket_per_source = false; + + void addChunk(Chunk chunk, size_t input); + void readFromAllInputs(); + bool tryPushSingleLevelData(); + bool tryPushTwoLevelData(); + bool tryPushOverflowData(); + void pushData(Chunks chunks, Int32 bucket, bool is_overflows); +}; + +/// Merge aggregated data from single bucket. +class MergingAggregatedBucketTransform : public ISimpleTransform +{ +public: + explicit MergingAggregatedBucketTransform(AggregatingTransformParamsPtr params); + String getName() const override { return "MergingAggregatedBucketTransform"; } + +protected: + void transform(Chunk & chunk) override; + +private: + AggregatingTransformParamsPtr params; +}; + +/// Has several inputs and single output. +/// Read from inputs merged bucket with aggregated data, sort them by bucket number and write to output. +/// Presumption: inputs return chunks with increasing bucket number, there is at most one chunk per bucket. +class SortingAggregatedTransform : public IProcessor +{ +public: + SortingAggregatedTransform(size_t num_inputs, AggregatingTransformParamsPtr params); + String getName() const override { return "SortingAggregatedTransform"; } + Status prepare() override; + +private: + size_t num_inputs; + AggregatingTransformParamsPtr params; + std::vector last_bucket_number; + std::vector is_input_finished; + std::map chunks; + Chunk overflow_chunk; + + bool tryPushChunk(); + void addChunk(Chunk chunk, size_t from_input); +}; + +/// Creates piece of pipeline which performs memory efficient merging of partially aggregated data from several sources. +/// First processor will have num_inputs, last - single output. You should connect them to create pipeline. +Processors createMergingAggregatedMemoryEfficientPipe( + Block header, + AggregatingTransformParamsPtr params, + size_t num_inputs, + size_t num_merging_processors); + +} + diff --git a/dbms/src/Processors/Transforms/MergingAggregatedTransform.cpp b/dbms/src/Processors/Transforms/MergingAggregatedTransform.cpp new file mode 100644 index 00000000000..32b833044cd --- /dev/null +++ b/dbms/src/Processors/Transforms/MergingAggregatedTransform.cpp @@ -0,0 +1,74 @@ +#include +#include + +namespace DB +{ + +MergingAggregatedTransform::MergingAggregatedTransform( + Block header, AggregatingTransformParamsPtr params, size_t max_threads) + : IAccumulatingTransform(std::move(header), params->getHeader()) + , params(std::move(params)), max_threads(max_threads) +{ +} + +void MergingAggregatedTransform::consume(Chunk chunk) +{ + if (!consume_started) + { + consume_started = true; + LOG_TRACE(log, "Reading blocks of partially aggregated data."); + } + + total_input_rows += chunk.getNumRows(); + ++total_input_blocks; + + auto & info = chunk.getChunkInfo(); + if (!info) + throw Exception("Chunk info was not set for chunk in MergingAggregatedTransform.", ErrorCodes::LOGICAL_ERROR); + + auto * agg_info = typeid_cast(info.get()); + if (!agg_info) + throw Exception("Chunk should have AggregatedChunkInfo in MergingAggregatedTransform.", ErrorCodes::LOGICAL_ERROR); + + auto block = getInputPort().getHeader().cloneWithColumns(chunk.getColumns()); + block.info.is_overflows = agg_info->is_overflows; + block.info.bucket_num = agg_info->bucket_num; + + bucket_to_blocks[agg_info->bucket_num].emplace_back(std::move(block)); +} + +Chunk MergingAggregatedTransform::generate() +{ + if (!generate_started) + { + generate_started = true; + LOG_TRACE(log, "Read " << total_input_blocks << " blocks of partially aggregated data, total " << total_input_rows + << " rows."); + + /// Exception safety. Make iterator valid in case any method below throws. + next_block = blocks.begin(); + + /// TODO: this operation can be made async. Add async for IAccumulatingTransform. + params->aggregator.mergeBlocks(std::move(bucket_to_blocks), data_variants, max_threads); + blocks = params->aggregator.convertToBlocks(data_variants, params->final, max_threads); + next_block = blocks.begin(); + } + + if (next_block == blocks.end()) + return {}; + + auto block = std::move(*next_block); + ++next_block; + + auto info = std::make_shared(); + info->bucket_num = block.info.bucket_num; + info->is_overflows = block.info.is_overflows; + + UInt64 num_rows = block.rows(); + Chunk chunk(block.getColumns(), num_rows); + chunk.setChunkInfo(std::move(info)); + + return chunk; +} + +} diff --git a/dbms/src/Processors/Transforms/MergingAggregatedTransform.h b/dbms/src/Processors/Transforms/MergingAggregatedTransform.h new file mode 100644 index 00000000000..a8c52f2b047 --- /dev/null +++ b/dbms/src/Processors/Transforms/MergingAggregatedTransform.h @@ -0,0 +1,40 @@ +#pragma once +#include +#include +#include + +namespace DB +{ + +/** A pre-aggregate stream of blocks in which each block is already aggregated. + * Aggregate functions in blocks should not be finalized so that their states can be merged. + */ +class MergingAggregatedTransform : public IAccumulatingTransform +{ +public: + MergingAggregatedTransform(Block header, AggregatingTransformParamsPtr params, size_t max_threads); + String getName() const override { return "MergingAggregatedTransform"; } + +protected: + void consume(Chunk chunk) override; + Chunk generate() override; + +private: + AggregatingTransformParamsPtr params; + Logger * log = &Logger::get("MergingAggregatedTransform"); + size_t max_threads; + + AggregatedDataVariants data_variants; + Aggregator::BucketToBlocks bucket_to_blocks; + + UInt64 total_input_rows = 0; + UInt64 total_input_blocks = 0; + + BlocksList blocks; + BlocksList::iterator next_block; + + bool consume_started = false; + bool generate_started = false; +}; + +} diff --git a/dbms/src/Processors/Transforms/MergingSortedTransform.cpp b/dbms/src/Processors/Transforms/MergingSortedTransform.cpp new file mode 100644 index 00000000000..b0283b0a56e --- /dev/null +++ b/dbms/src/Processors/Transforms/MergingSortedTransform.cpp @@ -0,0 +1,330 @@ +#include +#include +#include + +namespace DB +{ + +MergingSortedTransform::MergingSortedTransform( + const Block & header, + size_t num_inputs, + const SortDescription & description_, + size_t max_block_size, + UInt64 limit, + bool quiet, + bool have_all_inputs) + : IProcessor(InputPorts(num_inputs, header), {header}) + , description(description_), max_block_size(max_block_size), limit(limit), quiet(quiet) + , have_all_inputs(have_all_inputs) + , merged_data(header), source_chunks(num_inputs), cursors(num_inputs) +{ + auto & sample = outputs.front().getHeader(); + /// Replace column names in description to positions. + for (auto & column_description : description) + { + has_collation |= column_description.collator != nullptr; + if (!column_description.column_name.empty()) + { + column_description.column_number = sample.getPositionByName(column_description.column_name); + column_description.column_name.clear(); + } + } +} + +void MergingSortedTransform::addInput() +{ + if (have_all_inputs) + throw Exception("MergingSortedTransform already have all inputs.", ErrorCodes::LOGICAL_ERROR); + + inputs.emplace_back(outputs.front().getHeader(), this); + source_chunks.emplace_back(); + cursors.emplace_back(); +} + +void MergingSortedTransform::setHaveAllInputs() +{ + if (have_all_inputs) + throw Exception("MergingSortedTransform already have all inputs.", ErrorCodes::LOGICAL_ERROR); + + have_all_inputs = true; +} + +IProcessor::Status MergingSortedTransform::prepare() +{ + if (!have_all_inputs) + return Status::NeedData; + + auto & output = outputs.front(); + + /// Special case for no inputs. + if (inputs.empty()) + { + output.finish(); + return Status::Finished; + } + + /// Check can output. + + if (output.isFinished()) + { + for (auto & in : inputs) + in.close(); + + return Status::Finished; + } + + if (!output.isNeeded()) + { + for (auto & in : inputs) + in.setNotNeeded(); + + return Status::PortFull; + } + + if (output.hasData()) + return Status::PortFull; + + /// Special case for single input. + if (inputs.size() == 1) + { + auto & input = inputs.front(); + if (input.isFinished()) + { + output.finish(); + return Status::Finished; + } + + input.setNeeded(); + if (input.hasData()) + output.push(input.pull()); + + return Status::NeedData; + } + + /// Push if has data. + if (merged_data.mergedRows()) + output.push(merged_data.pull()); + + if (!is_initialized) + { + /// Check for inputs we need. + bool all_inputs_has_data = true; + auto it = inputs.begin(); + for (size_t i = 0; it != inputs.end(); ++i, ++it) + { + auto & input = *it; + if (input.isFinished()) + continue; + + if (!cursors[i].empty()) + { + input.setNotNeeded(); + continue; + } + + input.setNeeded(); + + if (!input.hasData()) + { + all_inputs_has_data = false; + continue; + } + + auto chunk = input.pull(); + if (chunk.hasNoRows()) + { + all_inputs_has_data = false; + continue; + } + + updateCursor(std::move(chunk), i); + } + + if (!all_inputs_has_data) + return Status::NeedData; + + if (has_collation) + initQueue(queue_with_collation); + else + initQueue(queue_without_collation); + + is_initialized = true; + return Status::Ready; + } + else + { + if (is_finished) + { + for (auto & input : inputs) + input.close(); + + outputs.front().finish(); + + return Status::Finished; + } + + if (need_data) + { + + auto & input = *std::next(inputs.begin(), next_input_to_read); + if (!input.isFinished()) + { + input.setNeeded(); + + if (!input.hasData()) + return Status::NeedData; + + auto chunk = input.pull(); + if (chunk.hasNoRows()) + return Status::NeedData; + + updateCursor(std::move(chunk), next_input_to_read); + pushToQueue(next_input_to_read); + need_data = false; + } + } + + return Status::Ready; + } +} + +void MergingSortedTransform::work() +{ + if (has_collation) + merge(queue_with_collation); + else + merge(queue_without_collation); +} + +template +void MergingSortedTransform::merge(std::priority_queue & queue) +{ + /// Returns MergeStatus which we should return if we are going to finish now. + auto can_read_another_row = [&, this]() + { + if (limit && merged_data.totalMergedRows() >= limit) + { + //std::cerr << "Limit reached\n"; + is_finished = true; + return false; + } + + if (merged_data.mergedRows() >= max_block_size) + { + //std::cerr << "max_block_size reached\n"; + return false; + } + + return true; + }; + + /// Take rows in required order and put them into `merged_data`, while the rows are no more than `max_block_size` + while (!queue.empty()) + { + /// Shouldn't happen at first iteration, but check just in case. + if (!can_read_another_row()) + return; + + TSortCursor current = queue.top(); + queue.pop(); + bool first_iteration = true; + + while (true) + { + if (!first_iteration && !can_read_another_row()) + { + queue.push(current); + return; + } + first_iteration = false; + + /** And what if the block is totally less or equal than the rest for the current cursor? + * Or is there only one data source left in the queue? Then you can take the entire block on current cursor. + */ + if (current.impl->isFirst() && (queue.empty() || current.totallyLessOrEquals(queue.top()))) + { + //std::cerr << "current block is totally less or equals\n"; + + /// If there are already data in the current block, we first return it. We'll get here again the next time we call the merge function. + if (merged_data.mergedRows() != 0) + { + //std::cerr << "merged rows is non-zero\n"; + queue.push(current); + return; + } + + /// Actually, current.impl->order stores source number (i.e. cursors[current.impl->order] == current.impl) + size_t source_num = current.impl->order; + insertFromChunk(source_num); + return; + } + + //std::cerr << "total_merged_rows: " << total_merged_rows << ", merged_rows: " << merged_rows << "\n"; + //std::cerr << "Inserting row\n"; + merged_data.insertRow(current->all_columns, current->pos); + + if (out_row_sources_buf) + { + /// Actually, current.impl->order stores source number (i.e. cursors[current.impl->order] == current.impl) + RowSourcePart row_source(current.impl->order); + out_row_sources_buf->write(row_source.data); + } + + if (current->isLast()) + { + need_data = true; + next_input_to_read = current.impl->order; + + if (limit && merged_data.totalMergedRows() >= limit) + is_finished = true; + + return; + } + + //std::cerr << "moving to next row\n"; + current->next(); + + if (!queue.empty() && current.greater(queue.top())) + { + //std::cerr << "next row is not least, pushing back to queue\n"; + queue.push(current); + break; + } + } + } + is_finished = true; +} + +void MergingSortedTransform::insertFromChunk(size_t source_num) +{ + if (source_num >= cursors.size()) + throw Exception("Logical error in MergingSortedTrandform", ErrorCodes::LOGICAL_ERROR); + + //std::cerr << "copied columns\n"; + + auto num_rows = source_chunks[source_num]->getNumRows(); + + UInt64 total_merged_rows_after_insertion = merged_data.mergedRows() + num_rows; + if (limit && total_merged_rows_after_insertion > limit) + { + num_rows = total_merged_rows_after_insertion - limit; + merged_data.insertFromChunk(std::move(*source_chunks[source_num]), num_rows); + is_finished = true; + } + else + { + merged_data.insertFromChunk(std::move(*source_chunks[source_num]), 0); + need_data = true; + next_input_to_read = source_num; + } + + if (out_row_sources_buf) + { + RowSourcePart row_source(source_num); + for (size_t i = 0; i < num_rows; ++i) + out_row_sources_buf->write(row_source.data); + } +} + + +} diff --git a/dbms/src/Processors/Transforms/MergingSortedTransform.h b/dbms/src/Processors/Transforms/MergingSortedTransform.h new file mode 100644 index 00000000000..5a1f417fdb6 --- /dev/null +++ b/dbms/src/Processors/Transforms/MergingSortedTransform.h @@ -0,0 +1,203 @@ +#pragma once +#include +#include +#include + +#include + +namespace DB +{ + +/// Allows you refer to the row in the block and hold the block ownership, +/// and thus avoid creating a temporary row object. +/// Do not use std::shared_ptr, since there is no need for a place for `weak_count` and `deleter`; +/// does not use Poco::SharedPtr, since you need to allocate a block and `refcount` in one piece; +/// does not use Poco::AutoPtr, since it does not have a `move` constructor and there are extra checks for nullptr; +/// The reference counter is not atomic, since it is used from one thread. +namespace detail +{ +struct SharedChunk : Chunk +{ + int refcount = 0; + + ColumnRawPtrs all_columns; + ColumnRawPtrs sort_columns; + + SharedChunk(Chunk && chunk) : Chunk(std::move(chunk)) {} +}; + +} + +using SharedChunkPtr = boost::intrusive_ptr; + + +inline void intrusive_ptr_add_ref(detail::SharedChunk * ptr) +{ + ++ptr->refcount; +} + +inline void intrusive_ptr_release(detail::SharedChunk * ptr) +{ + if (0 == --ptr->refcount) + delete ptr; +} + +class MergingSortedTransform : public IProcessor +{ +public: + MergingSortedTransform( + const Block & header, + size_t num_inputs, + const SortDescription & description_, + size_t max_block_size, + UInt64 limit = 0, + bool quiet = false, + bool have_all_inputs = true); + + String getName() const override { return "MergingSortedTransform"; } + Status prepare() override; + void work() override; + + void addInput(); + void setHaveAllInputs(); + +protected: + + class MergedData + { + public: + explicit MergedData(const Block & header) + { + columns.reserve(header.columns()); + for (const auto & column : header) + columns.emplace_back(column.type->createColumn()); + } + + void insertRow(const ColumnRawPtrs & raw_columns, size_t row) + { + size_t num_columns = raw_columns.size(); + for (size_t i = 0; i < num_columns; ++i) + columns[i]->insertFrom(*raw_columns[i], row); + + ++total_merged_rows; + ++merged_rows; + } + + void insertFromChunk(Chunk && chunk, size_t limit_rows) + { + if (merged_rows) + throw Exception("Cannot insert to MergedData from Chunk because MergedData is not empty.", + ErrorCodes::LOGICAL_ERROR); + + auto num_rows = chunk.getNumRows(); + columns = chunk.mutateColumns(); + if (limit_rows && num_rows > limit_rows) + for (auto & column : columns) + column = (*column->cut(0, limit_rows)).mutate(); + + total_merged_rows += num_rows; + merged_rows = num_rows; + } + + Chunk pull() + { + MutableColumns empty_columns; + empty_columns.reserve(columns.size()); + + for (const auto & column : columns) + empty_columns.emplace_back(column->cloneEmpty()); + + empty_columns.swap(columns); + Chunk chunk(std::move(empty_columns), merged_rows); + merged_rows = 0; + + return chunk; + } + + UInt64 totalMergedRows() const { return total_merged_rows; } + UInt64 mergedRows() const { return merged_rows; } + + private: + UInt64 total_merged_rows = 0; + UInt64 merged_rows = 0; + MutableColumns columns; + }; + + /// Settings + SortDescription description; + const size_t max_block_size; + UInt64 limit; + bool has_collation = false; + bool quiet = false; + + std::atomic have_all_inputs; + + MergedData merged_data; + + /// Used in Vertical merge algorithm to gather non-PK/non-index columns (on next step) + /// If it is not nullptr then it should be populated during execution + WriteBuffer * out_row_sources_buf = nullptr; + + /// Chunks currently being merged. + std::vector source_chunks; + + using CursorImpls = std::vector; + CursorImpls cursors; + + using Queue = std::priority_queue; + Queue queue_without_collation; + + using QueueWithCollation = std::priority_queue; + QueueWithCollation queue_with_collation; + +private: + + /// Processor state. + bool is_initialized = false; + bool is_finished = false; + bool need_data = false; + size_t next_input_to_read = 0; + + template + void merge(std::priority_queue & queue); + + void insertFromChunk(size_t source_num); + + void updateCursor(Chunk chunk, size_t source_num) + { + auto & shared_chunk_ptr = source_chunks[source_num]; + + if (!shared_chunk_ptr) + { + shared_chunk_ptr = new detail::SharedChunk(std::move(chunk)); + cursors[source_num] = SortCursorImpl(shared_chunk_ptr->getColumns(), description, source_num); + has_collation |= cursors[source_num].has_collation; + } + else + { + *shared_chunk_ptr = std::move(chunk); + cursors[source_num].reset(shared_chunk_ptr->getColumns(), {}); + } + + shared_chunk_ptr->all_columns = cursors[source_num].all_columns; + shared_chunk_ptr->sort_columns = cursors[source_num].sort_columns; + } + + void pushToQueue(size_t source_num) + { + if (has_collation) + queue_with_collation.push(SortCursorWithCollation(&cursors[source_num])); + else + queue_without_collation.push(SortCursor(&cursors[source_num])); + } + + template + void initQueue(std::priority_queue & queue) + { + for (auto & cursor : cursors) + if (!cursor.empty()) + queue.push(TSortCursor(&cursor)); + } +}; + +} diff --git a/dbms/src/Processors/Transforms/PartialSortingTransform.cpp b/dbms/src/Processors/Transforms/PartialSortingTransform.cpp new file mode 100644 index 00000000000..0f15c34c7ff --- /dev/null +++ b/dbms/src/Processors/Transforms/PartialSortingTransform.cpp @@ -0,0 +1,26 @@ +#include +#include + +namespace DB +{ + +PartialSortingTransform::PartialSortingTransform( + const Block & header, SortDescription & description, UInt64 limit, bool do_count_rows) + : ISimpleTransform(header, header, false) + , description(description), limit(limit), do_count_rows(do_count_rows) +{ +} + +void PartialSortingTransform::transform(Chunk & chunk) +{ + if (do_count_rows) + read_rows += chunk.getNumRows(); + + auto block = getInputPort().getHeader().cloneWithColumns(chunk.detachColumns()); + chunk.clear(); + + sortBlock(block, description, limit); + chunk.setColumns(block.getColumns(), block.rows()); +} + +} diff --git a/dbms/src/Processors/Transforms/PartialSortingTransform.h b/dbms/src/Processors/Transforms/PartialSortingTransform.h new file mode 100644 index 00000000000..645b4ebab07 --- /dev/null +++ b/dbms/src/Processors/Transforms/PartialSortingTransform.h @@ -0,0 +1,40 @@ +#pragma once +#include +#include + +namespace DB +{ + +/** Sorts each block individually by the values of the specified columns. + * At the moment, not very optimal algorithm is used. + */ +class PartialSortingTransform : public ISimpleTransform +{ +public: + /// limit - if not 0, then you can sort each block not completely, but only `limit` first rows by order. + /// When count_rows is false, getNumReadRows() will always return 0. + PartialSortingTransform( + const Block & header, + SortDescription & description, + UInt64 limit = 0, + bool do_count_rows = true); + + String getName() const override { return "PartialSortingTransform"; } + + /// Total num rows passed to transform. + UInt64 getNumReadRows() const { return read_rows; } + +protected: + void transform(Chunk & chunk) override; + +private: + SortDescription description; + UInt64 limit; + UInt64 read_rows = 0; + + /// Do we need calculate read_rows value? + /// Used to skip total row when count rows_before_limit_at_least. + bool do_count_rows; +}; + +} diff --git a/dbms/src/Processors/Transforms/RollupTransform.cpp b/dbms/src/Processors/Transforms/RollupTransform.cpp new file mode 100644 index 00000000000..9c8270ce091 --- /dev/null +++ b/dbms/src/Processors/Transforms/RollupTransform.cpp @@ -0,0 +1,49 @@ +#include +#include + +namespace DB +{ + +RollupTransform::RollupTransform(Block header, AggregatingTransformParamsPtr params_) + : IInflatingTransform(std::move(header), params_->getHeader()) + , params(std::move(params_)) + , keys(params->params.keys) +{ +} + +void RollupTransform::consume(Chunk chunk) +{ + consumed_chunk = std::move(chunk); + last_removed_key = keys.size(); +} + +bool RollupTransform::canGenerate() +{ + return consumed_chunk; +} + +Chunk RollupTransform::generate() +{ + auto gen_chunk = std::move(consumed_chunk); + + if (last_removed_key) + { + --last_removed_key; + auto key = keys[last_removed_key]; + + auto num_rows = gen_chunk.getNumRows(); + auto columns = gen_chunk.getColumns(); + columns[key] = columns[key]->cloneEmpty()->cloneResized(num_rows); + + BlocksList rollup_blocks = { getInputPort().getHeader().cloneWithColumns(columns) }; + auto rollup_block = params->aggregator.mergeBlocks(rollup_blocks, false); + + num_rows = rollup_block.rows(); + consumed_chunk = Chunk(rollup_block.getColumns(), num_rows); + } + + finalizeChunk(gen_chunk); + return gen_chunk; +} + +} diff --git a/dbms/src/Processors/Transforms/RollupTransform.h b/dbms/src/Processors/Transforms/RollupTransform.h new file mode 100644 index 00000000000..754e0237357 --- /dev/null +++ b/dbms/src/Processors/Transforms/RollupTransform.h @@ -0,0 +1,28 @@ +#pragma once +#include +#include + +namespace DB +{ + +/// Takes blocks after grouping, with non-finalized aggregate functions. +/// Calculates subtotals and grand totals values for a set of columns. +class RollupTransform : public IInflatingTransform +{ +public: + RollupTransform(Block header, AggregatingTransformParamsPtr params); + String getName() const override { return "RollupTransform"; } + +protected: + void consume(Chunk chunk) override; + bool canGenerate() override; + Chunk generate() override; + +private: + AggregatingTransformParamsPtr params; + ColumnNumbers keys; + Chunk consumed_chunk; + size_t last_removed_key = 0; +}; + +} diff --git a/dbms/src/Processors/Transforms/TotalsHavingTransform.cpp b/dbms/src/Processors/Transforms/TotalsHavingTransform.cpp new file mode 100644 index 00000000000..9b716aca03c --- /dev/null +++ b/dbms/src/Processors/Transforms/TotalsHavingTransform.cpp @@ -0,0 +1,276 @@ +#include +#include + +#include +#include + +#include +#include +#include + +namespace DB +{ + +void finalizeChunk(Chunk & chunk) +{ + auto num_rows = chunk.getNumRows(); + auto columns = chunk.detachColumns(); + + for (auto & column : columns) + if (auto * agg_function = typeid_cast(column.get())) + column = agg_function->convertToValues(); + + chunk.setColumns(std::move(columns), num_rows); +} + +static Block createOutputHeader(Block block, const ExpressionActionsPtr & expression, bool final) +{ + if (final) + finalizeBlock(block); + + if (expression) + expression->execute(block); + + return block; +} + +TotalsHavingTransform::TotalsHavingTransform( + const Block & header, + bool overflow_row_, + const ExpressionActionsPtr & expression_, + const std::string & filter_column_, + TotalsMode totals_mode_, + double auto_include_threshold_, + bool final_) + : ISimpleTransform(header, createOutputHeader(header, expression_, final_), true) + , overflow_row(overflow_row_) + , expression(expression_) + , filter_column_name(filter_column_) + , totals_mode(totals_mode_) + , auto_include_threshold(auto_include_threshold_) + , final(final_) + , arena(std::make_shared()) +{ + if (!filter_column_name.empty()) + filter_column_pos = outputs.front().getHeader().getPositionByName(filter_column_name); + + finalized_header = getInputPort().getHeader(); + finalizeBlock(finalized_header); + + /// Port for Totals. + if (expression) + { + auto totals_header = finalized_header; + expression->execute(totals_header); + outputs.emplace_back(totals_header, this); + } + else + outputs.emplace_back(finalized_header, this); + + /// Initialize current totals with initial state. + current_totals.reserve(header.columns()); + for (const auto & elem : header) + { + if (const auto * column = typeid_cast(elem.column.get())) + { + /// Create ColumnAggregateFunction with initial aggregate function state. + + IAggregateFunction * function = column->getAggregateFunction().get(); + auto target = ColumnAggregateFunction::create(column->getAggregateFunction(), Arenas(1, arena)); + AggregateDataPtr data = arena->alignedAlloc(function->sizeOfData(), function->alignOfData()); + function->create(data); + target->getData().push_back(data); + current_totals.emplace_back(std::move(target)); + } + else + { + /// Not an aggregate function state. Just create a column with default value. + + MutableColumnPtr new_column = elem.type->createColumn(); + elem.type->insertDefaultInto(*new_column); + current_totals.emplace_back(std::move(new_column)); + } + } +} + +IProcessor::Status TotalsHavingTransform::prepare() +{ + if (!finished_transform) + { + auto status = ISimpleTransform::prepare(); + + if (status != Status::Finished) + return status; + + finished_transform = true; + } + + auto & totals_output = getTotalsPort(); + + /// Check can output. + if (totals_output.isFinished()) + return Status::Finished; + + if (!totals_output.canPush()) + return Status::PortFull; + + if (!totals) + return Status::Ready; + + totals_output.push(std::move(totals)); + totals_output.finish(); + return Status::Finished; +} + +void TotalsHavingTransform::work() +{ + if (finished_transform) + prepareTotals(); + else + ISimpleTransform::work(); +} + +void TotalsHavingTransform::transform(Chunk & chunk) +{ + /// Block with values not included in `max_rows_to_group_by`. We'll postpone it. + if (overflow_row) + { + auto & info = chunk.getChunkInfo(); + if (!info) + throw Exception("Chunk info was not set for chunk in TotalsHavingTransform.", ErrorCodes::LOGICAL_ERROR); + + auto * agg_info = typeid_cast(info.get()); + if (!agg_info) + throw Exception("Chunk should have AggregatedChunkInfo in TotalsHavingTransform.", ErrorCodes::LOGICAL_ERROR); + + if (agg_info->is_overflows) + { + overflow_aggregates = std::move(chunk); + return; + } + } + + if (!chunk) + return; + + auto finalized = chunk.clone(); + if (final) + finalizeChunk(finalized); + + total_keys += finalized.getNumRows(); + + if (filter_column_name.empty()) + { + addToTotals(chunk, nullptr); + chunk = std::move(finalized); + } + else + { + /// Compute the expression in HAVING. + auto & cur_header = final ? finalized_header : getInputPort().getHeader(); + auto finalized_block = cur_header.cloneWithColumns(finalized.detachColumns()); + expression->execute(finalized_block); + auto columns = finalized_block.getColumns(); + + ColumnPtr filter_column_ptr = columns[filter_column_pos]; + ConstantFilterDescription const_filter_description(*filter_column_ptr); + + if (const_filter_description.always_true) + { + addToTotals(chunk, nullptr); + return; + } + + if (const_filter_description.always_false) + { + if (totals_mode == TotalsMode::BEFORE_HAVING) + addToTotals(chunk, nullptr); + + chunk.clear(); + return; + } + + FilterDescription filter_description(*filter_column_ptr); + + /// Add values to `totals` (if it was not already done). + if (totals_mode == TotalsMode::BEFORE_HAVING) + addToTotals(chunk, nullptr); + else + addToTotals(chunk, filter_description.data); + + /// Filter the block by expression in HAVING. + for (auto & column : columns) + { + column = column->filter(*filter_description.data, -1); + if (column->empty()) + { + chunk.clear(); + return; + } + } + + auto num_rows = columns.front()->size(); + chunk.setColumns(std::move(columns), num_rows); + } + + passed_keys += chunk.getNumRows(); +} + +void TotalsHavingTransform::addToTotals(const Chunk & chunk, const IColumn::Filter * filter) +{ + auto num_columns = chunk.getNumColumns(); + for (size_t col = 0; col < num_columns; ++col) + { + const auto & current = chunk.getColumns()[col]; + + if (const auto * column = typeid_cast(current.get())) + { + auto & target = typeid_cast(*current_totals[col]); + IAggregateFunction * function = target.getAggregateFunction().get(); + AggregateDataPtr data = target.getData()[0]; + + /// Accumulate all aggregate states into that value. + + const ColumnAggregateFunction::Container & vec = column->getData(); + size_t size = vec.size(); + + if (filter) + { + for (size_t row = 0; row < size; ++row) + if ((*filter)[row]) + function->merge(data, vec[row], arena.get()); + } + else + { + for (size_t row = 0; row < size; ++row) + function->merge(data, vec[row], arena.get()); + } + } + } +} + +void TotalsHavingTransform::prepareTotals() +{ + /// If totals_mode == AFTER_HAVING_AUTO, you need to decide whether to add aggregates to TOTALS for strings, + /// not passed max_rows_to_group_by. + if (overflow_aggregates) + { + if (totals_mode == TotalsMode::BEFORE_HAVING + || totals_mode == TotalsMode::AFTER_HAVING_INCLUSIVE + || (totals_mode == TotalsMode::AFTER_HAVING_AUTO + && static_cast(passed_keys) / total_keys >= auto_include_threshold)) + addToTotals(overflow_aggregates, nullptr); + } + + totals = Chunk(std::move(current_totals), 1); + finalizeChunk(totals); + + if (expression) + { + auto block = finalized_header.cloneWithColumns(totals.detachColumns()); + expression->execute(block); + totals = Chunk(block.getColumns(), 1); + } +} + +} diff --git a/dbms/src/Processors/Transforms/TotalsHavingTransform.h b/dbms/src/Processors/Transforms/TotalsHavingTransform.h new file mode 100644 index 00000000000..71d1a899307 --- /dev/null +++ b/dbms/src/Processors/Transforms/TotalsHavingTransform.h @@ -0,0 +1,75 @@ +#include + +#include +#include + +namespace DB +{ + +class Arena; +using ArenaPtr = std::shared_ptr; + +class ExpressionActions; +using ExpressionActionsPtr = std::shared_ptr; + +/** Takes blocks after grouping, with non-finalized aggregate functions. + * Calculates total values according to totals_mode. + * If necessary, evaluates the expression from HAVING and filters rows. Returns the finalized and filtered blocks. + */ +class TotalsHavingTransform : public ISimpleTransform +{ +public: + TotalsHavingTransform( + const Block & header, + bool overflow_row_, + const ExpressionActionsPtr & expression_, + const std::string & filter_column_, + TotalsMode totals_mode_, + double auto_include_threshold_, + bool final_); + + String getName() const override { return "TotalsHavingTransform"; } + + OutputPort & getTotalsPort() { return outputs.back(); } + + Status prepare() override; + void work() override; + +protected: + void transform(Chunk & chunk) override; + + bool finished_transform = false; + Chunk totals; + +private: + void addToTotals(const Chunk & block, const IColumn::Filter * filter); + void prepareTotals(); + + /// Params + bool overflow_row; + ExpressionActionsPtr expression; + String filter_column_name; + TotalsMode totals_mode; + double auto_include_threshold; + bool final; + + size_t passed_keys = 0; + size_t total_keys = 0; + + size_t filter_column_pos = 0; + + Block finalized_header; + + /// Here are the values that did not pass max_rows_to_group_by. + /// They are added or not added to the current_totals, depending on the totals_mode. + Chunk overflow_aggregates; + + /// Here, total values are accumulated. After the work is finished, they will be placed in IBlockInputStream::totals. + MutableColumns current_totals; + /// Arena for aggregate function states in totals. + ArenaPtr arena; +}; + +void finalizeChunk(Chunk & chunk); + +} diff --git a/dbms/src/Processors/printPipeline.h b/dbms/src/Processors/printPipeline.h new file mode 100644 index 00000000000..d3e8e58b2ce --- /dev/null +++ b/dbms/src/Processors/printPipeline.h @@ -0,0 +1,60 @@ +#pragma once + +#include +#include + +namespace DB +{ + +/** Print pipeline in "dot" format for GraphViz. + * You can render it with: + * dot -T png < pipeline.dot > pipeline.png + */ + +template +void printPipeline(const Processors & processors, const Statuses & statuses, WriteBuffer & out) +{ + out << "digraph\n{\n"; + + auto get_proc_id = [](const IProcessor & proc) -> UInt64 + { + return reinterpret_cast(&proc); + }; + + auto statuses_iter = statuses.begin(); + + /// Nodes // TODO quoting and escaping + for (const auto & processor : processors) + { + out << "n" << get_proc_id(*processor) << "[label=\"" << processor->getName() << processor->getDescription(); + + if (statuses_iter != statuses.end()) + { + out << " (" << IProcessor::statusToName(*statuses_iter) << ")"; + ++statuses_iter; + } + + out << "\"];\n"; + } + + /// Edges + for (const auto & processor : processors) + { + for (const auto & port : processor->getOutputs()) + { + const IProcessor & curr = *processor; + const IProcessor & next = port.getInputPort().getProcessor(); + + out << "n" << get_proc_id(curr) << " -> " << "n" << get_proc_id(next) << ";\n"; + } + } + out << "}\n"; +} + +template +void printPipeline(const Processors & processors, WriteBuffer & out) +{ + printPipeline(processors, std::vector(), out); +} + +} diff --git a/dbms/src/Processors/tests/CMakeLists.txt b/dbms/src/Processors/tests/CMakeLists.txt new file mode 100644 index 00000000000..5f44ec2a8fd --- /dev/null +++ b/dbms/src/Processors/tests/CMakeLists.txt @@ -0,0 +1,15 @@ +add_executable (processors_test processors_test.cpp) +add_executable (processors_test_chain processors_test_chain.cpp) +add_executable (processors_test_merge processors_test_merge.cpp) +add_executable (processors_test_merging_sorted_transform processors_test_merging_sorted_transform.cpp) +add_executable (processors_test_merge_sorting_transform processors_test_merge_sorting_transform.cpp) +add_executable (processors_test_expand_pipeline processors_test_expand_pipeline.cpp) +add_executable (processors_test_aggregation processors_test_aggregation.cpp) + +target_link_libraries (processors_test dbms) +target_link_libraries (processors_test_chain dbms) +target_link_libraries (processors_test_merge dbms) +target_link_libraries (processors_test_expand_pipeline dbms) +target_link_libraries (processors_test_merging_sorted_transform dbms) +target_link_libraries (processors_test_merge_sorting_transform dbms) +target_link_libraries (processors_test_aggregation dbms clickhouse_aggregate_functions) diff --git a/dbms/src/Processors/tests/processors_test.cpp b/dbms/src/Processors/tests/processors_test.cpp new file mode 100644 index 00000000000..b663cf319ad --- /dev/null +++ b/dbms/src/Processors/tests/processors_test.cpp @@ -0,0 +1,230 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + + +using namespace DB; + + +class NumbersSource : public ISource +{ +public: + String getName() const override { return "Numbers"; } + + NumbersSource(UInt64 start_number, unsigned sleep_useconds) + : ISource(Block({ColumnWithTypeAndName{ ColumnUInt64::create(), std::make_shared(), "number" }})), + current_number(start_number), sleep_useconds(sleep_useconds) + { + } + +private: + UInt64 current_number = 0; + unsigned sleep_useconds; + + Chunk generate() override + { + usleep(sleep_useconds); + + MutableColumns columns; + columns.emplace_back(ColumnUInt64::create(1, current_number)); + ++current_number; + return Chunk(std::move(columns), 1); + } +}; + + +class SleepyNumbersSource : public IProcessor +{ +protected: + OutputPort & output; + +public: + String getName() const override { return "SleepyNumbers"; } + + SleepyNumbersSource(UInt64 start_number, unsigned sleep_useconds) + : IProcessor({}, {Block({ColumnWithTypeAndName{ ColumnUInt64::create(), std::make_shared(), "number" }})}) + , output(outputs.front()), current_number(start_number), sleep_useconds(sleep_useconds) + { + } + + Status prepare() override + { + if (active) + return Status::Wait; + + if (output.isFinished()) + return Status::Finished; + + if (!output.canPush()) + return Status::PortFull; + + if (!current_chunk) + return Status::Async; + + output.push(std::move(current_chunk)); + return Status::Async; + } + + void schedule(EventCounter & watch) override + { + active = true; + pool.schedule([&watch, this] + { + usleep(sleep_useconds); + current_chunk = generate(); + active = false; + watch.notify(); + }); + } + + OutputPort & getPort() { return output; } + +private: + ThreadPool pool{1, 1, 0}; + Chunk current_chunk; + std::atomic_bool active {false}; + + UInt64 current_number = 0; + unsigned sleep_useconds; + + Chunk generate() + { + MutableColumns columns; + columns.emplace_back(ColumnUInt64::create(1, current_number)); + ++current_number; + return Chunk(std::move(columns), 1); + } +}; + + +class PrintSink : public ISink +{ +public: + String getName() const override { return "Print"; } + + PrintSink(String prefix) + : ISink(Block({ColumnWithTypeAndName{ ColumnUInt64::create(), std::make_shared(), "number" }})), + prefix(std::move(prefix)) + { + } + +private: + String prefix; + WriteBufferFromFileDescriptor out{STDOUT_FILENO}; + FormatSettings settings; + + void consume(Chunk chunk) override + { + size_t rows = chunk.getNumRows(); + size_t columns = chunk.getNumColumns(); + + for (size_t row_num = 0; row_num < rows; ++row_num) + { + writeString(prefix, out); + for (size_t column_num = 0; column_num < columns; ++column_num) + { + if (column_num != 0) + writeChar('\t', out); + getPort().getHeader().getByPosition(column_num).type->serializeAsText(*chunk.getColumns()[column_num], row_num, out, settings); + } + writeChar('\n', out); + } + + out.next(); + } +}; + + +int main(int, char **) +try +{ + auto source0 = std::make_shared(0, 300000); + auto header = source0->getPort().getHeader(); + auto limit0 = std::make_shared(header, 10, 0); + + connect(source0->getPort(), limit0->getInputPort()); + + auto queue = std::make_shared(header); + + connect(limit0->getOutputPort(), queue->getInputPort()); + + auto source1 = std::make_shared(100, 100000); + auto source2 = std::make_shared(1000, 200000); + + auto source3 = std::make_shared(10, 100000); + auto limit3 = std::make_shared(header, 5, 0); + + connect(source3->getPort(), limit3->getInputPort()); + + auto source4 = std::make_shared(10, 100000); + auto limit4 = std::make_shared(header, 5, 0); + + connect(source4->getPort(), limit4->getInputPort()); + + auto concat = std::make_shared(header, 2); + + connect(limit3->getOutputPort(), concat->getInputs().front()); + connect(limit4->getOutputPort(), concat->getInputs().back()); + + auto fork = std::make_shared(header, 2); + + connect(concat->getOutputPort(), fork->getInputPort()); + + auto print_after_concat = std::make_shared("---------- "); + + connect(fork->getOutputs().back(), print_after_concat->getPort()); + + auto resize = std::make_shared(header, 4, 1); + + auto input_it = resize->getInputs().begin(); + connect(queue->getOutputPort(), *(input_it++)); + connect(source1->getPort(), *(input_it++)); + connect(source2->getPort(), *(input_it++)); + connect(fork->getOutputs().front(), *(input_it++)); + + auto limit = std::make_shared(header, 100, 0); + + connect(resize->getOutputs().front(), limit->getInputPort()); + + auto sink = std::make_shared(""); + + connect(limit->getOutputPort(), sink->getPort()); + + WriteBufferFromOStream out(std::cout); + std::vector processors = {source0, source1, source2, source3, source4, limit0, limit3, limit4, limit, + queue, concat, fork, print_after_concat, resize, sink}; + printPipeline(processors, out); + + // ThreadPool pool(4, 4, 10); + PipelineExecutor executor(processors); + /// SequentialPipelineExecutor executor({sink}); + + executor.execute(1); + + return 0; +} +catch (...) +{ + std::cerr << getCurrentExceptionMessage(true) << '\n'; + throw; +} diff --git a/dbms/src/Processors/tests/processors_test_aggregation.cpp b/dbms/src/Processors/tests/processors_test_aggregation.cpp new file mode 100644 index 00000000000..116518391d6 --- /dev/null +++ b/dbms/src/Processors/tests/processors_test_aggregation.cpp @@ -0,0 +1,407 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +using namespace DB; + + +class NumbersSource : public ISource +{ +public: + String getName() const override { return "Numbers"; } + + NumbersSource(UInt64 start_number, UInt64 step, UInt64 block_size, unsigned sleep_useconds) + : ISource(Block({ColumnWithTypeAndName{ ColumnUInt64::create(), std::make_shared(), "number" }})), + current_number(start_number), step(step), block_size(block_size), sleep_useconds(sleep_useconds) + { + } + +private: + UInt64 current_number = 0; + UInt64 step; + UInt64 block_size; + unsigned sleep_useconds; + + Chunk generate() override + { + usleep(sleep_useconds); + + MutableColumns columns; + columns.emplace_back(ColumnUInt64::create()); + + for (UInt64 i = 0; i < block_size; ++i, current_number += step) + columns.back()->insert(Field(current_number)); + + return Chunk(std::move(columns), block_size); + } +}; + +class PrintSink : public ISink +{ +public: + String getName() const override { return "Print"; } + + PrintSink(String prefix, Block header) + : ISink(std::move(header)), + prefix(std::move(prefix)) + { + } + +private: + String prefix; + WriteBufferFromFileDescriptor out{STDOUT_FILENO}; + FormatSettings settings; + + void consume(Chunk chunk) override + { + size_t rows = chunk.getNumRows(); + size_t columns = chunk.getNumColumns(); + + for (size_t row_num = 0; row_num < rows; ++row_num) + { + writeString(prefix, out); + for (size_t column_num = 0; column_num < columns; ++column_num) + { + if (column_num != 0) + writeChar('\t', out); + getPort().getHeader().getByPosition(column_num).type->serializeAsText(*chunk.getColumns()[column_num], row_num, out, settings); + } + writeChar('\n', out); + } + + out.next(); + } +}; + +class CheckSink : public ISink +{ +public: + String getName() const override { return "Check"; } + + CheckSink(Block header, size_t num_rows) + : ISink(std::move(header)), read_rows(num_rows, false) + { + } + + void checkAllRead() + { + for (size_t i = 0; i < read_rows.size(); ++i) + { + if (!read_rows[i]) + { + throw Exception("Check Failed. Row " + toString(i) + " was not read.", ErrorCodes::LOGICAL_ERROR); + } + } + } + +private: + std::vector read_rows; + + void consume(Chunk chunk) override + { + size_t rows = chunk.getNumRows(); + size_t columns = chunk.getNumColumns(); + + for (size_t row_num = 0; row_num < rows; ++row_num) + { + std::vector values(columns); + for (size_t column_num = 0; column_num < columns; ++column_num) + { + values[column_num] = chunk.getColumns()[column_num]->getUInt(row_num); + } + + if (3 * values[0] != values[1]) + throw Exception("Check Failed. Got (" + toString(values[0]) + ", " + toString(values[1]) + ") in result," + + "but " + toString(values[0]) + " * 3 != " + toString(values[1]), + ErrorCodes::LOGICAL_ERROR); + + if (values[0] >= read_rows.size()) + throw Exception("Check Failed. Got string with number " + toString(values[0]) + + " (max " + toString(read_rows.size()), ErrorCodes::LOGICAL_ERROR); + + if (read_rows[values[0]]) + throw Exception("Row " + toString(values[0]) + " was already read.", ErrorCodes::LOGICAL_ERROR); + + read_rows[values[0]] = true; + } + } +}; + +template +struct measure +{ + template + static typename TimeT::rep execution(F&& func, Args&&... args) + { + auto start = std::chrono::steady_clock::now(); + std::forward(func)(std::forward(args)...); + auto duration = std::chrono::duration_cast< TimeT> + (std::chrono::steady_clock::now() - start); + return duration.count(); + } +}; + +int main(int, char **) +try +{ + ThreadStatus thread_status; + CurrentThread::initializeQuery(); + auto thread_group = CurrentThread::getGroup(); + + Poco::AutoPtr channel = new Poco::ConsoleChannel(std::cerr); + Logger::root().setChannel(channel); + Logger::root().setLevel("trace"); + + registerAggregateFunctions(); + auto & factory = AggregateFunctionFactory::instance(); + + auto cur_path = Poco::Path().absolute().toString(); + + auto execute_one_stream = [&](String msg, size_t num_threads, bool two_level, bool external) + { + std::cerr << '\n' << msg << "\n"; + + size_t num_rows = 1000000; + size_t block_size = 1000; + + auto source1 = std::make_shared(0, 1, block_size, 0); + auto source2 = std::make_shared(0, 1, block_size, 0); + auto source3 = std::make_shared(0, 1, block_size, 0); + + auto limit1 = std::make_shared(source1->getPort().getHeader(), num_rows, 0); + auto limit2 = std::make_shared(source2->getPort().getHeader(), num_rows, 0); + auto limit3 = std::make_shared(source3->getPort().getHeader(), num_rows, 0); + + auto resize = std::make_shared(source1->getPort().getHeader(), 3, 1); + + AggregateDescriptions aggregate_descriptions(1); + + DataTypes sum_types = { std::make_shared() }; + aggregate_descriptions[0].function = factory.get("sum", sum_types); + aggregate_descriptions[0].arguments = {0}; + + bool overflow_row = false; /// Without overflow row. + size_t max_rows_to_group_by = 0; /// All. + size_t group_by_two_level_threshold = two_level ? 10 : 0; + size_t group_by_two_level_threshold_bytes = two_level ? 128 : 0; + size_t max_bytes_before_external_group_by = external ? 10000000 : 0; + + Aggregator::Params params( + source1->getPort().getHeader(), + {0}, + aggregate_descriptions, + overflow_row, + max_rows_to_group_by, + OverflowMode::THROW, + nullptr, /// No compiler + 0, /// min_count_to_compile + group_by_two_level_threshold, + group_by_two_level_threshold_bytes, + max_bytes_before_external_group_by, + false, /// empty_result_for_aggregation_by_empty_set + cur_path, /// tmp_path + 1 /// max_threads + ); + + auto agg_params = std::make_shared(params, /* final =*/ false); + auto merge_params = std::make_shared(params, /* final =*/ true); + auto aggregating = std::make_shared(source1->getPort().getHeader(), agg_params); + auto merging = std::make_shared(aggregating->getOutputs().front().getHeader(), merge_params, 4); + auto sink = std::make_shared(merging->getOutputPort().getHeader(), num_rows); + + connect(source1->getPort(), limit1->getInputPort()); + connect(source2->getPort(), limit2->getInputPort()); + connect(source3->getPort(), limit3->getInputPort()); + + auto it = resize->getInputs().begin(); + connect(limit1->getOutputPort(), *(it++)); + connect(limit2->getOutputPort(), *(it++)); + connect(limit3->getOutputPort(), *(it++)); + + connect(resize->getOutputs().front(), aggregating->getInputs().front()); + connect(aggregating->getOutputs().front(), merging->getInputPort()); + connect(merging->getOutputPort(), sink->getPort()); + + std::vector processors = {source1, source2, source3, + limit1, limit2, limit3, + resize, aggregating, merging, sink}; +// WriteBufferFromOStream out(std::cout); +// printPipeline(processors, out); + + PipelineExecutor executor(processors); + executor.execute(num_threads); + sink->checkAllRead(); + }; + + auto execute_mult_streams = [&](String msg, size_t num_threads, bool two_level, bool external) + { + std::cerr << '\n' << msg << "\n"; + + size_t num_rows = 1000000; + size_t block_size = 1000; + + auto source1 = std::make_shared(0, 1, block_size, 0); + auto source2 = std::make_shared(0, 1, block_size, 0); + auto source3 = std::make_shared(0, 1, block_size, 0); + + auto limit1 = std::make_shared(source1->getPort().getHeader(), num_rows, 0); + auto limit2 = std::make_shared(source2->getPort().getHeader(), num_rows, 0); + auto limit3 = std::make_shared(source3->getPort().getHeader(), num_rows, 0); + + AggregateDescriptions aggregate_descriptions(1); + + DataTypes sum_types = { std::make_shared() }; + aggregate_descriptions[0].function = factory.get("sum", sum_types); + aggregate_descriptions[0].arguments = {0}; + + bool overflow_row = false; /// Without overflow row. + size_t max_rows_to_group_by = 0; /// All. + size_t group_by_two_level_threshold = two_level ? 10 : 0; + size_t group_by_two_level_threshold_bytes = two_level ? 128 : 0; + size_t max_bytes_before_external_group_by = external ? 10000000 : 0; + + Aggregator::Params params( + source1->getPort().getHeader(), + {0}, + aggregate_descriptions, + overflow_row, + max_rows_to_group_by, + OverflowMode::THROW, + nullptr, /// No compiler + 0, /// min_count_to_compile + group_by_two_level_threshold, + group_by_two_level_threshold_bytes, + max_bytes_before_external_group_by, + false, /// empty_result_for_aggregation_by_empty_set + cur_path, /// tmp_path + 1 /// max_threads + ); + + auto agg_params = std::make_shared(params, /* final =*/ false); + auto merge_params = std::make_shared(params, /* final =*/ true); + + ManyAggregatedDataPtr data = std::make_unique(3); + + auto aggregating1 = std::make_shared(source1->getPort().getHeader(), agg_params, data, 0, 4, 4); + auto aggregating2 = std::make_shared(source1->getPort().getHeader(), agg_params, data, 1, 4, 4); + auto aggregating3 = std::make_shared(source1->getPort().getHeader(), agg_params, data, 2, 4, 4); + + Processors merging_pipe = createMergingAggregatedMemoryEfficientPipe( + aggregating1->getOutputs().front().getHeader(), + merge_params, + 3, 2); + + auto sink = std::make_shared(merging_pipe.back()->getOutputs().back().getHeader(), num_rows); + + connect(source1->getPort(), limit1->getInputPort()); + connect(source2->getPort(), limit2->getInputPort()); + connect(source3->getPort(), limit3->getInputPort()); + + connect(limit1->getOutputPort(), aggregating1->getInputs().front()); + connect(limit2->getOutputPort(), aggregating2->getInputs().front()); + connect(limit3->getOutputPort(), aggregating3->getInputs().front()); + + auto it = merging_pipe.front()->getInputs().begin(); + connect(aggregating1->getOutputs().front(), *(it++)); + connect(aggregating2->getOutputs().front(), *(it++)); + connect(aggregating3->getOutputs().front(), *(it++)); + + connect(merging_pipe.back()->getOutputs().back(), sink->getPort()); + + std::vector processors = {source1, source2, source3, + limit1, limit2, limit3, + aggregating1, aggregating2, aggregating3, sink}; + + processors.insert(processors.end(), merging_pipe.begin(), merging_pipe.end()); +// WriteBufferFromOStream out(std::cout); +// printPipeline(processors, out); + + PipelineExecutor executor(processors); + executor.execute(num_threads); + sink->checkAllRead(); + }; + + std::vector messages; + std::vector times; + + auto exec = [&](auto func, String msg, size_t num_threads, bool two_level, bool external) + { + msg += ", two_level = " + toString(two_level) + ", external = " + toString(external); + Int64 time = 0; + + auto wrapper = [&]() + { + ThreadStatus cur_status; + + CurrentThread::attachToIfDetached(thread_group); + time = measure<>::execution(func, msg, num_threads, two_level, external); + }; + + std::thread thread(wrapper); + thread.join(); + + messages.emplace_back(msg); + times.emplace_back(time); + }; + + size_t num_threads = 4; + + exec(execute_one_stream, "One stream, single thread", 1, false, false); + exec(execute_one_stream, "One stream, multiple threads", num_threads, false, false); + + exec(execute_mult_streams, "Multiple streams, single thread", 1, false, false); + exec(execute_mult_streams, "Multiple streams, multiple threads", num_threads, false, false); + + exec(execute_one_stream, "One stream, single thread", 1, true, false); + exec(execute_one_stream, "One stream, multiple threads", num_threads, true, false); + + exec(execute_mult_streams, "Multiple streams, single thread", 1, true, false); + exec(execute_mult_streams, "Multiple streams, multiple threads", num_threads, true, false); + + exec(execute_one_stream, "One stream, single thread", 1, true, true); + exec(execute_one_stream, "One stream, multiple threads", num_threads, true, true); + + exec(execute_mult_streams, "Multiple streams, single thread", 1, true, true); + exec(execute_mult_streams, "Multiple streams, multiple threads", num_threads, true, true); + + for (size_t i = 0; i < messages.size(); ++i) + std::cout << messages[i] << " time: " << times[i] << " ms.\n"; + + return 0; +} +catch (...) +{ + std::cerr << getCurrentExceptionMessage(true) << '\n'; + throw; +} diff --git a/dbms/src/Processors/tests/processors_test_chain.cpp b/dbms/src/Processors/tests/processors_test_chain.cpp new file mode 100644 index 00000000000..dfcd2c6b5ee --- /dev/null +++ b/dbms/src/Processors/tests/processors_test_chain.cpp @@ -0,0 +1,165 @@ +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +#include +#include + + +using namespace DB; + + +class NumbersSource : public ISource +{ +public: + String getName() const override { return "Numbers"; } + + NumbersSource(UInt64 start_number, unsigned sleep_useconds) + : ISource(Block({ColumnWithTypeAndName{ ColumnUInt64::create(), std::make_shared(), "number" }})), + current_number(start_number), sleep_useconds(sleep_useconds) + { + } + +private: + UInt64 current_number = 0; + unsigned sleep_useconds; + + Chunk generate() override + { + usleep(sleep_useconds); + + MutableColumns columns; + columns.emplace_back(ColumnUInt64::create(1, current_number)); + ++current_number; + return Chunk(std::move(columns), 1); + } +}; + +class SleepyTransform : public ISimpleTransform +{ +public: + explicit SleepyTransform(unsigned sleep_useconds) + : ISimpleTransform( + Block({ColumnWithTypeAndName{ ColumnUInt64::create(), std::make_shared(), "number" }}), + Block({ColumnWithTypeAndName{ ColumnUInt64::create(), std::make_shared(), "number" }}), + /*skip_empty_chunks =*/ false) + , sleep_useconds(sleep_useconds) {} + + String getName() const override { return "SleepyTransform"; } + +protected: + void transform(Chunk &) override + { + usleep(sleep_useconds); + } + +private: + unsigned sleep_useconds; +}; + +class PrintSink : public ISink +{ +public: + String getName() const override { return "Print"; } + + PrintSink(String prefix) + : ISink(Block({ColumnWithTypeAndName{ ColumnUInt64::create(), std::make_shared(), "number" }})), + prefix(std::move(prefix)) + { + } + +private: + String prefix; + WriteBufferFromFileDescriptor out{STDOUT_FILENO}; + FormatSettings settings; + + void consume(Chunk chunk) override + { + size_t rows = chunk.getNumRows(); + size_t columns = chunk.getNumColumns(); + + for (size_t row_num = 0; row_num < rows; ++row_num) + { + writeString(prefix, out); + for (size_t column_num = 0; column_num < columns; ++column_num) + { + if (column_num != 0) + writeChar('\t', out); + getPort().getHeader().getByPosition(column_num).type->serializeAsText(*chunk.getColumns()[column_num], row_num, out, settings); + } + writeChar('\n', out); + } + + out.next(); + } +}; + +template +struct measure +{ + template + static typename TimeT::rep execution(F&& func, Args&&... args) + { + auto start = std::chrono::steady_clock::now(); + std::forward(func)(std::forward(args)...); + auto duration = std::chrono::duration_cast< TimeT> + (std::chrono::steady_clock::now() - start); + return duration.count(); + } +}; + +int main(int, char **) +try +{ + auto execute_chain = [](size_t num_threads) + { + std::cerr << "---------------------\n"; + + auto source = std::make_shared(0, 100000); + auto transform1 = std::make_shared(100000); + auto transform2 = std::make_shared(100000); + auto transform3 = std::make_shared(100000); + auto limit = std::make_shared(source->getPort().getHeader(), 20, 0); + auto sink = std::make_shared(""); + + connect(source->getPort(), transform1->getInputPort()); + connect(transform1->getOutputPort(), transform2->getInputPort()); + connect(transform2->getOutputPort(), transform3->getInputPort()); + connect(transform3->getOutputPort(), limit->getInputPort()); + connect(limit->getOutputPort(), sink->getPort()); + + std::vector processors = {source, transform1, transform2, transform3, limit, sink}; +// WriteBufferFromOStream out(std::cout); +// printPipeline(processors, out); + + PipelineExecutor executor(processors); + executor.execute(num_threads); + }; + + auto time_single = measure<>::execution(execute_chain, 1); + auto time_mt = measure<>::execution(execute_chain, 4); + + std::cout << "Single Thread time: " << time_single << " ms.\n"; + std::cout << "Multiple Threads time: " << time_mt << " ms.\n"; + + return 0; +} +catch (...) +{ + std::cerr << getCurrentExceptionMessage(true) << '\n'; + throw; +} diff --git a/dbms/src/Processors/tests/processors_test_expand_pipeline.cpp b/dbms/src/Processors/tests/processors_test_expand_pipeline.cpp new file mode 100644 index 00000000000..fa977dc7ba8 --- /dev/null +++ b/dbms/src/Processors/tests/processors_test_expand_pipeline.cpp @@ -0,0 +1,285 @@ +#include + +#include + +#include +#include +#include +#include +#include + + +#include +#include +#include + +#include + +#include +#include +#include + +using namespace DB; + +class PrintSink : public ISink +{ +public: + String getName() const override { return "Print"; } + + PrintSink(String prefix) + : ISink(Block({ColumnWithTypeAndName{ ColumnUInt64::create(), std::make_shared(), "number" }})), + prefix(std::move(prefix)) + { + } + +private: + String prefix; + WriteBufferFromFileDescriptor out{STDOUT_FILENO}; + FormatSettings settings; + + void consume(Chunk chunk) override + { + size_t rows = chunk.getNumRows(); + size_t columns = chunk.getNumColumns(); + + for (size_t row_num = 0; row_num < rows; ++row_num) + { + writeString(prefix, out); + for (size_t column_num = 0; column_num < columns; ++column_num) + { + if (column_num != 0) + writeChar('\t', out); + getPort().getHeader().getByPosition(column_num).type->serializeAsText(*chunk.getColumns()[column_num], row_num, out, settings); + } + writeChar('\n', out); + } + + out.next(); + } +}; + + +class OneNumberSource : public ISource +{ +public: + String getName() const override { return "OneNumber"; } + + OneNumberSource(UInt64 number) + : ISource(Block({ColumnWithTypeAndName{ ColumnUInt64::create(), std::make_shared(), "number" }})), + number(number) + { + } + +private: + UInt64 number; + bool done = false; + + Chunk generate() override + { + if (done) + return Chunk(); + + done = true; + + MutableColumns columns; + columns.emplace_back(ColumnUInt64::create(1, number)); + return Chunk(std::move(columns), 1); + } +}; + + +class ExpandingProcessor : public IProcessor +{ +public: + String getName() const override { return "Expanding"; } + ExpandingProcessor() + : IProcessor({Block({ColumnWithTypeAndName{ ColumnUInt64::create(), std::make_shared(), "number" }})}, + {Block({ColumnWithTypeAndName{ ColumnUInt64::create(), std::make_shared(), "number" }})}) + {} + + Status prepare() override + { + auto & main_input = inputs.front(); + auto & main_output = outputs.front(); + auto & additional_input = inputs.back(); + auto & additional_output = outputs.back(); + /// Check can output. + + + if (main_output.isFinished()) + { + main_input.close(); + additional_input.close(); + additional_output.finish(); + return Status::Finished; + } + + if (!main_output.canPush()) + { + main_input.setNotNeeded(); + additional_input.setNotNeeded(); + return Status::PortFull; + } + + if (chunk_from_add_inp && is_processed) + { + if (is_processed) + main_output.push(std::move(chunk_from_add_inp)); + else + return Status::Ready; + } + + if (expanded) + { + if (chunk_from_main_inp) + { + if (additional_output.isFinished()) + { + main_input.close(); + return Status::Finished; + } + + if (!additional_output.canPush()) + { + main_input.setNotNeeded(); + return Status::PortFull; + } + + additional_output.push(std::move(chunk_from_main_inp)); + main_input.close(); + } + + if (additional_input.isFinished()) + { + main_output.finish(); + return Status::Finished; + } + + additional_input.setNeeded(); + + if (!additional_input.hasData()) + return Status::NeedData; + + chunk_from_add_inp = additional_input.pull(); + is_processed = false; + return Status::Ready; + } + else + { + if (!chunk_from_main_inp) + { + + if (main_input.isFinished()) + { + main_output.finish(); + return Status::Finished; + } + + main_input.setNeeded(); + + if (!main_input.hasData()) + return Status::NeedData; + + chunk_from_main_inp = main_input.pull(); + main_input.close(); + } + + UInt64 val = chunk_from_main_inp.getColumns()[0]->getUInt(0); + if (val) + { + --val; + chunk_from_main_inp.setColumns(Columns{ColumnUInt64::create(1, val)}, 1); + return Status::ExpandPipeline; + } + + main_output.push(std::move(chunk_from_main_inp)); + main_output.finish(); + return Status::Finished; + } + } + + Processors expandPipeline() override + { + auto & main_input = inputs.front(); + auto & main_output = outputs.front(); + + Processors processors = {std::make_shared()}; + inputs.push_back({main_input.getHeader(), this}); + outputs.push_back({main_output.getHeader(), this}); + connect(outputs.back(), processors.back()->getInputs().front()); + connect(processors.back()->getOutputs().front(), inputs.back()); + inputs.back().setNeeded(); + + expanded = true; + return processors; + } + + void work() override + { + auto num_rows = chunk_from_add_inp.getNumRows(); + auto columns = chunk_from_add_inp.mutateColumns(); + columns.front()->insert(Field(num_rows)); + chunk_from_add_inp.setColumns(std::move(columns), num_rows + 1); + is_processed = true; + } + +private: + bool expanded = false; + Chunk chunk_from_main_inp; + Chunk chunk_from_add_inp; + bool is_processed = false; +}; + + +template +struct measure +{ + template + static typename TimeT::rep execution(F&& func, Args&&... args) + { + auto start = std::chrono::steady_clock::now(); + std::forward(func)(std::forward(args)...); + auto duration = std::chrono::duration_cast< TimeT> + (std::chrono::steady_clock::now() - start); + return duration.count(); + } +}; + +int main(int, char **) +try +{ + auto execute = [](String msg, size_t num, size_t num_threads) + { + std::cerr << msg << "\n"; + + auto source = std::make_shared(num); + auto expanding = std::make_shared(); + auto sink = std::make_shared(""); + + connect(source->getPort(), expanding->getInputs().front()); + connect(expanding->getOutputs().front(), sink->getPort()); + + std::vector processors = {source, expanding, sink}; + + PipelineExecutor executor(processors); + executor.execute(num_threads); + + WriteBufferFromOStream out(std::cout); + printPipeline(executor.getProcessors(), out); + }; + + ThreadPool pool(4, 4, 10); + + auto time_single = measure<>::execution(execute, "Single thread", 10, 1); + auto time_mt = measure<>::execution(execute, "Multiple threads", 10, 4); + + std::cout << "Single Thread time: " << time_single << " ms.\n"; + std::cout << "Multiple Threads time:" << time_mt << " ms.\n"; + + return 0; +} +catch (...) +{ + std::cerr << getCurrentExceptionMessage(true) << '\n'; + throw; +} diff --git a/dbms/src/Processors/tests/processors_test_merge.cpp b/dbms/src/Processors/tests/processors_test_merge.cpp new file mode 100644 index 00000000000..00e322430e5 --- /dev/null +++ b/dbms/src/Processors/tests/processors_test_merge.cpp @@ -0,0 +1,334 @@ +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +#include +#include + + +using namespace DB; + + +class MergingSortedProcessor : public IProcessor +{ +public: + MergingSortedProcessor(const Block & header, size_t num_inputs) + : IProcessor(InputPorts(num_inputs, header), OutputPorts{header}) + , chunks(num_inputs), positions(num_inputs, 0), finished(num_inputs, false) + { + } + + String getName() const override { return "MergingSortedProcessor"; } + + Status prepare() override + { + auto & output = outputs.front(); + + /// Check can output. + + if (output.isFinished()) + { + for (auto & in : inputs) + in.close(); + + return Status::Finished; + } + + if (!output.isNeeded()) + { + for (auto & in : inputs) + in.setNotNeeded(); + + return Status::PortFull; + } + + if (output.hasData()) + return Status::PortFull; + + /// Push if has data. + if (res) + { + output.push(std::move(res)); + return Status::PortFull; + } + + /// Check for inputs we need. + bool all_inputs_finished = true; + bool all_inputs_has_data = true; + auto it = inputs.begin(); + for (size_t i = 0; it != inputs.end(); ++it, ++i) + { + auto & input = *it; + if (!finished[i]) + { + if (!input.isFinished()) + { + all_inputs_finished = false; + bool needed = positions[i] >= chunks[i].getNumRows(); + if (needed) + { + input.setNeeded(); + if (input.hasData()) + { + chunks[i] = input.pull(); + positions[i] = 0; + } + else + all_inputs_has_data = false; + } + else + input.setNotNeeded(); + } + else + finished[i] = true; + } + } + + if (all_inputs_finished) + { + output.finish(); + return Status::Finished; + } + + if (!all_inputs_has_data) + return Status::NeedData; + + return Status::Ready; + } + + void work() override + { + using Key = std::pair; + std::priority_queue, std::greater<>> queue; + for (size_t i = 0; i < chunks.size(); ++i) + { + if (finished[i]) + continue; + + if (positions[i] >= chunks[i].getNumRows()) + return; + + queue.push({chunks[i].getColumns()[0]->getUInt(positions[i]), i}); + } + + auto col = ColumnUInt64::create(); + + while (!queue.empty()) + { + size_t ps = queue.top().second; + queue.pop(); + + auto & cur_col = chunks[ps].getColumns()[0]; + col->insertFrom(*cur_col, positions[ps]); + ++positions[ps]; + + if (positions[ps] == cur_col->size()) + break; + + queue.push({cur_col->getUInt(positions[ps]), ps}); + } + + UInt64 num_rows = col->size(); + res.setColumns(Columns({std::move(col)}), num_rows); + } + + OutputPort & getOutputPort() { return outputs.front(); } + +private: + Chunks chunks; + Chunk res; + std::vector positions; + std::vector finished; +}; + + +class NumbersSource : public ISource +{ +public: + String getName() const override { return "Numbers"; } + + NumbersSource(UInt64 start_number, UInt64 step, unsigned sleep_useconds) + : ISource(Block({ColumnWithTypeAndName{ ColumnUInt64::create(), std::make_shared(), "number" }})), + current_number(start_number), step(step), sleep_useconds(sleep_useconds) + { + } + +private: + UInt64 current_number = 0; + UInt64 step; + unsigned sleep_useconds; + + Chunk generate() override + { + usleep(sleep_useconds); + + MutableColumns columns; + columns.emplace_back(ColumnUInt64::create(1, current_number)); + current_number += step; + return Chunk(std::move(columns), 1); + } +}; + + +class SleepyTransform : public ISimpleTransform +{ +public: + explicit SleepyTransform(unsigned sleep_useconds) + : ISimpleTransform( + Block({ColumnWithTypeAndName{ ColumnUInt64::create(), std::make_shared(), "number" }}), + Block({ColumnWithTypeAndName{ ColumnUInt64::create(), std::make_shared(), "number" }}), + false) + , sleep_useconds(sleep_useconds) {} + + String getName() const override { return "SleepyTransform"; } + +protected: + void transform(Chunk &) override + { + usleep(sleep_useconds); + } + +private: + unsigned sleep_useconds; +}; + +class PrintSink : public ISink +{ +public: + String getName() const override { return "Print"; } + + PrintSink(String prefix) + : ISink(Block({ColumnWithTypeAndName{ ColumnUInt64::create(), std::make_shared(), "number" }})), + prefix(std::move(prefix)) + { + } + +private: + String prefix; + WriteBufferFromFileDescriptor out{STDOUT_FILENO}; + FormatSettings settings; + + void consume(Chunk chunk) override + { + size_t rows = chunk.getNumRows(); + size_t columns = chunk.getNumColumns(); + + for (size_t row_num = 0; row_num < rows; ++row_num) + { + writeString(prefix, out); + for (size_t column_num = 0; column_num < columns; ++column_num) + { + if (column_num != 0) + writeChar('\t', out); + getPort().getHeader().getByPosition(column_num).type->serializeAsText(*chunk.getColumns()[column_num], row_num, out, settings); + } + writeChar('\n', out); + } + + out.next(); + } +}; + +template +struct measure +{ + template + static typename TimeT::rep execution(F&& func, Args&&... args) + { + auto start = std::chrono::steady_clock::now(); + std::forward(func)(std::forward(args)...); + auto duration = std::chrono::duration_cast< TimeT> + (std::chrono::steady_clock::now() - start); + return duration.count(); + } +}; + +int main(int, char **) +try +{ + auto execute_chain = [](String msg, size_t start1, size_t start2, size_t start3, size_t num_threads) + { + std::cerr << msg << "\n"; + + auto source1 = std::make_shared(start1, 3, 100000); + auto source2 = std::make_shared(start2, 3, 100000); + auto source3 = std::make_shared(start3, 3, 100000); + + auto transform1 = std::make_shared(100000); + auto transform2 = std::make_shared(100000); + auto transform3 = std::make_shared(100000); + + auto limit1 = std::make_shared(source1->getPort().getHeader(), 20, 0); + auto limit2 = std::make_shared(source2->getPort().getHeader(), 20, 0); + auto limit3 = std::make_shared(source3->getPort().getHeader(), 20, 0); + + auto merge = std::make_shared(source1->getPort().getHeader(), 3); + auto limit_fin = std::make_shared(source1->getPort().getHeader(), 54, 0); + auto sink = std::make_shared(""); + + connect(source1->getPort(), transform1->getInputPort()); + connect(source2->getPort(), transform2->getInputPort()); + connect(source3->getPort(), transform3->getInputPort()); + + connect(transform1->getOutputPort(), limit1->getInputPort()); + connect(transform2->getOutputPort(), limit2->getInputPort()); + connect(transform3->getOutputPort(), limit3->getInputPort()); + + auto it = merge->getInputs().begin(); + connect(limit1->getOutputPort(), *(it++)); + connect(limit2->getOutputPort(), *(it++)); + connect(limit3->getOutputPort(), *(it++)); + + connect(merge->getOutputPort(), limit_fin->getInputPort()); + connect(limit_fin->getOutputPort(), sink->getPort()); + + std::vector processors = {source1, source2, source3, + transform1, transform2, transform3, + limit1, limit2, limit3, + merge, limit_fin, sink}; +// WriteBufferFromOStream out(std::cout); +// printPipeline(processors, out); + + PipelineExecutor executor(processors); + executor.execute(num_threads); + }; + + auto even_time_single = measure<>::execution(execute_chain, "Even distribution single thread", 0, 1, 2, 1); + auto even_time_mt = measure<>::execution(execute_chain, "Even distribution multiple threads", 0, 1, 2, 4); + + auto half_time_single = measure<>::execution(execute_chain, "Half distribution single thread", 0, 31, 62, 1); + auto half_time_mt = measure<>::execution(execute_chain, "Half distribution multiple threads", 0, 31, 62, 4); + + auto ordered_time_single = measure<>::execution(execute_chain, "Ordered distribution single thread", 0, 61, 122, 1); + auto ordered_time_mt = measure<>::execution(execute_chain, "Ordered distribution multiple threads", 0, 61, 122, 4); + + std::cout << "Single Thread [0:60:3] [1:60:3] [2:60:3] time: " << even_time_single << " ms.\n"; + std::cout << "Multiple Threads [0:60:3] [1:60:3] [2:60:3] time:" << even_time_mt << " ms.\n"; + + std::cout << "Single Thread [0:60:3] [31:90:3] [62:120:3] time: " << half_time_single << " ms.\n"; + std::cout << "Multiple Threads [0:60:3] [31:90:3] [62:120:3] time: " << half_time_mt << " ms.\n"; + + std::cout << "Single Thread [0:60:3] [61:120:3] [122:180:3] time: " << ordered_time_single << " ms.\n"; + std::cout << "Multiple Threads [0:60:3] [61:120:3] [122:180:3] time: " << ordered_time_mt << " ms.\n"; + + return 0; +} +catch (...) +{ + std::cerr << getCurrentExceptionMessage(true) << '\n'; + throw; +} diff --git a/dbms/src/Processors/tests/processors_test_merge_sorting_transform.cpp b/dbms/src/Processors/tests/processors_test_merge_sorting_transform.cpp new file mode 100644 index 00000000000..258e89e67e7 --- /dev/null +++ b/dbms/src/Processors/tests/processors_test_merge_sorting_transform.cpp @@ -0,0 +1,233 @@ +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +#include +#include +#include +#include + + +using namespace DB; + + +class NumbersSource : public ISource +{ +public: + String getName() const override { return "Numbers"; } + + NumbersSource(UInt64 count, UInt64 block_size, unsigned sleep_useconds) + : ISource(Block({ColumnWithTypeAndName{ ColumnUInt64::create(), std::make_shared(), "number" }})), + count(count), block_size(block_size), sleep_useconds(sleep_useconds) + { + } + +private: + UInt64 current_number = 0; + UInt64 count; + UInt64 block_size; + unsigned sleep_useconds; + + Chunk generate() override + { + if (current_number == count) + return {}; + + usleep(sleep_useconds); + + MutableColumns columns; + columns.emplace_back(ColumnUInt64::create()); + + UInt64 number = current_number++; + for (UInt64 i = 0; i < block_size; ++i, number += count) + columns.back()->insert(Field(number)); + + return Chunk(std::move(columns), block_size); + } +}; + +class CheckSortedSink : public ISink +{ +public: + String getName() const override { return "Print"; } + + CheckSortedSink() + : ISink(Block({ColumnWithTypeAndName{ ColumnUInt64::create(), std::make_shared(), "number" }})) + { + } + +private: + FormatSettings settings; + UInt64 current_number = 0; + + void consume(Chunk chunk) override + { + size_t rows = chunk.getNumRows(); + + UInt64 prev = current_number; + auto & col = chunk.getColumns().at(0); + for (size_t row_num = 0; row_num < rows; ++row_num) + { + UInt64 val = col->getUInt(row_num); + if (val != current_number) + throw Exception("Invalid value. Expected " + toString(current_number) + ", got " + toString(val), + ErrorCodes::LOGICAL_ERROR); + + ++current_number; + } + + std::cout << "CheckSortedSink: " << prev << " - " << current_number << std::endl; + } +}; + +template +struct measure +{ + template + static typename TimeT::rep execution(F&& func, Args&&... args) + { + auto start = std::chrono::steady_clock::now(); + std::forward(func)(std::forward(args)...); + auto duration = std::chrono::duration_cast< TimeT> + (std::chrono::steady_clock::now() - start); + return duration.count(); + } +}; + +int main(int, char **) +try +{ + Poco::AutoPtr channel = new Poco::ConsoleChannel(std::cerr); + Logger::root().setChannel(channel); + Logger::root().setLevel("trace"); + + auto execute_chain = []( + String msg, + UInt64 source_block_size, + UInt64 blocks_count, + size_t max_merged_block_size, + UInt64 limit, + size_t max_bytes_before_remerge, + size_t max_bytes_before_external_sort, + size_t num_threads) + { + std::cerr << "------------------------\n"; + std::cerr << msg << "\n"; + + auto source = std::make_shared(blocks_count, source_block_size, 100); + SortDescription description = {{0, 1, 1}}; + auto transform = std::make_shared( + source->getPort().getHeader(), description, + max_merged_block_size, limit, max_bytes_before_remerge, max_bytes_before_external_sort, "."); + auto sink = std::make_shared(); + + connect(source->getPort(), transform->getInputs().front()); + connect(transform->getOutputs().front(), sink->getPort()); + + std::vector processors = {source, transform, sink}; + PipelineExecutor executor(processors); + executor.execute(num_threads); + + WriteBufferFromOStream out(std::cout); + printPipeline(executor.getProcessors(), out); + }; + + std::map times; + + for (size_t num_threads : {1, 4}) + { + { + UInt64 source_block_size = 100; + UInt64 blocks_count = 10; + size_t max_merged_block_size = 100; + UInt64 limit = 0; + size_t max_bytes_before_remerge = 10000000; + size_t max_bytes_before_external_sort = 10000000; + std::string msg = num_threads > 1 ? "multiple threads" : "single thread"; + msg += ", " + toString(blocks_count) + " blocks per " + toString(source_block_size) + " numbers" + + ", no remerge and external sorts."; + + Int64 time = measure<>::execution(execute_chain, msg, + source_block_size, + blocks_count, + max_merged_block_size, + limit, + max_bytes_before_remerge, + max_bytes_before_external_sort, + num_threads); + + times[msg] = time; + } + + { + UInt64 source_block_size = 1024; + UInt64 blocks_count = 10; + size_t max_merged_block_size = 1024; + UInt64 limit = 2048; + size_t max_bytes_before_remerge = sizeof(UInt64) * source_block_size * 4; + size_t max_bytes_before_external_sort = 10000000; + std::string msg = num_threads > 1 ? "multiple threads" : "single thread"; + msg += ", " + toString(blocks_count) + " blocks per " + toString(source_block_size) + " numbers" + + ", with remerge, no external sorts."; + + Int64 time = measure<>::execution(execute_chain, msg, + source_block_size, + blocks_count, + max_merged_block_size, + limit, + max_bytes_before_remerge, + max_bytes_before_external_sort, + num_threads); + + times[msg] = time; + } + + { + UInt64 source_block_size = 1024; + UInt64 blocks_count = 10; + size_t max_merged_block_size = 1024; + UInt64 limit = 0; + size_t max_bytes_before_remerge = 0; + size_t max_bytes_before_external_sort = sizeof(UInt64) * source_block_size * 4; + std::string msg = num_threads > 1 ? "multiple threads" : "single thread"; + msg += ", " + toString(blocks_count) + " blocks per " + toString(source_block_size) + " numbers" + + ", no remerge, with external sorts."; + + Int64 time = measure<>::execution(execute_chain, msg, + source_block_size, + blocks_count, + max_merged_block_size, + limit, + max_bytes_before_remerge, + max_bytes_before_external_sort, + num_threads); + + times[msg] = time; + } + } + + for (auto & item : times) + std::cout << item.first << ' ' << item.second << " ms.\n"; + + return 0; +} +catch (...) +{ + std::cerr << getCurrentExceptionMessage(true) << '\n'; + throw; +} diff --git a/dbms/src/Processors/tests/processors_test_merging_sorted_transform.cpp b/dbms/src/Processors/tests/processors_test_merging_sorted_transform.cpp new file mode 100644 index 00000000000..214044dfd31 --- /dev/null +++ b/dbms/src/Processors/tests/processors_test_merging_sorted_transform.cpp @@ -0,0 +1,207 @@ +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +#include +#include + + +using namespace DB; + + +class NumbersSource : public ISource +{ +public: + String getName() const override { return "Numbers"; } + + NumbersSource(UInt64 start_number, UInt64 step, UInt64 block_size, unsigned sleep_useconds) + : ISource(Block({ColumnWithTypeAndName{ ColumnUInt64::create(), std::make_shared(), "number" }})), + current_number(start_number), step(step), block_size(block_size), sleep_useconds(sleep_useconds) + { + } + +private: + UInt64 current_number = 0; + UInt64 step; + UInt64 block_size; + unsigned sleep_useconds; + + Chunk generate() override + { + usleep(sleep_useconds); + + MutableColumns columns; + columns.emplace_back(ColumnUInt64::create()); + + for (UInt64 i = 0; i < block_size; ++i, current_number += step) + columns.back()->insert(Field(current_number)); + + return Chunk(std::move(columns), block_size); + } +}; + + +class SleepyTransform : public ISimpleTransform +{ +public: + explicit SleepyTransform(unsigned sleep_useconds) + : ISimpleTransform( + Block({ColumnWithTypeAndName{ ColumnUInt64::create(), std::make_shared(), "number" }}), + Block({ColumnWithTypeAndName{ ColumnUInt64::create(), std::make_shared(), "number" }}), + false) + , sleep_useconds(sleep_useconds) {} + + String getName() const override { return "SleepyTransform"; } + +protected: + void transform(Chunk &) override + { + usleep(sleep_useconds); + } + +private: + unsigned sleep_useconds; +}; + +class PrintSink : public ISink +{ +public: + String getName() const override { return "Print"; } + + PrintSink(String prefix) + : ISink(Block({ColumnWithTypeAndName{ ColumnUInt64::create(), std::make_shared(), "number" }})), + prefix(std::move(prefix)) + { + } + +private: + String prefix; + WriteBufferFromFileDescriptor out{STDOUT_FILENO}; + FormatSettings settings; + + void consume(Chunk chunk) override + { + size_t rows = chunk.getNumRows(); + size_t columns = chunk.getNumColumns(); + + for (size_t row_num = 0; row_num < rows; ++row_num) + { + writeString(prefix, out); + for (size_t column_num = 0; column_num < columns; ++column_num) + { + if (column_num != 0) + writeChar('\t', out); + getPort().getHeader().getByPosition(column_num).type->serializeAsText(*chunk.getColumns()[column_num], row_num, out, settings); + } + writeChar('\n', out); + } + + out.next(); + } +}; + +template +struct measure +{ + template + static typename TimeT::rep execution(F&& func, Args&&... args) + { + auto start = std::chrono::steady_clock::now(); + std::forward(func)(std::forward(args)...); + auto duration = std::chrono::duration_cast< TimeT> + (std::chrono::steady_clock::now() - start); + return duration.count(); + } +}; + +int main(int, char **) +try +{ + auto execute_chain = [](String msg, size_t start1, size_t start2, size_t start3, size_t num_threads) + { + std::cerr << msg << "\n"; + + auto source1 = std::make_shared(start1, 3, 2, 100000); + auto source2 = std::make_shared(start2, 3, 2, 100000); + auto source3 = std::make_shared(start3, 3, 2, 100000); + + auto transform1 = std::make_shared(100000); + auto transform2 = std::make_shared(100000); + auto transform3 = std::make_shared(100000); + + auto limit1 = std::make_shared(source1->getPort().getHeader(), 20, 0); + auto limit2 = std::make_shared(source2->getPort().getHeader(), 20, 0); + auto limit3 = std::make_shared(source3->getPort().getHeader(), 20, 0); + + SortDescription description = {{0, 1, 1}}; + auto merge = std::make_shared(source1->getPort().getHeader(), 3, description, 2); + auto limit_fin = std::make_shared(source1->getPort().getHeader(), 54, 0); + auto sink = std::make_shared(""); + + connect(source1->getPort(), transform1->getInputPort()); + connect(source2->getPort(), transform2->getInputPort()); + connect(source3->getPort(), transform3->getInputPort()); + + connect(transform1->getOutputPort(), limit1->getInputPort()); + connect(transform2->getOutputPort(), limit2->getInputPort()); + connect(transform3->getOutputPort(), limit3->getInputPort()); + + auto it = merge->getInputs().begin(); + connect(limit1->getOutputPort(), *(it++)); + connect(limit2->getOutputPort(), *(it++)); + connect(limit3->getOutputPort(), *(it++)); + + connect(merge->getOutputs().front(), limit_fin->getInputPort()); + connect(limit_fin->getOutputPort(), sink->getPort()); + + std::vector processors = {source1, source2, source3, + transform1, transform2, transform3, + limit1, limit2, limit3, + merge, limit_fin, sink}; +// WriteBufferFromOStream out(std::cout); +// printPipeline(processors, out); + + PipelineExecutor executor(processors); + executor.execute(num_threads); + }; + + auto even_time_single = measure<>::execution(execute_chain, "Even distribution single thread", 0, 1, 2, 1); + auto even_time_mt = measure<>::execution(execute_chain, "Even distribution multiple threads", 0, 1, 2, 4); + + auto half_time_single = measure<>::execution(execute_chain, "Half distribution single thread", 0, 31, 62, 1); + auto half_time_mt = measure<>::execution(execute_chain, "Half distribution multiple threads", 0, 31, 62, 4); + + auto ordered_time_single = measure<>::execution(execute_chain, "Ordered distribution single thread", 0, 61, 122, 1); + auto ordered_time_mt = measure<>::execution(execute_chain, "Ordered distribution multiple threads", 0, 61, 122, 4); + + std::cout << "Single Thread [0:60:3] [1:60:3] [2:60:3] time: " << even_time_single << " ms.\n"; + std::cout << "Multiple Threads [0:60:3] [1:60:3] [2:60:3] time:" << even_time_mt << " ms.\n"; + + std::cout << "Single Thread [0:60:3] [31:90:3] [62:120:3] time: " << half_time_single << " ms.\n"; + std::cout << "Multiple Threads [0:60:3] [31:90:3] [62:120:3] time: " << half_time_mt << " ms.\n"; + + std::cout << "Single Thread [0:60:3] [61:120:3] [122:180:3] time: " << ordered_time_single << " ms.\n"; + std::cout << "Multiple Threads [0:60:3] [61:120:3] [122:180:3] time: " << ordered_time_mt << " ms.\n"; + + return 0; +} +catch (...) +{ + std::cerr << getCurrentExceptionMessage(true) << '\n'; + throw; +} diff --git a/dbms/src/Storages/CMakeLists.txt b/dbms/src/Storages/CMakeLists.txt index 236d4d32524..ae47fba063a 100644 --- a/dbms/src/Storages/CMakeLists.txt +++ b/dbms/src/Storages/CMakeLists.txt @@ -1,5 +1,4 @@ add_subdirectory(System) -add_subdirectory(Kafka) if(ENABLE_TESTS) add_subdirectory(tests) diff --git a/dbms/src/Storages/CheckResults.h b/dbms/src/Storages/CheckResults.h new file mode 100644 index 00000000000..0f895fba3bc --- /dev/null +++ b/dbms/src/Storages/CheckResults.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include + +namespace DB +{ + +/// Result of CHECK TABLE query for single part of table +struct CheckResult +{ + /// Part name for merge tree or file name for simplier tables + String fs_path; + /// Does check passed + bool success = false; + /// Failure message if any + String failure_message; + + CheckResult() = default; + CheckResult(const String & fs_path_, bool success_, String failure_message_) + : fs_path(fs_path_), success(success_), failure_message(failure_message_) + {} +}; + +using CheckResults = std::vector; + +} diff --git a/dbms/src/Storages/IStorage.h b/dbms/src/Storages/IStorage.h index 5bfd8224372..477c4456b17 100644 --- a/dbms/src/Storages/IStorage.h +++ b/dbms/src/Storages/IStorage.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -37,6 +38,19 @@ class AlterCommands; class MutationCommands; class PartitionCommands; +struct ColumnSize +{ + size_t marks = 0; + size_t data_compressed = 0; + size_t data_uncompressed = 0; + + void add(const ColumnSize & other) + { + marks += other.marks; + data_compressed += other.data_compressed; + data_uncompressed += other.data_uncompressed; + } +}; /** Storage. Describes the table. Responsible for * - storage of the table data; @@ -61,7 +75,7 @@ public: /// The name of the table. virtual std::string getTableName() const = 0; - virtual std::string getDatabaseName() const { return {}; } // FIXME: should be an abstract method! + virtual std::string getDatabaseName() const = 0; /// Returns true if the storage receives data from a remote server or servers. virtual bool isRemote() const { return false; } @@ -81,6 +95,10 @@ public: /// Returns true if the storage supports deduplication of inserted data blocks. virtual bool supportsDeduplication() const { return false; } + /// Optional size information of each physical column. + /// Currently it's only used by the MergeTree family for query optimizations. + using ColumnSizeByName = std::unordered_map; + virtual ColumnSizeByName getColumnSizes() const { return {}; } public: /// thread-unsafe part. lockStructure must be acquired const ColumnsDescription & getColumns() const; /// returns combined set of columns @@ -285,7 +303,7 @@ public: virtual bool mayBenefitFromIndexForIn(const ASTPtr & /* left_in_operand */, const Context & /* query_context */) const { return false; } /// Checks validity of the data - virtual bool checkData() const { throw Exception("Check query is not supported for " + getName() + " storage", ErrorCodes::NOT_IMPLEMENTED); } + virtual CheckResults checkData(const ASTPtr & /* query */, const Context & /* context */) { throw Exception("Check query is not supported for " + getName() + " storage", ErrorCodes::NOT_IMPLEMENTED); } /// Checks that table could be dropped right now /// Otherwise - throws an exception with detailed information. diff --git a/dbms/src/Storages/Kafka/CMakeLists.txt b/dbms/src/Storages/Kafka/CMakeLists.txt deleted file mode 100644 index e581b379322..00000000000 --- a/dbms/src/Storages/Kafka/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -if(USE_RDKAFKA) - include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake) - add_headers_and_sources(clickhouse_storage_kafka .) - add_library(clickhouse_storage_kafka ${clickhouse_storage_kafka_sources}) - target_link_libraries(clickhouse_storage_kafka PRIVATE clickhouse_common_io ${CPPKAFKA_LIBRARY} ${RDKAFKA_LIBRARY}) - if(NOT USE_INTERNAL_RDKAFKA_LIBRARY) - target_include_directories(clickhouse_storage_kafka SYSTEM BEFORE PRIVATE ${RDKAFKA_INCLUDE_DIR}) - endif() -endif() diff --git a/dbms/src/Storages/MergeTree/KeyCondition.cpp b/dbms/src/Storages/MergeTree/KeyCondition.cpp index 8990f322ea0..f08a9c35c9e 100644 --- a/dbms/src/Storages/MergeTree/KeyCondition.cpp +++ b/dbms/src/Storages/MergeTree/KeyCondition.cpp @@ -178,6 +178,24 @@ const KeyCondition::AtomMap KeyCondition::atom_map return true; } }, + { + "empty", + [] (RPNElement & out, const Field &) + { + out.function = RPNElement::FUNCTION_IN_RANGE; + out.range = Range(""); + return true; + } + }, + { + "notEmpty", + [] (RPNElement & out, const Field &) + { + out.function = RPNElement::FUNCTION_NOT_IN_RANGE; + out.range = Range(""); + return true; + } + }, { "like", [] (RPNElement & out, const Field & value) @@ -191,6 +209,48 @@ const KeyCondition::AtomMap KeyCondition::atom_map String right_bound = firstStringThatIsGreaterThanAllStringsWithPrefix(prefix); + out.function = RPNElement::FUNCTION_IN_RANGE; + out.range = !right_bound.empty() + ? Range(prefix, true, right_bound, false) + : Range::createLeftBounded(prefix, true); + + return true; + } + }, + { + "notLike", + [] (RPNElement & out, const Field & value) + { + if (value.getType() != Field::Types::String) + return false; + + String prefix = extractFixedPrefixFromLikePattern(value.get()); + if (prefix.empty()) + return false; + + String right_bound = firstStringThatIsGreaterThanAllStringsWithPrefix(prefix); + + out.function = RPNElement::FUNCTION_NOT_IN_RANGE; + out.range = !right_bound.empty() + ? Range(prefix, true, right_bound, false) + : Range::createLeftBounded(prefix, true); + + return true; + } + }, + { + "startsWith", + [] (RPNElement & out, const Field & value) + { + if (value.getType() != Field::Types::String) + return false; + + String prefix = value.get(); + if (prefix.empty()) + return false; + + String right_bound = firstStringThatIsGreaterThanAllStringsWithPrefix(prefix); + out.function = RPNElement::FUNCTION_IN_RANGE; out.range = !right_bound.empty() ? Range(prefix, true, right_bound, false) @@ -624,92 +684,102 @@ bool KeyCondition::atomFromAST(const ASTPtr & node, const Context & context, Blo { const ASTs & args = func->arguments->children; - if (args.size() != 2) - return false; - DataTypePtr key_expr_type; /// Type of expression containing key column - size_t key_arg_pos; /// Position of argument with key column (non-const argument) size_t key_column_num = -1; /// Number of a key column (inside key_column_names array) MonotonicFunctionsChain chain; - bool is_set_const = false; - bool is_constant_transformed = false; + std::string func_name = func->name; - if (functionIsInOrGlobalInOperator(func->name) - && tryPrepareSetIndex(args, context, out, key_column_num)) + if (args.size() == 1) { - key_arg_pos = 0; - is_set_const = true; + if (!(isKeyPossiblyWrappedByMonotonicFunctions(args[0], context, key_column_num, key_expr_type, chain))) + return false; + + if (key_column_num == static_cast(-1)) + throw Exception("`key_column_num` wasn't initialized. It is a bug.", ErrorCodes::LOGICAL_ERROR); } - else if (getConstant(args[1], block_with_constants, const_value, const_type) - && isKeyPossiblyWrappedByMonotonicFunctions(args[0], context, key_column_num, key_expr_type, chain)) + else if (args.size() == 2) { - key_arg_pos = 0; - } - else if (getConstant(args[1], block_with_constants, const_value, const_type) - && canConstantBeWrappedByMonotonicFunctions(args[0], key_column_num, key_expr_type, const_value, const_type)) - { - key_arg_pos = 0; - is_constant_transformed = true; - } - else if (getConstant(args[0], block_with_constants, const_value, const_type) - && isKeyPossiblyWrappedByMonotonicFunctions(args[1], context, key_column_num, key_expr_type, chain)) - { - key_arg_pos = 1; - } - else if (getConstant(args[0], block_with_constants, const_value, const_type) - && canConstantBeWrappedByMonotonicFunctions(args[1], key_column_num, key_expr_type, const_value, const_type)) - { - key_arg_pos = 1; - is_constant_transformed = true; + size_t key_arg_pos; /// Position of argument with key column (non-const argument) + bool is_set_const = false; + bool is_constant_transformed = false; + + if (functionIsInOrGlobalInOperator(func_name) + && tryPrepareSetIndex(args, context, out, key_column_num)) + { + key_arg_pos = 0; + is_set_const = true; + } + else if (getConstant(args[1], block_with_constants, const_value, const_type) + && isKeyPossiblyWrappedByMonotonicFunctions(args[0], context, key_column_num, key_expr_type, chain)) + { + key_arg_pos = 0; + } + else if (getConstant(args[1], block_with_constants, const_value, const_type) + && canConstantBeWrappedByMonotonicFunctions(args[0], key_column_num, key_expr_type, const_value, const_type)) + { + key_arg_pos = 0; + is_constant_transformed = true; + } + else if (getConstant(args[0], block_with_constants, const_value, const_type) + && isKeyPossiblyWrappedByMonotonicFunctions(args[1], context, key_column_num, key_expr_type, chain)) + { + key_arg_pos = 1; + } + else if (getConstant(args[0], block_with_constants, const_value, const_type) + && canConstantBeWrappedByMonotonicFunctions(args[1], key_column_num, key_expr_type, const_value, const_type)) + { + key_arg_pos = 1; + is_constant_transformed = true; + } + else + return false; + + if (key_column_num == static_cast(-1)) + throw Exception("`key_column_num` wasn't initialized. It is a bug.", ErrorCodes::LOGICAL_ERROR); + + /// Transformed constant must weaken the condition, for example "x > 5" must weaken to "round(x) >= 5" + if (is_constant_transformed) + { + if (func_name == "less") + func_name = "lessOrEquals"; + else if (func_name == "greater") + func_name = "greaterOrEquals"; + } + + /// Replace on to <-sign> + if (key_arg_pos == 1) + { + if (func_name == "less") + func_name = "greater"; + else if (func_name == "greater") + func_name = "less"; + else if (func_name == "greaterOrEquals") + func_name = "lessOrEquals"; + else if (func_name == "lessOrEquals") + func_name = "greaterOrEquals"; + else if (func_name == "in" || func_name == "notIn" || func_name == "like") + { + /// "const IN data_column" doesn't make sense (unlike "data_column IN const") + return false; + } + } + + bool cast_not_needed = + is_set_const /// Set args are already casted inside Set::createFromAST + || (isNativeNumber(key_expr_type) && isNativeNumber(const_type)); /// Numbers are accurately compared without cast. + + if (!cast_not_needed) + castValueToType(key_expr_type, const_value, const_type, node); } else return false; - if (key_column_num == static_cast(-1)) - throw Exception("`key_column_num` wasn't initialized. It is a bug.", ErrorCodes::LOGICAL_ERROR); - - std::string func_name = func->name; - - /// Transformed constant must weaken the condition, for example "x > 5" must weaken to "round(x) >= 5" - if (is_constant_transformed) - { - if (func_name == "less") - func_name = "lessOrEquals"; - else if (func_name == "greater") - func_name = "greaterOrEquals"; - } - - /// Replace on to <-sign> - if (key_arg_pos == 1) - { - if (func_name == "less") - func_name = "greater"; - else if (func_name == "greater") - func_name = "less"; - else if (func_name == "greaterOrEquals") - func_name = "lessOrEquals"; - else if (func_name == "lessOrEquals") - func_name = "greaterOrEquals"; - else if (func_name == "in" || func_name == "notIn" || func_name == "like") - { - /// "const IN data_column" doesn't make sense (unlike "data_column IN const") - return false; - } - } - - out.key_column = key_column_num; - out.monotonic_functions_chain = std::move(chain); - const auto atom_it = atom_map.find(func_name); if (atom_it == std::end(atom_map)) return false; - bool cast_not_needed = - is_set_const /// Set args are already casted inside Set::createFromAST - || (isNativeNumber(key_expr_type) && isNativeNumber(const_type)); /// Numbers are accurately compared without cast. - - if (!cast_not_needed) - castValueToType(key_expr_type, const_value, const_type, node); + out.key_column = key_column_num; + out.monotonic_functions_chain = std::move(chain); return atom_it->second(out, const_value); } @@ -727,7 +797,6 @@ bool KeyCondition::atomFromAST(const ASTPtr & node, const Context & context, Blo return true; } } - return false; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index 96e1ba1429f..043adc7a259 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -119,7 +119,7 @@ void MergeTreeBlockSizePredictor::initialize(const Block & sample_block, const N ColumnInfo info; info.name = column_name; /// If column isn't fixed and doesn't have checksum, than take first - MergeTreeDataPart::ColumnSize column_size = data_part->getColumnSize( + ColumnSize column_size = data_part->getColumnSize( column_name, *column_with_type_and_name.type); info.bytes_per_row_global = column_size.data_uncompressed diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index d1e9e470e57..b32470f9f77 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -2378,8 +2378,8 @@ void MergeTreeData::addPartContributionToColumnSizes(const DataPartPtr & part) for (const auto & column : part->columns) { - DataPart::ColumnSize & total_column_size = column_sizes[column.name]; - DataPart::ColumnSize part_column_size = part->getColumnSize(column.name, *column.type); + ColumnSize & total_column_size = column_sizes[column.name]; + ColumnSize part_column_size = part->getColumnSize(column.name, *column.type); total_column_size.add(part_column_size); } } @@ -2390,8 +2390,8 @@ void MergeTreeData::removePartContributionToColumnSizes(const DataPartPtr & part for (const auto & column : part->columns) { - DataPart::ColumnSize & total_column_size = column_sizes[column.name]; - DataPart::ColumnSize part_column_size = part->getColumnSize(column.name, *column.type); + ColumnSize & total_column_size = column_sizes[column.name]; + ColumnSize part_column_size = part->getColumnSize(column.name, *column.type); auto log_subtract = [&](size_t & from, size_t value, const char * field) { diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 2ebddb886f6..29962382749 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -547,8 +547,7 @@ public: return it == std::end(column_sizes) ? 0 : it->second.data_compressed; } - using ColumnSizeByName = std::unordered_map; - ColumnSizeByName getColumnSizes() const + ColumnSizeByName getColumnSizes() const override { auto lock = lockParts(); return column_sizes; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp index fc4c5fef130..7b8be970e1d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp @@ -153,7 +153,7 @@ MergeTreeDataPart::MergeTreeDataPart(const MergeTreeData & storage_, const Strin /// Takes into account the fact that several columns can e.g. share their .size substreams. /// When calculating totals these should be counted only once. -MergeTreeDataPart::ColumnSize MergeTreeDataPart::getColumnSizeImpl( +ColumnSize MergeTreeDataPart::getColumnSizeImpl( const String & column_name, const IDataType & type, std::unordered_set * processed_substreams) const { ColumnSize size; @@ -182,12 +182,12 @@ MergeTreeDataPart::ColumnSize MergeTreeDataPart::getColumnSizeImpl( return size; } -MergeTreeDataPart::ColumnSize MergeTreeDataPart::getColumnSize(const String & column_name, const IDataType & type) const +ColumnSize MergeTreeDataPart::getColumnSize(const String & column_name, const IDataType & type) const { return getColumnSizeImpl(column_name, type, nullptr); } -MergeTreeDataPart::ColumnSize MergeTreeDataPart::getTotalColumnsSize() const +ColumnSize MergeTreeDataPart::getTotalColumnsSize() const { ColumnSize totals; std::unordered_set processed_substreams; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.h b/dbms/src/Storages/MergeTree/MergeTreeDataPart.h index f775e5bc085..f41ea8af424 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.h @@ -22,6 +22,7 @@ namespace DB { +struct ColumnSize; class MergeTreeData; @@ -39,20 +40,6 @@ struct MergeTreeDataPart /// If no checksums are present returns the name of the first physically existing column. String getColumnNameWithMinumumCompressedSize() const; - struct ColumnSize - { - size_t marks = 0; - size_t data_compressed = 0; - size_t data_uncompressed = 0; - - void add(const ColumnSize & other) - { - marks += other.marks; - data_compressed += other.data_compressed; - data_uncompressed += other.data_uncompressed; - } - }; - /// NOTE: Returns zeros if column files are not found in checksums. /// NOTE: You must ensure that no ALTERs are in progress when calculating ColumnSizes. /// (either by locking columns_lock, or by locking table structure). diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexFullText.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexFullText.cpp index 895764339e5..be9994ece64 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexFullText.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexFullText.cpp @@ -49,7 +49,17 @@ static void likeStringToBloomFilter( while (cur < data.size() && token_extractor->nextLike(data, &cur, token)) bloom_filter.add(token.c_str(), token.size()); } +/// Unified condition for equals, startsWith and endsWith +bool MergeTreeConditionFullText::createFunctionEqualsCondition(RPNElement & out, const Field & value, const MergeTreeIndexFullText & idx) +{ + out.function = RPNElement::FUNCTION_EQUALS; + out.bloom_filter = std::make_unique( + idx.bloom_filter_size, idx.bloom_filter_hashes, idx.seed); + const auto & str = value.get(); + stringToBloomFilter(str.c_str(), str.size(), idx.token_extractor_func, *out.bloom_filter); + return true; +} MergeTreeIndexGranuleFullText::MergeTreeIndexGranuleFullText(const MergeTreeIndexFullText & index) : IMergeTreeIndexGranule() @@ -129,20 +139,14 @@ const MergeTreeConditionFullText::AtomMap MergeTreeConditionFullText::atom_map "equals", [] (RPNElement & out, const Field & value, const MergeTreeIndexFullText & idx) { - out.function = RPNElement::FUNCTION_EQUALS; - out.bloom_filter = std::make_unique( - idx.bloom_filter_size, idx.bloom_filter_hashes, idx.seed); - - const auto & str = value.get(); - stringToBloomFilter(str.c_str(), str.size(), idx.token_extractor_func, *out.bloom_filter); - return true; + return createFunctionEqualsCondition(out, value, idx); } }, { "like", [] (RPNElement & out, const Field & value, const MergeTreeIndexFullText & idx) { - out.function = RPNElement::FUNCTION_LIKE; + out.function = RPNElement::FUNCTION_EQUALS; out.bloom_filter = std::make_unique( idx.bloom_filter_size, idx.bloom_filter_hashes, idx.seed); @@ -151,6 +155,55 @@ const MergeTreeConditionFullText::AtomMap MergeTreeConditionFullText::atom_map return true; } }, + { + "notLike", + [] (RPNElement & out, const Field & value, const MergeTreeIndexFullText & idx) + { + out.function = RPNElement::FUNCTION_NOT_EQUALS; + out.bloom_filter = std::make_unique( + idx.bloom_filter_size, idx.bloom_filter_hashes, idx.seed); + + const auto & str = value.get(); + likeStringToBloomFilter(str, idx.token_extractor_func, *out.bloom_filter); + return true; + } + }, + { + "startsWith", + [] (RPNElement & out, const Field & value, const MergeTreeIndexFullText & idx) + { + return createFunctionEqualsCondition(out, value, idx); + } + }, + { + "endsWith", + [] (RPNElement & out, const Field & value, const MergeTreeIndexFullText & idx) + { + return createFunctionEqualsCondition(out, value, idx); + } + }, + { + "multiSearchAny", + [] (RPNElement & out, const Field & value, const MergeTreeIndexFullText & idx) + { + out.function = RPNElement::FUNCTION_MULTI_SEARCH; + + /// 2d vector is not needed here but is used because already exists for FUNCTION_IN + std::vector> bloom_filters; + bloom_filters.emplace_back(); + for (const auto & element : value.get()) + { + if (element.getType() != Field::Types::String) + return false; + + bloom_filters.back().emplace_back(idx.bloom_filter_size, idx.bloom_filter_hashes, idx.seed); + const auto & str = element.get(); + stringToBloomFilter(str.c_str(), str.size(), idx.token_extractor_func, bloom_filters.back().back()); + } + out.set_bloom_filters = std::move(bloom_filters); + return true; + } + }, { "notIn", [] (RPNElement & out, const Field &, const MergeTreeIndexFullText &) @@ -197,10 +250,9 @@ bool MergeTreeConditionFullText::alwaysUnknownOrTrue() const } else if (element.function == RPNElement::FUNCTION_EQUALS || element.function == RPNElement::FUNCTION_NOT_EQUALS - || element.function == RPNElement::FUNCTION_LIKE - || element.function == RPNElement::FUNCTION_NOT_LIKE || element.function == RPNElement::FUNCTION_IN || element.function == RPNElement::FUNCTION_NOT_IN + || element.function == RPNElement::FUNCTION_MULTI_SEARCH || element.function == RPNElement::ALWAYS_FALSE) { rpn_stack.push_back(false); @@ -255,17 +307,8 @@ bool MergeTreeConditionFullText::mayBeTrueOnGranule(MergeTreeIndexGranulePtr idx if (element.function == RPNElement::FUNCTION_NOT_EQUALS) rpn_stack.back() = !rpn_stack.back(); } - else if (element.function == RPNElement::FUNCTION_LIKE - || element.function == RPNElement::FUNCTION_NOT_LIKE) - { - rpn_stack.emplace_back( - granule->bloom_filters[element.key_column].contains(*element.bloom_filter), true); - - if (element.function == RPNElement::FUNCTION_NOT_LIKE) - rpn_stack.back() = !rpn_stack.back(); - } else if (element.function == RPNElement::FUNCTION_IN - || element.function == RPNElement::FUNCTION_NOT_IN) + || element.function == RPNElement::FUNCTION_NOT_IN) { std::vector result(element.set_bloom_filters.back().size(), true); @@ -283,6 +326,18 @@ bool MergeTreeConditionFullText::mayBeTrueOnGranule(MergeTreeIndexGranulePtr idx if (element.function == RPNElement::FUNCTION_NOT_IN) rpn_stack.back() = !rpn_stack.back(); } + else if (element.function == RPNElement::FUNCTION_MULTI_SEARCH) + { + std::vector result(element.set_bloom_filters.back().size(), true); + + const auto & bloom_filters = element.set_bloom_filters[0]; + + for (size_t row = 0; row < bloom_filters.size(); ++row) + result[row] = result[row] && granule->bloom_filters[element.key_column].contains(bloom_filters[row]); + + rpn_stack.emplace_back( + std::find(std::cbegin(result), std::cend(result), true) != std::end(result), true); + } else if (element.function == RPNElement::FUNCTION_NOT) { rpn_stack.back() = !rpn_stack.back(); @@ -343,8 +398,9 @@ bool MergeTreeConditionFullText::atomFromAST( size_t key_arg_pos; /// Position of argument with key column (non-const argument) size_t key_column_num = -1; /// Number of a key column (inside key_column_names array) + const auto & func_name = func->name; - if (functionIsInOrGlobalInOperator(func->name) && tryPrepareSetBloomFilter(args, out)) + if (functionIsInOrGlobalInOperator(func_name) && tryPrepareSetBloomFilter(args, out)) { key_arg_pos = 0; } @@ -359,17 +415,17 @@ bool MergeTreeConditionFullText::atomFromAST( else return false; - if (const_type && const_type->getTypeId() != TypeIndex::String && const_type->getTypeId() != TypeIndex::FixedString) + if (const_type && const_type->getTypeId() != TypeIndex::String + && const_type->getTypeId() != TypeIndex::FixedString + && const_type->getTypeId() != TypeIndex::Array) return false; - if (key_arg_pos == 1 && (func->name != "equals" || func->name != "notEquals")) + if (key_arg_pos == 1 && (func_name != "equals" || func_name != "notEquals")) return false; - else if (!index.token_extractor_func->supportLike() && (func->name == "like" || func->name == "notLike")) + else if (!index.token_extractor_func->supportLike() && (func_name == "like" || func_name == "notLike")) return false; - else - key_arg_pos = 0; - const auto atom_it = atom_map.find(func->name); + const auto atom_it = atom_map.find(func_name); if (atom_it == std::end(atom_map)) return false; @@ -380,8 +436,8 @@ bool MergeTreeConditionFullText::atomFromAST( { /// Check constant like in KeyCondition if (const_value.getType() == Field::Types::UInt64 - || const_value.getType() == Field::Types::Int64 - || const_value.getType() == Field::Types::Float64) + || const_value.getType() == Field::Types::Int64 + || const_value.getType() == Field::Types::Float64) { /// Zero in all types is represented in memory the same way as in UInt64. out.function = const_value.get() @@ -475,7 +531,6 @@ bool MergeTreeConditionFullText::tryPrepareSetBloomFilter( return true; } - MergeTreeIndexGranulePtr MergeTreeIndexFullText::createIndexGranule() const { return std::make_shared(*this); diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexFullText.h b/dbms/src/Storages/MergeTree/MergeTreeIndexFullText.h index cd8ac534e64..f6230134596 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexFullText.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexFullText.h @@ -78,10 +78,9 @@ private: /// Atoms of a Boolean expression. FUNCTION_EQUALS, FUNCTION_NOT_EQUALS, - FUNCTION_LIKE, - FUNCTION_NOT_LIKE, FUNCTION_IN, FUNCTION_NOT_IN, + FUNCTION_MULTI_SEARCH, FUNCTION_UNKNOWN, /// Can take any value. /// Operators of the logical expression. FUNCTION_NOT, @@ -93,15 +92,20 @@ private: }; RPNElement( - Function function_ = FUNCTION_UNKNOWN, size_t key_column_ = 0, std::unique_ptr && const_bloom_filter_ = nullptr) - : function(function_), key_column(key_column_), bloom_filter(std::move(const_bloom_filter_)) {} + Function function_ = FUNCTION_UNKNOWN, size_t key_column_ = 0, std::unique_ptr && const_bloom_filter_ = nullptr) + : function(function_), key_column(key_column_), bloom_filter(std::move(const_bloom_filter_)) {} Function function = FUNCTION_UNKNOWN; - /// For FUNCTION_EQUALS, FUNCTION_NOT_EQUALS, FUNCTION_LIKE, FUNCTION_NOT_LIKE. + /// For FUNCTION_EQUALS, FUNCTION_NOT_EQUALS and FUNCTION_MULTI_SEARCH size_t key_column; + + /// For FUNCTION_EQUALS, FUNCTION_NOT_EQUALS std::unique_ptr bloom_filter; - /// For FUNCTION_IN and FUNCTION_NOT_IN + + /// For FUNCTION_IN, FUNCTION_NOT_IN and FUNCTION_MULTI_SEARCH std::vector> set_bloom_filters; + + /// For FUNCTION_IN and FUNCTION_NOT_IN std::vector set_key_position; }; @@ -113,6 +117,8 @@ private: bool getKey(const ASTPtr & node, size_t & key_column_num); bool tryPrepareSetBloomFilter(const ASTs & args, RPNElement & out); + static bool createFunctionEqualsCondition(RPNElement & out, const Field & value, const MergeTreeIndexFullText & idx); + static const AtomMap atom_map; const MergeTreeIndexFullText & index; diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexSet.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexSet.cpp index 9527e6a0a67..fc3b905d1eb 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexSet.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexSet.cpp @@ -411,7 +411,10 @@ static bool checkAtomName(const String & name) "greaterOrEquals", "in", "notIn", - "like" + "like", + "startsWith", + "endsWith", + "multiSearchAny" }; return atoms.find(name) != atoms.end(); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeSelectBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeSelectBlockInputStream.cpp index a6d2b32fb19..e1c5e5239b5 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSelectBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSelectBlockInputStream.cpp @@ -46,13 +46,13 @@ MergeTreeSelectBlockInputStream::MergeTreeSelectBlockInputStream( for (const auto & range : all_mark_ranges) total_marks_count += range.end - range.begin; - size_t total_rows = data_part->index_granularity.getTotalRows(); + size_t total_rows = data_part->index_granularity.getRowsCountInRanges(all_mark_ranges); if (!quiet) LOG_TRACE(log, "Reading " << all_mark_ranges.size() << " ranges from part " << data_part->name << ", approx. " << total_rows << (all_mark_ranges.size() > 1 - ? ", up to " + toString(data_part->index_granularity.getRowsCountInRanges(all_mark_ranges)) + ? ", up to " + toString(total_rows) : "") << " rows starting from " << data_part->index_granularity.getMarkStartingRow(all_mark_ranges.front().begin)); diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 38c2a612b1d..8b38f1ff32e 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -392,7 +392,7 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm auto & stream = *skip_indices_streams[i]; size_t prev_pos = 0; - size_t current_mark = 0; + size_t skip_index_current_mark = 0; while (prev_pos < rows) { UInt64 limit = 0; @@ -402,7 +402,7 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm } else { - limit = index_granularity.getMarkRows(current_mark); + limit = index_granularity.getMarkRows(skip_index_current_mark); if (skip_indices_aggregators[i]->empty()) { skip_indices_aggregators[i] = index->createIndexAggregator(); @@ -435,7 +435,7 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm } } prev_pos = pos; - current_mark++; + ++skip_index_current_mark; } } } diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index b06dea30052..045aa8a6461 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -181,7 +181,7 @@ void ReplicatedMergeTreePartCheckThread::searchForMissingPart(const String & par } -void ReplicatedMergeTreePartCheckThread::checkPart(const String & part_name) +CheckResult ReplicatedMergeTreePartCheckThread::checkPart(const String & part_name) { LOG_WARNING(log, "Checking part " << part_name); ProfileEvents::increment(ProfileEvents::ReplicatedPartChecks); @@ -197,6 +197,7 @@ void ReplicatedMergeTreePartCheckThread::checkPart(const String & part_name) if (!part) { searchForMissingPart(part_name); + return {part_name, false, "Part is missing, will search for it"}; } /// We have this part, and it's active. We will check whether we need this part and whether it has the right data. else if (part->name == part_name) @@ -242,7 +243,7 @@ void ReplicatedMergeTreePartCheckThread::checkPart(const String & part_name) if (need_stop) { LOG_INFO(log, "Checking part was cancelled."); - return; + return {part_name, false, "Checking part was cancelled"}; } LOG_INFO(log, "Part " << part_name << " looks good."); @@ -253,13 +254,15 @@ void ReplicatedMergeTreePartCheckThread::checkPart(const String & part_name) tryLogCurrentException(log, __PRETTY_FUNCTION__); - LOG_ERROR(log, "Part " << part_name << " looks broken. Removing it and queueing a fetch."); + String message = "Part " + part_name + " looks broken. Removing it and queueing a fetch."; + LOG_ERROR(log, message); ProfileEvents::increment(ProfileEvents::ReplicatedPartChecksFailed); storage.removePartAndEnqueueFetch(part_name); /// Delete part locally. storage.forgetPartAndMoveToDetached(part, "broken"); + return {part_name, false, message}; } } else if (part->modification_time + MAX_AGE_OF_LOCAL_PART_THAT_WASNT_ADDED_TO_ZOOKEEPER < time(nullptr)) @@ -269,8 +272,10 @@ void ReplicatedMergeTreePartCheckThread::checkPart(const String & part_name) /// Therefore, delete only if the part is old (not very reliable). ProfileEvents::increment(ProfileEvents::ReplicatedPartChecksFailed); - LOG_ERROR(log, "Unexpected part " << part_name << " in filesystem. Removing."); + String message = "Unexpected part " + part_name + " in filesystem. Removing."; + LOG_ERROR(log, message); storage.forgetPartAndMoveToDetached(part, "unexpected"); + return {part_name, false, message}; } else { @@ -290,6 +295,8 @@ void ReplicatedMergeTreePartCheckThread::checkPart(const String & part_name) /// In the worst case, errors will still appear `old_parts_lifetime` seconds in error log until the part is removed as the old one. LOG_WARNING(log, "We have part " << part->name << " covering part " << part_name); } + + return {part_name, true, ""}; } diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.h b/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.h index 432fb0f4bb6..322ee593c46 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.h +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.h @@ -11,6 +11,7 @@ #include #include #include +#include namespace DB { @@ -66,12 +67,12 @@ public: /// Get the number of parts in the queue for check. size_t size() const; - + /// Check part by name + CheckResult checkPart(const String & part_name); private: void run(); - void checkPart(const String & part_name); void searchForMissingPart(const String & part_name); StorageReplicatedMergeTree & storage; diff --git a/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h index 53573ed523a..833bad47ecd 100644 --- a/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h @@ -17,6 +17,7 @@ class StorageFromMergeTreeDataPart : public ext::shared_ptr_helperstorage.getTableName() + " (part " + part->name + ")"; } + String getDatabaseName() const override { return part->storage.getDatabaseName(); } BlockInputStreams read( const Names & column_names, diff --git a/dbms/src/Storages/MergeTree/checkDataPart.cpp b/dbms/src/Storages/MergeTree/checkDataPart.cpp index 2b17a675144..2ae83b5076a 100644 --- a/dbms/src/Storages/MergeTree/checkDataPart.cpp +++ b/dbms/src/Storages/MergeTree/checkDataPart.cpp @@ -53,6 +53,20 @@ public: private: ReadBufferFromFile mrk_file_buf; + + std::pair readMarkFromFile() + { + size_t mrk_rows; + MarkInCompressedFile mrk_mark; + readIntBinary(mrk_mark.offset_in_compressed_file, mrk_hashing_buf); + readIntBinary(mrk_mark.offset_in_decompressed_block, mrk_hashing_buf); + if (mrk_file_extension == ".mrk2") + readIntBinary(mrk_rows, mrk_hashing_buf); + else + mrk_rows = index_granularity.getMarkRows(mark_position); + + return {mrk_mark, mrk_rows}; + } public: HashingReadBuffer mrk_hashing_buf; @@ -78,15 +92,8 @@ public: void assertMark(bool only_read=false) { - MarkInCompressedFile mrk_mark; - readIntBinary(mrk_mark.offset_in_compressed_file, mrk_hashing_buf); - readIntBinary(mrk_mark.offset_in_decompressed_block, mrk_hashing_buf); - size_t mrk_rows; - if (mrk_file_extension == ".mrk2") - readIntBinary(mrk_rows, mrk_hashing_buf); - else - mrk_rows = index_granularity.getMarkRows(mark_position); + auto [mrk_mark, mrk_rows] = readMarkFromFile(); bool has_alternative_mark = false; MarkInCompressedFile alternative_data_mark = {}; MarkInCompressedFile data_mark = {}; @@ -136,6 +143,12 @@ public: + toString(compressed_hashing_buf.count()) + " (compressed), " + toString(uncompressed_hashing_buf.count()) + " (uncompressed)", ErrorCodes::CORRUPTED_DATA); + if (index_granularity.hasFinalMark()) + { + auto final_mark_rows = readMarkFromFile().second; + if (final_mark_rows != 0) + throw Exception("Incorrect final mark at the end of " + mrk_file_path + " expected 0 rows, got " + toString(final_mark_rows), ErrorCodes::CORRUPTED_DATA); + } if (!mrk_hashing_buf.eof()) throw Exception("EOF expected in " + mrk_file_path + " file" + " at position " diff --git a/dbms/src/Storages/StorageBuffer.cpp b/dbms/src/Storages/StorageBuffer.cpp index 24e1ecef2e3..3c334b2a48b 100644 --- a/dbms/src/Storages/StorageBuffer.cpp +++ b/dbms/src/Storages/StorageBuffer.cpp @@ -55,17 +55,17 @@ namespace ErrorCodes } -StorageBuffer::StorageBuffer(const std::string & name_, const ColumnsDescription & columns_, +StorageBuffer::StorageBuffer(const std::string & database_name_, const std::string & table_name_, const ColumnsDescription & columns_, Context & context_, size_t num_shards_, const Thresholds & min_thresholds_, const Thresholds & max_thresholds_, const String & destination_database_, const String & destination_table_, bool allow_materialized_) : IStorage{columns_}, - name(name_), global_context(context_), + table_name(table_name_), database_name(database_name_), global_context(context_), num_shards(num_shards_), buffers(num_shards_), min_thresholds(min_thresholds_), max_thresholds(max_thresholds_), destination_database(destination_database_), destination_table(destination_table_), no_destination(destination_database.empty() && destination_table.empty()), - allow_materialized(allow_materialized_), log(&Logger::get("StorageBuffer (" + name + ")")) + allow_materialized(allow_materialized_), log(&Logger::get("StorageBuffer (" + table_name + ")")) { } @@ -692,7 +692,7 @@ void StorageBuffer::flushThread() } -void StorageBuffer::alter(const AlterCommands & params, const String & database_name, const String & table_name, const Context & context, TableStructureWriteLockHolder & table_lock_holder) +void StorageBuffer::alter(const AlterCommands & params, const String & database_name_, const String & table_name_, const Context & context, TableStructureWriteLockHolder & table_lock_holder) { lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); @@ -702,7 +702,7 @@ void StorageBuffer::alter(const AlterCommands & params, const String & database_ auto new_columns = getColumns(); auto new_indices = getIndices(); params.apply(new_columns); - context.getDatabase(database_name)->alterTable(context, table_name, new_columns, new_indices, {}); + context.getDatabase(database_name_)->alterTable(context, table_name_, new_columns, new_indices, {}); setColumns(std::move(new_columns)); } @@ -741,6 +741,7 @@ void registerStorageBuffer(StorageFactory & factory) UInt64 max_bytes = applyVisitor(FieldVisitorConvertToNumber(), engine_args[8]->as().value); return StorageBuffer::create( + args.database_name, args.table_name, args.columns, args.context, num_buckets, diff --git a/dbms/src/Storages/StorageBuffer.h b/dbms/src/Storages/StorageBuffer.h index 9e7127f8417..4c317a7a102 100644 --- a/dbms/src/Storages/StorageBuffer.h +++ b/dbms/src/Storages/StorageBuffer.h @@ -52,7 +52,8 @@ public: }; std::string getName() const override { return "Buffer"; } - std::string getTableName() const override { return name; } + std::string getTableName() const override { return table_name; } + std::string getDatabaseName() const override { return database_name; } QueryProcessingStage::Enum getQueryProcessingStage(const Context & context) const override; @@ -71,7 +72,7 @@ public: void shutdown() override; bool optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & context) override; - void rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & new_table_name) override { name = new_table_name; } + void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name) override { table_name = new_table_name; database_name = new_database_name; } bool supportsSampling() const override { return true; } bool supportsPrewhere() const override @@ -94,7 +95,8 @@ public: const Context & context, TableStructureWriteLockHolder & table_lock_holder) override; private: - String name; + String table_name; + String database_name; Context global_context; @@ -138,7 +140,7 @@ protected: /** num_shards - the level of internal parallelism (the number of independent buffers) * The buffer is flushed if all minimum thresholds or at least one of the maximum thresholds are exceeded. */ - StorageBuffer(const std::string & name_, const ColumnsDescription & columns_, + StorageBuffer(const std::string & database_name_, const std::string & table_name_, const ColumnsDescription & columns_, Context & context_, size_t num_shards_, const Thresholds & min_thresholds_, const Thresholds & max_thresholds_, const String & destination_database_, const String & destination_table_, bool allow_materialized_); diff --git a/dbms/src/Storages/StorageCatBoostPool.cpp b/dbms/src/Storages/StorageCatBoostPool.cpp index b76150611c4..a9e2acedcce 100644 --- a/dbms/src/Storages/StorageCatBoostPool.cpp +++ b/dbms/src/Storages/StorageCatBoostPool.cpp @@ -90,11 +90,16 @@ static void checkCreationIsAllowed(const String & base_path, const String & path } -StorageCatBoostPool::StorageCatBoostPool(const Context & context, - String column_description_file_name_, - String data_description_file_name_) - : column_description_file_name(std::move(column_description_file_name_)), - data_description_file_name(std::move(data_description_file_name_)) +StorageCatBoostPool::StorageCatBoostPool( + const String & database_name_, + const String & table_name_, + const Context & context, + String column_description_file_name_, + String data_description_file_name_) + : table_name(table_name_) + , database_name(database_name_) + , column_description_file_name(std::move(column_description_file_name_)) + , data_description_file_name(std::move(data_description_file_name_)) { auto base_path = canonicalPath(context.getPath()); column_description_file_name = resolvePath(base_path, std::move(column_description_file_name)); diff --git a/dbms/src/Storages/StorageCatBoostPool.h b/dbms/src/Storages/StorageCatBoostPool.h index 54a7bf1b655..0cc457fabc0 100644 --- a/dbms/src/Storages/StorageCatBoostPool.h +++ b/dbms/src/Storages/StorageCatBoostPool.h @@ -11,8 +11,8 @@ class StorageCatBoostPool : public ext::shared_ptr_helper, { public: std::string getName() const override { return "CatBoostPool"; } - std::string getTableName() const override { return table_name; } + std::string getDatabaseName() const override { return database_name; } BlockInputStreams read(const Names & column_names, const SelectQueryInfo & query_info, @@ -23,6 +23,7 @@ public: private: String table_name; + String database_name; String column_description_file_name; String data_description_file_name; @@ -75,7 +76,7 @@ private: void createSampleBlockAndColumns(); protected: - StorageCatBoostPool(const Context & context, String column_description_file_name, String data_description_file_name); + StorageCatBoostPool(const String & database_name_, const String & table_name_, const Context & context, String column_description_file_name, String data_description_file_name); }; } diff --git a/dbms/src/Storages/StorageDictionary.cpp b/dbms/src/Storages/StorageDictionary.cpp index ea9276766ba..923a309f1f6 100644 --- a/dbms/src/Storages/StorageDictionary.cpp +++ b/dbms/src/Storages/StorageDictionary.cpp @@ -24,12 +24,14 @@ namespace ErrorCodes StorageDictionary::StorageDictionary( + const String & database_name_, const String & table_name_, const ColumnsDescription & columns_, const Context & context, bool attach, const String & dictionary_name_) : IStorage{columns_}, table_name(table_name_), + database_name(database_name_), dictionary_name(dictionary_name_), logger(&Poco::Logger::get("StorageDictionary")) { @@ -104,7 +106,7 @@ void registerStorageDictionary(StorageFactory & factory) String dictionary_name = args.engine_args[0]->as().value.safeGet(); return StorageDictionary::create( - args.table_name, args.columns, args.context, args.attach, dictionary_name); + args.database_name, args.table_name, args.columns, args.context, args.attach, dictionary_name); }); } diff --git a/dbms/src/Storages/StorageDictionary.h b/dbms/src/Storages/StorageDictionary.h index 96798022ebf..cf5fd647e74 100644 --- a/dbms/src/Storages/StorageDictionary.h +++ b/dbms/src/Storages/StorageDictionary.h @@ -24,6 +24,8 @@ class StorageDictionary : public ext::shared_ptr_helper, publ public: std::string getName() const override { return "Dictionary"; } std::string getTableName() const override { return table_name; } + std::string getDatabaseName() const override { return database_name; } + BlockInputStreams read(const Names & column_names, const SelectQueryInfo & query_info, const Context & context, @@ -58,13 +60,16 @@ private: using Ptr = MultiVersion::Version; String table_name; + String database_name; String dictionary_name; Poco::Logger * logger; void checkNamesAndTypesCompatibleWithDictionary(const DictionaryStructure & dictionary_structure) const; protected: - StorageDictionary(const String & table_name_, + StorageDictionary( + const String & database_name_, + const String & table_name_, const ColumnsDescription & columns_, const Context & context, bool attach, diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index 826407e0599..27ceb1f45db 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -189,7 +189,7 @@ StorageDistributed::StorageDistributed( const ASTPtr & sharding_key_, const String & data_path_, bool attach) - : IStorage{columns_}, table_name(table_name), + : IStorage{columns_}, table_name(table_name), database_name(database_name), remote_database(remote_database_), remote_table(remote_table_), global_context(context_), cluster_name(global_context.getMacros()->expand(cluster_name_)), has_sharding_key(sharding_key_), sharding_key_expr(sharding_key_ ? buildShardingKeyExpression(sharding_key_, global_context, getColumns().getAllPhysical(), false) : nullptr), @@ -343,7 +343,7 @@ BlockOutputStreamPtr StorageDistributed::write(const ASTPtr &, const Context & c void StorageDistributed::alter( - const AlterCommands & params, const String & database_name, const String & current_table_name, + const AlterCommands & params, const String & current_database_name, const String & current_table_name, const Context & context, TableStructureWriteLockHolder & table_lock_holder) { lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); @@ -351,7 +351,7 @@ void StorageDistributed::alter( auto new_columns = getColumns(); auto new_indices = getIndices(); params.apply(new_columns); - context.getDatabase(database_name)->alterTable(context, current_table_name, new_columns, new_indices, {}); + context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, {}); setColumns(std::move(new_columns)); } diff --git a/dbms/src/Storages/StorageDistributed.h b/dbms/src/Storages/StorageDistributed.h index fee3ba78d8d..86fe80f575f 100644 --- a/dbms/src/Storages/StorageDistributed.h +++ b/dbms/src/Storages/StorageDistributed.h @@ -52,6 +52,8 @@ public: std::string getName() const override { return "Distributed"; } std::string getTableName() const override { return table_name; } + std::string getDatabaseName() const override { return database_name; } + bool supportsSampling() const override { return true; } bool supportsFinal() const override { return true; } bool supportsPrewhere() const override { return true; } @@ -79,7 +81,7 @@ public: /// Removes temporary data in local filesystem. void truncate(const ASTPtr &, const Context &) override; - void rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & new_table_name) override { table_name = new_table_name; } + void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name) override { table_name = new_table_name; database_name = new_database_name; } /// in the sub-tables, you need to manually add and delete columns /// the structure of the sub-table is not checked void alter( @@ -113,6 +115,7 @@ public: ActionLock getActionLock(StorageActionBlockType type) override; String table_name; + String database_name; String remote_database; String remote_table; ASTPtr remote_table_function_ptr; diff --git a/dbms/src/Storages/StorageFile.cpp b/dbms/src/Storages/StorageFile.cpp index 28c81c4902f..35bc1747dee 100644 --- a/dbms/src/Storages/StorageFile.cpp +++ b/dbms/src/Storages/StorageFile.cpp @@ -68,12 +68,13 @@ StorageFile::StorageFile( const std::string & table_path_, int table_fd_, const std::string & db_dir_path, + const std::string & database_name_, const std::string & table_name_, const std::string & format_name_, const ColumnsDescription & columns_, Context & context_) : IStorage(columns_), - table_name(table_name_), format_name(format_name_), context_global(context_), table_fd(table_fd_) + table_name(table_name_), database_name(database_name_), format_name(format_name_), context_global(context_), table_fd(table_fd_) { if (table_fd < 0) /// Will use file { @@ -265,7 +266,7 @@ void StorageFile::drop() } -void StorageFile::rename(const String & new_path_to_db, const String & /*new_database_name*/, const String & new_table_name) +void StorageFile::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) { if (!is_db_table) throw Exception("Can't rename table '" + table_name + "' binded to user-defined file (or FD)", ErrorCodes::DATABASE_ACCESS_DENIED); @@ -277,6 +278,8 @@ void StorageFile::rename(const String & new_path_to_db, const String & /*new_dat Poco::File(path).renameTo(path_new); path = std::move(path_new); + table_name = new_table_name; + database_name = new_database_name; } @@ -327,7 +330,7 @@ void registerStorageFile(StorageFactory & factory) return StorageFile::create( source_path, source_fd, args.data_path, - args.table_name, format_name, args.columns, + args.database_name, args.table_name, format_name, args.columns, args.context); }); } diff --git a/dbms/src/Storages/StorageFile.h b/dbms/src/Storages/StorageFile.h index eb74ad615a7..cc5878520ce 100644 --- a/dbms/src/Storages/StorageFile.h +++ b/dbms/src/Storages/StorageFile.h @@ -21,15 +21,9 @@ class StorageFileBlockOutputStream; class StorageFile : public ext::shared_ptr_helper, public IStorage { public: - std::string getName() const override - { - return "File"; - } - - std::string getTableName() const override - { - return table_name; - } + std::string getName() const override { return "File"; } + std::string getTableName() const override { return table_name; } + std::string getDatabaseName() const override { return database_name; } BlockInputStreams read( const Names & column_names, @@ -62,14 +56,15 @@ protected: const std::string & table_path_, int table_fd_, const std::string & db_dir_path, + const std::string & database_name_, const std::string & table_name_, const std::string & format_name_, const ColumnsDescription & columns_, Context & context_); private: - std::string table_name; + std::string database_name; std::string format_name; Context & context_global; diff --git a/dbms/src/Storages/StorageHDFS.cpp b/dbms/src/Storages/StorageHDFS.cpp index d7bd942527b..8c87f4ccd6a 100644 --- a/dbms/src/Storages/StorageHDFS.cpp +++ b/dbms/src/Storages/StorageHDFS.cpp @@ -26,6 +26,7 @@ namespace ErrorCodes } StorageHDFS::StorageHDFS(const String & uri_, + const std::string & database_name_, const std::string & table_name_, const String & format_name_, const ColumnsDescription & columns_, @@ -34,6 +35,7 @@ StorageHDFS::StorageHDFS(const String & uri_, , uri(uri_) , format_name(format_name_) , table_name(table_name_) + , database_name(database_name_) , context(context_) { } @@ -144,7 +146,11 @@ BlockInputStreams StorageHDFS::read( max_block_size)}; } -void StorageHDFS::rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & /*new_table_name*/) {} +void StorageHDFS::rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name) +{ + table_name = new_table_name; + database_name = new_database_name; +} BlockOutputStreamPtr StorageHDFS::write(const ASTPtr & /*query*/, const Context & /*context*/) { @@ -169,7 +175,7 @@ void registerStorageHDFS(StorageFactory & factory) String format_name = engine_args[1]->as().value.safeGet(); - return StorageHDFS::create(url, args.table_name, format_name, args.columns, args.context); + return StorageHDFS::create(url, args.database_name, args.table_name, format_name, args.columns, args.context); }); } diff --git a/dbms/src/Storages/StorageHDFS.h b/dbms/src/Storages/StorageHDFS.h index 73342cf3671..44f0286f97e 100644 --- a/dbms/src/Storages/StorageHDFS.h +++ b/dbms/src/Storages/StorageHDFS.h @@ -16,15 +16,9 @@ namespace DB class StorageHDFS : public ext::shared_ptr_helper, public IStorage { public: - String getName() const override - { - return "HDFS"; - } - - String getTableName() const override - { - return table_name; - } + String getName() const override { return "HDFS"; } + String getTableName() const override { return table_name; } + String getDatabaseName() const override { return database_name; } BlockInputStreams read(const Names & column_names, const SelectQueryInfo & query_info, @@ -39,6 +33,7 @@ public: protected: StorageHDFS(const String & uri_, + const String & database_name_, const String & table_name_, const String & format_name_, const ColumnsDescription & columns_, @@ -48,6 +43,7 @@ private: String uri; String format_name; String table_name; + String database_name; Context & context; Logger * log = &Logger::get("StorageHDFS"); diff --git a/dbms/src/Storages/StorageJoin.cpp b/dbms/src/Storages/StorageJoin.cpp index 443e9b958a7..8a38c6fa8ff 100644 --- a/dbms/src/Storages/StorageJoin.cpp +++ b/dbms/src/Storages/StorageJoin.cpp @@ -27,7 +27,8 @@ namespace ErrorCodes StorageJoin::StorageJoin( const String & path_, - const String & name_, + const String & database_name_, + const String & table_name_, const Names & key_names_, bool use_nulls_, SizeLimits limits_, @@ -35,7 +36,7 @@ StorageJoin::StorageJoin( ASTTableJoin::Strictness strictness_, const ColumnsDescription & columns_, bool overwrite) - : StorageSetOrJoinBase{path_, name_, columns_} + : StorageSetOrJoinBase{path_, database_name_, table_name_, columns_} , key_names(key_names_) , use_nulls(use_nulls_) , limits(limits_) @@ -160,6 +161,7 @@ void registerStorageJoin(StorageFactory & factory) return StorageJoin::create( args.data_path, + args.database_name, args.table_name, key_names, join_use_nulls.value, diff --git a/dbms/src/Storages/StorageJoin.h b/dbms/src/Storages/StorageJoin.h index 3f5341d2822..356f4b0e7d9 100644 --- a/dbms/src/Storages/StorageJoin.h +++ b/dbms/src/Storages/StorageJoin.h @@ -57,7 +57,8 @@ private: protected: StorageJoin( const String & path_, - const String & name_, + const String & database_name_, + const String & table_name_, const Names & key_names_, bool use_nulls_, SizeLimits limits_, diff --git a/dbms/src/Storages/StorageLog.cpp b/dbms/src/Storages/StorageLog.cpp index afef7af57e9..12e45ca96bc 100644 --- a/dbms/src/Storages/StorageLog.cpp +++ b/dbms/src/Storages/StorageLog.cpp @@ -211,7 +211,7 @@ Block LogBlockInputStream::readImpl() } catch (Exception & e) { - e.addMessage("while reading column " + name_type.name + " at " + storage.path + escapeForFileName(storage.name)); + e.addMessage("while reading column " + name_type.name + " at " + storage.path + escapeForFileName(storage.table_name)); throw; } @@ -419,24 +419,25 @@ void LogBlockOutputStream::writeMarks(MarksForColumns && marks) StorageLog::StorageLog( const std::string & path_, - const std::string & name_, + const std::string & database_name_, + const std::string & table_name_, const ColumnsDescription & columns_, size_t max_compress_block_size_) : IStorage{columns_}, - path(path_), name(name_), + path(path_), table_name(table_name_), database_name(database_name_), max_compress_block_size(max_compress_block_size_), - file_checker(path + escapeForFileName(name) + '/' + "sizes.json") + file_checker(path + escapeForFileName(table_name) + '/' + "sizes.json") { if (path.empty()) throw Exception("Storage " + getName() + " requires data path", ErrorCodes::INCORRECT_FILE_NAME); /// create files if they do not exist - Poco::File(path + escapeForFileName(name) + '/').createDirectories(); + Poco::File(path + escapeForFileName(table_name) + '/').createDirectories(); for (const auto & column : getColumns().getAllPhysical()) addFiles(column.name, *column.type); - marks_file = Poco::File(path + escapeForFileName(name) + '/' + DBMS_STORAGE_LOG_MARKS_FILE_NAME); + marks_file = Poco::File(path + escapeForFileName(table_name) + '/' + DBMS_STORAGE_LOG_MARKS_FILE_NAME); } @@ -455,7 +456,7 @@ void StorageLog::addFiles(const String & column_name, const IDataType & type) ColumnData & column_data = files[stream_name]; column_data.column_index = file_count; column_data.data_file = Poco::File{ - path + escapeForFileName(name) + '/' + stream_name + DBMS_STORAGE_LOG_DATA_FILE_EXTENSION}; + path + escapeForFileName(table_name) + '/' + stream_name + DBMS_STORAGE_LOG_DATA_FILE_EXTENSION}; column_names_by_idx.push_back(stream_name); ++file_count; @@ -508,28 +509,29 @@ void StorageLog::loadMarks() } -void StorageLog::rename(const String & new_path_to_db, const String & /*new_database_name*/, const String & new_table_name) +void StorageLog::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) { std::unique_lock lock(rwlock); /// Rename directory with data. - Poco::File(path + escapeForFileName(name)).renameTo(new_path_to_db + escapeForFileName(new_table_name)); + Poco::File(path + escapeForFileName(table_name)).renameTo(new_path_to_db + escapeForFileName(new_table_name)); path = new_path_to_db; - name = new_table_name; - file_checker.setPath(path + escapeForFileName(name) + '/' + "sizes.json"); + table_name = new_table_name; + database_name = new_database_name; + file_checker.setPath(path + escapeForFileName(table_name) + '/' + "sizes.json"); for (auto & file : files) - file.second.data_file = Poco::File(path + escapeForFileName(name) + '/' + Poco::Path(file.second.data_file.path()).getFileName()); + file.second.data_file = Poco::File(path + escapeForFileName(table_name) + '/' + Poco::Path(file.second.data_file.path()).getFileName()); - marks_file = Poco::File(path + escapeForFileName(name) + '/' + DBMS_STORAGE_LOG_MARKS_FILE_NAME); + marks_file = Poco::File(path + escapeForFileName(table_name) + '/' + DBMS_STORAGE_LOG_MARKS_FILE_NAME); } void StorageLog::truncate(const ASTPtr &, const Context &) { std::shared_lock lock(rwlock); - String table_dir = path + escapeForFileName(name); + String table_dir = path + escapeForFileName(table_name); files.clear(); file_count = 0; @@ -625,7 +627,7 @@ BlockOutputStreamPtr StorageLog::write( return std::make_shared(*this); } -bool StorageLog::checkData() const +CheckResults StorageLog::checkData(const ASTPtr & /* query */, const Context & /* context */) { std::shared_lock lock(rwlock); return file_checker.check(); @@ -642,7 +644,7 @@ void registerStorageLog(StorageFactory & factory) ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); return StorageLog::create( - args.data_path, args.table_name, args.columns, + args.data_path, args.database_name, args.table_name, args.columns, args.context.getSettings().max_compress_block_size); }); } diff --git a/dbms/src/Storages/StorageLog.h b/dbms/src/Storages/StorageLog.h index cf0d07a3bfe..f612d0ab29a 100644 --- a/dbms/src/Storages/StorageLog.h +++ b/dbms/src/Storages/StorageLog.h @@ -24,7 +24,8 @@ friend class LogBlockOutputStream; public: std::string getName() const override { return "Log"; } - std::string getTableName() const override { return name; } + std::string getTableName() const override { return table_name; } + std::string getDatabaseName() const override { return database_name; } BlockInputStreams read( const Names & column_names, @@ -38,11 +39,11 @@ public: void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override; - bool checkData() const override; + CheckResults checkData(const ASTPtr & /* query */, const Context & /* context */) override; void truncate(const ASTPtr &, const Context &) override; - std::string full_path() const { return path + escapeForFileName(name) + '/';} + std::string full_path() const { return path + escapeForFileName(table_name) + '/';} String getDataPath() const override { return full_path(); } @@ -53,13 +54,15 @@ protected: */ StorageLog( const std::string & path_, - const std::string & name_, + const std::string & database_name_, + const std::string & table_name_, const ColumnsDescription & columns_, size_t max_compress_block_size_); private: String path; - String name; + String table_name; + String database_name; mutable std::shared_mutex rwlock; @@ -119,7 +122,7 @@ private: */ const Marks & getMarksWithRealRowCount() const; - std::string getFullPath() const { return path + escapeForFileName(name) + '/'; } + std::string getFullPath() const { return path + escapeForFileName(table_name) + '/'; } }; } diff --git a/dbms/src/Storages/StorageMaterializedView.cpp b/dbms/src/Storages/StorageMaterializedView.cpp index c843312e3d7..8a33e8af030 100644 --- a/dbms/src/Storages/StorageMaterializedView.cpp +++ b/dbms/src/Storages/StorageMaterializedView.cpp @@ -297,7 +297,7 @@ static void executeRenameQuery(Context & global_context, const String & database } -void StorageMaterializedView::rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & new_table_name) +void StorageMaterializedView::rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name) { if (has_inner_table && tryGetTargetTable()) { @@ -313,6 +313,7 @@ void StorageMaterializedView::rename(const String & /*new_path_to_db*/, const St DatabaseAndTableName(database_name, table_name)); table_name = new_table_name; + database_name = new_database_name; global_context.addDependencyUnsafe( DatabaseAndTableName(select_database_name, select_table_name), diff --git a/dbms/src/Storages/StorageMaterializedView.h b/dbms/src/Storages/StorageMaterializedView.h index 5b14b90d77b..452ee7d51c0 100644 --- a/dbms/src/Storages/StorageMaterializedView.h +++ b/dbms/src/Storages/StorageMaterializedView.h @@ -14,6 +14,8 @@ class StorageMaterializedView : public ext::shared_ptr_helperclone(); } NameAndTypePair getColumn(const String & column_name) const override; diff --git a/dbms/src/Storages/StorageMemory.cpp b/dbms/src/Storages/StorageMemory.cpp index ddcf649a726..5e8f100f2f6 100644 --- a/dbms/src/Storages/StorageMemory.cpp +++ b/dbms/src/Storages/StorageMemory.cpp @@ -74,8 +74,8 @@ private: }; -StorageMemory::StorageMemory(String table_name_, ColumnsDescription columns_description_) - : IStorage{std::move(columns_description_)}, table_name(std::move(table_name_)) +StorageMemory::StorageMemory(String database_name_, String table_name_, ColumnsDescription columns_description_) + : IStorage{std::move(columns_description_)}, database_name(std::move(database_name_)), table_name(std::move(table_name_)) { } @@ -143,7 +143,7 @@ void registerStorageMemory(StorageFactory & factory) "Engine " + args.engine_name + " doesn't support any arguments (" + toString(args.engine_args.size()) + " given)", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - return StorageMemory::create(args.table_name, args.columns); + return StorageMemory::create(args.database_name, args.table_name, args.columns); }); } diff --git a/dbms/src/Storages/StorageMemory.h b/dbms/src/Storages/StorageMemory.h index 89947b50b8f..e4a0f0cf9d4 100644 --- a/dbms/src/Storages/StorageMemory.h +++ b/dbms/src/Storages/StorageMemory.h @@ -23,8 +23,9 @@ friend class MemoryBlockInputStream; friend class MemoryBlockOutputStream; public: - std::string getName() const override { return "Memory"; } - std::string getTableName() const override { return table_name; } + String getName() const override { return "Memory"; } + String getTableName() const override { return table_name; } + String getDatabaseName() const override { return database_name; } size_t getSize() const { return data.size(); } @@ -42,9 +43,10 @@ public: void truncate(const ASTPtr &, const Context &) override; - void rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & new_table_name) override { table_name = new_table_name; } + void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name) override { table_name = new_table_name; database_name = new_database_name; } private: + String database_name; String table_name; /// The data itself. `list` - so that when inserted to the end, the existing iterators are not invalidated. @@ -53,7 +55,7 @@ private: std::mutex mutex; protected: - StorageMemory(String table_name_, ColumnsDescription columns_description_); + StorageMemory(String database_name_, String table_name_, ColumnsDescription columns_description_); }; } diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp index 713ca9b7be9..4c029fab677 100644 --- a/dbms/src/Storages/StorageMerge.cpp +++ b/dbms/src/Storages/StorageMerge.cpp @@ -46,13 +46,15 @@ namespace ErrorCodes StorageMerge::StorageMerge( - const std::string & name_, + const std::string & database_name_, + const std::string & table_name_, const ColumnsDescription & columns_, const String & source_database_, const String & table_name_regexp_, const Context & context_) : IStorage(columns_, ColumnsDescription({{"_table", std::make_shared()}}, true)) - , name(name_) + , table_name(table_name_) + , database_name(database_name_) , source_database(source_database_) , table_name_regexp(table_name_regexp_) , global_context(context_) @@ -165,7 +167,7 @@ BlockInputStreams StorageMerge::read( const Context & context, QueryProcessingStage::Enum processed_stage, const size_t max_block_size, - const unsigned num_streams) + unsigned num_streams) { BlockInputStreams res; @@ -201,8 +203,10 @@ BlockInputStreams StorageMerge::read( return createSourceStreams( query_info, processed_stage, max_block_size, header, {}, {}, real_column_names, modified_context, 0, has_table_virtual_column); - size_t remaining_streams = num_streams; size_t tables_count = selected_tables.size(); + Float64 num_streams_multiplier = std::min(unsigned(tables_count), std::max(1U, unsigned(context.getSettingsRef().max_streams_multiplier_for_merge_tables))); + num_streams *= num_streams_multiplier; + size_t remaining_streams = num_streams; for (auto it = selected_tables.begin(); it != selected_tables.end(); ++it) { @@ -384,13 +388,13 @@ StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables(const ASTPtr DatabaseIteratorPtr StorageMerge::getDatabaseIterator(const Context & context) const { auto database = context.getDatabase(source_database); - auto table_name_match = [this](const String & table_name) { return table_name_regexp.match(table_name); }; + auto table_name_match = [this](const String & table_name_) { return table_name_regexp.match(table_name_); }; return database->getIterator(global_context, table_name_match); } void StorageMerge::alter( - const AlterCommands & params, const String & database_name, const String & table_name, + const AlterCommands & params, const String & database_name_, const String & table_name_, const Context & context, TableStructureWriteLockHolder & table_lock_holder) { lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); @@ -398,7 +402,7 @@ void StorageMerge::alter( auto new_columns = getColumns(); auto new_indices = getIndices(); params.apply(new_columns); - context.getDatabase(database_name)->alterTable(context, table_name, new_columns, new_indices, {}); + context.getDatabase(database_name_)->alterTable(context, table_name_, new_columns, new_indices, {}); setColumns(new_columns); } @@ -489,7 +493,7 @@ void registerStorageMerge(StorageFactory & factory) String table_name_regexp = engine_args[1]->as().value.safeGet(); return StorageMerge::create( - args.table_name, args.columns, + args.database_name, args.table_name, args.columns, source_database, table_name_regexp, args.context); }); } diff --git a/dbms/src/Storages/StorageMerge.h b/dbms/src/Storages/StorageMerge.h index e51f89e93f8..6d02ad029cc 100644 --- a/dbms/src/Storages/StorageMerge.h +++ b/dbms/src/Storages/StorageMerge.h @@ -16,7 +16,8 @@ class StorageMerge : public ext::shared_ptr_helper, public IStorag { public: std::string getName() const override { return "Merge"; } - std::string getTableName() const override { return name; } + std::string getTableName() const override { return table_name; } + std::string getDatabaseName() const override { return database_name; } bool isRemote() const override; @@ -41,7 +42,7 @@ public: unsigned num_streams) override; void drop() override {} - void rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & new_table_name) override { name = new_table_name; } + void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name) override { table_name = new_table_name; database_name = new_database_name; } /// you need to add and remove columns in the sub-tables manually /// the structure of sub-tables is not checked @@ -52,7 +53,8 @@ public: bool mayBenefitFromIndexForIn(const ASTPtr & left_in_operand, const Context & query_context) const override; private: - String name; + String table_name; + String database_name; String source_database; OptimizedRegularExpression table_name_regexp; Context global_context; @@ -70,7 +72,8 @@ private: protected: StorageMerge( - const std::string & name_, + const std::string & database_name_, + const std::string & table_name_, const ColumnsDescription & columns_, const String & source_database_, const String & table_name_regexp_, diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index df936c167d3..d021866487c 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -7,8 +7,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -17,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -174,7 +177,7 @@ void StorageMergeTree::truncate(const ASTPtr &, const Context &) clearOldPartsFromFilesystem(); } -void StorageMergeTree::rename(const String & new_path_to_db, const String & /*new_database_name*/, const String & new_table_name) +void StorageMergeTree::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) { std::string new_full_path = new_path_to_db + escapeForFileName(new_table_name) + '/'; @@ -182,6 +185,7 @@ void StorageMergeTree::rename(const String & new_path_to_db, const String & /*ne path = new_path_to_db; table_name = new_table_name; + database_name = new_database_name; full_path = new_full_path; /// NOTE: Logger names are not updated. @@ -1121,4 +1125,59 @@ ActionLock StorageMergeTree::getActionLock(StorageActionBlockType action_type) return {}; } +CheckResults StorageMergeTree::checkData(const ASTPtr & query, const Context & context) +{ + CheckResults results; + DataPartsVector data_parts; + if (const auto & check_query = query->as(); check_query.partition) + { + String partition_id = getPartitionIDFromQuery(check_query.partition, context); + data_parts = getDataPartsVectorInPartition(MergeTreeDataPartState::Committed, partition_id); + } + else + data_parts = getDataPartsVector(); + + for (auto & part : data_parts) + { + String full_part_path = part->getFullPath(); + /// If the checksums file is not present, calculate the checksums and write them to disk. + String checksums_path = full_part_path + "checksums.txt"; + String tmp_checksums_path = full_part_path + "checksums.txt.tmp"; + if (!Poco::File(checksums_path).exists()) + { + try + { + auto calculated_checksums = checkDataPart(part, false, primary_key_data_types, skip_indices); + calculated_checksums.checkEqual(part->checksums, true); + WriteBufferFromFile out(tmp_checksums_path, 4096); + part->checksums.write(out); + Poco::File(tmp_checksums_path).renameTo(checksums_path); + results.emplace_back(part->name, true, "Checksums recounted and written to disk."); + } + catch (const Exception & ex) + { + Poco::File tmp_file(tmp_checksums_path); + if (tmp_file.exists()) + tmp_file.remove(); + + results.emplace_back(part->name, false, + "Check of part finished with error: '" + ex.message() + "'"); + } + } + else + { + try + { + checkDataPart(part, true, primary_key_data_types, skip_indices); + results.emplace_back(part->name, true, ""); + } + catch (const Exception & ex) + { + results.emplace_back(part->name, false, ex.message()); + } + } + } + return results; +} + } diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index b5156ce7137..0de9618d915 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -70,6 +70,8 @@ public: String getDataPath() const override { return full_path; } + CheckResults checkData(const ASTPtr & query, const Context & context) override; + private: String path; diff --git a/dbms/src/Storages/StorageMySQL.cpp b/dbms/src/Storages/StorageMySQL.cpp index 37eaa5e7cd2..25a06a8bd4e 100644 --- a/dbms/src/Storages/StorageMySQL.cpp +++ b/dbms/src/Storages/StorageMySQL.cpp @@ -36,7 +36,9 @@ String backQuoteMySQL(const String & x) return res; } -StorageMySQL::StorageMySQL(const std::string & name, +StorageMySQL::StorageMySQL( + const std::string & database_name_, + const std::string & table_name_, mysqlxx::Pool && pool, const std::string & remote_database_name, const std::string & remote_table_name, @@ -45,7 +47,8 @@ StorageMySQL::StorageMySQL(const std::string & name, const ColumnsDescription & columns_, const Context & context) : IStorage{columns_} - , name(name) + , table_name(table_name_) + , database_name(database_name_) , remote_database_name(remote_database_name) , remote_table_name(remote_table_name) , replace_query{replace_query} @@ -230,6 +233,7 @@ void registerStorageMySQL(StorageFactory & factory) ErrorCodes::BAD_ARGUMENTS); return StorageMySQL::create( + args.database_name, args.table_name, std::move(pool), remote_database, diff --git a/dbms/src/Storages/StorageMySQL.h b/dbms/src/Storages/StorageMySQL.h index 312fca75c9e..f28bb2aaa55 100644 --- a/dbms/src/Storages/StorageMySQL.h +++ b/dbms/src/Storages/StorageMySQL.h @@ -20,7 +20,8 @@ class StorageMySQL : public ext::shared_ptr_helper, public IStorag { public: StorageMySQL( - const std::string & name, + const std::string & database_name_, + const std::string & table_name_, mysqlxx::Pool && pool, const std::string & remote_database_name, const std::string & remote_table_name, @@ -30,7 +31,8 @@ public: const Context & context); std::string getName() const override { return "MySQL"; } - std::string getTableName() const override { return name; } + std::string getTableName() const override { return table_name; } + std::string getDatabaseName() const override { return database_name; } BlockInputStreams read( const Names & column_names, @@ -44,7 +46,8 @@ public: private: friend class StorageMySQLBlockOutputStream; - std::string name; + std::string table_name; + std::string database_name; std::string remote_database_name; std::string remote_table_name; diff --git a/dbms/src/Storages/StorageNull.cpp b/dbms/src/Storages/StorageNull.cpp index 1762c8372f5..f6de6e87e37 100644 --- a/dbms/src/Storages/StorageNull.cpp +++ b/dbms/src/Storages/StorageNull.cpp @@ -26,7 +26,7 @@ void registerStorageNull(StorageFactory & factory) "Engine " + args.engine_name + " doesn't support any arguments (" + toString(args.engine_args.size()) + " given)", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - return StorageNull::create(args.table_name, args.columns); + return StorageNull::create(args.database_name, args.table_name, args.columns); }); } diff --git a/dbms/src/Storages/StorageNull.h b/dbms/src/Storages/StorageNull.h index 14ce1140680..fec7638706f 100644 --- a/dbms/src/Storages/StorageNull.h +++ b/dbms/src/Storages/StorageNull.h @@ -19,6 +19,7 @@ class StorageNull : public ext::shared_ptr_helper, public IStorage public: std::string getName() const override { return "Null"; } std::string getTableName() const override { return table_name; } + std::string getDatabaseName() const override { return database_name; } BlockInputStreams read( const Names & column_names, @@ -36,9 +37,10 @@ public: return std::make_shared(getSampleBlock()); } - void rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & new_table_name) override + void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name) override { table_name = new_table_name; + database_name = new_database_name; } void alter( @@ -47,10 +49,11 @@ public: private: String table_name; + String database_name; protected: - StorageNull(String table_name_, ColumnsDescription columns_description_) - : IStorage{std::move(columns_description_)}, table_name(std::move(table_name_)) + StorageNull(String database_name_, String table_name_, ColumnsDescription columns_description_) + : IStorage{std::move(columns_description_)}, table_name(std::move(table_name_)), database_name(std::move(database_name_)) { } }; diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index b02f22b987e..b51da168192 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include @@ -2380,7 +2381,7 @@ void StorageReplicatedMergeTree::removePartAndEnqueueFetch(const String & part_n auto results = zookeeper->multi(ops); - String path_created = dynamic_cast(*results[0]).path_created; + String path_created = dynamic_cast(*results.back()).path_created; log_entry->znode_name = path_created.substr(path_created.find_last_of('/') + 1); queue.insert(zookeeper, log_entry); } @@ -5107,4 +5108,31 @@ bool StorageReplicatedMergeTree::dropPartsInPartition( return true; } + +CheckResults StorageReplicatedMergeTree::checkData(const ASTPtr & query, const Context & context) +{ + CheckResults results; + DataPartsVector data_parts; + if (const auto & check_query = query->as(); check_query.partition) + { + String partition_id = getPartitionIDFromQuery(check_query.partition, context); + data_parts = getDataPartsVectorInPartition(MergeTreeDataPartState::Committed, partition_id); + } + else + data_parts = getDataPartsVector(); + + for (auto & part : data_parts) + { + try + { + results.push_back(part_check_thread.checkPart(part->name)); + } + catch (const Exception & ex) + { + results.emplace_back(part->name, false, "Check of part finished with error: '" + ex.message() + "'"); + } + } + return results; +} + } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index eba0511e15e..59afb96a523 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -170,6 +170,8 @@ public: String getDataPath() const override { return full_path; } + CheckResults checkData(const ASTPtr & query, const Context & context) override; + private: /// Delete old parts from disk and from ZooKeeper. void clearOldPartsAndRemoveFromZK(); diff --git a/dbms/src/Storages/StorageSet.cpp b/dbms/src/Storages/StorageSet.cpp index f1f4f039c8c..0d649728096 100644 --- a/dbms/src/Storages/StorageSet.cpp +++ b/dbms/src/Storages/StorageSet.cpp @@ -88,9 +88,10 @@ BlockOutputStreamPtr StorageSetOrJoinBase::write(const ASTPtr & /*query*/, const StorageSetOrJoinBase::StorageSetOrJoinBase( const String & path_, + const String & database_name_, const String & table_name_, const ColumnsDescription & columns_) - : IStorage{columns_}, table_name(table_name_) + : IStorage{columns_}, table_name(table_name_), database_name(database_name_) { if (path_.empty()) throw Exception("Join and Set storages require data path", ErrorCodes::INCORRECT_FILE_NAME); @@ -102,9 +103,10 @@ StorageSetOrJoinBase::StorageSetOrJoinBase( StorageSet::StorageSet( const String & path_, - const String & name_, + const String & database_name_, + const String & table_name_, const ColumnsDescription & columns_) - : StorageSetOrJoinBase{path_, name_, columns_}, + : StorageSetOrJoinBase{path_, database_name_, table_name_, columns_}, set(std::make_shared(SizeLimits(), false)) { Block header = getSampleBlock(); @@ -186,7 +188,7 @@ void StorageSetOrJoinBase::restoreFromFile(const String & file_path) } -void StorageSetOrJoinBase::rename(const String & new_path_to_db, const String & /*new_database_name*/, const String & new_table_name) +void StorageSetOrJoinBase::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) { /// Rename directory with data. String new_path = new_path_to_db + escapeForFileName(new_table_name); @@ -194,6 +196,7 @@ void StorageSetOrJoinBase::rename(const String & new_path_to_db, const String & path = new_path + "/"; table_name = new_table_name; + database_name = new_database_name; } @@ -206,7 +209,7 @@ void registerStorageSet(StorageFactory & factory) "Engine " + args.engine_name + " doesn't support any arguments (" + toString(args.engine_args.size()) + " given)", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - return StorageSet::create(args.data_path, args.table_name, args.columns); + return StorageSet::create(args.data_path, args.database_name, args.table_name, args.columns); }); } diff --git a/dbms/src/Storages/StorageSet.h b/dbms/src/Storages/StorageSet.h index 0585dc271c6..1c2891b78e1 100644 --- a/dbms/src/Storages/StorageSet.h +++ b/dbms/src/Storages/StorageSet.h @@ -20,6 +20,7 @@ class StorageSetOrJoinBase : public IStorage public: String getTableName() const override { return table_name; } + String getDatabaseName() const override { return database_name; } void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override; @@ -30,11 +31,13 @@ public: protected: StorageSetOrJoinBase( const String & path_, + const String & database_name_, const String & table_name_, const ColumnsDescription & columns_); String path; String table_name; + String database_name; std::atomic increment = 0; /// For the backup file names. @@ -76,7 +79,8 @@ private: protected: StorageSet( const String & path_, - const String & name_, + const String & database_name_, + const String & table_name_, const ColumnsDescription & columns_); }; diff --git a/dbms/src/Storages/StorageStripeLog.cpp b/dbms/src/Storages/StorageStripeLog.cpp index dba2e64a88f..3052962606d 100644 --- a/dbms/src/Storages/StorageStripeLog.cpp +++ b/dbms/src/Storages/StorageStripeLog.cpp @@ -195,20 +195,21 @@ private: StorageStripeLog::StorageStripeLog( const std::string & path_, - const std::string & name_, + const std::string & database_name_, + const std::string & table_name_, const ColumnsDescription & columns_, bool attach, size_t max_compress_block_size_) : IStorage{columns_}, - path(path_), name(name_), + path(path_), table_name(table_name_), database_name(database_name_), max_compress_block_size(max_compress_block_size_), - file_checker(path + escapeForFileName(name) + '/' + "sizes.json"), + file_checker(path + escapeForFileName(table_name) + '/' + "sizes.json"), log(&Logger::get("StorageStripeLog")) { if (path.empty()) throw Exception("Storage " + getName() + " requires data path", ErrorCodes::INCORRECT_FILE_NAME); - String full_path = path + escapeForFileName(name) + '/'; + String full_path = path + escapeForFileName(table_name) + '/'; if (!attach) { /// create files if they do not exist @@ -218,16 +219,17 @@ StorageStripeLog::StorageStripeLog( } -void StorageStripeLog::rename(const String & new_path_to_db, const String & /*new_database_name*/, const String & new_table_name) +void StorageStripeLog::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) { std::unique_lock lock(rwlock); /// Rename directory with data. - Poco::File(path + escapeForFileName(name)).renameTo(new_path_to_db + escapeForFileName(new_table_name)); + Poco::File(path + escapeForFileName(table_name)).renameTo(new_path_to_db + escapeForFileName(new_table_name)); path = new_path_to_db; - name = new_table_name; - file_checker.setPath(path + escapeForFileName(name) + "/" + "sizes.json"); + table_name = new_table_name; + database_name = new_database_name; + file_checker.setPath(path + escapeForFileName(table_name) + "/" + "sizes.json"); } @@ -282,7 +284,7 @@ BlockOutputStreamPtr StorageStripeLog::write( } -bool StorageStripeLog::checkData() const +CheckResults StorageStripeLog::checkData(const ASTPtr & /* query */, const Context & /* context */) { std::shared_lock lock(rwlock); return file_checker.check(); @@ -290,16 +292,16 @@ bool StorageStripeLog::checkData() const void StorageStripeLog::truncate(const ASTPtr &, const Context &) { - if (name.empty()) + if (table_name.empty()) throw Exception("Logical error: table name is empty", ErrorCodes::LOGICAL_ERROR); std::shared_lock lock(rwlock); - auto file = Poco::File(path + escapeForFileName(name)); + auto file = Poco::File(path + escapeForFileName(table_name)); file.remove(true); file.createDirectories(); - file_checker = FileChecker{path + escapeForFileName(name) + '/' + "sizes.json"}; + file_checker = FileChecker{path + escapeForFileName(table_name) + '/' + "sizes.json"}; } @@ -313,7 +315,7 @@ void registerStorageStripeLog(StorageFactory & factory) ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); return StorageStripeLog::create( - args.data_path, args.table_name, args.columns, + args.data_path, args.database_name, args.table_name, args.columns, args.attach, args.context.getSettings().max_compress_block_size); }); } diff --git a/dbms/src/Storages/StorageStripeLog.h b/dbms/src/Storages/StorageStripeLog.h index 6489c82873e..6d528353dbf 100644 --- a/dbms/src/Storages/StorageStripeLog.h +++ b/dbms/src/Storages/StorageStripeLog.h @@ -26,7 +26,8 @@ friend class StripeLogBlockOutputStream; public: std::string getName() const override { return "StripeLog"; } - std::string getTableName() const override { return name; } + std::string getTableName() const override { return table_name; } + std::string getDatabaseName() const override { return database_name; } BlockInputStreams read( const Names & column_names, @@ -40,7 +41,7 @@ public: void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override; - bool checkData() const override; + CheckResults checkData(const ASTPtr & /* query */, const Context & /* context */) override; /// Data of the file. struct ColumnData @@ -49,7 +50,7 @@ public: }; using Files_t = std::map; - std::string full_path() const { return path + escapeForFileName(name) + '/';} + std::string full_path() const { return path + escapeForFileName(table_name) + '/';} String getDataPath() const override { return full_path(); } @@ -57,7 +58,8 @@ public: private: String path; - String name; + String table_name; + String database_name; size_t max_compress_block_size; @@ -69,7 +71,8 @@ private: protected: StorageStripeLog( const std::string & path_, - const std::string & name_, + const std::string & database_name_, + const std::string & table_name_, const ColumnsDescription & columns_, bool attach, size_t max_compress_block_size_); diff --git a/dbms/src/Storages/StorageTinyLog.cpp b/dbms/src/Storages/StorageTinyLog.cpp index 4690ab925e8..24a75496a24 100644 --- a/dbms/src/Storages/StorageTinyLog.cpp +++ b/dbms/src/Storages/StorageTinyLog.cpp @@ -32,6 +32,7 @@ #include #include +#include #include @@ -320,20 +321,21 @@ void TinyLogBlockOutputStream::write(const Block & block) StorageTinyLog::StorageTinyLog( const std::string & path_, - const std::string & name_, + const std::string & database_name_, + const std::string & table_name_, const ColumnsDescription & columns_, bool attach, size_t max_compress_block_size_) : IStorage{columns_}, - path(path_), name(name_), + path(path_), table_name(table_name_), database_name(database_name_), max_compress_block_size(max_compress_block_size_), - file_checker(path + escapeForFileName(name) + '/' + "sizes.json"), + file_checker(path + escapeForFileName(table_name) + '/' + "sizes.json"), log(&Logger::get("StorageTinyLog")) { if (path.empty()) throw Exception("Storage " + getName() + " requires data path", ErrorCodes::INCORRECT_FILE_NAME); - String full_path = path + escapeForFileName(name) + '/'; + String full_path = path + escapeForFileName(table_name) + '/'; if (!attach) { /// create files if they do not exist @@ -360,7 +362,7 @@ void StorageTinyLog::addFiles(const String & column_name, const IDataType & type ColumnData column_data; files.insert(std::make_pair(stream_name, column_data)); files[stream_name].data_file = Poco::File( - path + escapeForFileName(name) + '/' + stream_name + DBMS_STORAGE_LOG_DATA_FILE_EXTENSION); + path + escapeForFileName(table_name) + '/' + stream_name + DBMS_STORAGE_LOG_DATA_FILE_EXTENSION); } }; @@ -369,17 +371,18 @@ void StorageTinyLog::addFiles(const String & column_name, const IDataType & type } -void StorageTinyLog::rename(const String & new_path_to_db, const String & /*new_database_name*/, const String & new_table_name) +void StorageTinyLog::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) { /// Rename directory with data. - Poco::File(path + escapeForFileName(name)).renameTo(new_path_to_db + escapeForFileName(new_table_name)); + Poco::File(path + escapeForFileName(table_name)).renameTo(new_path_to_db + escapeForFileName(new_table_name)); path = new_path_to_db; - name = new_table_name; - file_checker.setPath(path + escapeForFileName(name) + "/" + "sizes.json"); + table_name = new_table_name; + database_name = new_database_name; + file_checker.setPath(path + escapeForFileName(table_name) + "/" + "sizes.json"); for (Files_t::iterator it = files.begin(); it != files.end(); ++it) - it->second.data_file = Poco::File(path + escapeForFileName(name) + '/' + Poco::Path(it->second.data_file.path()).getFileName()); + it->second.data_file = Poco::File(path + escapeForFileName(table_name) + '/' + Poco::Path(it->second.data_file.path()).getFileName()); } @@ -404,22 +407,22 @@ BlockOutputStreamPtr StorageTinyLog::write( } -bool StorageTinyLog::checkData() const +CheckResults StorageTinyLog::checkData(const ASTPtr & /* query */, const Context & /* context */) { return file_checker.check(); } void StorageTinyLog::truncate(const ASTPtr &, const Context &) { - if (name.empty()) + if (table_name.empty()) throw Exception("Logical error: table name is empty", ErrorCodes::LOGICAL_ERROR); - auto file = Poco::File(path + escapeForFileName(name)); + auto file = Poco::File(path + escapeForFileName(table_name)); file.remove(true); file.createDirectories(); files.clear(); - file_checker = FileChecker{path + escapeForFileName(name) + '/' + "sizes.json"}; + file_checker = FileChecker{path + escapeForFileName(table_name) + '/' + "sizes.json"}; for (const auto &column : getColumns().getAllPhysical()) addFiles(column.name, *column.type); @@ -436,7 +439,7 @@ void registerStorageTinyLog(StorageFactory & factory) ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); return StorageTinyLog::create( - args.data_path, args.table_name, args.columns, + args.data_path, args.database_name, args.table_name, args.columns, args.attach, args.context.getSettings().max_compress_block_size); }); } diff --git a/dbms/src/Storages/StorageTinyLog.h b/dbms/src/Storages/StorageTinyLog.h index 5b8e4bc90ac..7a360af632e 100644 --- a/dbms/src/Storages/StorageTinyLog.h +++ b/dbms/src/Storages/StorageTinyLog.h @@ -25,7 +25,8 @@ friend class TinyLogBlockOutputStream; public: std::string getName() const override { return "TinyLog"; } - std::string getTableName() const override { return name; } + std::string getTableName() const override { return table_name; } + std::string getDatabaseName() const override { return database_name; } BlockInputStreams read( const Names & column_names, @@ -39,7 +40,7 @@ public: void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override; - bool checkData() const override; + CheckResults checkData(const ASTPtr & /* query */, const Context & /* context */) override; /// Column data struct ColumnData @@ -48,7 +49,7 @@ public: }; using Files_t = std::map; - std::string full_path() const { return path + escapeForFileName(name) + '/';} + std::string full_path() const { return path + escapeForFileName(table_name) + '/';} String getDataPath() const override { return full_path(); } @@ -56,7 +57,8 @@ public: private: String path; - String name; + String table_name; + String database_name; size_t max_compress_block_size; @@ -72,7 +74,8 @@ private: protected: StorageTinyLog( const std::string & path_, - const std::string & name_, + const std::string & database_name_, + const std::string & table_name_, const ColumnsDescription & columns_, bool attach, size_t max_compress_block_size_); diff --git a/dbms/src/Storages/StorageURL.cpp b/dbms/src/Storages/StorageURL.cpp index 7ebae3f66db..8a36c008be9 100644 --- a/dbms/src/Storages/StorageURL.cpp +++ b/dbms/src/Storages/StorageURL.cpp @@ -26,10 +26,11 @@ namespace ErrorCodes IStorageURLBase::IStorageURLBase(const Poco::URI & uri_, const Context & context_, + const std::string & database_name_, const std::string & table_name_, const String & format_name_, const ColumnsDescription & columns_) - : IStorage(columns_), uri(uri_), context_global(context_), format_name(format_name_), table_name(table_name_) + : IStorage(columns_), uri(uri_), context_global(context_), format_name(format_name_), table_name(table_name_), database_name(database_name_) { } @@ -182,7 +183,11 @@ BlockInputStreams IStorageURLBase::read(const Names & column_names, return {std::make_shared(block_input, column_defaults, context)}; } -void IStorageURLBase::rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & /*new_table_name*/) {} +void IStorageURLBase::rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name) +{ + table_name = new_table_name; + database_name = new_database_name; +} BlockOutputStreamPtr IStorageURLBase::write(const ASTPtr & /*query*/, const Context & /*context*/) { @@ -209,7 +214,7 @@ void registerStorageURL(StorageFactory & factory) String format_name = engine_args[1]->as().value.safeGet(); - return StorageURL::create(uri, args.table_name, format_name, args.columns, args.context); + return StorageURL::create(uri, args.database_name, args.table_name, format_name, args.columns, args.context); }); } } diff --git a/dbms/src/Storages/StorageURL.h b/dbms/src/Storages/StorageURL.h index b84898c696d..361b0f6c35e 100644 --- a/dbms/src/Storages/StorageURL.h +++ b/dbms/src/Storages/StorageURL.h @@ -16,10 +16,8 @@ namespace DB class IStorageURLBase : public IStorage { public: - String getTableName() const override - { - return table_name; - } + String getTableName() const override { return table_name; } + String getDatabaseName() const override { return database_name; } BlockInputStreams read(const Names & column_names, const SelectQueryInfo & query_info, @@ -35,6 +33,7 @@ public: protected: IStorageURLBase(const Poco::URI & uri_, const Context & context_, + const std::string & database_name_, const std::string & table_name_, const String & format_name_, const ColumnsDescription & columns_); @@ -45,6 +44,7 @@ protected: private: String format_name; String table_name; + String database_name; virtual std::string getReadMethod() const; @@ -67,11 +67,12 @@ class StorageURL : public ext::shared_ptr_helper, public IStorageURL { public: StorageURL(const Poco::URI & uri_, + const std::string & database_name_, const std::string & table_name_, const String & format_name_, const ColumnsDescription & columns_, Context & context_) - : IStorageURLBase(uri_, context_, table_name_, format_name_, columns_) + : IStorageURLBase(uri_, context_, database_name_, table_name_, format_name_, columns_) { } diff --git a/dbms/src/Storages/StorageView.cpp b/dbms/src/Storages/StorageView.cpp index 09d54f40938..9d4855b586a 100644 --- a/dbms/src/Storages/StorageView.cpp +++ b/dbms/src/Storages/StorageView.cpp @@ -27,10 +27,11 @@ namespace ErrorCodes StorageView::StorageView( + const String & database_name_, const String & table_name_, const ASTCreateQuery & query, const ColumnsDescription & columns_) - : IStorage{columns_}, table_name(table_name_) + : IStorage{columns_}, table_name(table_name_), database_name(database_name_) { if (!query.select) throw Exception("SELECT query is not specified for " + getName(), ErrorCodes::INCORRECT_QUERY); @@ -101,7 +102,7 @@ void registerStorageView(StorageFactory & factory) if (args.query.storage) throw Exception("Specifying ENGINE is not allowed for a View", ErrorCodes::INCORRECT_QUERY); - return StorageView::create(args.table_name, args.query, args.columns); + return StorageView::create(args.database_name, args.table_name, args.query, args.columns); }); } diff --git a/dbms/src/Storages/StorageView.h b/dbms/src/Storages/StorageView.h index afd9b5ce326..cda128027c2 100644 --- a/dbms/src/Storages/StorageView.h +++ b/dbms/src/Storages/StorageView.h @@ -15,6 +15,7 @@ class StorageView : public ext::shared_ptr_helper, public IStorage public: std::string getName() const override { return "View"; } std::string getTableName() const override { return table_name; } + std::string getDatabaseName() const override { return database_name; } /// It is passed inside the query and solved at its level. bool supportsSampling() const override { return true; } @@ -28,19 +29,22 @@ public: size_t max_block_size, unsigned num_streams) override; - void rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & new_table_name) override + void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name) override { table_name = new_table_name; + database_name = new_database_name; } private: String table_name; + String database_name; ASTPtr inner_query; void replaceTableNameWithSubquery(ASTSelectQuery * select_query, ASTPtr & subquery); protected: StorageView( + const String & database_name_, const String & table_name_, const ASTCreateQuery & query, const ColumnsDescription & columns_); diff --git a/dbms/src/Storages/StorageXDBC.cpp b/dbms/src/Storages/StorageXDBC.cpp index 6e57f8a4286..57327fc4dcd 100644 --- a/dbms/src/Storages/StorageXDBC.cpp +++ b/dbms/src/Storages/StorageXDBC.cpp @@ -22,13 +22,15 @@ namespace ErrorCodes } -StorageXDBC::StorageXDBC(const std::string & table_name_, +StorageXDBC::StorageXDBC( + const std::string & database_name_, + const std::string & table_name_, const std::string & remote_database_name_, const std::string & remote_table_name_, const ColumnsDescription & columns_, const Context & context_, const BridgeHelperPtr bridge_helper_) - : IStorageURLBase(Poco::URI(), context_, table_name_, IXDBCBridgeHelper::DEFAULT_FORMAT, columns_) + : IStorageURLBase(Poco::URI(), context_, database_name_, table_name_, IXDBCBridgeHelper::DEFAULT_FORMAT, columns_) , bridge_helper(bridge_helper_) , remote_database_name(remote_database_name_) , remote_table_name(remote_table_name_) @@ -116,7 +118,7 @@ namespace BridgeHelperPtr bridge_helper = std::make_shared>(args.context, args.context.getSettingsRef().http_receive_timeout.value, engine_args[0]->as().value.safeGet()); - return std::make_shared(args.table_name, + return std::make_shared(args.database_name, args.table_name, engine_args[1]->as().value.safeGet(), engine_args[2]->as().value.safeGet(), args.columns, diff --git a/dbms/src/Storages/StorageXDBC.h b/dbms/src/Storages/StorageXDBC.h index 749e8910a24..23e108519b4 100644 --- a/dbms/src/Storages/StorageXDBC.h +++ b/dbms/src/Storages/StorageXDBC.h @@ -23,7 +23,8 @@ namespace DB unsigned num_streams) override; - StorageXDBC(const std::string & table_name_, + StorageXDBC(const std::string & database_name_, + const std::string & table_name_, const std::string & remote_database_name, const std::string & remote_table_name, const ColumnsDescription & columns_, diff --git a/dbms/src/Storages/System/CMakeLists.txt b/dbms/src/Storages/System/CMakeLists.txt index 8edfad534ec..18c452caf7b 100644 --- a/dbms/src/Storages/System/CMakeLists.txt +++ b/dbms/src/Storages/System/CMakeLists.txt @@ -12,7 +12,7 @@ include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake) add_headers_and_sources(storages_system .) list (APPEND storages_system_sources ${CONFIG_BUILD}) add_library(clickhouse_storages_system ${storages_system_headers} ${storages_system_sources}) -target_link_libraries(clickhouse_storages_system PRIVATE dbms common string_utils clickhouse_common_zookeeper) +target_link_libraries(clickhouse_storages_system PRIVATE dbms common string_utils clickhouse_common_zookeeper clickhouse_parsers) add_custom_target(generate-contributors ./StorageSystemContributors.sh SOURCES StorageSystemContributors.sh WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} # BYPRODUCTS StorageSystemContributors.generated.cpp diff --git a/dbms/src/Storages/System/IStorageSystemOneBlock.h b/dbms/src/Storages/System/IStorageSystemOneBlock.h index 8cdbd1804dc..b5c8c2ad78f 100644 --- a/dbms/src/Storages/System/IStorageSystemOneBlock.h +++ b/dbms/src/Storages/System/IStorageSystemOneBlock.h @@ -25,10 +25,8 @@ public: setColumns(ColumnsDescription(Self::getNamesAndTypes())); } - std::string getTableName() const override - { - return name; - } + std::string getTableName() const override { return name; } + std::string getDatabaseName() const override { return "system"; } BlockInputStreams read(const Names & column_names, const SelectQueryInfo & query_info, diff --git a/dbms/src/Storages/System/StorageSystemClusters.cpp b/dbms/src/Storages/System/StorageSystemClusters.cpp index b33b2d86d0e..d9403aba688 100644 --- a/dbms/src/Storages/System/StorageSystemClusters.cpp +++ b/dbms/src/Storages/System/StorageSystemClusters.cpp @@ -10,7 +10,8 @@ namespace DB NamesAndTypesList StorageSystemClusters::getNamesAndTypes() { - return { + return + { {"cluster", std::make_shared()}, {"shard_num", std::make_shared()}, {"shard_weight", std::make_shared()}, @@ -48,7 +49,8 @@ void StorageSystemClusters::fillData(MutableColumns & res_columns, const Context res_columns[i++]->insert(shard_info.weight); res_columns[i++]->insert(replica_index + 1); res_columns[i++]->insert(address.host_name); - res_columns[i++]->insert(DNSResolver::instance().resolveHost(address.host_name).toString()); + auto resolved = address.getResolvedAddress(); + res_columns[i++]->insert(resolved ? resolved->host().toString() : String()); res_columns[i++]->insert(address.port); res_columns[i++]->insert(shard_info.isLocal()); res_columns[i++]->insert(address.user); diff --git a/dbms/src/Storages/System/StorageSystemColumns.cpp b/dbms/src/Storages/System/StorageSystemColumns.cpp index 26749775d23..3ba1128c245 100644 --- a/dbms/src/Storages/System/StorageSystemColumns.cpp +++ b/dbms/src/Storages/System/StorageSystemColumns.cpp @@ -121,11 +121,7 @@ protected: cols_required_for_primary_key = storage->getColumnsRequiredForPrimaryKey(); cols_required_for_sampling = storage->getColumnsRequiredForSampling(); - /** Info about sizes of columns for tables of MergeTree family. - * NOTE: It is possible to add getter for this info to IStorage interface. - */ - if (auto storage_concrete = dynamic_cast(storage.get())) - column_sizes = storage_concrete->getColumnSizes(); + column_sizes = storage->getColumnSizes(); } for (const auto & column : columns) diff --git a/dbms/src/Storages/System/StorageSystemColumns.h b/dbms/src/Storages/System/StorageSystemColumns.h index e1814c96ef1..b9aa04b0b25 100644 --- a/dbms/src/Storages/System/StorageSystemColumns.h +++ b/dbms/src/Storages/System/StorageSystemColumns.h @@ -15,8 +15,8 @@ class StorageSystemColumns : public ext::shared_ptr_helper { public: std::string getName() const override { return "SystemColumns"; } - std::string getTableName() const override { return name; } + std::string getDatabaseName() const override { return "system"; } BlockInputStreams read( const Names & column_names, diff --git a/dbms/src/Storages/System/StorageSystemDetachedParts.cpp b/dbms/src/Storages/System/StorageSystemDetachedParts.cpp index 088e14903df..9ae6f7b607a 100644 --- a/dbms/src/Storages/System/StorageSystemDetachedParts.cpp +++ b/dbms/src/Storages/System/StorageSystemDetachedParts.cpp @@ -23,6 +23,7 @@ class StorageSystemDetachedParts : public: std::string getName() const override { return "SystemDetachedParts"; } std::string getTableName() const override { return "detached_parts"; } + std::string getDatabaseName() const override { return "system"; } protected: explicit StorageSystemDetachedParts() diff --git a/dbms/src/Storages/System/StorageSystemNumbers.h b/dbms/src/Storages/System/StorageSystemNumbers.h index 99d9599dda2..452ec5a9ef5 100644 --- a/dbms/src/Storages/System/StorageSystemNumbers.h +++ b/dbms/src/Storages/System/StorageSystemNumbers.h @@ -25,6 +25,7 @@ class StorageSystemNumbers : public ext::shared_ptr_helper public: std::string getName() const override { return "SystemNumbers"; } std::string getTableName() const override { return name; } + std::string getDatabaseName() const override { return "system"; } BlockInputStreams read( const Names & column_names, diff --git a/dbms/src/Storages/System/StorageSystemOne.h b/dbms/src/Storages/System/StorageSystemOne.h index 721684b7802..974435e99f0 100644 --- a/dbms/src/Storages/System/StorageSystemOne.h +++ b/dbms/src/Storages/System/StorageSystemOne.h @@ -20,6 +20,7 @@ class StorageSystemOne : public ext::shared_ptr_helper, public public: std::string getName() const override { return "SystemOne"; } std::string getTableName() const override { return name; } + std::string getDatabaseName() const override { return "system"; } BlockInputStreams read( const Names & column_names, diff --git a/dbms/src/Storages/System/StorageSystemParts.cpp b/dbms/src/Storages/System/StorageSystemParts.cpp index 0240baf9604..f8fffd2d9c9 100644 --- a/dbms/src/Storages/System/StorageSystemParts.cpp +++ b/dbms/src/Storages/System/StorageSystemParts.cpp @@ -9,6 +9,7 @@ #include #include #include +#include namespace DB { @@ -45,6 +46,10 @@ StorageSystemParts::StorageSystemParts(const std::string & name) {"table", std::make_shared()}, {"engine", std::make_shared()}, {"path", std::make_shared()}, + + {"hash_of_all_files", std::make_shared()}, + {"hash_of_uncompressed_files", std::make_shared()}, + {"uncompressed_hash_of_compressed_files", std::make_shared()} } ) { @@ -63,7 +68,7 @@ void StorageSystemParts::processNextStorage(MutableColumns & columns, const Stor const auto & part = all_parts[part_number]; auto part_state = all_parts_state[part_number]; - MergeTreeDataPart::ColumnSize columns_size = part->getTotalColumnsSize(); + ColumnSize columns_size = part->getTotalColumnsSize(); size_t i = 0; { @@ -107,6 +112,18 @@ void StorageSystemParts::processNextStorage(MutableColumns & columns, const Stor if (has_state_column) columns[i++]->insert(part->stateString()); + + MinimalisticDataPartChecksums helper; + helper.computeTotalChecksums(part->checksums); + + auto checksum = helper.hash_of_all_files; + columns[i++]->insert(getHexUIntLowercase(checksum.first) + getHexUIntLowercase(checksum.second)); + + checksum = helper.hash_of_uncompressed_files; + columns[i++]->insert(getHexUIntLowercase(checksum.first) + getHexUIntLowercase(checksum.second)); + + checksum = helper.uncompressed_hash_of_compressed_files; + columns[i++]->insert(getHexUIntLowercase(checksum.first) + getHexUIntLowercase(checksum.second)); } } diff --git a/dbms/src/Storages/System/StorageSystemPartsBase.h b/dbms/src/Storages/System/StorageSystemPartsBase.h index da90d7524e7..080153a2a91 100644 --- a/dbms/src/Storages/System/StorageSystemPartsBase.h +++ b/dbms/src/Storages/System/StorageSystemPartsBase.h @@ -54,6 +54,7 @@ class StorageSystemPartsBase : public IStorage { public: std::string getTableName() const override { return name; } + std::string getDatabaseName() const override { return "system"; } NameAndTypePair getColumn(const String & column_name) const override; diff --git a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp index 5d20e5797f4..ab688b514e7 100644 --- a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp +++ b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp @@ -46,13 +46,13 @@ StorageSystemPartsColumns::StorageSystemPartsColumns(const std::string & name) {"path", std::make_shared()}, {"column", std::make_shared()}, - {"type", std::make_shared() }, - {"default_kind", std::make_shared() }, - {"default_expression", std::make_shared() }, - {"column_bytes_on_disk", std::make_shared() }, - {"column_data_compressed_bytes", std::make_shared() }, - {"column_data_uncompressed_bytes", std::make_shared() }, - {"column_marks_bytes", std::make_shared() }, + {"type", std::make_shared()}, + {"default_kind", std::make_shared()}, + {"default_expression", std::make_shared()}, + {"column_bytes_on_disk", std::make_shared()}, + {"column_data_compressed_bytes", std::make_shared()}, + {"column_data_uncompressed_bytes", std::make_shared()}, + {"column_marks_bytes", std::make_shared()} } ) { @@ -100,6 +100,7 @@ void StorageSystemPartsColumns::processNextStorage(MutableColumns & columns, con using State = MergeTreeDataPart::State; for (const auto & column : part->columns) + { size_t j = 0; { @@ -150,7 +151,7 @@ void StorageSystemPartsColumns::processNextStorage(MutableColumns & columns, con columns[j++]->insertDefault(); } - MergeTreeDataPart::ColumnSize column_size = part->getColumnSize(column.name, *column.type); + ColumnSize column_size = part->getColumnSize(column.name, *column.type); columns[j++]->insert(column_size.data_compressed + column_size.marks); columns[j++]->insert(column_size.data_compressed); columns[j++]->insert(column_size.data_uncompressed); diff --git a/dbms/src/Storages/System/StorageSystemReplicas.h b/dbms/src/Storages/System/StorageSystemReplicas.h index 53ba5ebd9bd..49865ad869a 100644 --- a/dbms/src/Storages/System/StorageSystemReplicas.h +++ b/dbms/src/Storages/System/StorageSystemReplicas.h @@ -17,6 +17,7 @@ class StorageSystemReplicas : public ext::shared_ptr_helper, public: std::string getName() const override { return "SystemTables"; } std::string getTableName() const override { return name; } + std::string getDatabaseName() const override { return "system"; } BlockInputStreams read( const Names & column_names, diff --git a/dbms/src/Storages/tests/gtest_aux_funcs_for_adaptive_granularity.cpp b/dbms/src/Storages/tests/gtest_aux_funcs_for_adaptive_granularity.cpp index 85df83be2de..95c56c74132 100644 --- a/dbms/src/Storages/tests/gtest_aux_funcs_for_adaptive_granularity.cpp +++ b/dbms/src/Storages/tests/gtest_aux_funcs_for_adaptive_granularity.cpp @@ -1,8 +1,3 @@ -#pragma GCC diagnostic ignored "-Wsign-compare" -#ifdef __clang__ -#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" -#pragma clang diagnostic ignored "-Wundef" -#endif #include #include #include diff --git a/dbms/src/Storages/tests/gtest_row_source_bits_test.cpp b/dbms/src/Storages/tests/gtest_row_source_bits_test.cpp index 7b2f25061b6..a6d9179c106 100644 --- a/dbms/src/Storages/tests/gtest_row_source_bits_test.cpp +++ b/dbms/src/Storages/tests/gtest_row_source_bits_test.cpp @@ -1,8 +1,3 @@ -#pragma GCC diagnostic ignored "-Wsign-compare" -#ifdef __clang__ - #pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" - #pragma clang diagnostic ignored "-Wundef" -#endif #include #include diff --git a/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp b/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp index bcee0b8d8e1..c17676bc655 100644 --- a/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp +++ b/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp @@ -1,8 +1,3 @@ -#pragma GCC diagnostic ignored "-Wsign-compare" -#ifdef __clang__ - #pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" - #pragma clang diagnostic ignored "-Wundef" -#endif #include #include @@ -28,7 +23,8 @@ struct State { registerFunctions(); DatabasePtr database = std::make_shared("test"); - database->attachTable("table", StorageMemory::create("table", ColumnsDescription{columns})); + database->attachTable("table", StorageMemory::create("test", "table", ColumnsDescription{columns})); + context.makeGlobalContext(); context.addDatabase("test", database); context.setCurrentDatabase("test"); } diff --git a/dbms/src/Storages/tests/storage_log.cpp b/dbms/src/Storages/tests/storage_log.cpp index db905731e6f..bcf7b1d4d03 100644 --- a/dbms/src/Storages/tests/storage_log.cpp +++ b/dbms/src/Storages/tests/storage_log.cpp @@ -25,10 +25,11 @@ try names_and_types.emplace_back("a", std::make_shared()); names_and_types.emplace_back("b", std::make_shared()); - StoragePtr table = StorageLog::create("./", "test", ColumnsDescription{names_and_types}, 1048576); + StoragePtr table = StorageLog::create("./", "test", "test", ColumnsDescription{names_and_types}, 1048576); table->startup(); auto context = Context::createGlobal(); + context.makeGlobalContext(); /// write into it { diff --git a/dbms/src/Storages/tests/system_numbers.cpp b/dbms/src/Storages/tests/system_numbers.cpp index 92d1f9d65e2..0ba94fb5d0a 100644 --- a/dbms/src/Storages/tests/system_numbers.cpp +++ b/dbms/src/Storages/tests/system_numbers.cpp @@ -26,9 +26,10 @@ try WriteBufferFromOStream out_buf(std::cout); - QueryProcessingStage::Enum stage = table->getQueryProcessingStage(Context::createGlobal()); - auto context = Context::createGlobal(); + context.makeGlobalContext(); + QueryProcessingStage::Enum stage = table->getQueryProcessingStage(context); + LimitBlockInputStream input(table->read(column_names, {}, context, stage, 10, 1)[0], 10, 96); BlockOutputStreamPtr out = FormatFactory::instance().getOutput("TabSeparated", out_buf, sample, context); diff --git a/dbms/src/TableFunctions/CMakeLists.txt b/dbms/src/TableFunctions/CMakeLists.txt index 9bd1374cd4d..26f9b7ddeb2 100644 --- a/dbms/src/TableFunctions/CMakeLists.txt +++ b/dbms/src/TableFunctions/CMakeLists.txt @@ -5,4 +5,4 @@ list(REMOVE_ITEM clickhouse_table_functions_sources ITableFunction.cpp TableFunc list(REMOVE_ITEM clickhouse_table_functions_headers ITableFunction.h TableFunctionFactory.h) add_library(clickhouse_table_functions ${clickhouse_table_functions_sources}) -target_link_libraries(clickhouse_table_functions PRIVATE clickhouse_storages_system dbms ${Poco_Foundation_LIBRARY}) +target_link_libraries(clickhouse_table_functions PRIVATE clickhouse_parsers clickhouse_storages_system dbms ${Poco_Foundation_LIBRARY}) diff --git a/dbms/src/TableFunctions/ITableFunction.h b/dbms/src/TableFunctions/ITableFunction.h index 026ff944976..fe08e56106b 100644 --- a/dbms/src/TableFunctions/ITableFunction.h +++ b/dbms/src/TableFunctions/ITableFunction.h @@ -26,6 +26,8 @@ class Context; class ITableFunction { public: + static inline std::string getDatabaseName() { return "_table_function"; } + /// Get the main function name. virtual std::string getName() const = 0; diff --git a/dbms/src/TableFunctions/ITableFunctionXDBC.cpp b/dbms/src/TableFunctions/ITableFunctionXDBC.cpp index 32011dc8f8a..6548dcc1acb 100644 --- a/dbms/src/TableFunctions/ITableFunctionXDBC.cpp +++ b/dbms/src/TableFunctions/ITableFunctionXDBC.cpp @@ -76,7 +76,7 @@ StoragePtr ITableFunctionXDBC::executeImpl(const ASTPtr & ast_function, const Co readStringBinary(columns_info, buf); NamesAndTypesList columns = NamesAndTypesList::parse(columns_info); - auto result = std::make_shared(table_name, schema_name, table_name, ColumnsDescription{columns}, context, helper); + auto result = std::make_shared(getDatabaseName(), table_name, schema_name, table_name, ColumnsDescription{columns}, context, helper); if (!result) throw Exception("Failed to instantiate storage from table function " + getName(), ErrorCodes::UNKNOWN_EXCEPTION); diff --git a/dbms/src/TableFunctions/TableFunctionCatBoostPool.cpp b/dbms/src/TableFunctions/TableFunctionCatBoostPool.cpp index 23ff34be827..da926dcc906 100644 --- a/dbms/src/TableFunctions/TableFunctionCatBoostPool.cpp +++ b/dbms/src/TableFunctions/TableFunctionCatBoostPool.cpp @@ -45,7 +45,7 @@ StoragePtr TableFunctionCatBoostPool::executeImpl(const ASTPtr & ast_function, c String column_descriptions_file = getStringLiteral(*args[0], "Column descriptions file"); String dataset_description_file = getStringLiteral(*args[1], "Dataset description file"); - return StorageCatBoostPool::create(context, column_descriptions_file, dataset_description_file); + return StorageCatBoostPool::create(getDatabaseName(), getName(), context, column_descriptions_file, dataset_description_file); } void registerTableFunctionCatBoostPool(TableFunctionFactory & factory) diff --git a/dbms/src/TableFunctions/TableFunctionFile.cpp b/dbms/src/TableFunctions/TableFunctionFile.cpp index 89531096d35..c1fed19101a 100644 --- a/dbms/src/TableFunctions/TableFunctionFile.cpp +++ b/dbms/src/TableFunctions/TableFunctionFile.cpp @@ -10,6 +10,7 @@ StoragePtr TableFunctionFile::getStorage( return StorageFile::create(source, -1, global_context.getUserFilesPath(), + getDatabaseName(), getName(), format, ColumnsDescription{sample_block.getNamesAndTypesList()}, diff --git a/dbms/src/TableFunctions/TableFunctionHDFS.cpp b/dbms/src/TableFunctions/TableFunctionHDFS.cpp index 9c09ad9313c..7396b477cd1 100644 --- a/dbms/src/TableFunctions/TableFunctionHDFS.cpp +++ b/dbms/src/TableFunctions/TableFunctionHDFS.cpp @@ -11,6 +11,7 @@ StoragePtr TableFunctionHDFS::getStorage( const String & source, const String & format, const Block & sample_block, Context & global_context) const { return StorageHDFS::create(source, + getDatabaseName(), getName(), format, ColumnsDescription{sample_block.getNamesAndTypesList()}, diff --git a/dbms/src/TableFunctions/TableFunctionMerge.cpp b/dbms/src/TableFunctions/TableFunctionMerge.cpp index cd8d906b94d..d446c8d723e 100644 --- a/dbms/src/TableFunctions/TableFunctionMerge.cpp +++ b/dbms/src/TableFunctions/TableFunctionMerge.cpp @@ -70,6 +70,7 @@ StoragePtr TableFunctionMerge::executeImpl(const ASTPtr & ast_function, const Co String table_name_regexp = args[1]->as().value.safeGet(); auto res = StorageMerge::create( + getDatabaseName(), getName(), ColumnsDescription{chooseColumns(source_database, table_name_regexp, context)}, source_database, diff --git a/dbms/src/TableFunctions/TableFunctionMySQL.cpp b/dbms/src/TableFunctions/TableFunctionMySQL.cpp index 71d195e95ed..8bba0388933 100644 --- a/dbms/src/TableFunctions/TableFunctionMySQL.cpp +++ b/dbms/src/TableFunctions/TableFunctionMySQL.cpp @@ -119,6 +119,7 @@ StoragePtr TableFunctionMySQL::executeImpl(const ASTPtr & ast_function, const Co throw Exception("MySQL table " + backQuoteIfNeed(database_name) + "." + backQuoteIfNeed(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); auto res = StorageMySQL::create( + getDatabaseName(), table_name, std::move(pool), database_name, diff --git a/dbms/src/TableFunctions/TableFunctionURL.cpp b/dbms/src/TableFunctions/TableFunctionURL.cpp index f33e5a92cb3..d5d9168aed8 100644 --- a/dbms/src/TableFunctions/TableFunctionURL.cpp +++ b/dbms/src/TableFunctions/TableFunctionURL.cpp @@ -9,7 +9,7 @@ StoragePtr TableFunctionURL::getStorage( const String & source, const String & format, const Block & sample_block, Context & global_context) const { Poco::URI uri(source); - return StorageURL::create(uri, getName(), format, ColumnsDescription{sample_block.getNamesAndTypesList()}, global_context); + return StorageURL::create(uri, getDatabaseName(), getName(), format, ColumnsDescription{sample_block.getNamesAndTypesList()}, global_context); } void registerTableFunctionURL(TableFunctionFactory & factory) diff --git a/dbms/tests/clickhouse-test b/dbms/tests/clickhouse-test index 8e4e0bcb8fc..2e99b74ffda 100755 --- a/dbms/tests/clickhouse-test +++ b/dbms/tests/clickhouse-test @@ -16,7 +16,10 @@ from subprocess import CalledProcessError from datetime import datetime from time import sleep from errno import ESRCH -import termcolor +try: + import termcolor +except ImportError: + termcolor = None from random import random import commands import multiprocessing @@ -93,7 +96,7 @@ def get_server_pid(server_tcp_port): return None def colored(text, args, color=None, on_color=None, attrs=None): - if sys.stdout.isatty() or args.force_color: + if termcolor and (sys.stdout.isatty() or args.force_color): return termcolor.colored(text, color, on_color, attrs) else: return text diff --git a/dbms/tests/config/query_masking_rules.xml b/dbms/tests/config/query_masking_rules.xml new file mode 100644 index 00000000000..5a854848f3d --- /dev/null +++ b/dbms/tests/config/query_masking_rules.xml @@ -0,0 +1,10 @@ + + + + + + TOPSECRET.TOPSECRET + [hidden] + + + diff --git a/dbms/tests/instructions/sanitizers.md b/dbms/tests/instructions/sanitizers.md index f71d469182d..c21ff9b0a9b 100644 --- a/dbms/tests/instructions/sanitizers.md +++ b/dbms/tests/instructions/sanitizers.md @@ -66,4 +66,6 @@ sudo -u clickhouse UBSAN_OPTIONS='print_stacktrace=1' ./clickhouse-ubsan server # How to use Memory Sanitizer -TODO +``` +CC=clang-8 CXX=clang++-8 cmake -D ENABLE_HDFS=0 -D ENABLE_CAPNP=0 -D ENABLE_RDKAFKA=0 -D ENABLE_ICU=0 -D ENABLE_POCO_MONGODB=0 -D ENABLE_POCO_NETSSL=0 -D ENABLE_POCO_ODBC=0 -D ENABLE_ODBC=0 -D ENABLE_MYSQL=0 -D ENABLE_EMBEDDED_COMPILER=0 -D USE_INTERNAL_CAPNP_LIBRARY=0 -D USE_INTERNAL_SSL_LIBRARY=0 -D USE_SIMDJSON=0 -DENABLE_READLINE=0 -D SANITIZE=memory .. +``` diff --git a/dbms/tests/integration/helpers/client.py b/dbms/tests/integration/helpers/client.py index 9df53c40e67..fd59166b137 100644 --- a/dbms/tests/integration/helpers/client.py +++ b/dbms/tests/integration/helpers/client.py @@ -44,6 +44,9 @@ class Client: return self.get_query_request(sql, stdin=stdin, timeout=timeout, settings=settings, user=user).get_error() + def query_and_get_answer_with_error(self, sql, stdin=None, timeout=None, settings=None, user=None): + return self.get_query_request(sql, stdin=stdin, timeout=timeout, settings=settings, user=user).get_answer_and_error() + class QueryTimeoutExceedException(Exception): pass @@ -110,3 +113,17 @@ class CommandRequest: raise QueryRuntimeException('Client expected to be failed but succeeded! stdout: {}'.format(stdout)) return stderr + + + def get_answer_and_error(self): + self.process.wait() + self.stdout_file.seek(0) + self.stderr_file.seek(0) + + stdout = self.stdout_file.read() + stderr = self.stderr_file.read() + + if self.timer is not None and not self.process_finished_before_timeout and not self.ignore_error: + raise QueryTimeoutExceedException('Client timed out!') + + return (stdout, stderr) diff --git a/dbms/tests/integration/helpers/cluster.py b/dbms/tests/integration/helpers/cluster.py index fbc6591eaa5..bd3ecb9ae9c 100644 --- a/dbms/tests/integration/helpers/cluster.py +++ b/dbms/tests/integration/helpers/cluster.py @@ -527,6 +527,10 @@ class ClickHouseInstance: def query_and_get_error(self, sql, stdin=None, timeout=None, settings=None, user=None): return self.client.query_and_get_error(sql, stdin, timeout, settings, user) + # The same as query_and_get_error but ignores successful query. + def query_and_get_answer_with_error(self, sql, stdin=None, timeout=None, settings=None, user=None): + return self.client.query_and_get_answer_with_error(sql, stdin, timeout, settings, user) + # Connects to the instance via HTTP interface, sends a query and returns the answer def http_query(self, sql, data=None): return urllib.urlopen("http://"+self.ip_address+":8123/?query="+urllib.quote(sql,safe=''), data).read() diff --git a/dbms/tests/integration/test_check_table/__init__.py b/dbms/tests/integration/test_check_table/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_check_table/test.py b/dbms/tests/integration/test_check_table/test.py new file mode 100644 index 00000000000..83df59b44a0 --- /dev/null +++ b/dbms/tests/integration/test_check_table/test.py @@ -0,0 +1,129 @@ +import time + +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import assert_eq_with_retry + +cluster = ClickHouseCluster(__file__) + +node1 = cluster.add_instance('node1', with_zookeeper=True) +node2 = cluster.add_instance('node2', with_zookeeper=True) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + for node in [node1, node2]: + node.query(''' + CREATE TABLE replicated_mt(date Date, id UInt32, value Int32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/replicated_mt', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY id; + '''.format(replica=node.name)) + + node1.query(''' + CREATE TABLE non_replicated_mt(date Date, id UInt32, value Int32) + ENGINE = MergeTree() PARTITION BY toYYYYMM(date) ORDER BY id; + ''') + + yield cluster + + finally: + cluster.shutdown() + + +def corrupt_data_part_on_disk(node, table, part_name): + part_path = node.query("SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format(table, part_name)).strip() + node.exec_in_container(['bash', '-c', 'cd {p} && ls *.bin | head -n 1 | xargs -I{{}} sh -c \'echo "1" >> $1\' -- {{}}'.format(p=part_path)], privileged=True) + +def remove_checksums_on_disk(node, table, part_name): + part_path = node.query("SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format(table, part_name)).strip() + node.exec_in_container(['bash', '-c', 'rm -r {p}/checksums.txt'.format(p=part_path)], privileged=True) + +def remove_part_from_disk(node, table, part_name): + part_path = node.query("SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format(table, part_name)).strip() + if not part_path: + raise Exception("Part " + part_name + "doesn't exist") + node.exec_in_container(['bash', '-c', 'rm -r {p}/*'.format(p=part_path)], privileged=True) + + +def test_check_normal_table_corruption(started_cluster): + node1.query("INSERT INTO non_replicated_mt VALUES (toDate('2019-02-01'), 1, 10), (toDate('2019-02-01'), 2, 12)") + assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201902", settings={"check_query_single_value_result": 0}) == "201902_1_1_0\t1\t\n" + + remove_checksums_on_disk(node1, "non_replicated_mt", "201902_1_1_0") + + assert node1.query("CHECK TABLE non_replicated_mt", settings={"check_query_single_value_result": 0}).strip() == "201902_1_1_0\t1\tChecksums recounted and written to disk." + + assert node1.query("SELECT COUNT() FROM non_replicated_mt") == "2\n" + + remove_checksums_on_disk(node1, "non_replicated_mt", "201902_1_1_0") + + assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201902", settings={"check_query_single_value_result": 0}).strip() == "201902_1_1_0\t1\tChecksums recounted and written to disk." + + assert node1.query("SELECT COUNT() FROM non_replicated_mt") == "2\n" + + corrupt_data_part_on_disk(node1, "non_replicated_mt", "201902_1_1_0") + + assert node1.query("CHECK TABLE non_replicated_mt", settings={"check_query_single_value_result": 0}).strip() == "201902_1_1_0\t0\tCannot read all data. Bytes read: 2. Bytes expected: 16." + + assert node1.query("CHECK TABLE non_replicated_mt", settings={"check_query_single_value_result": 0}).strip() == "201902_1_1_0\t0\tCannot read all data. Bytes read: 2. Bytes expected: 16." + + node1.query("INSERT INTO non_replicated_mt VALUES (toDate('2019-01-01'), 1, 10), (toDate('2019-01-01'), 2, 12)") + + assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "201901_2_2_0\t1\t\n" + + corrupt_data_part_on_disk(node1, "non_replicated_mt", "201901_2_2_0") + + remove_checksums_on_disk(node1, "non_replicated_mt", "201901_2_2_0") + + assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "201901_2_2_0\t0\tCheck of part finished with error: \\'Cannot read all data. Bytes read: 2. Bytes expected: 16.\\'\n" + + +def test_check_replicated_table_simple(started_cluster): + node1.query("TRUNCATE TABLE replicated_mt") + node2.query("SYSTEM SYNC REPLICA replicated_mt") + node1.query("INSERT INTO replicated_mt VALUES (toDate('2019-02-01'), 1, 10), (toDate('2019-02-01'), 2, 12)") + node2.query("SYSTEM SYNC REPLICA replicated_mt") + + assert node1.query("SELECT count() from replicated_mt") == "2\n" + assert node2.query("SELECT count() from replicated_mt") == "2\n" + + assert node1.query("CHECK TABLE replicated_mt", settings={"check_query_single_value_result": 0}) == "201902_0_0_0\t1\t\n" + assert node2.query("CHECK TABLE replicated_mt", settings={"check_query_single_value_result": 0}) == "201902_0_0_0\t1\t\n" + + node2.query("INSERT INTO replicated_mt VALUES (toDate('2019-01-02'), 3, 10), (toDate('2019-01-02'), 4, 12)") + node1.query("SYSTEM SYNC REPLICA replicated_mt") + assert node1.query("SELECT count() from replicated_mt") == "4\n" + assert node2.query("SELECT count() from replicated_mt") == "4\n" + + assert node1.query("CHECK TABLE replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "201901_0_0_0\t1\t\n" + assert node2.query("CHECK TABLE replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "201901_0_0_0\t1\t\n" + + +def test_check_replicated_table_corruption(started_cluster): + node1.query("TRUNCATE TABLE replicated_mt") + node2.query("SYSTEM SYNC REPLICA replicated_mt") + node1.query("INSERT INTO replicated_mt VALUES (toDate('2019-02-01'), 1, 10), (toDate('2019-02-01'), 2, 12)") + node1.query("INSERT INTO replicated_mt VALUES (toDate('2019-01-02'), 3, 10), (toDate('2019-01-02'), 4, 12)") + node2.query("SYSTEM SYNC REPLICA replicated_mt") + + assert node1.query("SELECT count() from replicated_mt") == "4\n" + assert node2.query("SELECT count() from replicated_mt") == "4\n" + + part_name = node1.query("SELECT name from system.parts where table = 'replicated_mt' and partition_id = '201901' and active = 1").strip() + + corrupt_data_part_on_disk(node1, "replicated_mt", part_name) + assert node1.query("CHECK TABLE replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "{p}\t0\tPart {p} looks broken. Removing it and queueing a fetch.\n".format(p=part_name) + + node1.query("SYSTEM SYNC REPLICA replicated_mt") + assert node1.query("CHECK TABLE replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "{}\t1\t\n".format(part_name) + assert node1.query("SELECT count() from replicated_mt") == "4\n" + + remove_part_from_disk(node2, "replicated_mt", part_name) + assert node2.query("CHECK TABLE replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "{p}\t0\tPart {p} looks broken. Removing it and queueing a fetch.\n".format(p=part_name) + + node1.query("SYSTEM SYNC REPLICA replicated_mt") + assert node1.query("CHECK TABLE replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "{}\t1\t\n".format(part_name) + assert node1.query("SELECT count() from replicated_mt") == "4\n" diff --git a/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_dep_x.xml b/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_dep_x.xml index 36a421f1908..6eed3fbc891 100644 --- a/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_dep_x.xml +++ b/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_dep_x.xml @@ -12,7 +12,7 @@ dep_z
- 60 + 5 @@ -21,7 +21,7 @@ id - String_ + a String XX diff --git a/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_dep_y.xml b/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_dep_y.xml index 771ab889660..7891e945566 100644 --- a/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_dep_y.xml +++ b/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_dep_y.xml @@ -9,10 +9,10 @@ default test - dictionary_source
+ small_dict_source
- 60 + 5 @@ -21,17 +21,17 @@ id - Int64_ - Int64 - 121 + b + Int32 + -1 - Float32_ - Float32 - 121 + c + Float64 + -2 - String_ + a String YY diff --git a/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_dep_z.xml b/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_dep_z.xml index 875d55c39f9..e17107d2ac2 100644 --- a/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_dep_z.xml +++ b/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_dep_z.xml @@ -10,9 +10,10 @@ dict dep_y
+ SELECT intDiv(count(), 5) from dict.dep_y - 60 + 5 @@ -21,12 +22,12 @@ id - Int64_ - Int64 - 122 + b + Int32 + -3 - String_ + a String ZZ diff --git a/dbms/tests/integration/test_dictionaries/test.py b/dbms/tests/integration/test_dictionaries/test.py index bf698bae452..dfb27cd2ed7 100644 --- a/dbms/tests/integration/test_dictionaries/test.py +++ b/dbms/tests/integration/test_dictionaries/test.py @@ -3,7 +3,7 @@ import os import time from helpers.cluster import ClickHouseCluster -from helpers.test_tools import TSV +from helpers.test_tools import TSV, assert_eq_with_retry from generate_dictionaries import generate_structure, generate_dictionaries, DictionaryTestTable SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -54,6 +54,13 @@ def started_cluster(): for line in TSV(instance.query('select name from system.dictionaries')).lines: print line, + # Create table `test.small_dict_source` + instance.query(''' + drop table if exists test.small_dict_source; + create table test.small_dict_source (id UInt64, a String, b Int32, c Float64) engine=Log; + insert into test.small_dict_source values (0, 'water', 10, 1), (1, 'air', 40, 0.01), (2, 'earth', 100, 1.7); + ''') + yield cluster finally: @@ -166,17 +173,37 @@ def test_dictionary_dependency(started_cluster): # Dictionary 'dep_x' depends on 'dep_z', which depends on 'dep_y'. # So they all should be loaded at once. - assert query("SELECT dictGetString('dep_x', 'String_', toUInt64(1))") == "10577349846663553072\n" + assert query("SELECT dictGetString('dep_x', 'a', toUInt64(1))") == "air\n" assert get_status('dep_x') == 'LOADED' assert get_status('dep_y') == 'LOADED' assert get_status('dep_z') == 'LOADED' # Other dictionaries should work too. - assert query("SELECT dictGetString('dep_y', 'String_', toUInt64(1))") == "10577349846663553072\n" - assert query("SELECT dictGetString('dep_z', 'String_', toUInt64(1))") == "10577349846663553072\n" - assert query("SELECT dictGetString('dep_x', 'String_', toUInt64(12121212))") == "XX\n" - assert query("SELECT dictGetString('dep_y', 'String_', toUInt64(12121212))") == "YY\n" - assert query("SELECT dictGetString('dep_z', 'String_', toUInt64(12121212))") == "ZZ\n" + assert query("SELECT dictGetString('dep_y', 'a', toUInt64(1))") == "air\n" + assert query("SELECT dictGetString('dep_z', 'a', toUInt64(1))") == "air\n" + + assert query("SELECT dictGetString('dep_x', 'a', toUInt64(3))") == "XX\n" + assert query("SELECT dictGetString('dep_y', 'a', toUInt64(3))") == "YY\n" + assert query("SELECT dictGetString('dep_z', 'a', toUInt64(3))") == "ZZ\n" + + # Update the source table. + query("insert into test.small_dict_source values (3, 'fire', 30, 8)") + + # Wait for dictionaries to be reloaded. + assert_eq_with_retry(instance, "SELECT dictHas('dep_y', toUInt64(3))", "1", sleep_time = 2, retry_count = 10) + assert query("SELECT dictGetString('dep_x', 'a', toUInt64(3))") == "XX\n" + assert query("SELECT dictGetString('dep_y', 'a', toUInt64(3))") == "fire\n" + assert query("SELECT dictGetString('dep_z', 'a', toUInt64(3))") == "ZZ\n" + + # dep_x and dep_z are updated only when there `intDiv(count(), 4)` is changed. + query("insert into test.small_dict_source values (4, 'ether', 404, 0.001)") + assert_eq_with_retry(instance, "SELECT dictHas('dep_x', toUInt64(4))", "1", sleep_time = 2, retry_count = 10) + assert query("SELECT dictGetString('dep_x', 'a', toUInt64(3))") == "fire\n" + assert query("SELECT dictGetString('dep_y', 'a', toUInt64(3))") == "fire\n" + assert query("SELECT dictGetString('dep_z', 'a', toUInt64(3))") == "fire\n" + assert query("SELECT dictGetString('dep_x', 'a', toUInt64(4))") == "ether\n" + assert query("SELECT dictGetString('dep_y', 'a', toUInt64(4))") == "ether\n" + assert query("SELECT dictGetString('dep_z', 'a', toUInt64(4))") == "ether\n" def test_reload_while_loading(started_cluster): diff --git a/dbms/tests/integration/test_external_dictionaries/configs/config.xml b/dbms/tests/integration/test_external_dictionaries/configs/config.xml index 97cbaf97d4a..fb8f0e1312e 100644 --- a/dbms/tests/integration/test_external_dictionaries/configs/config.xml +++ b/dbms/tests/integration/test_external_dictionaries/configs/config.xml @@ -1,12 +1,14 @@ - trace - /var/log/clickhouse-server/clickhouse-server.log - /var/log/clickhouse-server/clickhouse-server.err.log - 1000M - 10 - + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + /var/log/clickhouse-server/stderr.log + /var/log/clickhouse-server/stdout.log + 9000 127.0.0.1 diff --git a/dbms/tests/integration/test_logs_level/__init__.py b/dbms/tests/integration/test_logs_level/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_logs_level/configs/config_information.xml b/dbms/tests/integration/test_logs_level/configs/config_information.xml new file mode 100644 index 00000000000..b99f57402fd --- /dev/null +++ b/dbms/tests/integration/test_logs_level/configs/config_information.xml @@ -0,0 +1,26 @@ + + + + information + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + + + + + true + none + + AcceptCertificateHandler + + + + + 500 + 5368709120 + users.xml + + /etc/clickhouse-server/config.d/*.xml + diff --git a/dbms/tests/integration/test_logs_level/test.py b/dbms/tests/integration/test_logs_level/test.py new file mode 100644 index 00000000000..302686b6fa0 --- /dev/null +++ b/dbms/tests/integration/test_logs_level/test.py @@ -0,0 +1,18 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance('node', main_configs=['configs/config_information.xml']) + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + +def test_check_client_logs_level(start_cluster): + logs = node.query_and_get_answer_with_error("SELECT 1", settings={"send_logs_level": 'trace'})[1] + assert logs.count('Trace') != 0 diff --git a/dbms/tests/integration/test_old_versions_client/test.py b/dbms/tests/integration/test_old_versions_client/test.py index 2977d82ee9a..59a4aaca31c 100644 --- a/dbms/tests/integration/test_old_versions_client/test.py +++ b/dbms/tests/integration/test_old_versions_client/test.py @@ -11,13 +11,14 @@ node18_14 = cluster.add_instance('node18_14', image='yandex/clickhouse-server:18 node19_1 = cluster.add_instance('node19_1', image='yandex/clickhouse-server:19.1.16', with_installed_binary=True) node19_4 = cluster.add_instance('node19_4', image='yandex/clickhouse-server:19.4.5.35', with_installed_binary=True) node19_6 = cluster.add_instance('node19_6', image='yandex/clickhouse-server:19.6.3.18', with_installed_binary=True) +node19_8 = cluster.add_instance('node19_8', image='yandex/clickhouse-server:19.8.3.8', with_installed_binary=True) node_new = cluster.add_instance('node_new') @pytest.fixture(scope="module") def setup_nodes(): try: cluster.start() - for n in (node18_14, node19_1, node19_4, node19_6, node_new): + for n in (node18_14, node19_1, node19_4, node19_6, node19_8, node_new): n.query('''CREATE TABLE test_table (id UInt32, value UInt64) ENGINE = MergeTree() ORDER BY tuple()''') yield cluster @@ -29,7 +30,7 @@ def query_from_one_node_to_another(client_node, server_node, query): client_node.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --host {} --query '{}'".format(server_node.name, query)]) def test_client_from_different_versions(setup_nodes): - old_nodes = (node18_14, node19_1, node19_4, node19_6,) + old_nodes = (node18_14, node19_1, node19_4, node19_6, node19_8) # from new to old for n in old_nodes: query_from_one_node_to_another(node_new, n, "INSERT INTO test_table VALUES (1, 1)") diff --git a/dbms/tests/performance/count.xml b/dbms/tests/performance/count.xml new file mode 100644 index 00000000000..9ce3c0a98e4 --- /dev/null +++ b/dbms/tests/performance/count.xml @@ -0,0 +1,27 @@ + + count + + loop + + + + 30000 + + + 6000 + 60000 + + + + + + + + CREATE TABLE data(k UInt64, v UInt64) ENGINE = MergeTree ORDER BY k + + INSERT INTO data SELECT number, 1 from numbers(10000000) + + SELECT count() FROM data + + DROP TABLE IF EXISTS data + diff --git a/dbms/tests/performance/jit_large_requests.xml b/dbms/tests/performance/jit_large_requests.xml new file mode 100644 index 00000000000..54aa2af65b1 --- /dev/null +++ b/dbms/tests/performance/jit_large_requests.xml @@ -0,0 +1,64 @@ + + loop + + + + 100 + + + + + CREATE TABLE jit_test ( + a UInt64, + b UInt64, + c UInt64, + d UInt64, + e UInt64, + f UInt64, + g UInt64, + h UInt64, + i UInt64, + j UInt64 + ) Engine = Memory + + + + INSERT INTO jit_test + SELECT + number, + number, + number, + number, + number, + number, + number, + number, + number, + number + FROM + system.numbers + LIMIT 10000000 + + + SELECT + COUNT() + FROM + jit_test + WHERE + NOT ignore(a / b + c / d + e / f + g / h + i / j) + SETTINGS + compile_expressions = 0; + + + SELECT + COUNT() + FROM + jit_test + WHERE + NOT ignore(a / b + c / d + e / f + g / h + i / j) + SETTINGS + compile_expressions = 1, + min_count_to_compile = 1 + + DROP TABLE IF EXISTS jit_test + diff --git a/dbms/tests/performance/small_requests.xml b/dbms/tests/performance/jit_small_requests.xml similarity index 100% rename from dbms/tests/performance/small_requests.xml rename to dbms/tests/performance/jit_small_requests.xml diff --git a/dbms/tests/performance/joins_in_memory.xml b/dbms/tests/performance/joins_in_memory.xml index 23b009a6027..1da400c48f4 100644 --- a/dbms/tests/performance/joins_in_memory.xml +++ b/dbms/tests/performance/joins_in_memory.xml @@ -13,11 +13,11 @@ CREATE TABLE ints (i64 Int64, i32 Int32, i16 Int16, i8 Int8) ENGINE = Memory - INSERT INTO ints SELECT number AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(10000) - INSERT INTO ints SELECT 10000 + number % 1000 AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(10000) - INSERT INTO ints SELECT 20000 + number % 100 AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(10000) - INSERT INTO ints SELECT 30000 + number % 10 AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(10000) - INSERT INTO ints SELECT 40000 + number % 1 AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(10000) + INSERT INTO ints SELECT number AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(5000) + INSERT INTO ints SELECT 10000 + number % 1000 AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(5000) + INSERT INTO ints SELECT 20000 + number % 100 AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(5000) + INSERT INTO ints SELECT 30000 + number % 10 AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(5000) + INSERT INTO ints SELECT 40000 + number % 1 AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(5000) SELECT COUNT() FROM ints l ANY LEFT JOIN ints r USING i64 WHERE i32 = 200042 SELECT COUNT() FROM ints l ANY LEFT JOIN ints r USING i64,i32,i16,i8 WHERE i32 = 200042 diff --git a/dbms/tests/performance/merge_table_streams.xml b/dbms/tests/performance/merge_table_streams.xml new file mode 100644 index 00000000000..3f19c21109e --- /dev/null +++ b/dbms/tests/performance/merge_table_streams.xml @@ -0,0 +1,42 @@ + + loop + + + hits_100m_single + + + + + 60000 + 3 + + + 30 + + + + + + + + + 5 + + + +CREATE TABLE merge_table_streams_1 AS hits_100m_single +CREATE TABLE merge_table_streams_2 AS hits_100m_single +CREATE TABLE merge_table_streams_3 AS hits_100m_single +CREATE TABLE merge_table_streams_4 AS hits_100m_single + +SELECT UserID FROM merge(default, '^(hits_100m_single|merge_table_streams_\\d)$') WHERE UserID = 12345678901234567890 + +DROP TABLE merge_table_streams_1 +DROP TABLE merge_table_streams_2 +DROP TABLE merge_table_streams_3 +DROP TABLE merge_table_streams_4 + + diff --git a/dbms/tests/performance/uniq.xml b/dbms/tests/performance/uniq.xml index af41d1c79e5..c44a4e2ca58 100644 --- a/dbms/tests/performance/uniq.xml +++ b/dbms/tests/performance/uniq.xml @@ -46,8 +46,6 @@ uniqCombined(16) uniqCombined(17) uniqCombined(18) - uniqCombined(19) - uniqCombined(20) uniqUpTo(3) uniqUpTo(5) uniqUpTo(10) diff --git a/dbms/tests/queries/0_stateless/00063_check_query.sql b/dbms/tests/queries/0_stateless/00063_check_query.sql index c9493d19d34..2b806cb3bf2 100644 --- a/dbms/tests/queries/0_stateless/00063_check_query.sql +++ b/dbms/tests/queries/0_stateless/00063_check_query.sql @@ -1,3 +1,5 @@ +SET check_query_single_value_result = 1; + DROP TABLE IF EXISTS check_query_tiny_log; CREATE TABLE check_query_tiny_log (N UInt32, S String) Engine = TinyLog; diff --git a/dbms/tests/queries/0_stateless/00110_external_sort.sql b/dbms/tests/queries/0_stateless/00110_external_sort.sql index 91459d2dabb..e7d6b41837b 100644 --- a/dbms/tests/queries/0_stateless/00110_external_sort.sql +++ b/dbms/tests/queries/0_stateless/00110_external_sort.sql @@ -1,3 +1,3 @@ -SET max_memory_usage = 100000000; +SET max_memory_usage = 300000000; SET max_bytes_before_external_sort = 20000000; SELECT number FROM (SELECT number FROM system.numbers LIMIT 10000000) ORDER BY number * 1234567890123456789 LIMIT 9999990, 10; diff --git a/dbms/tests/queries/0_stateless/00111_shard_external_sort_distributed.sql b/dbms/tests/queries/0_stateless/00111_shard_external_sort_distributed.sql index 676be19852b..5f5fa5443fd 100644 --- a/dbms/tests/queries/0_stateless/00111_shard_external_sort_distributed.sql +++ b/dbms/tests/queries/0_stateless/00111_shard_external_sort_distributed.sql @@ -1,4 +1,4 @@ -SET max_memory_usage = 100000000; +SET max_memory_usage = 300000000; SET max_bytes_before_external_sort = 20000000; DROP TABLE IF EXISTS numbers10m; diff --git a/dbms/tests/queries/0_stateless/00153_transform.sql b/dbms/tests/queries/0_stateless/00153_transform.sql index a5b79eeecec..a5e531d36a4 100644 --- a/dbms/tests/queries/0_stateless/00153_transform.sql +++ b/dbms/tests/queries/0_stateless/00153_transform.sql @@ -12,3 +12,7 @@ SELECT transform(1, [2, 3], ['Яндекс', 'Google'], 'Остальные') AS SELECT transform(2, [2, 3], ['Яндекс', 'Google'], 'Остальные') AS title; SELECT transform(3, [2, 3], ['Яндекс', 'Google'], 'Остальные') AS title; SELECT transform(4, [2, 3], ['Яндекс', 'Google'], 'Остальные') AS title; +SELECT transform('hello', 'wrong', 1); -- { serverError 43 } +SELECT transform('hello', ['wrong'], 1); -- { serverError 43 } +SELECT transform('hello', ['wrong'], [1]); -- { serverError 43 } +SELECT transform(tuple(1), ['sdf'], [1]); -- { serverError 43 } diff --git a/dbms/tests/queries/0_stateless/00254_tuple_extremes.sql b/dbms/tests/queries/0_stateless/00254_tuple_extremes.sql index f87fdf93d3b..9c78463bc38 100644 --- a/dbms/tests/queries/0_stateless/00254_tuple_extremes.sql +++ b/dbms/tests/queries/0_stateless/00254_tuple_extremes.sql @@ -1 +1,8 @@ -SELECT number, (number, toDate('2015-01-01') + number) FROM system.numbers LIMIT 10 SETTINGS extremes = 1; +drop table if exists numbers_10; + +create table numbers_10 (number UInt64) engine = MergeTree order by number; +insert into numbers_10 select number from system.numbers limit 10; + +SELECT number, (number, toDate('2015-01-01') + number) FROM numbers_10 LIMIT 10 SETTINGS extremes = 1; + +drop table if exists numbers_10; diff --git a/dbms/tests/queries/0_stateless/00365_statistics_in_formats.sh b/dbms/tests/queries/0_stateless/00365_statistics_in_formats.sh index 202611821de..f0e23337806 100755 --- a/dbms/tests/queries/0_stateless/00365_statistics_in_formats.sh +++ b/dbms/tests/queries/0_stateless/00365_statistics_in_formats.sh @@ -3,10 +3,16 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh -$CLICKHOUSE_CLIENT --query="SELECT number FROM system.numbers LIMIT 10 FORMAT JSON" | grep 'rows_read'; -$CLICKHOUSE_CLIENT --query="SELECT number FROM system.numbers LIMIT 10 FORMAT JSONCompact" | grep 'rows_read'; -$CLICKHOUSE_CLIENT --query="SELECT number FROM system.numbers LIMIT 10 FORMAT XML" | grep 'rows_read'; +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.numbers"; +$CLICKHOUSE_CLIENT --query="CREATE TABLE test.numbers (number UInt64) engine = MergeTree order by number"; +$CLICKHOUSE_CLIENT --query="INSERT INTO test.numbers select * from system.numbers limit 10"; -${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "SELECT number FROM system.numbers LIMIT 10 FORMAT JSON" | grep 'rows_read'; -${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "SELECT number FROM system.numbers LIMIT 10 FORMAT JSONCompact" | grep 'rows_read'; -${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "SELECT number FROM system.numbers LIMIT 10 FORMAT XML" | grep 'rows_read'; +$CLICKHOUSE_CLIENT --query="SELECT number FROM test.numbers LIMIT 10 FORMAT JSON" | grep 'rows_read'; +$CLICKHOUSE_CLIENT --query="SELECT number FROM test.numbers LIMIT 10 FORMAT JSONCompact" | grep 'rows_read'; +$CLICKHOUSE_CLIENT --query="SELECT number FROM test.numbers LIMIT 10 FORMAT XML" | grep 'rows_read'; + +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "SELECT number FROM test.numbers LIMIT 10 FORMAT JSON" | grep 'rows_read'; +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "SELECT number FROM test.numbers LIMIT 10 FORMAT JSONCompact" | grep 'rows_read'; +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "SELECT number FROM test.numbers LIMIT 10 FORMAT XML" | grep 'rows_read'; + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.numbers"; \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh b/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh index 76ca56efca8..2ae5d905fbe 100755 --- a/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh +++ b/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh @@ -6,8 +6,9 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) ${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}?max_block_size=5&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0" -d 'SELECT max(number) FROM numbers(10)' 2>&1 | grep -E 'Content-Encoding|X-ClickHouse-Progress|^[0-9]' # This test will fail with external poco (progress not supported) -${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}?max_block_size=1&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0" -d 'SELECT number FROM system.numbers LIMIT 10' 2>&1 | grep -E 'Content-Encoding|X-ClickHouse-Progress|^[0-9]' -${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?max_block_size=1&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0&enable_http_compression=1" -H 'Accept-Encoding: gzip' -d 'SELECT number FROM system.numbers LIMIT 10' | gzip -d +# "grep -v 11" in order to skip extra progress header for 11-th row (for processors pipeline) +${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}?max_block_size=1&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0&experimental_use_processors=0" -d 'SELECT number FROM system.numbers LIMIT 10' 2>&1 | grep -E 'Content-Encoding|X-ClickHouse-Progress|^[0-9]' | grep -v 11 +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?max_block_size=1&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0&enable_http_compression=1experimental_use_processors=0" -H 'Accept-Encoding: gzip' -d 'SELECT number FROM system.numbers LIMIT 10' | gzip -d # 'send_progress_in_http_headers' is false by default ${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}?max_block_size=1&http_headers_progress_interval_ms=0" -d 'SELECT number FROM system.numbers LIMIT 10' 2>&1 | grep -q 'X-ClickHouse-Progress' && echo 'Fail' || true diff --git a/dbms/tests/queries/0_stateless/00584_view_union_all.sql b/dbms/tests/queries/0_stateless/00584_view_union_all.sql index 87242842b23..2e4d7ea66db 100644 --- a/dbms/tests/queries/0_stateless/00584_view_union_all.sql +++ b/dbms/tests/queries/0_stateless/00584_view_union_all.sql @@ -23,7 +23,7 @@ CREATE VIEW TestView AS FROM Test_00584 GROUP BY str; -SELECT * FROM TestView; +SELECT * FROM TestView ORDER BY key; DROP TABLE TestView; DROP TABLE Test_00584; diff --git a/dbms/tests/queries/0_stateless/00587_union_all_type_conversions.reference b/dbms/tests/queries/0_stateless/00587_union_all_type_conversions.reference index be82f93ae83..326729c6098 100644 --- a/dbms/tests/queries/0_stateless/00587_union_all_type_conversions.reference +++ b/dbms/tests/queries/0_stateless/00587_union_all_type_conversions.reference @@ -7,12 +7,12 @@ 1 Nullable(UInt8) \N Nullable(UInt8) 1 -\N 1 +\N 1 Nullable(Float64) 2 \N Nullable(Float64) 1 -1 -2 3 +2 +1 nan \N diff --git a/dbms/tests/queries/0_stateless/00587_union_all_type_conversions.sql b/dbms/tests/queries/0_stateless/00587_union_all_type_conversions.sql index ad7b0a81149..6bc6d98d171 100644 --- a/dbms/tests/queries/0_stateless/00587_union_all_type_conversions.sql +++ b/dbms/tests/queries/0_stateless/00587_union_all_type_conversions.sql @@ -1,12 +1,12 @@ SET max_threads = 1; -SELECT 1 UNION ALL SELECT -1; -SELECT x, toTypeName(x) FROM (SELECT 1 AS x UNION ALL SELECT -1); +SELECT * FROM (SELECT 1 as x UNION ALL SELECT -1) ORDER BY x DESC; +SELECT * FROM (SELECT x, toTypeName(x) FROM (SELECT 1 AS x UNION ALL SELECT -1)) ORDER BY x DESC; -SELECT 1 UNION ALL SELECT NULL; -SELECT x, toTypeName(x) FROM (SELECT 1 AS x UNION ALL SELECT NULL); +SELECT x FROM (SELECT 1 as x UNION ALL SELECT NULL) ORDER BY x DESC; +SELECT * FROM (SELECT x, toTypeName(x) FROM (SELECT 1 AS x UNION ALL SELECT NULL)) ORDER BY x DESC; -SELECT 1 AS x UNION ALL SELECT NULL UNION ALL SELECT 1.0; -SELECT x, toTypeName(x), count() FROM (SELECT 1 AS x UNION ALL SELECT NULL UNION ALL SELECT 1.0) GROUP BY x; +SELECT x FROM (SELECT 1 AS x UNION ALL SELECT NULL UNION ALL SELECT 1.0) ORDER BY x DESC; +SELECT * FROM (SELECT x, toTypeName(x), count() FROM (SELECT 1 AS x UNION ALL SELECT NULL UNION ALL SELECT 1.0) GROUP BY x) ORDER BY x DESC; -SELECT arrayJoin(x) AS res FROM (SELECT [1, 2, 3] AS x UNION ALL SELECT [nan, NULL]) ORDER BY res; +SELECT res FROM (SELECT arrayJoin(x) AS res FROM (SELECT [1, 2, 3] AS x UNION ALL SELECT [nan, NULL]) ORDER BY res) ORDER BY res DESC; diff --git a/dbms/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh b/dbms/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh index 47acdd34a19..eb418277c8f 100755 --- a/dbms/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh +++ b/dbms/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh @@ -13,7 +13,7 @@ server_logs_file=${CLICKHOUSE_TMP}/$cur_name"_server.logs" server_logs="--server_logs_file=$server_logs_file" rm -f "$server_logs_file" -settings="$server_logs --log_queries=1 --log_query_threads=1 --log_profile_events=1 --log_query_settings=1" +settings="$server_logs --log_queries=1 --log_query_threads=1 --log_profile_events=1 --log_query_settings=1 --experimental_use_processors=0" # Test insert logging on each block and checkPacket() method diff --git a/dbms/tests/queries/0_stateless/00640_endsWith.sql b/dbms/tests/queries/0_stateless/00640_endsWith.sql index cd2e6a08e5c..c497f529954 100644 --- a/dbms/tests/queries/0_stateless/00640_endsWith.sql +++ b/dbms/tests/queries/0_stateless/00640_endsWith.sql @@ -12,4 +12,6 @@ INSERT INTO endsWith_test values ('11', '22', '33'), ('a', 'a', 'bb'), ('abc', ' SELECT COUNT() FROM endsWith_test WHERE endsWith(S1, S1); SELECT COUNT() FROM endsWith_test WHERE endsWith(S1, S2); SELECT COUNT() FROM endsWith_test WHERE endsWith(S2, S3); + +SELECT endsWith([], 'str'); -- { serverError 43 } DROP TABLE endsWith_test; diff --git a/dbms/tests/queries/0_stateless/00734_timeslot.sql b/dbms/tests/queries/0_stateless/00734_timeslot.sql index 51e6a3f5330..f9422ee8f16 100644 --- a/dbms/tests/queries/0_stateless/00734_timeslot.sql +++ b/dbms/tests/queries/0_stateless/00734_timeslot.sql @@ -2,3 +2,6 @@ SELECT timeSlot(toDateTime('2000-01-02 03:04:05', 'UTC')); SELECT timeSlots(toDateTime('2000-01-02 03:04:05', 'UTC'), toUInt32(10000)); SELECT timeSlots(toDateTime('2000-01-02 03:04:05', 'UTC'), toUInt32(10000), 600); SELECT timeSlots(toDateTime('2000-01-02 03:04:05', 'UTC'), toUInt32(600), 30); +SELECT timeSlots(toDateTime('2000-01-02 03:04:05', 'UTC'), 'wrong argument'); -- { serverError 43 } +SELECT timeSlots(toDateTime('2000-01-02 03:04:05', 'UTC'), toUInt32(600), 'wrong argument'); -- { serverError 43 } +SELECT timeSlots(toDateTime('2000-01-02 03:04:05', 'UTC'), toUInt32(600), 0); -- { serverError 44 } \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/00753_quantile_format.reference b/dbms/tests/queries/0_stateless/00753_quantile_format.reference index e61cee21c64..04bb0000b3e 100644 --- a/dbms/tests/queries/0_stateless/00753_quantile_format.reference +++ b/dbms/tests/queries/0_stateless/00753_quantile_format.reference @@ -6,6 +6,10 @@ ['2016-06-15 23:00:00'] 2016-06-15 23:00:00 ['2016-06-15 23:00:00'] +30000 +[30000] +30000 +[30000] 2016-06-15 23:01:04 ['2016-06-15 23:01:04'] 2016-06-15 23:01:04 diff --git a/dbms/tests/queries/0_stateless/00753_quantile_format.sql b/dbms/tests/queries/0_stateless/00753_quantile_format.sql index 3a0947015ea..24cf0e4bf90 100644 --- a/dbms/tests/queries/0_stateless/00753_quantile_format.sql +++ b/dbms/tests/queries/0_stateless/00753_quantile_format.sql @@ -15,11 +15,11 @@ SELECT quantilesExact(0.2)(d) FROM datetime; SELECT quantileExactWeighted(0.2)(d, 1) FROM datetime; SELECT quantilesExactWeighted(0.2)(d, 1) FROM datetime; -SELECT quantileTiming(0.2)(d) FROM datetime; -- { serverError 43 } -SELECT quantilesTiming(0.2)(d) FROM datetime; -- { serverError 43 } +SELECT quantileTiming(0.2)(d) FROM datetime; +SELECT quantilesTiming(0.2)(d) FROM datetime; -SELECT quantileTimingWeighted(0.2)(d, 1) FROM datetime; -- { serverError 43 } -SELECT quantilesTimingWeighted(0.2)(d, 1) FROM datetime; -- { serverError 43 } +SELECT quantileTimingWeighted(0.2)(d, 1) FROM datetime; +SELECT quantilesTimingWeighted(0.2)(d, 1) FROM datetime; SELECT quantileTDigest(0.2)(d) FROM datetime; SELECT quantilesTDigest(0.2)(d) FROM datetime; diff --git a/dbms/tests/queries/0_stateless/00876_wrong_arraj_join_column.reference b/dbms/tests/queries/0_stateless/00876_wrong_arraj_join_column.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/00876_wrong_arraj_join_column.sql b/dbms/tests/queries/0_stateless/00876_wrong_arraj_join_column.sql new file mode 100644 index 00000000000..0e72f9a67ce --- /dev/null +++ b/dbms/tests/queries/0_stateless/00876_wrong_arraj_join_column.sql @@ -0,0 +1,9 @@ +DROP TABLE IF EXISTS visits; +CREATE TABLE visits (str String) ENGINE = MergeTree ORDER BY (str); + +SELECT 1 +FROM visits +ARRAY JOIN arrayFilter(t -> 1, arrayMap(x -> tuple(x), [42])) AS i +WHERE ((str, i.1) IN ('x', 0)); + +DROP TABLE visits; diff --git a/dbms/tests/queries/0_stateless/00877_memory_limit_for_new_delete.reference b/dbms/tests/queries/0_stateless/00877_memory_limit_for_new_delete.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/00877_memory_limit_for_new_delete.sql b/dbms/tests/queries/0_stateless/00877_memory_limit_for_new_delete.sql new file mode 100644 index 00000000000..111104bb06e --- /dev/null +++ b/dbms/tests/queries/0_stateless/00877_memory_limit_for_new_delete.sql @@ -0,0 +1,7 @@ +SET max_memory_usage = 1000000000; + +SELECT sum(ignore(*)) FROM ( + SELECT number, argMax(number, (number, toFixedString(toString(number), 1024))) + FROM numbers(1000000) + GROUP BY number +) -- { serverError 241 } diff --git a/dbms/tests/queries/0_stateless/00898_quantile_timing_parameter_check.reference b/dbms/tests/queries/0_stateless/00898_quantile_timing_parameter_check.reference index 7ed6ff82de6..47586a86a32 100644 --- a/dbms/tests/queries/0_stateless/00898_quantile_timing_parameter_check.reference +++ b/dbms/tests/queries/0_stateless/00898_quantile_timing_parameter_check.reference @@ -1 +1,2 @@ 5 +2 diff --git a/dbms/tests/queries/0_stateless/00898_quantile_timing_parameter_check.sql b/dbms/tests/queries/0_stateless/00898_quantile_timing_parameter_check.sql index 8279804a22c..ce1f3e897d3 100644 --- a/dbms/tests/queries/0_stateless/00898_quantile_timing_parameter_check.sql +++ b/dbms/tests/queries/0_stateless/00898_quantile_timing_parameter_check.sql @@ -1,2 +1,2 @@ SELECT quantileTiming(0.5)(number) FROM numbers(10); -SELECT quantileTiming(0.5)(number / 2) FROM numbers(10); -- { serverError 43 } +SELECT quantileTiming(0.5)(number / 2) FROM numbers(10); diff --git a/dbms/tests/queries/0_stateless/00918_json_functions.reference b/dbms/tests/queries/0_stateless/00918_json_functions.reference index 631d421b66f..44d4dc1c9bf 100644 --- a/dbms/tests/queries/0_stateless/00918_json_functions.reference +++ b/dbms/tests/queries/0_stateless/00918_json_functions.reference @@ -1,3 +1,79 @@ +--allow_simdjson=1-- +--JSONLength-- +2 +3 +0 +--JSONHas-- +1 +1 +0 +--JSONKey-- +a +b +b +a +--JSONType-- +Object +Array +--JSONExtract-- +-100 +200 +300 +1 +0 +--JSONExtractString-- +hello +hello +\n\0 +☺ + + +--JSONExtract (generic)-- +('hello',[-100,200,300]) +('hello',[-100,200,300]) +([-100,200,300],'hello') +('hello\0',0) +hello +[-100,200,300] +(-100,200,300) +[-100,0,0] +[-100,NULL,NULL] +[0,200,0] +[NULL,200,NULL] +-100 +200 +\N +1 +Thursday +Friday +--JSONExtractKeysAndValues-- +[('a','hello')] +[('b',[-100,200,300])] +[('a','hello'),('b','world')] +[('a',5),('b',7),('c',11)] +--JSONExtractRaw-- +{"a":"hello","b":[-100,200,300]} +"hello" +[-100,200,300] +-100 +{"a":"hello","b":[-100,200,300],"c":{"d":[121,144]}} +{"d":[121,144]} +[121,144] +144 + +{"passed":true} +{} +"\\n\\u0000" +"☺" +--const/non-const mixed-- +a +b +c +d +e +u +v +--allow_simdjson=0-- --JSONLength-- 2 3 diff --git a/dbms/tests/queries/0_stateless/00918_json_functions.sql b/dbms/tests/queries/0_stateless/00918_json_functions.sql index 1a9ce2bbc11..83f6d1578f9 100644 --- a/dbms/tests/queries/0_stateless/00918_json_functions.sql +++ b/dbms/tests/queries/0_stateless/00918_json_functions.sql @@ -1,3 +1,4 @@ +SELECT '--allow_simdjson=1--'; SET allow_simdjson=1; SELECT '--JSONLength--'; @@ -78,3 +79,87 @@ SELECT JSONExtractRaw('{"abc":"\\u263a"}', 'abc'); SELECT '--const/non-const mixed--'; SELECT JSONExtractString('["a", "b", "c", "d", "e"]', idx) FROM (SELECT arrayJoin([1,2,3,4,5]) AS idx); SELECT JSONExtractString(json, 's') FROM (SELECT arrayJoin(['{"s":"u"}', '{"s":"v"}']) AS json); + + + +SELECT '--allow_simdjson=0--'; +SET allow_simdjson=0; + +SELECT '--JSONLength--'; +SELECT JSONLength('{"a": "hello", "b": [-100, 200.0, 300]}'); +SELECT JSONLength('{"a": "hello", "b": [-100, 200.0, 300]}', 'b'); +SELECT JSONLength('{}'); + +SELECT '--JSONHas--'; +SELECT JSONHas('{"a": "hello", "b": [-100, 200.0, 300]}', 'a'); +SELECT JSONHas('{"a": "hello", "b": [-100, 200.0, 300]}', 'b'); +SELECT JSONHas('{"a": "hello", "b": [-100, 200.0, 300]}', 'c'); + +SELECT '--JSONKey--'; +SELECT JSONKey('{"a": "hello", "b": [-100, 200.0, 300]}', 1); +SELECT JSONKey('{"a": "hello", "b": [-100, 200.0, 300]}', 2); +SELECT JSONKey('{"a": "hello", "b": [-100, 200.0, 300]}', -1); +SELECT JSONKey('{"a": "hello", "b": [-100, 200.0, 300]}', -2); + +SELECT '--JSONType--'; +SELECT JSONType('{"a": "hello", "b": [-100, 200.0, 300]}'); +SELECT JSONType('{"a": "hello", "b": [-100, 200.0, 300]}', 'b'); + +SELECT '--JSONExtract--'; +SELECT JSONExtractInt('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 1); +SELECT JSONExtractFloat('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 2); +SELECT JSONExtractUInt('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', -1); +SELECT JSONExtractBool('{"passed": true}', 'passed'); +SELECT JSONExtractBool('"HX-='); + +SELECT '--JSONExtractString--'; +SELECT JSONExtractString('{"a": "hello", "b": [-100, 200.0, 300]}', 'a'); +SELECT JSONExtractString('{"a": "hello", "b": [-100, 200.0, 300]}', 1); +select JSONExtractString('{"abc":"\\n\\u0000"}', 'abc'); +select JSONExtractString('{"abc":"\\u263a"}', 'abc'); +select JSONExtractString('{"abc":"\\u263"}', 'abc'); +select JSONExtractString('{"abc":"hello}', 'abc'); + +SELECT '--JSONExtract (generic)--'; +SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'Tuple(String, Array(Float64))'); +SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'Tuple(a String, b Array(Float64))'); +SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'Tuple(b Array(Float64), a String)'); +SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'Tuple(a FixedString(6), c UInt8)'); +SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'a', 'String'); +SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 'Array(Float32)'); +SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 'Tuple(Int8, Float32, UInt16)'); +SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 'Array(Int8)'); +SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 'Array(Nullable(Int8))'); +SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 'Array(UInt8)'); +SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 'Array(Nullable(UInt8))'); +SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 1, 'Int8'); +SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 2, 'Int32'); +SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 4, 'Nullable(Int64)'); +SELECT JSONExtract('{"passed": true}', 'passed', 'UInt8'); +SELECT JSONExtract('{"day": "Thursday"}', 'day', 'Enum8(\'Sunday\' = 0, \'Monday\' = 1, \'Tuesday\' = 2, \'Wednesday\' = 3, \'Thursday\' = 4, \'Friday\' = 5, \'Saturday\' = 6)'); +SELECT JSONExtract('{"day": 5}', 'day', 'Enum8(\'Sunday\' = 0, \'Monday\' = 1, \'Tuesday\' = 2, \'Wednesday\' = 3, \'Thursday\' = 4, \'Friday\' = 5, \'Saturday\' = 6)'); + +SELECT '--JSONExtractKeysAndValues--'; +SELECT JSONExtractKeysAndValues('{"a": "hello", "b": [-100, 200.0, 300]}', 'String'); +SELECT JSONExtractKeysAndValues('{"a": "hello", "b": [-100, 200.0, 300]}', 'Array(Float64)'); +SELECT JSONExtractKeysAndValues('{"a": "hello", "b": "world"}', 'String'); +SELECT JSONExtractKeysAndValues('{"x": {"a": 5, "b": 7, "c": 11}}', 'x', 'Int8'); + +SELECT '--JSONExtractRaw--'; +SELECT JSONExtractRaw('{"a": "hello", "b": [-100, 200.0, 300]}'); +SELECT JSONExtractRaw('{"a": "hello", "b": [-100, 200.0, 300]}', 'a'); +SELECT JSONExtractRaw('{"a": "hello", "b": [-100, 200.0, 300]}', 'b'); +SELECT JSONExtractRaw('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 1); +SELECT JSONExtractRaw('{"a": "hello", "b": [-100, 200.0, 300], "c":{"d":[121,144]}}'); +SELECT JSONExtractRaw('{"a": "hello", "b": [-100, 200.0, 300], "c":{"d":[121,144]}}', 'c'); +SELECT JSONExtractRaw('{"a": "hello", "b": [-100, 200.0, 300], "c":{"d":[121,144]}}', 'c', 'd'); +SELECT JSONExtractRaw('{"a": "hello", "b": [-100, 200.0, 300], "c":{"d":[121,144]}}', 'c', 'd', 2); +SELECT JSONExtractRaw('{"a": "hello", "b": [-100, 200.0, 300], "c":{"d":[121,144]}}', 'c', 'd', 3); +SELECT JSONExtractRaw('{"passed": true}'); +SELECT JSONExtractRaw('{}'); +SELECT JSONExtractRaw('{"abc":"\\n\\u0000"}', 'abc'); +SELECT JSONExtractRaw('{"abc":"\\u263a"}', 'abc'); + +SELECT '--const/non-const mixed--'; +SELECT JSONExtractString('["a", "b", "c", "d", "e"]', idx) FROM (SELECT arrayJoin([1,2,3,4,5]) AS idx); +SELECT JSONExtractString(json, 's') FROM (SELECT arrayJoin(['{"s":"u"}', '{"s":"v"}']) AS json); diff --git a/dbms/tests/queries/0_stateless/00938_ipv6_cidr_range.reference b/dbms/tests/queries/0_stateless/00938_ipv6_cidr_range.reference index 496b48a70ac..fe2a43fbda5 100644 --- a/dbms/tests/queries/0_stateless/00938_ipv6_cidr_range.reference +++ b/dbms/tests/queries/0_stateless/00938_ipv6_cidr_range.reference @@ -15,3 +15,4 @@ ffff:: 4 ('f000::','ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff') ('ffff:ffff:ffff:ffff::','ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff') ('::','ff:ffff:ffff:ffff:ffff:ffff:ffff:ffff') ('f000::','ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff') +1 diff --git a/dbms/tests/queries/0_stateless/00938_ipv6_cidr_range.sql b/dbms/tests/queries/0_stateless/00938_ipv6_cidr_range.sql index 624b996200f..5f69710b220 100644 --- a/dbms/tests/queries/0_stateless/00938_ipv6_cidr_range.sql +++ b/dbms/tests/queries/0_stateless/00938_ipv6_cidr_range.sql @@ -22,3 +22,4 @@ SELECT IPv6CIDRToRange(IPv6StringToNum('2001:0db8:0000:85a3:0000:0000:ac1f:8001' SELECT IPv6CIDRToRange(IPv6StringToNum('ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff'), 64); SELECT IPv6CIDRToRange(IPv6StringToNum('0000:0000:0000:0000:0000:0000:0000:0000'), 8); SELECT IPv6CIDRToRange(IPv6StringToNum('ffff:0000:0000:0000:0000:0000:0000:0000'), 4); +SELECT IPv6CIDRToRange(IPv6StringToNum('2001:0db8:0000:85a3:0000:0000:ac1f:8001'), 128) = IPv6CIDRToRange(IPv6StringToNum('2001:0db8:0000:85a3:0000:0000:ac1f:8001'), 200) ; diff --git a/dbms/tests/queries/0_stateless/00941_to_custom_week.reference b/dbms/tests/queries/0_stateless/00941_to_custom_week.reference new file mode 100644 index 00000000000..6171bc3937f --- /dev/null +++ b/dbms/tests/queries/0_stateless/00941_to_custom_week.reference @@ -0,0 +1,73 @@ +0 0 1 1 +52 52 53 53 +1 0 +198153 198153 198252 198252 +198701 198652 +0 0 0 0 0 0 1 +1 0 1 1 1 1 1 +0 1 1 1 1 0 0 +1 1 1 2 2 1 1 +199952 200053 200152 200252 200352 200452 200601 +200001 200053 200201 200301 200401 200501 200601 +199952 200101 200201 200301 200401 200453 200552 +200001 200101 200201 200302 200402 200501 200601 +52 53 52 52 +53 52 +52 53 52 53 52 52 52 52 +0 0 52 52 0 0 52 52 +1 1 1 1 1 1 1 1 +53 52 53 52 53 52 1 52 +0 1 53 1 1 1 1 1 +200053 200052 200053 200052 200101 200052 200101 200052 +2016-12-21 52 52 201652 201652 +2016-12-22 52 52 201652 201652 +2016-12-23 52 52 201652 201652 +2016-12-24 52 52 201652 201652 +2016-12-25 53 52 201653 201652 +2016-12-26 53 1 201653 201701 +2016-12-27 53 1 201653 201701 +2016-12-28 53 1 201653 201701 +2016-12-29 53 1 201653 201701 +2016-12-30 53 1 201653 201701 +2016-12-31 53 1 201653 201701 +2017-01-01 1 1 201701 201701 +2017-01-02 1 2 201701 201702 +2017-01-03 1 2 201701 201702 +2017-01-04 1 2 201701 201702 +2017-01-05 1 2 201701 201702 +2017-01-06 1 2 201701 201702 +2017-01-07 1 2 201701 201702 +2017-01-08 2 2 201702 201702 +2017-01-09 2 3 201702 201703 +2017-01-10 2 3 201702 201703 +2016-12-22 00:00:00 52 52 201652 201652 +2016-12-23 00:00:00 52 52 201652 201652 +2016-12-24 00:00:00 52 52 201652 201652 +2016-12-25 00:00:00 53 52 201653 201652 +2016-12-26 00:00:00 53 1 201653 201701 +2016-12-27 00:00:00 53 1 201653 201701 +2016-12-28 00:00:00 53 1 201653 201701 +2016-12-29 00:00:00 53 1 201653 201701 +2016-12-30 00:00:00 53 1 201653 201701 +2016-12-31 00:00:00 53 1 201653 201701 +2017-01-01 00:00:00 1 1 201701 201701 +2017-01-02 00:00:00 1 2 201701 201702 +2017-01-03 00:00:00 1 2 201701 201702 +2017-01-04 00:00:00 1 2 201701 201702 +2017-01-05 00:00:00 1 2 201701 201702 +2017-01-06 00:00:00 1 2 201701 201702 +2017-01-07 00:00:00 1 2 201701 201702 +2017-01-08 00:00:00 2 2 201702 201702 +2017-01-09 00:00:00 2 3 201702 201703 +2017-01-10 00:00:00 2 3 201702 201703 +2017-01-11 00:00:00 2 3 201702 201703 +2018-12-25 2018-12-25 00:00:00 2018-12-23 2018-12-23 2018-12-24 2018-12-24 +2018-12-26 2018-12-26 00:00:00 2018-12-23 2018-12-23 2018-12-24 2018-12-24 +2018-12-27 2018-12-27 00:00:00 2018-12-23 2018-12-23 2018-12-24 2018-12-24 +2018-12-28 2018-12-28 00:00:00 2018-12-23 2018-12-23 2018-12-24 2018-12-24 +2018-12-29 2018-12-29 00:00:00 2018-12-23 2018-12-23 2018-12-24 2018-12-24 +2018-12-30 2018-12-30 00:00:00 2018-12-30 2018-12-30 2018-12-24 2018-12-24 +2018-12-31 2018-12-31 00:00:00 2018-12-30 2018-12-30 2018-12-31 2018-12-31 +2019-01-01 2019-01-01 00:00:00 2018-12-30 2018-12-30 2018-12-31 2018-12-31 +2019-01-02 2019-01-02 00:00:00 2018-12-30 2018-12-30 2018-12-31 2018-12-31 +2019-01-03 2019-01-03 00:00:00 2018-12-30 2018-12-30 2018-12-31 2018-12-31 diff --git a/dbms/tests/queries/0_stateless/00941_to_custom_week.sql b/dbms/tests/queries/0_stateless/00941_to_custom_week.sql new file mode 100644 index 00000000000..a6ff40a6d3f --- /dev/null +++ b/dbms/tests/queries/0_stateless/00941_to_custom_week.sql @@ -0,0 +1,52 @@ +-- week mode [0,7], week test case. refer to the mysql test case +SELECT toWeek(toDate('1998-01-01')), toWeek(toDate('1997-01-01')), toWeek(toDate('1998-01-01'), 1), toWeek(toDate('1997-01-01'), 1); +SELECT toWeek(toDate('1998-12-31')), toWeek(toDate('1997-12-31')), toWeek(toDate('1998-12-31'), 1), toWeek(toDate('1997-12-31'), 1); +SELECT toWeek(toDate('1995-01-01')), toWeek(toDate('1995-01-01'), 1); +SELECT toYearWeek(toDate('1981-12-31'), 1), toYearWeek(toDate('1982-01-01'), 1), toYearWeek(toDate('1982-12-31'), 1), toYearWeek(toDate('1983-01-01'), 1); +SELECT toYearWeek(toDate('1987-01-01'), 1), toYearWeek(toDate('1987-01-01')); + +SELECT toWeek(toDate('2000-01-01'),0) AS w2000, toWeek(toDate('2001-01-01'),0) AS w2001, toWeek(toDate('2002-01-01'),0) AS w2002,toWeek(toDate('2003-01-01'),0) AS w2003, toWeek(toDate('2004-01-01'),0) AS w2004, toWeek(toDate('2005-01-01'),0) AS w2005, toWeek(toDate('2006-01-01'),0) AS w2006; +SELECT toWeek(toDate('2000-01-06'),0) AS w2000, toWeek(toDate('2001-01-06'),0) AS w2001, toWeek(toDate('2002-01-06'),0) AS w2002,toWeek(toDate('2003-01-06'),0) AS w2003, toWeek(toDate('2004-01-06'),0) AS w2004, toWeek(toDate('2005-01-06'),0) AS w2005, toWeek(toDate('2006-01-06'),0) AS w2006; +SELECT toWeek(toDate('2000-01-01'),1) AS w2000, toWeek(toDate('2001-01-01'),1) AS w2001, toWeek(toDate('2002-01-01'),1) AS w2002,toWeek(toDate('2003-01-01'),1) AS w2003, toWeek(toDate('2004-01-01'),1) AS w2004, toWeek(toDate('2005-01-01'),1) AS w2005, toWeek(toDate('2006-01-01'),1) AS w2006; +SELECT toWeek(toDate('2000-01-06'),1) AS w2000, toWeek(toDate('2001-01-06'),1) AS w2001, toWeek(toDate('2002-01-06'),1) AS w2002,toWeek(toDate('2003-01-06'),1) AS w2003, toWeek(toDate('2004-01-06'),1) AS w2004, toWeek(toDate('2005-01-06'),1) AS w2005, toWeek(toDate('2006-01-06'),1) AS w2006; +SELECT toYearWeek(toDate('2000-01-01'),0) AS w2000, toYearWeek(toDate('2001-01-01'),0) AS w2001, toYearWeek(toDate('2002-01-01'),0) AS w2002,toYearWeek(toDate('2003-01-01'),0) AS w2003, toYearWeek(toDate('2004-01-01'),0) AS w2004, toYearWeek(toDate('2005-01-01'),0) AS w2005, toYearWeek(toDate('2006-01-01'),0) AS w2006; +SELECT toYearWeek(toDate('2000-01-06'),0) AS w2000, toYearWeek(toDate('2001-01-06'),0) AS w2001, toYearWeek(toDate('2002-01-06'),0) AS w2002,toYearWeek(toDate('2003-01-06'),0) AS w2003, toYearWeek(toDate('2004-01-06'),0) AS w2004, toYearWeek(toDate('2005-01-06'),0) AS w2005, toYearWeek(toDate('2006-01-06'),0) AS w2006; +SELECT toYearWeek(toDate('2000-01-01'),1) AS w2000, toYearWeek(toDate('2001-01-01'),1) AS w2001, toYearWeek(toDate('2002-01-01'),1) AS w2002,toYearWeek(toDate('2003-01-01'),1) AS w2003, toYearWeek(toDate('2004-01-01'),1) AS w2004, toYearWeek(toDate('2005-01-01'),1) AS w2005, toYearWeek(toDate('2006-01-01'),1) AS w2006; +SELECT toYearWeek(toDate('2000-01-06'),1) AS w2000, toYearWeek(toDate('2001-01-06'),1) AS w2001, toYearWeek(toDate('2002-01-06'),1) AS w2002,toYearWeek(toDate('2003-01-06'),1) AS w2003, toYearWeek(toDate('2004-01-06'),1) AS w2004, toYearWeek(toDate('2005-01-06'),1) AS w2005, toYearWeek(toDate('2006-01-06'),1) AS w2006; +SELECT toWeek(toDate('1998-12-31'),2),toWeek(toDate('1998-12-31'),3), toWeek(toDate('2000-01-01'),2), toWeek(toDate('2000-01-01'),3); +SELECT toWeek(toDate('2000-12-31'),2),toWeek(toDate('2000-12-31'),3); + +SELECT toWeek(toDate('1998-12-31'),0) AS w0, toWeek(toDate('1998-12-31'),1) AS w1, toWeek(toDate('1998-12-31'),2) AS w2, toWeek(toDate('1998-12-31'),3) AS w3, toWeek(toDate('1998-12-31'),4) AS w4, toWeek(toDate('1998-12-31'),5) AS w5, toWeek(toDate('1998-12-31'),6) AS w6, toWeek(toDate('1998-12-31'),7) AS w7; +SELECT toWeek(toDate('2000-01-01'),0) AS w0, toWeek(toDate('2000-01-01'),1) AS w1, toWeek(toDate('2000-01-01'),2) AS w2, toWeek(toDate('2000-01-01'),3) AS w3, toWeek(toDate('2000-01-01'),4) AS w4, toWeek(toDate('2000-01-01'),5) AS w5, toWeek(toDate('2000-01-01'),6) AS w6, toWeek(toDate('2000-01-01'),7) AS w7; +SELECT toWeek(toDate('2000-01-06'),0) AS w0, toWeek(toDate('2000-01-06'),1) AS w1, toWeek(toDate('2000-01-06'),2) AS w2, toWeek(toDate('2000-01-06'),3) AS w3, toWeek(toDate('2000-01-06'),4) AS w4, toWeek(toDate('2000-01-06'),5) AS w5, toWeek(toDate('2000-01-06'),6) AS w6, toWeek(toDate('2000-01-06'),7) AS w7; +SELECT toWeek(toDate('2000-12-31'),0) AS w0, toWeek(toDate('2000-12-31'),1) AS w1, toWeek(toDate('2000-12-31'),2) AS w2, toWeek(toDate('2000-12-31'),3) AS w3, toWeek(toDate('2000-12-31'),4) AS w4, toWeek(toDate('2000-12-31'),5) AS w5, toWeek(toDate('2000-12-31'),6) AS w6, toWeek(toDate('2000-12-31'),7) AS w7; +SELECT toWeek(toDate('2001-01-01'),0) AS w0, toWeek(toDate('2001-01-01'),1) AS w1, toWeek(toDate('2001-01-01'),2) AS w2, toWeek(toDate('2001-01-01'),3) AS w3, toWeek(toDate('2001-01-01'),4) AS w4, toWeek(toDate('2001-01-01'),5) AS w5, toWeek(toDate('2001-01-01'),6) AS w6, toWeek(toDate('2001-01-01'),7) AS w7; + +SELECT toYearWeek(toDate('2000-12-31'),0), toYearWeek(toDate('2000-12-31'),1), toYearWeek(toDate('2000-12-31'),2), toYearWeek(toDate('2000-12-31'),3), toYearWeek(toDate('2000-12-31'),4), toYearWeek(toDate('2000-12-31'),5), toYearWeek(toDate('2000-12-31'),6), toYearWeek(toDate('2000-12-31'),7); + +-- week mode 8,9 +SELECT + toDate('2016-12-21') + number AS d, + toWeek(d, 8) AS week8, + toWeek(d, 9) AS week9, + toYearWeek(d, 8) AS yearWeek8, + toYearWeek(d, 9) AS yearWeek9 +FROM numbers(21); + +SELECT toDateTime(toDate('2016-12-22') + number, 'Europe/Moscow' ) AS d, + toWeek(d, 8, 'Europe/Moscow') AS week8, + toWeek(d, 9, 'Europe/Moscow') AS week9, + toYearWeek(d, 8, 'Europe/Moscow') AS yearWeek8, + toYearWeek(d, 9, 'Europe/Moscow') AS yearWeek9 +FROM numbers(21); + +-- toStartOfWeek +SELECT + toDate('2018-12-25') + number AS x, + toDateTime(x) AS x_t, + toStartOfWeek(x) AS w0, + toStartOfWeek(x_t) AS wt0, + toStartOfWeek(x, 3) AS w3, + toStartOfWeek(x_t, 3) AS wt3 +FROM numbers(10); + diff --git a/dbms/tests/queries/0_stateless/00944_ml_test.sql b/dbms/tests/queries/0_stateless/00944_ml_test.sql index 3f731731d42..657d312229b 100644 --- a/dbms/tests/queries/0_stateless/00944_ml_test.sql +++ b/dbms/tests/queries/0_stateless/00944_ml_test.sql @@ -16,7 +16,7 @@ select ans < -61.374 and ans > -61.375 from (with (select state from remote('127.0.0.1', currentDatabase(), model)) as model select evalMLMethod(model, predict1, predict2) as ans from remote('127.0.0.1', currentDatabase(), defaults)); SELECT 0 < ans[1] and ans[1] < 0.15 and 0.95 < ans[2] and ans[2] < 1.0 and 0 < ans[3] and ans[3] < 0.05 FROM -(SELECT stochasticLinearRegression(0.000001, 0.01, 100)(number, rand() % 100, number) AS ans FROM numbers(1000)); +(SELECT stochasticLinearRegression(0.000001, 0.01, 100, 'SGD')(number, rand() % 100, number) AS ans FROM numbers(1000)); DROP TABLE model; DROP TABLE defaults; diff --git a/dbms/tests/queries/0_stateless/00946_ml_test.reference b/dbms/tests/queries/0_stateless/00946_ml_test.reference index d7d900a7713..800c6e7d6a5 100644 --- a/dbms/tests/queries/0_stateless/00946_ml_test.reference +++ b/dbms/tests/queries/0_stateless/00946_ml_test.reference @@ -11,43 +11,43 @@ 0.6542885368159769 0.6542885368159769 0.6542885368159769 -0.8444267125384497 -0.8444267125384497 -0.8444267125384497 -0.8444267125384497 -0.8444267125384497 -0.8444267125384497 -0.8444267125384497 -0.8444267125384497 -0.8444267125384497 -0.8444267125384497 -0.8444267125384497 -0.8444267125384497 -0.9683751248474649 -0.9683751248474649 -0.9683751248474649 -0.9683751248474649 -0.9683751248474649 -0.9683751248474649 -0.9683751248474649 -0.9683751248474649 -0.9683751248474649 -0.9683751248474649 -0.9683751248474649 -0.9683751248474649 -0.7836319925339996 -0.7836319925339996 -0.7836319925339996 -0.7836319925339996 -0.7836319925339996 -0.7836319925339996 -0.7836319925339996 -0.7836319925339996 -0.7836319925339996 -0.7836319925339996 -0.7836319925339996 -0.7836319925339996 -0.7836319925339996 +0.8444267125384498 +0.8444267125384498 +0.8444267125384498 +0.8444267125384498 +0.8444267125384498 +0.8444267125384498 +0.8444267125384498 +0.8444267125384498 +0.8444267125384498 +0.8444267125384498 +0.8444267125384498 +0.8444267125384498 +0.968375124847465 +0.968375124847465 +0.968375124847465 +0.968375124847465 +0.968375124847465 +0.968375124847465 +0.968375124847465 +0.968375124847465 +0.968375124847465 +0.968375124847465 +0.968375124847465 +0.968375124847465 +0.7836319925339997 +0.7836319925339997 +0.7836319925339997 +0.7836319925339997 +0.7836319925339997 +0.7836319925339997 +0.7836319925339997 +0.7836319925339997 +0.7836319925339997 +0.7836319925339997 +0.7836319925339997 +0.7836319925339997 +0.7836319925339997 0.6375535053362572 0.6375535053362572 0.6375535053362572 @@ -60,18 +60,18 @@ 0.6375535053362572 0.6375535053362572 0.6375535053362572 -0.5871709781307677 -0.5871709781307677 -0.5871709781307677 -0.5871709781307677 -0.5871709781307677 -0.5871709781307677 -0.5871709781307677 -0.5871709781307677 -0.5871709781307677 -0.5871709781307677 -0.5871709781307677 -0.5871709781307677 +0.5871709781307678 +0.5871709781307678 +0.5871709781307678 +0.5871709781307678 +0.5871709781307678 +0.5871709781307678 +0.5871709781307678 +0.5871709781307678 +0.5871709781307678 +0.5871709781307678 +0.5871709781307678 +0.5871709781307678 0.5202091999924413 0.5202091999924413 0.5202091999924413 @@ -97,18 +97,18 @@ 0.5130525169352177 0.5130525169352177 0.5130525169352177 -0.5581075117249047 -0.5581075117249047 -0.5581075117249047 -0.5581075117249047 -0.5581075117249047 -0.5581075117249047 -0.5581075117249047 -0.5581075117249047 -0.5581075117249047 -0.5581075117249047 -0.5581075117249047 -0.5581075117249047 +0.5581075117249046 +0.5581075117249046 +0.5581075117249046 +0.5581075117249046 +0.5581075117249046 +0.5581075117249046 +0.5581075117249046 +0.5581075117249046 +0.5581075117249046 +0.5581075117249046 +0.5581075117249046 +0.5581075117249046 0.6798311701936688 0.6798311701936688 0.6798311701936688 @@ -146,43 +146,43 @@ 0.7773509726916165 0.7773509726916165 0.7773509726916165 -0.8606987912604607 -0.8606987912604607 -0.8606987912604607 -0.8606987912604607 -0.8606987912604607 -0.8606987912604607 -0.8606987912604607 -0.8606987912604607 -0.8606987912604607 -0.8606987912604607 -0.8606987912604607 -0.8606987912604607 -0.8606987912604607 -0.6352934050115681 -0.6352934050115681 -0.6352934050115681 -0.6352934050115681 -0.6352934050115681 -0.6352934050115681 -0.6352934050115681 -0.6352934050115681 -0.6352934050115681 -0.6352934050115681 -0.6352934050115681 -0.6352934050115681 -0.9771089703353684 -0.9771089703353684 -0.9771089703353684 -0.9771089703353684 -0.9771089703353684 -0.9771089703353684 -0.9771089703353684 -0.9771089703353684 -0.9771089703353684 -0.9771089703353684 -0.9771089703353684 -0.9771089703353684 +0.8606987912604608 +0.8606987912604608 +0.8606987912604608 +0.8606987912604608 +0.8606987912604608 +0.8606987912604608 +0.8606987912604608 +0.8606987912604608 +0.8606987912604608 +0.8606987912604608 +0.8606987912604608 +0.8606987912604608 +0.8606987912604608 +0.635293405011568 +0.635293405011568 +0.635293405011568 +0.635293405011568 +0.635293405011568 +0.635293405011568 +0.635293405011568 +0.635293405011568 +0.635293405011568 +0.635293405011568 +0.635293405011568 +0.635293405011568 +0.9771089703353683 +0.9771089703353683 +0.9771089703353683 +0.9771089703353683 +0.9771089703353683 +0.9771089703353683 +0.9771089703353683 +0.9771089703353683 +0.9771089703353683 +0.9771089703353683 +0.9771089703353683 +0.9771089703353683 0.9955717835823895 0.9955717835823895 0.9955717835823895 @@ -196,18 +196,18 @@ 0.9955717835823895 0.9955717835823895 0.9955717835823895 -0.6124539775938347 -0.6124539775938347 -0.6124539775938347 -0.6124539775938347 -0.6124539775938347 -0.6124539775938347 -0.6124539775938347 -0.6124539775938347 -0.6124539775938347 -0.6124539775938347 -0.6124539775938347 -0.6124539775938347 +0.6124539775938348 +0.6124539775938348 +0.6124539775938348 +0.6124539775938348 +0.6124539775938348 +0.6124539775938348 +0.6124539775938348 +0.6124539775938348 +0.6124539775938348 +0.6124539775938348 +0.6124539775938348 +0.6124539775938348 0.6564358792397615 0.6564358792397615 0.6564358792397615 @@ -220,19 +220,19 @@ 0.6564358792397615 0.6564358792397615 0.6564358792397615 -0.552111558999158 -0.552111558999158 -0.552111558999158 -0.552111558999158 -0.552111558999158 -0.552111558999158 -0.552111558999158 -0.552111558999158 -0.552111558999158 -0.552111558999158 -0.552111558999158 -0.552111558999158 -0.552111558999158 +0.5521115589991579 +0.5521115589991579 +0.5521115589991579 +0.5521115589991579 +0.5521115589991579 +0.5521115589991579 +0.5521115589991579 +0.5521115589991579 +0.5521115589991579 +0.5521115589991579 +0.5521115589991579 +0.5521115589991579 +0.5521115589991579 0.7792659923782862 0.7792659923782862 0.7792659923782862 @@ -257,31 +257,31 @@ 0.6656871036437929 0.6656871036437929 0.6656871036437929 -0.7435137743371989 -0.7435137743371989 -0.7435137743371989 -0.7435137743371989 -0.7435137743371989 -0.7435137743371989 -0.7435137743371989 -0.7435137743371989 -0.7435137743371989 -0.7435137743371989 -0.7435137743371989 -0.7435137743371989 -0.7435137743371989 -0.8688023472919777 -0.8688023472919777 -0.8688023472919777 -0.8688023472919777 -0.8688023472919777 -0.8688023472919777 -0.8688023472919777 -0.8688023472919777 -0.8688023472919777 -0.8688023472919777 -0.8688023472919777 -0.8688023472919777 +0.743513774337199 +0.743513774337199 +0.743513774337199 +0.743513774337199 +0.743513774337199 +0.743513774337199 +0.743513774337199 +0.743513774337199 +0.743513774337199 +0.743513774337199 +0.743513774337199 +0.743513774337199 +0.743513774337199 +0.8688023472919778 +0.8688023472919778 +0.8688023472919778 +0.8688023472919778 +0.8688023472919778 +0.8688023472919778 +0.8688023472919778 +0.8688023472919778 +0.8688023472919778 +0.8688023472919778 +0.8688023472919778 +0.8688023472919778 0.7225690042828818 0.7225690042828818 0.7225690042828818 @@ -307,18 +307,18 @@ 0.8866100282141612 0.8866100282141612 0.8866100282141612 -0.8374461350184257 -0.8374461350184257 -0.8374461350184257 -0.8374461350184257 -0.8374461350184257 -0.8374461350184257 -0.8374461350184257 -0.8374461350184257 -0.8374461350184257 -0.8374461350184257 -0.8374461350184257 -0.8374461350184257 +0.8374461350184258 +0.8374461350184258 +0.8374461350184258 +0.8374461350184258 +0.8374461350184258 +0.8374461350184258 +0.8374461350184258 +0.8374461350184258 +0.8374461350184258 +0.8374461350184258 +0.8374461350184258 +0.8374461350184258 0.8365104788783658 0.8365104788783658 0.8365104788783658 @@ -344,18 +344,18 @@ 0.928892180915439 0.928892180915439 0.928892180915439 -0.7275019293899534 -0.7275019293899534 -0.7275019293899534 -0.7275019293899534 -0.7275019293899534 -0.7275019293899534 -0.7275019293899534 -0.7275019293899534 -0.7275019293899534 -0.7275019293899534 -0.7275019293899534 -0.7275019293899534 +0.7275019293899533 +0.7275019293899533 +0.7275019293899533 +0.7275019293899533 +0.7275019293899533 +0.7275019293899533 +0.7275019293899533 +0.7275019293899533 +0.7275019293899533 +0.7275019293899533 +0.7275019293899533 +0.7275019293899533 0.9516437185963472 0.9516437185963472 0.9516437185963472 diff --git a/dbms/tests/queries/0_stateless/00947_ml_test.sql b/dbms/tests/queries/0_stateless/00947_ml_test.sql index 25b2199b384..94e4f3b4626 100644 --- a/dbms/tests/queries/0_stateless/00947_ml_test.sql +++ b/dbms/tests/queries/0_stateless/00947_ml_test.sql @@ -12,7 +12,7 @@ insert into defaults values (-3.273, -1.452, 4.267, 20.0, 40.0), (0.121, -0.615, DROP TABLE IF EXISTS model; create table model engine = Memory as select stochasticLinearRegressionState(0.03, 0.00001, 2, 'Nesterov')(target, param1, param2) as state from defaults; -select ans > -67.0 and ans < -66.9 from +select ans > -67.01 and ans < -66.9 from (with (select state from model) as model select evalMLMethod(model, predict1, predict2) as ans from defaults limit 1); -- Check that returned weights are close to real @@ -40,10 +40,10 @@ INSERT INTO grouptest VALUES (1, 1.732, 3.653, 11.422), (1, 2.150, 2.103, 7.609), (1, 0.061, 3.310, 7.052), (1, 1.030, 3.671, 10.075), (1, 1.879, 0.578, 2.492), (1, 0.922, 2.552, 6.499), (1, 1.145, -0.095, -0.993), (1, 1.920, 0.373, 1.959), (1, 0.458, 0.094, -1.801), (1, -0.118, 3.273, 6.582), (1, 2.667, 1.472, 6.752), (1, -0.387, -0.529, -5.360), (1, 2.219, 1.790, 6.810), (1, -0.754, 2.139, 1.908), (1, -0.446, -0.668, -5.896), (1, 1.729, 0.914, 3.199), (1, 2.908, -0.420, 1.556), (1, 1.645, 3.581, 11.034), (1, 0.358, -0.950, -5.136), (1, -0.467, 2.339, 3.084), (1, 3.629, 2.959, 13.135), (1, 2.393, 0.926, 4.563), (1, -0.945, 0.281, -4.047), (1, 3.688, -0.570, 2.667), (1, 3.016, 1.775, 8.356), (1, 2.571, 0.139, 2.559), (1, 2.999, 0.956, 5.866), (1, 1.754, -0.809, -1.920), (1, 3.943, 0.382, 6.030), (1, -0.970, 2.315, 2.004), (1, 1.503, 0.790, 2.376), (1, -0.775, 2.563, 3.139), (1, 1.211, 0.113, -0.240), (1, 3.058, 0.977, 6.048), (1, 2.729, 1.634, 7.360), (1, 0.307, 2.759, 5.893), (1, 3.272, 0.181, 4.089), (1, 1.192, 1.963, 5.273), (1, 0.931, 1.447, 3.203), (1, 3.835, 3.447, 15.011), (1, 0.709, 0.008, -1.559), (1, 3.155, -0.676, 1.283), (1, 2.342, 1.047, 4.824), (1, 2.059, 1.262, 4.903), (1, 2.797, 0.855, 5.159), (1, 0.387, 0.645, -0.292), (1, 1.418, 0.408, 1.060), (1, 2.719, -0.826, -0.039), (1, 2.735, 3.736, 13.678), (1, 0.205, 0.777, -0.260), (1, 3.117, 2.063, 9.424), (1, 0.601, 0.178, -1.263), (1, 0.064, 0.157, -2.401), (1, 3.104, -0.455, 1.842), (1, -0.253, 0.672, -1.490), (1, 2.592, -0.408, 0.961), (1, -0.909, 1.314, -0.878), (1, 0.625, 2.594, 6.031), (1, 2.749, -0.210, 1.869), (1, -0.469, 1.532, 0.657), (1, 1.954, 1.827, 6.388), (1, -0.528, 1.136, -0.647), (1, 0.802, -0.583, -3.146), (1, -0.176, 1.584, 1.400), (1, -0.705, -0.785, -6.766), (1, 1.660, 2.365, 7.416), (1, 2.278, 3.977, 13.485), (1, 2.846, 3.845, 14.229), (1, 3.588, -0.401, 2.974), (1, 3.525, 3.831, 15.542), (1, 0.191, 3.312, 7.318), (1, 2.615, -0.287, 1.370), (1, 2.701, -0.446, 1.064), (1, 2.065, -0.556, -0.538), (1, 2.572, 3.618, 12.997), (1, 3.743, -0.708, 2.362), (1, 3.734, 2.319, 11.425), (1, 3.768, 2.777, 12.866), (1, 3.203, 0.958, 6.280), (1, 1.512, 2.635, 7.927), (1, 2.194, 2.323, 8.356), (1, -0.726, 2.729, 3.735), (1, 0.020, 1.704, 2.152), (1, 2.173, 2.856, 9.912), (1, 3.124, 1.705, 8.364), (1, -0.834, 2.142, 1.759), (1, -0.702, 3.024, 4.666), (1, 1.393, 0.583, 1.535), (1, 2.136, 3.770, 12.581), (1, -0.445, 0.991, -0.917), (1, 0.244, -0.835, -5.016), (1, 2.789, 0.691, 4.652), (1, 0.246, 2.661, 5.475), (1, 3.793, 2.671, 12.601), (1, 1.645, -0.973, -2.627), (1, 2.405, 1.842, 7.336), (1, 3.221, 3.109, 12.769), (1, -0.638, 3.220, 5.385), (1, 1.836, 3.025, 9.748), (1, -0.660, 1.818, 1.133), (1, 0.901, 0.981, 1.744), (1, -0.236, 3.087, 5.789), (1, 1.744, 3.864, 12.078), (1, -0.166, 3.186, 6.226), (1, 3.536, -0.090, 3.803), (1, 3.284, 2.026, 9.648), (1, 1.327, 2.822, 8.119), (1, -0.709, 0.105, -4.104), (1, 0.509, -0.989, -4.949), (1, 0.180, -0.934, -5.440), (1, 3.522, 1.374, 8.168), (1, 1.497, -0.764, -2.297), (1, 1.696, 2.364, 7.482), (1, -0.202, -0.032, -3.500), (1, 3.109, -0.138, 2.804), (1, -0.238, 2.992, 5.501), (1, 1.639, 1.634, 5.181), (1, 1.919, 0.341, 1.859), (1, -0.563, 1.750, 1.124), (1, 0.886, 3.589, 9.539), (1, 3.619, 3.020, 13.299), (1, 1.703, -0.493, -1.073), (1, 2.364, 3.764, 13.022), (1, 1.820, 1.854, 6.201), (1, 1.437, -0.765, -2.421), (1, 1.396, 0.959, 2.668), (1, 2.608, 2.032, 8.312), (1, 0.333, -0.040, -2.455), (1, 3.441, 0.824, 6.355), (1, 1.303, 2.767, 7.908), (1, 1.359, 2.404, 6.932), (1, 0.674, 0.241, -0.930), (1, 2.708, -0.077, 2.183), (1, 3.821, 3.215, 14.287), (1, 3.316, 1.591, 8.404), (1, -0.848, 1.145, -1.259), (1, 3.455, 3.081, 13.153), (1, 2.568, 0.259, 2.914), (1, 2.866, 2.636, 10.642), (1, 2.776, -0.309, 1.626), (1, 2.087, 0.619, 3.031), (1, 1.682, 1.201, 3.967), (1, 3.800, 2.600, 12.399), (1, 3.344, -0.780, 1.347), (1, 1.053, -0.817, -3.346), (1, 0.805, 3.085, 7.865), (1, 0.173, 0.069, -2.449), (1, 2.018, 1.309, 4.964), (1, 3.713, 3.804, 15.838), (1, 3.805, -0.063, 4.421), (1, 3.587, 2.854, 12.738), (1, 2.426, -0.179, 1.315), (1, 0.535, 0.572, -0.213), (1, -0.558, 0.142, -3.690), (1, -0.875, 2.700, 3.349), (1, 2.405, 3.933, 13.610), (1, 1.633, 1.222, 3.934), (1, 0.049, 2.853, 5.657), (1, 1.146, 0.907, 2.015), (1, 0.300, 0.219, -1.744), (1, 2.226, 2.526, 9.029), (1, 2.545, -0.762, -0.198), (1, 2.553, 3.956, 13.974), (1, -0.898, 2.836, 3.713), (1, 3.796, -0.202, 3.985), (1, -0.810, 2.963, 4.268), (1, 0.511, 2.104, 4.334), (1, 3.527, 3.741, 15.275), (1, -0.921, 3.094, 4.440), (1, 0.856, 3.108, 8.036), (1, 0.815, 0.565, 0.323), (1, 3.717, 0.693, 6.512), (1, 3.052, 3.558, 13.778), (1, 2.942, 3.034, 11.986), (1, 0.765, 3.177, 8.061), (1, 3.175, -0.525, 1.776), (1, 0.309, 1.006, 0.638), (1, 1.922, 0.835, 3.349), (1, 3.678, 3.314, 14.297), (1, 2.840, -0.486, 1.221), (1, 1.195, 3.396, 9.578), (1, -0.157, 3.122, 6.053), (1, 2.404, 1.434, 6.110), (1, 3.108, 2.210, 9.845), (1, 2.289, 1.188, 5.142), (1, -0.319, -0.044, -3.769), (1, -0.625, 3.701, 6.854), (1, 2.269, -0.276, 0.710), (1, 0.777, 1.963, 4.442), (1, 0.411, 1.893, 3.501), (1, 1.173, 0.461, 0.728), (1, 1.767, 3.077, 9.765), (1, 0.853, 3.076, 7.933), (1, -0.013, 3.149, 6.421), (1, 3.841, 1.526, 9.260), (1, -0.950, 0.277, -4.070), (1, -0.644, -0.747, -6.527), (1, -0.923, 1.733, 0.353), (1, 0.044, 3.037, 6.201), (1, 2.074, 2.494, 8.631), (1, 0.016, 0.961, -0.085), (1, -0.780, -0.448, -5.904), (1, 0.170, 1.936, 3.148), (1, -0.420, 3.730, 7.349), (1, -0.630, 1.504, 0.254), (1, -0.006, 0.045, -2.879), (1, 1.101, -0.985, -3.753), (1, 1.618, 0.555, 1.900), (1, -0.336, 1.408, 0.552), (1, 1.086, 3.284, 9.024), (1, -0.815, 2.032, 1.466), (1, 3.144, -0.380, 2.148), (1, 2.326, 2.077, 7.883), (1, -0.571, 0.964, -1.251), (1, 2.416, 1.255, 5.595), (1, 3.964, 1.379, 9.065), (1, 3.897, 1.553, 9.455), (1, 1.806, 2.667, 8.611), (1, 0.323, 3.809, 9.073), (1, 0.501, 3.256, 7.769), (1, -0.679, 3.539, 6.259), (1, 2.825, 3.856, 14.219), (1, 0.288, -0.536, -4.032), (1, 3.009, 0.725, 5.193), (1, -0.763, 1.140, -1.105), (1, 1.124, 3.807, 10.670), (1, 2.478, 0.204, 2.570), (1, 2.825, 2.639, 10.566), (1, 1.878, -0.883, -1.892), (1, 3.380, 2.942, 12.587), (1, 2.202, 1.739, 6.621), (1, -0.711, -0.680, -6.463), (1, -0.266, 1.827, 1.951), (1, -0.846, 1.003, -1.683), (1, 3.201, 0.132, 3.798), (1, 2.797, 0.085, 2.849), (1, 1.632, 3.269, 10.072), (1, 2.410, 2.727, 10.003), (1, -0.624, 0.853, -1.690), (1, 1.314, 3.268, 9.433), (1, -0.395, 0.450, -2.440), (1, 0.992, 3.168, 8.489), (1, 3.355, 2.106, 10.028), (1, 0.509, -0.888, -4.647), (1, 1.007, 0.797, 1.405), (1, 0.045, 0.211, -2.278), (1, -0.911, 1.093, -1.544), (1, 2.409, 0.273, 2.637), (1, 2.640, 3.540, 12.899), (1, 2.668, -0.433, 1.038), (1, -0.014, 0.341, -2.005), (1, -0.525, -0.344, -5.083), (1, 2.278, 3.517, 12.105), (1, 3.712, 0.901, 7.128), (1, -0.689, 2.842, 4.149), (1, -0.467, 1.263, -0.147), (1, 0.963, -0.653, -3.034), (1, 2.559, 2.590, 9.889), (1, 1.566, 1.393, 4.312), (1, -1.000, 1.809, 0.429), (1, -0.297, 3.221, 6.070), (1, 2.199, 3.820, 12.856), (1, 3.096, 3.251, 12.944), (1, 1.479, 1.835, 5.461), (1, 0.276, 0.773, -0.130), (1, 0.607, 1.382, 2.360), (1, 1.169, -0.108, -0.985), (1, 3.429, 0.475, 5.282), (1, 2.626, 0.104, 2.563), (1, 1.156, 3.512, 9.850), (1, 3.947, 0.796, 7.282), (1, -0.462, 2.425, 3.351), (1, 3.957, 0.366, 6.014), (1, 3.763, -0.330, 3.536), (1, 0.667, 3.361, 8.417), (1, -0.583, 0.892, -1.492), (1, -0.505, 1.344, 0.021), (1, -0.474, 2.714, 4.195), (1, 3.455, 0.014, 3.950), (1, 1.016, 1.828, 4.516), (1, 1.845, 0.193, 1.269), (1, -0.529, 3.930, 7.731), (1, 2.636, 0.045, 2.408), (1, 3.757, -0.918, 1.760), (1, -0.808, 1.160, -1.137), (1, 0.744, 1.435, 2.793), (1, 3.457, 3.566, 14.613), (1, 1.061, 3.140, 8.544), (1, 3.733, 3.368, 14.570), (1, -0.969, 0.879, -2.301), (1, 3.940, 3.136, 14.287), (1, -0.730, 2.107, 1.860), (1, 3.699, 2.820, 12.858), (1, 2.197, -0.636, -0.514), (1, 0.775, -0.979, -4.387), (1, 2.019, 2.828, 9.521), (1, 1.415, 0.113, 0.170), (1, 1.567, 3.410, 10.363), (1, 0.984, -0.960, -3.913), (1, 1.809, 2.487, 8.079), (1, 1.550, 1.130, 3.489), (1, -0.770, 3.027, 4.542), (1, -0.358, 3.326, 6.262), (1, 3.140, 0.096, 3.567), (1, -0.685, 2.213, 2.270), (1, 0.916, 0.692, 0.907), (1, 1.526, 1.159, 3.527), (1, 2.675, -0.568, 0.645), (1, 1.740, 3.019, 9.538), (1, 1.223, 2.088, 5.709), (1, 1.572, -0.125, -0.230), (1, 3.641, 0.362, 5.369), (1, 2.944, 3.897, 14.578), (1, 2.775, 2.461, 9.932), (1, -0.200, 2.492, 4.076), (1, 0.065, 2.055, 3.296), (1, 2.375, -0.639, -0.167), (1, -0.133, 1.138, 0.149), (1, -0.385, 0.163, -3.281), (1, 2.200, 0.863, 3.989), (1, -0.470, 3.492, 6.536), (1, -0.916, -0.547, -6.472), (1, 0.634, 0.927, 1.049), (1, 2.930, 2.655, 10.825), (1, 3.094, 2.802, 11.596), (1, 0.457, 0.539, -0.470), (1, 1.277, 2.229, 6.240), (1, -0.157, 1.270, 0.496), (1, 3.320, 0.640, 5.559), (1, 2.836, 1.067, 5.872), (1, 0.921, -0.716, -3.307), (1, 3.886, 1.487, 9.233), (1, 0.306, -0.142, -2.815), (1, 3.727, -0.410, 3.225), (1, 1.268, -0.801, -2.866), (1, 2.302, 2.493, 9.084), (1, 0.331, 0.373, -1.220), (1, 3.224, -0.857, 0.879), (1, 1.328, 2.786, 8.014), (1, 3.639, 1.601, 9.081), (1, 3.201, -0.484, 1.949), (1, 3.447, -0.734, 1.692), (1, 2.773, -0.143, 2.117), (1, 1.517, -0.493, -1.445), (1, 1.778, -0.428, -0.728), (1, 3.989, 0.099, 5.274), (1, 1.126, 3.985, 11.206), (1, 0.348, 0.756, -0.035), (1, 2.399, 2.576, 9.525), (1, 0.866, 1.800, 4.132), (1, 3.612, 1.598, 9.017), (1, 0.495, 2.239, 4.707), (1, 2.442, 3.712, 13.019), (1, 0.238, -0.844, -5.057), (1, 1.404, 3.095, 9.093), (1, 2.842, 2.044, 8.816), (1, 0.622, 0.322, -0.791), (1, -0.561, 1.242, -0.395), (1, 0.679, 3.822, 9.823), (1, 1.875, 3.526, 11.327), (1, 3.587, 1.050, 7.324), (1, 1.467, 0.588, 1.699), (1, 3.180, 1.571, 8.074), (1, 1.402, 0.430, 1.093), (1, 1.834, 2.209, 7.294), (1, 3.542, -0.259, 3.306), (1, -0.517, 0.174, -3.513), (1, 3.549, 2.210, 10.729), (1, 2.260, 3.393, 11.699), (1, 0.036, 1.893, 2.751), (1, 0.680, 2.815, 6.804), (1, 0.219, 0.368, -1.459), (1, -0.519, 3.987, 7.924), (1, 0.974, 0.761, 1.231), (1, 0.107, 0.620, -0.927), (1, 1.513, 1.910, 5.755), (1, 3.114, 0.894, 5.910), (1, 3.061, 3.052, 12.276), (1, 2.556, 3.779, 13.448), (1, 1.964, 2.692, 9.002), (1, 3.894, -0.032, 4.690), (1, -0.693, 0.910, -1.655), (1, 2.692, 2.908, 11.108), (1, -0.824, 1.190, -1.078), (1, 3.621, 0.918, 6.997), (1, 3.190, 2.442, 10.707), (1, 1.424, -0.546, -1.791), (1, 2.061, -0.427, -0.158), (1, 1.532, 3.158, 9.540), (1, 0.648, 3.557, 8.967), (1, 2.511, 1.665, 7.017), (1, 1.903, -0.168, 0.302), (1, -0.186, -0.718, -5.528), (1, 2.421, 3.896, 13.531), (1, 3.063, 1.841, 8.650), (1, 0.636, 1.699, 3.367), (1, 1.555, 0.688, 2.174), (1, -0.412, 0.454, -2.462), (1, 1.645, 3.207, 9.911), (1, 3.396, 3.766, 15.090), (1, 0.375, -0.256, -3.017), (1, 3.636, 0.732, 6.469), (1, 2.503, 3.133, 11.405), (1, -0.253, 0.693, -1.429), (1, 3.178, 3.110, 12.686), (1, 3.282, -0.725, 1.388), (1, -0.297, 1.222, 0.070), (1, 1.872, 3.211, 10.377), (1, 3.471, 1.446, 8.278), (1, 2.891, 0.197, 3.374), (1, -0.896, 2.198, 1.802), (1, 1.178, -0.717, -2.796), (1, 0.650, 3.371, 8.412), (1, 0.447, 3.248, 7.637), (1, 1.616, -0.109, -0.097), (1, 1.837, 1.092, 3.951), (1, 0.767, 1.384, 2.684), (1, 3.466, -0.600, 2.133), (1, -0.800, -0.734, -6.802), (1, -0.534, 0.068, -3.865), (1, 3.416, -0.459, 2.455), (1, 0.800, -0.132, -1.795), (1, 2.150, 1.190, 4.869), (1, 0.830, 1.220, 2.319), (1, 2.656, 2.587, 10.072), (1, 0.375, -0.219, -2.906), (1, 0.582, -0.637, -3.749), (1, 0.588, -0.723, -3.992), (1, 3.875, 2.126, 11.127), (1, -0.476, 1.909, 1.775), (1, 0.963, 3.597, 9.716), (1, -0.888, 3.933, 7.021), (1, 1.711, -0.868, -2.184), (1, 3.244, 1.990, 9.460), (1, -0.057, 1.537, 1.497), (1, -0.015, 3.511, 7.504), (1, 0.280, 0.582, -0.695), (1, 2.402, 2.731, 9.998), (1, 2.053, 2.253, 7.865), (1, 1.955, 0.172, 1.424), (1, 3.746, 0.872, 7.107), (1, -0.157, 2.381, 3.829), (1, 3.548, -0.918, 1.340), (1, 2.449, 3.195, 11.482), (1, 1.582, 1.055, 3.329), (1, 1.908, -0.839, -1.700), (1, 2.341, 3.137, 11.091), (1, -0.043, 3.873, 8.532), (1, 0.528, -0.752, -4.198), (1, -0.940, 0.261, -4.098), (1, 2.609, 3.531, 12.812), (1, 2.439, 2.486, 9.336), (1, -0.659, -0.150, -4.768), (1, 2.131, 1.973, 7.181), (1, 0.253, 0.304, -1.583), (1, -0.169, 2.273, 3.480), (1, 1.855, 3.974, 12.631), (1, 0.092, 1.160, 0.666), (1, 3.990, 0.402, 6.187), (1, -0.455, 0.932, -1.113), (1, 2.365, 1.152, 5.185), (1, -0.058, 1.244, 0.618), (1, 0.674, 0.481, -0.209), (1, 3.002, 0.246, 3.743), (1, 1.804, 3.765, 11.902), (1, 3.567, -0.752, 1.876), (1, 0.098, 2.257, 3.968), (1, 0.130, -0.889, -5.409), (1, 0.633, 1.891, 3.940), (1, 0.421, 2.533, 5.440), (1, 2.252, 1.853, 7.063), (1, 3.191, -0.980, 0.443), (1, -0.776, 3.241, 5.171), (1, 0.509, 1.737, 3.229), (1, 3.583, 1.274, 7.986), (1, 1.101, 2.896, 7.891), (1, 3.072, -0.008, 3.120), (1, 2.945, -0.295, 2.006), (1, 3.621, -0.161, 3.760), (1, 1.399, 3.759, 11.075), (1, 3.783, -0.866, 1.968), (1, -0.241, 2.902, 5.225), (1, 1.323, 1.934, 5.449), (1, 1.449, 2.855, 8.464), (1, 0.088, 1.526, 1.753), (1, -1.000, 2.161, 1.485), (1, -0.214, 3.358, 6.647), (1, -0.384, 3.230, 5.921), (1, 3.146, 1.228, 6.975), (1, 1.917, 0.860, 3.415), (1, 1.982, 1.735, 6.167), (1, 1.404, 1.851, 5.360), (1, 2.428, -0.674, -0.166), (1, 2.081, -0.505, -0.352), (1, 0.914, -0.543, -2.802), (1, -0.029, -0.482, -4.506), (1, 0.671, 0.184, -1.105), (1, 1.641, -0.524, -1.292), (1, 1.005, 0.361, 0.094), (1, -0.493, 3.582, 6.760), (2, 3.876, 2.563, 21.500), (2, 0.159, -0.309, 7.986), (2, -0.496, 0.417, 12.998), (2, -0.164, -0.512, 7.092), (2, 0.632, 3.200, 28.571), (2, 3.772, 0.493, 9.188), (2, 2.430, -0.797, 2.789), (2, 3.872, -0.775, 1.475), (2, -0.031, -0.256, 8.495), (2, 2.726, 3.000, 25.271), (2, 1.116, -0.269, 7.269), (2, 0.551, 3.402, 29.860), (2, 0.820, 2.500, 24.179), (2, 1.153, -0.453, 6.131), (2, -0.717, -0.360, 8.556), (2, 0.532, 0.531, 12.654), (2, 2.096, 0.981, 13.791), (2, 0.146, -0.433, 7.259), (2, 1.000, 1.075, 15.452), (2, 2.963, -0.090, 6.495), (2, 1.047, 2.052, 21.267), (2, 0.882, 1.778, 19.785), (2, 1.380, 2.702, 24.832), (2, 1.853, 0.401, 10.554), (2, 2.004, 1.770, 18.618), (2, 3.377, 0.772, 11.253), (2, 1.227, -0.169, 7.759), (2, 0.428, 2.052, 21.885), (2, 0.070, 3.648, 31.816), (2, 0.128, -0.938, 4.244), (2, 2.061, 0.753, 12.454), (2, 1.207, -0.301, 6.989), (2, -0.168, 3.765, 32.757), (2, 3.450, 1.801, 17.353), (2, -0.483, 3.344, 30.547), (2, 1.847, 1.884, 19.455), (2, 3.241, 2.369, 20.975), (2, 0.628, 3.590, 30.912), (2, 2.183, 1.741, 18.263), (2, 0.774, 2.638, 25.057), (2, 3.292, 2.867, 23.912), (2, 0.056, 2.651, 25.850), (2, -0.506, 0.300, 12.308), (2, 0.524, 1.182, 16.570), (2, -0.267, 2.563, 25.647), (2, 3.953, -0.334, 4.040), (2, 2.507, 2.319, 21.408), (2, -0.770, 1.017, 16.875), (2, 0.481, 1.591, 19.062), (2, 3.243, 1.060, 13.114), (2, 2.178, -0.325, 5.873), (2, 2.510, 1.235, 14.900), (2, 2.684, 2.370, 21.535), (2, 3.466, 3.656, 28.469), (2, 2.994, 3.960, 30.764), (2, -0.363, 3.592, 31.917), (2, 1.738, 0.074, 8.708), (2, 1.462, 3.727, 30.902), (2, 0.059, 0.180, 11.021), (2, 2.980, 2.317, 20.925), (2, 1.248, 0.965, 14.545), (2, 0.776, -0.229, 7.850), (2, -0.562, 2.839, 27.598), (2, 3.581, 0.244, 7.883), (2, -0.958, 0.901, 16.362), (2, 3.257, 0.364, 8.925), (2, 1.478, 1.718, 18.827), (2, -0.121, -0.436, 7.507), (2, 0.966, 1.444, 17.697), (2, 3.631, 3.463, 27.144), (2, 0.174, -0.663, 5.848), (2, 2.783, 0.124, 7.959), (2, 1.106, -0.936, 3.276), (2, 0.186, -0.942, 4.162), (2, 3.513, 2.456, 21.222), (2, 0.339, 2.316, 23.558), (2, 0.566, 2.515, 24.523), (2, -0.134, 0.746, 14.607), (2, 1.554, 0.106, 9.084), (2, -0.846, 2.748, 27.337), (2, 3.934, 0.564, 9.451), (2, 2.840, -0.966, 1.366), (2, 1.379, 0.307, 10.463), (2, 1.065, -0.780, 4.253), (2, 3.324, 2.145, 19.546), (2, 0.974, -0.543, 5.767), (2, 2.469, 3.976, 31.385), (2, -0.434, 3.689, 32.570), (2, 0.261, 0.481, 12.624), (2, 3.786, 2.605, 21.843), (2, -0.460, -0.536, 7.243), (2, 2.576, 2.880, 24.702), (2, -0.501, 3.551, 31.810), (2, 2.946, 3.263, 26.633), (2, 2.959, -0.813, 2.162), (2, -0.749, 0.490, 13.686), (2, 2.821, 0.335, 9.187), (2, 3.964, 0.272, 7.667), (2, 0.808, -0.700, 4.994), (2, 0.415, 2.183, 22.682), (2, 2.551, 3.785, 30.156), (2, 0.821, 1.120, 15.897), (2, 1.714, 3.019, 26.400), (2, 2.265, 1.950, 19.438), (2, 1.493, 3.317, 28.409), (2, -0.445, 2.282, 24.134), (2, -0.508, 2.508, 25.553), (2, 1.017, -0.621, 5.255), (2, 1.053, 2.246, 22.422), (2, 0.441, 1.637, 19.382), (2, 3.657, 1.246, 13.816), (2, 0.756, 0.808, 14.095), (2, 1.849, 1.599, 17.742), (2, 1.782, -0.000, 8.215), (2, 1.136, 3.940, 32.506), (2, 2.814, 3.288, 26.916), (2, 3.180, 3.198, 26.008), (2, 0.728, -0.054, 8.946), (2, 0.801, 0.775, 13.852), (2, 1.399, -0.546, 5.322), (2, 1.415, 1.753, 19.103), (2, 2.860, 1.796, 17.913), (2, 0.712, 2.902, 26.699), (2, -0.389, 3.093, 28.945), (2, 3.661, 3.666, 28.333), (2, 3.944, 0.996, 12.030), (2, 1.655, 1.385, 16.657), (2, 0.122, -0.662, 5.906), (2, 3.667, 2.763, 22.912), (2, 2.606, 0.630, 11.172), (2, -0.291, 1.492, 19.242), (2, -0.787, 1.223, 18.125), (2, 2.405, 0.325, 9.545), (2, 3.129, -0.412, 4.398), (2, 0.588, 3.964, 33.194), (2, -0.177, 3.636, 31.993), (2, 2.079, 3.280, 27.603), (2, 3.055, 3.958, 30.692), (2, -0.164, 3.188, 29.292), (2, 3.803, 3.151, 25.105), (2, 3.123, -0.891, 1.531), (2, 3.070, -0.824, 1.988), (2, 3.103, -0.931, 1.309), (2, 0.589, 3.353, 29.529), (2, 1.095, 1.973, 20.744), (2, -0.557, 0.370, 12.775), (2, 1.223, 0.307, 10.620), (2, 3.255, -0.768, 2.136), (2, 0.508, 2.157, 22.435), (2, 0.373, 0.319, 11.544), (2, 1.240, 1.736, 19.177), (2, 1.846, 0.970, 13.972), (2, 3.352, -0.534, 3.445), (2, -0.352, -0.290, 8.610), (2, 0.281, 0.193, 10.880), (2, 3.450, -0.059, 6.193), (2, 0.310, 2.575, 25.140), (2, 1.791, 1.127, 14.970), (2, 1.992, 2.347, 22.087), (2, -0.288, 2.881, 27.576), (2, 3.464, 3.664, 28.518), (2, 0.573, 2.789, 26.159), (2, 2.265, 1.583, 17.233), (2, 3.203, 0.730, 11.177), (2, 3.345, 1.368, 14.862), (2, 0.891, 3.690, 31.248), (2, 2.252, -0.311, 5.884), (2, -0.087, 0.804, 14.912), (2, 0.153, 2.510, 24.905), (2, 3.533, -0.965, 0.675), (2, 2.035, 1.953, 19.683), (2, 0.316, 2.448, 24.373), (2, 2.199, 3.858, 30.946), (2, -0.519, 3.647, 32.399), (2, 0.867, 1.961, 20.901), (2, 2.739, 2.268, 20.866), (2, 2.462, -0.664, 3.551), (2, 1.372, 3.419, 29.144), (2, -0.628, 2.723, 26.968), (2, 3.989, -0.225, 4.659), (2, 0.166, 3.190, 28.976), (2, 1.681, 2.937, 25.943), (2, 2.979, 2.263, 20.600), (2, 3.896, -0.419, 3.590), (2, 3.861, 2.224, 19.485), (2, -0.087, -0.861, 4.918), (2, 1.182, 1.886, 20.133), (2, 3.622, 2.320, 20.301), (2, 3.560, 0.008, 6.491), (2, 3.082, -0.605, 3.285), (2, 1.777, 1.324, 16.169), (2, 2.269, 2.436, 22.348), (2, 0.019, 3.074, 28.423), (2, -0.560, 3.868, 33.765), (2, 1.568, 2.886, 25.749), (2, 2.045, 0.222, 9.286), (2, 1.391, 0.352, 10.723), (2, 0.172, 1.908, 21.276), (2, 1.173, -0.726, 4.474), (2, 1.642, 2.576, 23.814), (2, 3.346, 1.377, 14.918), (2, 0.120, 0.411, 12.344), (2, 3.913, 0.820, 11.008), (2, 1.054, 3.732, 31.340), (2, 2.284, 0.108, 8.362), (2, 2.266, 0.066, 8.131), (2, 3.204, 1.156, 13.735), (2, 3.243, 2.032, 18.947), (2, 3.052, -0.121, 6.221), (2, 1.131, 2.189, 22.000), (2, 2.958, 0.658, 10.990), (2, 1.717, 3.708, 30.530), (2, 2.417, 2.070, 20.004), (2, 2.175, 0.881, 13.110), (2, 0.333, 3.494, 30.629), (2, 3.598, 3.940, 30.044), (2, 3.683, -0.110, 5.660), (2, 2.555, 1.196, 14.620), (2, 1.511, 0.453, 11.206), (2, 0.903, 1.390, 17.439), (2, -0.897, 3.303, 30.716), (2, 0.245, 2.129, 22.527), (2, 1.370, 2.715, 24.923), (2, 1.822, -0.917, 2.676), (2, 2.690, -0.109, 6.657), (2, 0.206, 1.561, 19.162), (2, 3.905, 2.710, 22.357), (2, -0.438, 3.207, 29.678), (2, 0.898, 3.445, 29.772), (2, 1.838, 2.871, 25.385), (2, 0.116, 1.401, 18.292), (2, -0.408, 2.375, 24.656), (2, 1.681, 3.338, 28.349), (2, 1.177, -0.318, 6.914), (2, 1.004, 0.626, 12.753), (2, 2.840, 2.589, 22.691), (2, 1.258, 3.993, 32.700), (2, 2.016, 3.489, 28.920), (2, -0.728, 0.164, 11.713), (2, 0.193, 1.479, 18.682), (2, 2.647, -0.969, 1.541), (2, 3.837, 2.602, 21.773), (2, 0.541, 0.205, 10.690), (2, 0.026, 2.756, 26.511), (2, 0.924, 0.909, 14.530), (2, 0.974, -0.074, 8.581), (2, 0.081, 0.005, 9.948), (2, 1.331, 2.942, 26.320), (2, 2.498, 3.405, 27.934), (2, 3.741, 1.554, 15.581), (2, 3.502, -0.089, 5.964), (2, 3.069, 1.768, 17.539), (2, 3.115, -0.008, 6.839), (2, 3.237, -0.503, 3.745), (2, 0.768, -0.135, 8.420), (2, 0.410, 3.974, 33.437), (2, 0.238, -0.700, 5.564), (2, 3.619, 0.350, 8.482), (2, 3.563, 3.059, 24.788), (2, 2.916, 3.101, 25.691), (2, 0.144, 3.282, 29.549), (2, 1.288, 2.642, 24.565), (2, -0.859, 0.229, 12.234), (2, 1.507, -0.711, 4.229), (2, -0.634, 2.608, 26.281), (2, 2.054, -0.834, 2.942), (2, 0.453, 1.072, 15.980), (2, 3.914, 1.159, 13.039), (2, 0.254, 1.835, 20.758), (2, 1.577, 0.428, 10.991), (2, 1.990, 3.569, 29.421), (2, 1.584, 1.803, 19.234), (2, 0.835, 3.603, 30.785), (2, 0.900, 3.033, 27.296), (2, 1.180, 0.280, 10.499), (2, 2.400, 2.802, 24.409), (2, 0.924, 2.462, 23.851), (2, 2.138, 0.722, 12.192), (2, -0.253, -0.809, 5.401), (2, 3.570, -0.116, 5.733), (2, 0.201, -0.182, 8.708), (2, 2.457, 0.454, 10.267), (2, -0.053, 0.443, 12.709), (2, 2.108, 2.069, 20.309), (2, -0.964, -0.441, 8.318), (2, 1.802, 0.403, 10.614), (2, 3.704, 3.902, 29.711), (2, 1.904, 2.418, 22.603), (2, 2.965, 3.429, 27.606), (2, -0.801, -0.072, 10.370), (2, 3.009, 0.491, 9.937), (2, 2.781, 1.026, 13.376), (2, -0.421, 0.744, 14.883), (2, 3.639, -0.148, 5.476), (2, 0.584, 2.041, 21.663), (2, 1.547, -0.391, 6.107), (2, -0.204, 0.727, 14.564), (2, 0.372, 0.464, 12.410), (2, 1.185, 1.732, 19.207), (2, 3.574, 0.755, 10.954), (2, 2.164, 1.425, 16.385), (2, 1.895, 1.374, 16.351), (2, 2.352, 2.188, 20.779), (2, 0.187, 0.677, 13.874), (2, -0.589, 3.686, 32.703), (2, 3.081, 0.414, 9.403), (2, 3.341, 3.246, 26.137), (2, 0.617, -0.201, 8.174), (2, 1.518, 3.833, 31.481), (2, 2.613, -0.350, 5.286), (2, 3.426, 0.751, 11.082), (2, 2.726, 3.586, 28.787), (2, 2.834, -0.219, 5.855), (2, 1.038, 3.607, 30.605), (2, 0.479, 1.226, 16.874), (2, 1.729, 0.297, 10.053), (2, 0.050, 1.815, 20.841), (2, -0.554, 3.538, 31.782), (2, 2.773, 0.973, 13.064), (2, -0.239, 3.425, 30.786), (2, 3.611, 3.700, 28.590), (2, 1.418, 3.625, 30.332), (2, 1.599, 1.626, 18.156), (2, 1.841, 1.518, 17.269), (2, 1.119, 1.996, 20.856), (2, 2.810, 2.293, 20.947), (2, 1.174, 2.062, 21.198), (2, -0.326, -0.279, 8.655), (2, -0.365, 0.816, 15.259), (2, 1.296, -0.095, 8.132), (2, -0.263, 0.511, 13.327), (2, 1.757, 3.012, 26.314), (2, 1.849, 1.065, 14.539), (2, 1.651, 2.244, 21.814), (2, 3.942, 1.026, 12.214), (2, 2.314, 1.944, 19.353), (2, 3.055, -0.002, 6.930), (2, 0.402, 1.350, 17.698), (2, 0.004, 2.288, 23.724), (2, 3.265, 2.962, 24.509), (2, 1.044, -0.684, 4.850), (2, -0.280, 2.278, 23.948), (2, 1.216, 0.726, 13.142), (2, 3.181, 3.518, 27.925), (2, 3.199, -0.124, 6.055), (2, 0.510, -0.622, 5.755), (2, 2.920, 1.067, 13.484), (2, 2.573, 1.844, 18.492), (2, 1.155, 3.505, 29.878), (2, 2.033, 1.756, 18.502), (2, 1.312, 0.114, 9.373), (2, -0.823, 3.339, 30.854), (2, 0.287, 3.891, 33.060), (2, -0.621, -0.210, 9.363), (2, 3.734, 1.574, 15.712), (2, -0.932, 0.772, 15.561), (2, -0.719, 1.604, 20.345), (2, -0.555, 0.773, 15.190), (2, -0.744, 3.934, 34.348), (2, 1.671, -0.425, 5.778), (2, 2.754, 2.690, 23.385), (2, 1.826, 2.185, 21.283), (2, 1.970, 0.021, 8.159), (2, 2.882, 3.494, 28.081), (2, 1.668, -0.030, 8.150), (2, 0.472, 2.184, 22.633), (2, 1.656, 3.393, 28.701), (2, -0.069, 2.331, 24.057), (2, 0.075, 1.341, 17.973), (2, 1.836, 0.565, 11.554), (2, -0.235, 0.520, 13.357), (2, 3.620, 3.169, 25.393), (2, 0.401, -0.062, 9.224), (2, 1.503, 1.667, 18.501), (2, 3.727, 1.149, 13.166), (2, 2.777, -0.081, 6.737), (2, 3.914, -0.234, 4.680), (2, 1.765, 0.750, 12.737), (2, 1.746, 1.818, 19.161), (2, 0.019, 2.819, 26.893), (2, 1.068, 1.917, 20.434), (2, 3.035, 3.158, 25.915), (2, 2.012, 0.724, 12.330), (2, 2.597, 2.264, 20.986), (2, 3.428, 3.239, 26.005), (2, -0.016, -0.529, 6.842), (2, 1.314, 0.735, 13.095), (2, 2.832, -0.567, 3.768), (2, -0.296, 2.641, 26.141), (2, 2.863, 3.889, 30.470), (2, 2.849, 3.997, 31.130), (2, 1.660, 1.813, 19.216), (2, 2.798, 0.977, 13.062), (2, 3.935, 0.549, 9.359), (2, 1.002, 3.557, 30.342), (2, 3.052, 2.207, 20.193), (2, 3.455, 0.458, 9.294), (2, 3.312, 2.138, 19.515), (2, 0.292, 0.058, 10.056), (2, 0.050, -0.211, 8.682), (2, -0.215, 1.108, 16.866), (2, -0.169, 0.647, 14.048), (2, 2.546, 0.876, 12.709), (2, -0.911, -0.209, 9.659), (2, 0.950, 2.894, 26.413), (2, -0.512, -0.167, 9.508), (2, 1.821, -0.747, 3.696), (2, 2.257, 3.945, 31.415), (2, 2.398, -0.586, 4.087), (2, 3.051, 0.815, 11.836), (2, 3.399, 2.131, 19.389), (2, 2.982, 1.549, 16.314), (2, -0.790, -0.329, 8.819), (2, 3.797, 0.327, 8.167), (2, 1.838, 0.290, 9.902), (2, 1.906, 1.782, 18.785), (2, 1.330, -0.208, 7.422), (2, -0.217, 0.854, 15.344), (2, 3.310, 1.582, 16.180), (2, 2.965, 0.917, 12.537), (2, 3.558, -0.164, 5.460), (2, -0.841, 2.060, 23.203), (2, 2.892, 2.621, 22.834), (2, -0.011, -0.198, 8.821), (2, -0.430, 2.999, 28.424), (2, -0.584, 0.894, 15.946), (2, 0.033, 1.310, 17.829), (2, 3.044, 0.410, 9.418), (2, 3.932, 0.295, 7.836), (2, 0.394, 1.315, 17.494), (2, 1.424, -0.167, 7.573), (2, 1.676, 1.118, 15.031), (2, 1.821, 0.714, 12.462), (2, 2.688, 1.497, 16.292), (2, 3.960, 2.344, 20.103), (2, -0.787, -0.161, 9.819), (2, 3.538, 3.651, 28.366), (2, -0.338, 0.458, 13.088), (2, -0.146, 3.162, 29.120), (2, 3.124, 3.352, 26.989), (2, -0.189, 3.685, 32.301), (2, 0.396, 1.004, 15.626), (2, -0.171, 2.114, 22.858), (2, 3.736, 0.732, 10.659), (2, 1.259, 2.564, 24.127), (2, -0.263, 2.426, 24.820), (2, 1.558, -0.858, 3.292), (2, 2.882, 1.110, 13.776), (2, 0.039, 1.284, 17.666), (2, 3.074, 2.379, 21.201), (2, -0.523, 0.303, 12.344), (2, 0.363, 1.082, 16.132), (2, 2.925, 2.187, 20.195), (2, 0.595, -0.335, 7.397), (2, 0.062, -0.232, 8.544), (2, 0.877, 2.155, 22.050), (2, -0.256, 2.922, 27.788), (2, 1.813, 3.161, 27.152), (2, 2.177, 2.532, 23.016), (2, -0.051, 0.035, 10.263), (2, 2.688, 3.599, 28.906), (2, 2.539, -0.076, 7.008), (2, 2.563, 1.467, 16.240), (2, -0.755, 2.276, 24.410), (2, 3.092, 0.660, 10.868), (2, 2.403, 2.693, 23.756), (2, -0.170, 2.178, 23.239), (2, 2.672, -0.603, 3.712), (2, -0.077, -0.493, 7.116), (2, 1.997, 1.934, 19.608), (2, 1.913, -0.792, 3.335), (2, 0.171, -0.329, 7.857), (2, 2.488, 0.171, 8.540), (2, -0.514, 0.331, 12.500), (2, -0.201, 2.484, 25.103), (2, 2.436, 0.032, 7.759), (2, -0.094, 2.530, 25.275), (2, 2.186, 2.591, 23.358), (2, 3.171, -0.766, 2.231), (2, 2.410, 0.183, 8.687), (2, -0.699, -0.329, 8.728), (2, 3.285, 2.252, 20.228), (2, 1.928, -0.059, 7.720), (2, 3.460, 0.399, 8.931), (2, 2.542, 0.224, 8.801), (2, 2.902, 2.101, 19.702), (2, 3.808, 2.528, 21.358), (2, 0.330, 0.642, 13.522), (2, -0.088, 1.286, 17.804), (2, 3.025, 2.354, 21.100), (2, 3.306, 2.049, 18.986), (2, 1.477, 1.720, 18.845), (2, 2.676, 3.601, 28.931), (2, 1.577, 0.170, 9.443), (2, 1.362, 3.534, 29.843), (2, 2.616, 3.106, 26.018), (2, 3.773, 0.378, 8.496), (2, -0.125, 2.057, 22.465), (2, 3.174, 1.382, 15.120), (2, 0.844, 2.058, 21.503); SELECT ANS[1] > -1.1 AND ANS[1] < -0.9 AND ANS[2] > 5.9 AND ANS[2] < 6.1 AND ANS[3] > 9.9 AND ANS[3] < 10.1 FROM -(SELECT stochasticLinearRegression(0.05, 0, 1)(target, p1, p2) AS ANS FROM grouptest GROUP BY user_id limit 0,1); +(SELECT stochasticLinearRegression(0.05, 0, 1, 'SGD')(target, p1, p2) AS ANS FROM grouptest GROUP BY user_id LIMIT 0, 1); SELECT ANS[1] > 1.9 AND ANS[1] < 2.1 AND ANS[2] > 2.9 AND ANS[2] < 3.1 AND ANS[3] > -3.1 AND ANS[3] < -2.9 FROM -(SELECT stochasticLinearRegression(0.05, 0, 1)(target, p1, p2) AS ANS FROM grouptest GROUP BY user_id limit 1, 1); +(SELECT stochasticLinearRegression(0.05, 0, 1, 'SGD')(target, p1, p2) AS ANS FROM grouptest GROUP BY user_id LIMIT 1, 1); DROP TABLE defaults; DROP TABLE model; diff --git a/dbms/tests/queries/0_stateless/00961_check_table.reference b/dbms/tests/queries/0_stateless/00961_check_table.reference new file mode 100644 index 00000000000..d85c66db622 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00961_check_table.reference @@ -0,0 +1,11 @@ +201901_1_1_0 1 +======== +201901_1_1_0 1 +201901_2_2_0 1 +======== +201901_1_2_1 1 +======== +201901_1_2_1 1 +201902_3_3_0 1 +======== +201902_3_4_1 1 diff --git a/dbms/tests/queries/0_stateless/00961_check_table.sql b/dbms/tests/queries/0_stateless/00961_check_table.sql new file mode 100644 index 00000000000..0e0b2c3b483 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00961_check_table.sql @@ -0,0 +1,38 @@ +SET check_query_single_value_result = 0; +DROP TABLE IF EXISTS mt_table; + +CREATE TABLE mt_table (d Date, key UInt64, data String) ENGINE = MergeTree() PARTITION BY toYYYYMM(d) ORDER BY key; + +CHECK TABLE mt_table; + +INSERT INTO mt_table VALUES (toDate('2019-01-02'), 1, 'Hello'), (toDate('2019-01-02'), 2, 'World'); + +CHECK TABLE mt_table; + +INSERT INTO mt_table VALUES (toDate('2019-01-02'), 3, 'quick'), (toDate('2019-01-02'), 4, 'brown'); + +SELECT '========'; + +CHECK TABLE mt_table; + +OPTIMIZE TABLE mt_table FINAL; + +SELECT '========'; + +CHECK TABLE mt_table; + +SELECT '========'; + +INSERT INTO mt_table VALUES (toDate('2019-02-03'), 5, '!'), (toDate('2019-02-03'), 6, '?'); + +CHECK TABLE mt_table; + +SELECT '========'; + +INSERT INTO mt_table VALUES (toDate('2019-02-03'), 7, 'jump'), (toDate('2019-02-03'), 8, 'around'); + +OPTIMIZE TABLE mt_table FINAL; + +CHECK TABLE mt_table PARTITION 201902; + +DROP TABLE IF EXISTS mt_table; diff --git a/dbms/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.reference b/dbms/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.reference new file mode 100644 index 00000000000..282b0ddca7b --- /dev/null +++ b/dbms/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.reference @@ -0,0 +1 @@ +20000101_20000101_1_1_0 test_00961 1c63ae7a38eb76e2a71c28aaf0b3ae4d 0053df9b467cc5483e752ec62e91cfd4 da96ff1e527a8a1f908ddf2b1d0af239 diff --git a/dbms/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.sql b/dbms/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.sql new file mode 100644 index 00000000000..5f7774ac799 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.sql @@ -0,0 +1,17 @@ +DROP TABLE IF EXISTS test_00961; + +CREATE TABLE test_00961 (d Date, a String, b UInt8, x String, y Int8, z UInt32) ENGINE = MergeTree(d, (a, b), 111); + +INSERT INTO test_00961 VALUES ('2000-01-01', 'Hello, world!', 123, 'xxx yyy', -123, 123456789); + +SELECT + name, + table, + hash_of_all_files, + hash_of_uncompressed_files, + uncompressed_hash_of_compressed_files +FROM system.parts +WHERE table = 'test_00961'; + +DROP TABLE test_00961; + diff --git a/dbms/tests/queries/0_stateless/00961_visit_param_buffer_underflow.reference b/dbms/tests/queries/0_stateless/00961_visit_param_buffer_underflow.reference new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00961_visit_param_buffer_underflow.reference @@ -0,0 +1 @@ + diff --git a/dbms/tests/queries/0_stateless/00961_visit_param_buffer_underflow.sql b/dbms/tests/queries/0_stateless/00961_visit_param_buffer_underflow.sql new file mode 100644 index 00000000000..92b7501d700 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00961_visit_param_buffer_underflow.sql @@ -0,0 +1 @@ +SELECT visitParamExtractRaw('\"a\":', 'a'); diff --git a/dbms/tests/queries/0_stateless/00962_enumNotExect.reference b/dbms/tests/queries/0_stateless/00962_enumNotExect.reference new file mode 100644 index 00000000000..06e02da1be8 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00962_enumNotExect.reference @@ -0,0 +1,14 @@ +hello +world +hello +1 +2 +1 +hello +world +hello +1 +128 +1 +Enum16(\'a\' = 2, \'b\' = 128) +Enum8(\'a\' = 2, \'b\' = 127) diff --git a/dbms/tests/queries/0_stateless/00962_enumNotExect.sql b/dbms/tests/queries/0_stateless/00962_enumNotExect.sql new file mode 100644 index 00000000000..3431f238ad5 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00962_enumNotExect.sql @@ -0,0 +1,14 @@ +DROP TABLE IF EXISTS t_enum8; +CREATE TABLE t_enum8( x Enum('hello' = 1, 'world' = 2) ) ENGINE = TinyLog; +INSERT INTO t_enum8 Values('hello'),('world'),('hello'); +SELECT * FROM t_enum8; +SELECT CAST(x, 'Int8') FROM t_enum8; +DROP TABLE t_enum8; +DROP TABLE IF EXISTS t_enum16; +CREATE TABLE t_enum16( x Enum('hello' = 1, 'world' = 128) ) ENGINE = TinyLog; +INSERT INTO t_enum16 Values('hello'),('world'),('hello'); +SELECT * FROM t_enum16; +SELECT CAST(x, 'Int16') FROM t_enum16; +DROP TABLE t_enum16; +SELECT toTypeName(CAST('a', 'Enum(\'a\' = 2, \'b\' = 128)')); +SELECT toTypeName(CAST('a', 'Enum(\'a\' = 2, \'b\' = 127)')); diff --git a/dbms/tests/queries/0_stateless/00962_visit_param_various.reference b/dbms/tests/queries/0_stateless/00962_visit_param_various.reference new file mode 100644 index 00000000000..e1b066a6135 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00962_visit_param_various.reference @@ -0,0 +1,4 @@ +123 +Hello +Hello +0 diff --git a/dbms/tests/queries/0_stateless/00962_visit_param_various.sql b/dbms/tests/queries/0_stateless/00962_visit_param_various.sql new file mode 100644 index 00000000000..d65cb88c392 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00962_visit_param_various.sql @@ -0,0 +1,5 @@ +SELECT visitParamExtractUInt('"a":123', 'a'); +SELECT visitParamExtractString('"a":"Hello"', 'a'); +SELECT visitParamExtractRaw('"a":Hello}', 'a'); + +SELECT sum(ignore(visitParamExtractRaw(concat('{"a":', reinterpretAsString(rand64())), 'a'))) FROM numbers(1000000); diff --git a/dbms/tests/queries/0_stateless/00963_achimbab.reference b/dbms/tests/queries/0_stateless/00963_achimbab.reference new file mode 100644 index 00000000000..5248223efdd --- /dev/null +++ b/dbms/tests/queries/0_stateless/00963_achimbab.reference @@ -0,0 +1,57 @@ +{ + "meta": + [ + { + "name": "total", + "type": "Nullable(UInt8)" + }, + { + "name": "arrayElement(k, 1)", + "type": "Nullable(UInt16)" + }, + { + "name": "arrayElement(k, 2)", + "type": "Nullable(UInt16)" + } + ], + + "data": + [ + { + "total": 1, + "arrayElement(k, 1)": null, + "arrayElement(k, 2)": 4 + }, + { + "total": 1, + "arrayElement(k, 1)": null, + "arrayElement(k, 2)": 2 + }, + { + "total": 1, + "arrayElement(k, 1)": null, + "arrayElement(k, 2)": 1 + }, + { + "total": 1, + "arrayElement(k, 1)": null, + "arrayElement(k, 2)": 3 + }, + { + "total": 1, + "arrayElement(k, 1)": null, + "arrayElement(k, 2)": 5 + } + ], + + "totals": + { + "total": 1, + "arrayElement(k, 1)": null, + "arrayElement(k, 2)": null + }, + + "rows": 5, + + "rows_before_limit_at_least": 5 +} diff --git a/dbms/tests/queries/0_stateless/00963_achimbab.sql b/dbms/tests/queries/0_stateless/00963_achimbab.sql new file mode 100644 index 00000000000..d9d27429d39 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00963_achimbab.sql @@ -0,0 +1,18 @@ +SET output_format_write_statistics = 0; + +select + sum(cnt) > 0 as total, + k[1], k[2] + from + ( + select + arrayMap( x -> x % 3 ? toNullable(number%5 + x) : null, range(3)) as k, + number % 4 ? toNullable( rand() ) : Null as cnt + from system.numbers_mt + where number < 1000000 + limit 1000000 + ) +group by k with totals +order by total desc +SETTINGS max_threads = 100, max_execution_time = 120 +format JSON; diff --git a/dbms/tests/queries/0_stateless/00963_startsWith_force_primary_key.reference b/dbms/tests/queries/0_stateless/00963_startsWith_force_primary_key.reference new file mode 100644 index 00000000000..f1e5eeed2d9 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00963_startsWith_force_primary_key.reference @@ -0,0 +1,2 @@ +3 +2 diff --git a/dbms/tests/queries/0_stateless/00963_startsWith_force_primary_key.sql b/dbms/tests/queries/0_stateless/00963_startsWith_force_primary_key.sql new file mode 100644 index 00000000000..b3895a93b64 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00963_startsWith_force_primary_key.sql @@ -0,0 +1,6 @@ +DROP TABLE IF EXISTS test_startsWith; +CREATE TABLE test_startsWith (a String) Engine = MergeTree PARTITION BY tuple() ORDER BY a; +INSERT INTO test_startsWith (a) values ('a'), ('abcd'), ('bbb'), (''), ('abc'); +SELECT count() from test_startsWith where startsWith(a, 'a') settings force_primary_key=1; +SELECT count() from test_startsWith where startsWith(a, 'abc') settings force_primary_key=1; +DROP TABLE test_startsWith; diff --git a/dbms/tests/queries/0_stateless/00964_bloom_index_string_functions.reference b/dbms/tests/queries/0_stateless/00964_bloom_index_string_functions.reference new file mode 100644 index 00000000000..4a8c828426e --- /dev/null +++ b/dbms/tests/queries/0_stateless/00964_bloom_index_string_functions.reference @@ -0,0 +1,53 @@ +9 abra +14 abracadabra + "rows_read": 6, +8 computer science + "rows_read": 2, +9 abra +10 cadabra +11 crabacadabra +14 abracadabra +15 cadabraabra + "rows_read": 6, +6 some string +7 another string + "rows_read": 2, +9 abra +14 abracadabra + "rows_read": 6, +8 computer science + "rows_read": 2, +1 ClickHouse is a column-oriented database management system (DBMS) +2 column-oriented database management system +13 basement + "rows_read": 6, +6 some string +7 another string + "rows_read": 2, +6 some string +7 another string +8 computer science + "rows_read": 4, +1 ClickHouse is a column-oriented database management system (DBMS) +2 column-oriented database management system +13 basement + "rows_read": 6, +9 abra +10 cadabra +11 crabacadabra +14 abracadabra +15 cadabraabra + "rows_read": 6, +4 какая-то строка +5 еще строка +6 some string +7 another string + "rows_read": 4, +14 abracadabra + "rows_read": 4, +1 ClickHouse is a column-oriented database management system (DBMS) +2 column-oriented database management system +10 cadabra +11 crabacadabra +15 cadabraabra + "rows_read": 8, diff --git a/dbms/tests/queries/0_stateless/00964_bloom_index_string_functions.sh b/dbms/tests/queries/0_stateless/00964_bloom_index_string_functions.sh new file mode 100755 index 00000000000..28120e782c1 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00964_bloom_index_string_functions.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS bloom_filter_idx;" + +# NGRAM BF +$CLICKHOUSE_CLIENT -n --query=" +SET allow_experimental_data_skipping_indices = 1; +CREATE TABLE bloom_filter_idx +( + k UInt64, + s String, + INDEX bf (s, lower(s)) TYPE ngrambf_v1(3, 512, 2, 0) GRANULARITY 1 +) ENGINE = MergeTree() +ORDER BY k +SETTINGS index_granularity = 2;" + +$CLICKHOUSE_CLIENT --query="INSERT INTO bloom_filter_idx VALUES +(0, 'ClickHouse - столбцовая система управления базами данных (СУБД)'), +(1, 'ClickHouse is a column-oriented database management system (DBMS)'), +(2, 'column-oriented database management system'), +(3, 'columns'), +(4, 'какая-то строка'), +(5, 'еще строка'), +(6, 'some string'), +(7, 'another string'), +(8, 'computer science'), +(9, 'abra'), +(10, 'cadabra'), +(11, 'crabacadabra'), +(12, 'crab'), +(13, 'basement'), +(14, 'abracadabra'), +(15, 'cadabraabra')" + +# STARTS_WITH +$CLICKHOUSE_CLIENT --query="SELECT * FROM bloom_filter_idx WHERE startsWith(s, 'abra') ORDER BY k" +$CLICKHOUSE_CLIENT --query="SELECT * FROM bloom_filter_idx WHERE startsWith(s, 'abra') ORDER BY k FORMAT JSON" | grep "rows_read" + +$CLICKHOUSE_CLIENT --query="SELECT * FROM bloom_filter_idx WHERE startsWith(s, 'computer') ORDER BY k" +$CLICKHOUSE_CLIENT --query="SELECT * FROM bloom_filter_idx WHERE startsWith(s, 'computer') ORDER BY k FORMAT JSON" | grep "rows_read" + +# ENDS_WITH +$CLICKHOUSE_CLIENT --query="SELECT * FROM bloom_filter_idx WHERE endsWith(s, 'abra') ORDER BY k" +$CLICKHOUSE_CLIENT --query="SELECT * FROM bloom_filter_idx WHERE endsWith(s, 'abra') ORDER BY k FORMAT JSON" | grep "rows_read" + +$CLICKHOUSE_CLIENT --query="SELECT * FROM bloom_filter_idx WHERE endsWith(s, 'ring') ORDER BY k" +$CLICKHOUSE_CLIENT --query="SELECT * FROM bloom_filter_idx WHERE endsWith(s, 'ring') ORDER BY k FORMAT JSON" | grep "rows_read" + +# COMBINED +$CLICKHOUSE_CLIENT --query="SELECT * FROM bloom_filter_idx WHERE startsWith(s, 'abra') AND endsWith(s, 'abra')" +$CLICKHOUSE_CLIENT --query="SELECT * FROM bloom_filter_idx WHERE startsWith(s, 'abra') AND endsWith(s, 'abra') FORMAT JSON" | grep "rows_read" + +$CLICKHOUSE_CLIENT --query="SELECT * FROM bloom_filter_idx WHERE startsWith(s, 'c') AND endsWith(s, 'science')" +$CLICKHOUSE_CLIENT --query="SELECT * FROM bloom_filter_idx WHERE startsWith(s, 'c') AND endsWith(s, 'science') FORMAT JSON" | grep "rows_read" + +# MULTY_SEARCH_ANY +$CLICKHOUSE_CLIENT --query="SELECT * FROM bloom_filter_idx WHERE multiSearchAny(s, ['data', 'base'])" +$CLICKHOUSE_CLIENT --query="SELECT * FROM bloom_filter_idx WHERE multiSearchAny(s, ['data', 'base']) FORMAT JSON" | grep "rows_read" + +$CLICKHOUSE_CLIENT --query="SELECT * FROM bloom_filter_idx WHERE multiSearchAny(s, ['string'])" +$CLICKHOUSE_CLIENT --query="SELECT * FROM bloom_filter_idx WHERE multiSearchAny(s, ['string']) FORMAT JSON" | grep "rows_read" + +$CLICKHOUSE_CLIENT --query="SELECT * FROM bloom_filter_idx WHERE multiSearchAny(s, ['string', 'computer'])" +$CLICKHOUSE_CLIENT --query="SELECT * FROM bloom_filter_idx WHERE multiSearchAny(s, ['string', 'computer']) FORMAT JSON" | grep "rows_read" + +$CLICKHOUSE_CLIENT --query="SELECT * FROM bloom_filter_idx WHERE multiSearchAny(s, ['base', 'seme', 'gement'])" +$CLICKHOUSE_CLIENT --query="SELECT * FROM bloom_filter_idx WHERE multiSearchAny(s, ['base', 'seme', 'gement']) FORMAT JSON" | grep "rows_read" + +$CLICKHOUSE_CLIENT --query="SELECT * FROM bloom_filter_idx WHERE multiSearchAny(s, ['abra', 'cadabra', 'cab', 'extra'])" +$CLICKHOUSE_CLIENT --query="SELECT * FROM bloom_filter_idx WHERE multiSearchAny(s, ['abra', 'cadabra', 'cab', 'extra']) FORMAT JSON" | grep "rows_read" + +$CLICKHOUSE_CLIENT --query="SELECT * FROM bloom_filter_idx WHERE multiSearchAny(s, ['строка', 'string'])" +$CLICKHOUSE_CLIENT --query="SELECT * FROM bloom_filter_idx WHERE multiSearchAny(s, ['строка', 'string']) FORMAT JSON" | grep "rows_read" + +# MULTY_SEARCH_ANY + OTHER + +$CLICKHOUSE_CLIENT --query="SELECT * FROM bloom_filter_idx WHERE multiSearchAny(s, ['adab', 'cad', 'aba']) AND startsWith(s, 'abra')" +$CLICKHOUSE_CLIENT --query="SELECT * FROM bloom_filter_idx WHERE multiSearchAny(s, ['adab', 'cad', 'aba']) AND startsWith(s, 'abra') FORMAT JSON" | grep "rows_read" + +$CLICKHOUSE_CLIENT --query="SELECT * FROM bloom_filter_idx WHERE multiSearchAny(s, ['adab', 'cad', 'aba']) AND (startsWith(s, 'c') OR startsWith(s, 'C'))" +$CLICKHOUSE_CLIENT --query="SELECT * FROM bloom_filter_idx WHERE multiSearchAny(s, ['adab', 'cad', 'aba']) AND (startsWith(s, 'c') OR startsWith(s, 'C')) FORMAT JSON" | grep "rows_read" + +$CLICKHOUSE_CLIENT --query="DROP TABLE bloom_filter_idx;" diff --git a/dbms/tests/queries/0_stateless/00964_os_thread_priority.reference b/dbms/tests/queries/0_stateless/00964_os_thread_priority.reference new file mode 100644 index 00000000000..83b33d238da --- /dev/null +++ b/dbms/tests/queries/0_stateless/00964_os_thread_priority.reference @@ -0,0 +1 @@ +1000 diff --git a/dbms/tests/queries/0_stateless/00964_os_thread_priority.sql b/dbms/tests/queries/0_stateless/00964_os_thread_priority.sql new file mode 100644 index 00000000000..4244de8e909 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00964_os_thread_priority.sql @@ -0,0 +1,3 @@ +-- the setting exists and server does not crash +SET os_thread_priority = 10; +SELECT count() FROM numbers(1000); diff --git a/dbms/tests/queries/0_stateless/00965_pocopatch_logs_level_bugfix.reference b/dbms/tests/queries/0_stateless/00965_pocopatch_logs_level_bugfix.reference new file mode 100644 index 00000000000..233d012cabe --- /dev/null +++ b/dbms/tests/queries/0_stateless/00965_pocopatch_logs_level_bugfix.reference @@ -0,0 +1,24 @@ +1 +1 +1 +1 +1 +1 + + + + + + + + + + + + + + + + + + diff --git a/dbms/tests/queries/0_stateless/00965_pocopatch_logs_level_bugfix.sh b/dbms/tests/queries/0_stateless/00965_pocopatch_logs_level_bugfix.sh new file mode 100755 index 00000000000..99de8cfb2d5 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00965_pocopatch_logs_level_bugfix.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +> 00965_logs_level_bugfix.tmp + +clickhouse-client --send_logs_level="trace" --query="SELECT 1;" 2>> 00965_logs_level_bugfix.tmp +clickhouse-client --send_logs_level="debug" --query="SELECT 1;" 2>> 00965_logs_level_bugfix.tmp +clickhouse-client --send_logs_level="information" --query="SELECT 1;" 2>> 00965_logs_level_bugfix.tmp +clickhouse-client --send_logs_level="warning" --query="SELECT 1;" 2>> 00965_logs_level_bugfix.tmp +clickhouse-client --send_logs_level="error" --query="SELECT 1;" 2>> 00965_logs_level_bugfix.tmp +clickhouse-client --send_logs_level="none" --query="SELECT 1;" 2>> 00965_logs_level_bugfix.tmp + +awk '{ print $8 }' 00965_logs_level_bugfix.tmp + + diff --git a/dbms/tests/queries/0_stateless/00965_pocopatch_send_logs_level_concurrent_queries.reference b/dbms/tests/queries/0_stateless/00965_pocopatch_send_logs_level_concurrent_queries.reference new file mode 100644 index 00000000000..45c133ba43b --- /dev/null +++ b/dbms/tests/queries/0_stateless/00965_pocopatch_send_logs_level_concurrent_queries.reference @@ -0,0 +1,12 @@ + + + + + + + + + + +***** + diff --git a/dbms/tests/queries/0_stateless/00965_pocopatch_send_logs_level_concurrent_queries.sh b/dbms/tests/queries/0_stateless/00965_pocopatch_send_logs_level_concurrent_queries.sh new file mode 100755 index 00000000000..48b6b41282c --- /dev/null +++ b/dbms/tests/queries/0_stateless/00965_pocopatch_send_logs_level_concurrent_queries.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +> 00965_send_logs_level_concurrent_queries_first.tmp +> 00965_send_logs_level_concurrent_queries_second.tmp + +clickhouse-client --send_logs_level="trace" --query="SELECT * from numbers(100000);" >> /dev/null 2>> 00965_send_logs_level_concurrent_queries_first.tmp & +clickhouse-client --send_logs_level="information" --query="SELECT * from numbers(100000);" >> /dev/null 2>> 00965_send_logs_level_concurrent_queries_second.tmp + +sleep 2 + +awk '{ print $8 }' 00965_send_logs_level_concurrent_queries_first.tmp +echo "*****" +awk '{ print $8 }' 00965_send_logs_level_concurrent_queries_second.tmp + diff --git a/dbms/tests/queries/0_stateless/00965_set_index_string_functions.reference b/dbms/tests/queries/0_stateless/00965_set_index_string_functions.reference new file mode 100644 index 00000000000..a9c9f97bdec --- /dev/null +++ b/dbms/tests/queries/0_stateless/00965_set_index_string_functions.reference @@ -0,0 +1,16 @@ +9 abra +14 abracadabra + "rows_read": 4, +9 abra +10 cadabra +11 crabacadabra +14 abracadabra +15 cadabraabra + "rows_read": 6, +9 abra +14 abracadabra + "rows_read": 4, +1 ClickHouse is a column-oriented database management system (DBMS) +2 column-oriented database management system +13 basement + "rows_read": 6, diff --git a/dbms/tests/queries/0_stateless/00965_set_index_string_functions.sh b/dbms/tests/queries/0_stateless/00965_set_index_string_functions.sh new file mode 100755 index 00000000000..056915c0e7c --- /dev/null +++ b/dbms/tests/queries/0_stateless/00965_set_index_string_functions.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS set_idx;" + +$CLICKHOUSE_CLIENT -n --query=" +SET allow_experimental_data_skipping_indices = 1; +CREATE TABLE set_idx +( + k UInt64, + s String, + INDEX idx (s) TYPE set(2) GRANULARITY 1 +) ENGINE = MergeTree() +ORDER BY k +SETTINGS index_granularity = 2;" + +$CLICKHOUSE_CLIENT --query="INSERT INTO set_idx VALUES +(0, 'ClickHouse - столбцовая система управления базами данных (СУБД)'), +(1, 'ClickHouse is a column-oriented database management system (DBMS)'), +(2, 'column-oriented database management system'), +(3, 'columns'), +(4, 'какая-то строка'), +(5, 'еще строка'), +(6, 'some string'), +(7, 'another string'), +(8, 'computer science'), +(9, 'abra'), +(10, 'cadabra'), +(11, 'crabacadabra'), +(12, 'crab'), +(13, 'basement'), +(14, 'abracadabra'), +(15, 'cadabraabra')" + +# STARTS_WITH +$CLICKHOUSE_CLIENT --query="SELECT * FROM set_idx WHERE startsWith(s, 'abra')" +$CLICKHOUSE_CLIENT --query="SELECT * FROM set_idx WHERE startsWith(s, 'abra') FORMAT JSON" | grep "rows_read" + +# ENDS_WITH +$CLICKHOUSE_CLIENT --query="SELECT * FROM set_idx WHERE endsWith(s, 'abra')" +$CLICKHOUSE_CLIENT --query="SELECT * FROM set_idx WHERE endsWith(s, 'abra') FORMAT JSON" | grep "rows_read" + +# COMBINED +$CLICKHOUSE_CLIENT --query="SELECT * FROM set_idx WHERE startsWith(s, 'abra') AND endsWith(s, 'abra')" +$CLICKHOUSE_CLIENT --query="SELECT * FROM set_idx WHERE startsWith(s, 'abra') AND endsWith(s, 'abra') FORMAT JSON" | grep "rows_read" + +# MULTY_SEARCH_ANY +$CLICKHOUSE_CLIENT --query="SELECT * FROM set_idx WHERE multiSearchAny(s, ['data', 'base'])" +$CLICKHOUSE_CLIENT --query="SELECT * FROM set_idx WHERE multiSearchAny(s, ['data', 'base']) FORMAT JSON" | grep "rows_read" + +$CLICKHOUSE_CLIENT --query="DROP TABLE set_idx;" diff --git a/dbms/tests/queries/0_stateless/00965_shard_unresolvable_addresses.reference b/dbms/tests/queries/0_stateless/00965_shard_unresolvable_addresses.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00965_shard_unresolvable_addresses.reference @@ -0,0 +1 @@ +1 diff --git a/dbms/tests/queries/0_stateless/00965_shard_unresolvable_addresses.sql b/dbms/tests/queries/0_stateless/00965_shard_unresolvable_addresses.sql new file mode 100644 index 00000000000..b6b981c7d00 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00965_shard_unresolvable_addresses.sql @@ -0,0 +1,2 @@ +SELECT count() FROM remote('127.0.0.1,localhos', system.one); -- { serverError 279 } +SELECT count() FROM remote('127.0.0.1|localhos', system.one); diff --git a/dbms/tests/queries/0_stateless/00966_invalid_json_must_not_parse.reference b/dbms/tests/queries/0_stateless/00966_invalid_json_must_not_parse.reference new file mode 100644 index 00000000000..f7eb44d66e0 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00966_invalid_json_must_not_parse.reference @@ -0,0 +1,6 @@ +0 +0 +0 +0 +0 +0 diff --git a/dbms/tests/queries/0_stateless/00966_invalid_json_must_not_parse.sql b/dbms/tests/queries/0_stateless/00966_invalid_json_must_not_parse.sql new file mode 100644 index 00000000000..afcbc78cfd5 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00966_invalid_json_must_not_parse.sql @@ -0,0 +1,12 @@ +SET allow_simdjson=1; + +SELECT JSONLength('"HX-='); +SELECT JSONLength('[9]\0\x42\xD3\x36\xE3'); +SELECT JSONLength(unhex('5B30000E06D7AA5D')); + + +SET allow_simdjson=0; + +SELECT JSONLength('"HX-='); +SELECT JSONLength('[9]\0\x42\xD3\x36\xE3'); +SELECT JSONLength(unhex('5B30000E06D7AA5D')); diff --git a/dbms/tests/queries/0_stateless/00967_ubsan_bit_test.reference b/dbms/tests/queries/0_stateless/00967_ubsan_bit_test.reference new file mode 100644 index 00000000000..573541ac970 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00967_ubsan_bit_test.reference @@ -0,0 +1 @@ +0 diff --git a/dbms/tests/queries/0_stateless/00967_ubsan_bit_test.sql b/dbms/tests/queries/0_stateless/00967_ubsan_bit_test.sql new file mode 100644 index 00000000000..1682e725670 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00967_ubsan_bit_test.sql @@ -0,0 +1 @@ +SELECT sum(ignore(bitTest(number, 65))) FROM numbers(10); diff --git a/dbms/tests/queries/0_stateless/00968_file_engine_in_subquery.reference b/dbms/tests/queries/0_stateless/00968_file_engine_in_subquery.reference new file mode 100644 index 00000000000..8b1acc12b63 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00968_file_engine_in_subquery.reference @@ -0,0 +1,10 @@ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 diff --git a/dbms/tests/queries/0_stateless/00968_file_engine_in_subquery.sql b/dbms/tests/queries/0_stateless/00968_file_engine_in_subquery.sql new file mode 100644 index 00000000000..5ed3be77cd6 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00968_file_engine_in_subquery.sql @@ -0,0 +1,9 @@ +DROP TABLE IF EXISTS tableFile_00968; +DROP TABLE IF EXISTS tableMergeTree_00968; +CREATE TABLE tableFile_00968(number UInt64) ENGINE = File('TSV'); +CREATE TABLE tableMergeTree_00968(id UInt64) ENGINE = MergeTree() PARTITION BY id ORDER BY id; + +INSERT INTO tableFile_00968 SELECT number FROM system.numbers LIMIT 10; +INSERT INTO tableMergeTree_00968 SELECT number FROM system.numbers LIMIT 100; + +SELECT id FROM tableMergeTree_00968 WHERE id IN (SELECT number FROM tableFile_00968) ORDER BY id; diff --git a/dbms/tests/queries/0_stateless/00968_roundAge.reference b/dbms/tests/queries/0_stateless/00968_roundAge.reference new file mode 100644 index 00000000000..e85c792b528 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00968_roundAge.reference @@ -0,0 +1,7 @@ +0 +18 +25 +35 +45 +55 +55 diff --git a/dbms/tests/queries/0_stateless/00968_roundAge.sql b/dbms/tests/queries/0_stateless/00968_roundAge.sql new file mode 100644 index 00000000000..c8e5a5579f2 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00968_roundAge.sql @@ -0,0 +1,7 @@ +SELECT roundAge(0); +SELECT roundAge(18); +SELECT roundAge(25); +SELECT roundAge(35); +SELECT roundAge(45); +SELECT roundAge(55); +SELECT roundAge(56); \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/00969_roundDuration.reference b/dbms/tests/queries/0_stateless/00969_roundDuration.reference new file mode 100644 index 00000000000..463d6f92125 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00969_roundDuration.reference @@ -0,0 +1,16 @@ +0 +10 +30 +60 +120 +180 +240 +300 +600 +1200 +1800 +3600 +7200 +18000 +36000 +36000 diff --git a/dbms/tests/queries/0_stateless/00969_roundDuration.sql b/dbms/tests/queries/0_stateless/00969_roundDuration.sql new file mode 100644 index 00000000000..200ae2b78bd --- /dev/null +++ b/dbms/tests/queries/0_stateless/00969_roundDuration.sql @@ -0,0 +1,16 @@ +SELECT roundDuration(0); +SELECT roundDuration(10); +SELECT roundDuration(30); +SELECT roundDuration(60); +SELECT roundDuration(120); +SELECT roundDuration(180); +SELECT roundDuration(240); +SELECT roundDuration(300); +SELECT roundDuration(600); +SELECT roundDuration(1200); +SELECT roundDuration(1800); +SELECT roundDuration(3600); +SELECT roundDuration(7200); +SELECT roundDuration(18000); +SELECT roundDuration(36000); +SELECT roundDuration(37000); \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/00970_substring_arg_validation.reference b/dbms/tests/queries/0_stateless/00970_substring_arg_validation.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/00970_substring_arg_validation.sql b/dbms/tests/queries/0_stateless/00970_substring_arg_validation.sql new file mode 100644 index 00000000000..7d8320a1d64 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00970_substring_arg_validation.sql @@ -0,0 +1,4 @@ +SELECT substring('hello', []); -- { serverError 43 } +SELECT substring('hello', 1, []); -- { serverError 43 } +SELECT substring(materialize('hello'), -1, -1); -- { serverError 69 } +SELECT substring(materialize('hello'), 0); -- { serverError 135 } \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/00971_merge_tree_uniform_read_distribution_and_max_rows_to_read.reference b/dbms/tests/queries/0_stateless/00971_merge_tree_uniform_read_distribution_and_max_rows_to_read.reference new file mode 100644 index 00000000000..fcd78da1283 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00971_merge_tree_uniform_read_distribution_and_max_rows_to_read.reference @@ -0,0 +1,2 @@ +1000000 +1000000 diff --git a/dbms/tests/queries/0_stateless/00971_merge_tree_uniform_read_distribution_and_max_rows_to_read.sql b/dbms/tests/queries/0_stateless/00971_merge_tree_uniform_read_distribution_and_max_rows_to_read.sql new file mode 100644 index 00000000000..37d09a3d3dd --- /dev/null +++ b/dbms/tests/queries/0_stateless/00971_merge_tree_uniform_read_distribution_and_max_rows_to_read.sql @@ -0,0 +1,22 @@ +DROP TABLE IF EXISTS merge_tree; +CREATE TABLE merge_tree (x UInt8) ENGINE = MergeTree ORDER BY x; +INSERT INTO merge_tree SELECT 0 FROM numbers(1000000); + +SET max_threads = 4; +SET max_rows_to_read = 1100000; + +SET merge_tree_uniform_read_distribution = 1; +SELECT count() FROM merge_tree; + +SET merge_tree_uniform_read_distribution = 0; +SELECT count() FROM merge_tree; + +SET max_rows_to_read = 900000; + +SET merge_tree_uniform_read_distribution = 1; +SELECT count() FROM merge_tree; -- { serverError 158 } + +SET merge_tree_uniform_read_distribution = 0; +SELECT count() FROM merge_tree; -- { serverError 158 } + +DROP TABLE merge_tree; diff --git a/dbms/tests/queries/0_stateless/00971_query_id_in_logs.reference b/dbms/tests/queries/0_stateless/00971_query_id_in_logs.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/00971_query_id_in_logs.sh b/dbms/tests/queries/0_stateless/00971_query_id_in_logs.sh new file mode 100755 index 00000000000..a4ef7671f48 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00971_query_id_in_logs.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +set -e + +# No log lines without query id +$CLICKHOUSE_CLIENT --send_logs_level=trace --query_id=hello --query="SELECT count() FROM numbers(10)" 2>&1 | grep -vF ' {hello} ' | grep -P '<\w+>' ||: diff --git a/dbms/tests/queries/0_stateless/00972_desc_table_virtual_columns.reference b/dbms/tests/queries/0_stateless/00972_desc_table_virtual_columns.reference new file mode 100644 index 00000000000..b244da814f9 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00972_desc_table_virtual_columns.reference @@ -0,0 +1 @@ +x UInt64 diff --git a/dbms/tests/queries/0_stateless/00972_desc_table_virtual_columns.sql b/dbms/tests/queries/0_stateless/00972_desc_table_virtual_columns.sql new file mode 100644 index 00000000000..920025a844d --- /dev/null +++ b/dbms/tests/queries/0_stateless/00972_desc_table_virtual_columns.sql @@ -0,0 +1,9 @@ +-- No virtual columns should be output in DESC TABLE query. + +DROP TABLE IF EXISTS upyachka; +CREATE TABLE upyachka (x UInt64) ENGINE = Memory; + +-- Merge table has virtual column `_table` +DESC TABLE merge(currentDatabase(), 'upyachka'); + +DROP TABLE upyachka; diff --git a/dbms/tests/queries/0_stateless/00973_uniq_non_associativity.reference b/dbms/tests/queries/0_stateless/00973_uniq_non_associativity.reference new file mode 100644 index 00000000000..9e0c113389f --- /dev/null +++ b/dbms/tests/queries/0_stateless/00973_uniq_non_associativity.reference @@ -0,0 +1,2 @@ +80041 +80041 diff --git a/dbms/tests/queries/0_stateless/00973_uniq_non_associativity.sql b/dbms/tests/queries/0_stateless/00973_uniq_non_associativity.sql new file mode 100644 index 00000000000..91d62a134f8 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00973_uniq_non_associativity.sql @@ -0,0 +1,133 @@ +/* Aggregate function 'uniq' is intended to be associative and provide deterministic results regardless to the schedule of query execution threads and remote servers in a cluster. + * But due to subtle bug in implementation it is not associative in very rare cases. + * In this test we fill data structure with specific pattern that reproduces this behaviour. + */ + +DROP TABLE IF EXISTS part_a; +DROP TABLE IF EXISTS part_b; +DROP TABLE IF EXISTS part_c; +DROP TABLE IF EXISTS part_d; + +/* Create values that will resize hash table to the maximum (131072 cells) and fill it with less than max_fill (65536 cells) + * and occupy cells near the end except last 10 cells: + * [ ----------- ] + * Pick values that will vanish if table will be rehashed. + */ +CREATE TABLE part_a ENGINE = TinyLog AS SELECT * FROM +( +WITH + number AS k1, + bitXor(k1, bitShiftRight(k1, 33)) AS k2, + k2 * 0xff51afd7ed558ccd AS k3, + bitXor(k3, bitShiftRight(k3, 33)) AS k4, + k4 * 0xc4ceb9fe1a85ec53 AS k5, + bitXor(k5, bitShiftRight(k5, 33)) AS k6, + k6 AS hash, + bitShiftRight(hash, 15) % 0x20000 AS place, + hash % 2 = 0 AS will_remain +SELECT hash, number, place FROM system.numbers WHERE place >= 90000 AND place < 131062 AND NOT will_remain LIMIT 1 BY place LIMIT 41062 +) ORDER BY place; + +/* Create values that will resize hash table to the maximum (131072 cells) and fill it with less than max_fill (65536 cells), + * but if we use both "a" and "b", it will force rehash. + * [ ----------- ] + * Pick values that will remain after rehash. + */ +CREATE TABLE part_b ENGINE = TinyLog AS SELECT * FROM +( +WITH + number AS k1, + bitXor(k1, bitShiftRight(k1, 33)) AS k2, + k2 * 0xff51afd7ed558ccd AS k3, + bitXor(k3, bitShiftRight(k3, 33)) AS k4, + k4 * 0xc4ceb9fe1a85ec53 AS k5, + bitXor(k5, bitShiftRight(k5, 33)) AS k6, + k6 AS hash, + bitShiftRight(hash, 15) % 0x20000 AS place, + hash % 2 = 0 AS will_remain +SELECT hash, number, place FROM system.numbers WHERE place >= 50000 AND place < 90000 AND will_remain LIMIT 1 BY place LIMIT 40000 +) ORDER BY place; + +/* Occupy 10 cells near the end of "a": + * a: [ ----------- ] + * c: [ -- ] + * If we insert "a" then "c", these values will be placed at the end of hash table due to collision resolution: + * a + c: [ aaaaaaaaaaacc] + */ +CREATE TABLE part_c ENGINE = TinyLog AS SELECT * FROM +( +WITH + number AS k1, + bitXor(k1, bitShiftRight(k1, 33)) AS k2, + k2 * 0xff51afd7ed558ccd AS k3, + bitXor(k3, bitShiftRight(k3, 33)) AS k4, + k4 * 0xc4ceb9fe1a85ec53 AS k5, + bitXor(k5, bitShiftRight(k5, 33)) AS k6, + k6 AS hash, + bitShiftRight(hash, 15) % 0x20000 AS place, + hash % 2 = 0 AS will_remain +SELECT hash, number, place FROM system.numbers WHERE place >= 131052 AND place < 131062 AND will_remain AND hash NOT IN (SELECT hash FROM part_a) LIMIT 1 BY place LIMIT 10 +) ORDER BY place; + +/* Occupy 10 cells at the end of hash table, after "a": + * a: [ ----------- ] + * d: [ --] + * a + d: [ aaaaaaaaaaadd] + * But if we insert "a" then "c" then "d", these values will be placed at the beginning of the hash table due to collision resolution: + * a+c+d: [dd aaaaaaaaaaacc] + */ +CREATE TABLE part_d ENGINE = TinyLog AS SELECT * FROM +( +WITH + number AS k1, + bitXor(k1, bitShiftRight(k1, 33)) AS k2, + k2 * 0xff51afd7ed558ccd AS k3, + bitXor(k3, bitShiftRight(k3, 33)) AS k4, + k4 * 0xc4ceb9fe1a85ec53 AS k5, + bitXor(k5, bitShiftRight(k5, 33)) AS k6, + k6 AS hash, + bitShiftRight(hash, 15) % 0x20000 AS place, + hash % 2 = 0 AS will_remain +SELECT hash, number, place FROM system.numbers WHERE place >= 131062 AND will_remain LIMIT 1 BY place LIMIT 10 +) ORDER BY place; + +/** What happens if we insert a then c then d then b? + * Insertion of b forces rehash. + * a will be removed, but c, d, b remain: + * [dd bbbbbbbbbb cc] + * Then we go through hash table and move elements to better places in collision resolution chain. + * c will be moved left to their right place: + * [dd bbbbbbbbbb cc ] + * + * And d must be moved also: + * [ bbbbbbbbbb ccdd] + * But our algorithm was incorrect and it doesn't happen. + * + * If we insert d again, it will be placed twice because original d will not found: + * [dd bbbbbbbbbb ccdd] + * This will lead to slightly higher return value of "uniq" aggregate function and it is dependent on insertion order. + */ + + +SET max_threads = 1; + +/** Results of these two queries must match: */ + +SELECT uniq(number) FROM ( + SELECT * FROM part_a +UNION ALL SELECT * FROM part_c +UNION ALL SELECT * FROM part_d +UNION ALL SELECT * FROM part_b); + +SELECT uniq(number) FROM ( + SELECT * FROM part_a +UNION ALL SELECT * FROM part_c +UNION ALL SELECT * FROM part_d +UNION ALL SELECT * FROM part_b +UNION ALL SELECT * FROM part_d); + + +DROP TABLE part_a; +DROP TABLE part_b; +DROP TABLE part_c; +DROP TABLE part_d; diff --git a/dbms/tests/queries/1_stateful/00077_log_tinylog_stripelog.sql b/dbms/tests/queries/1_stateful/00077_log_tinylog_stripelog.sql index d745203ea6b..2324ad69165 100644 --- a/dbms/tests/queries/1_stateful/00077_log_tinylog_stripelog.sql +++ b/dbms/tests/queries/1_stateful/00077_log_tinylog_stripelog.sql @@ -1,3 +1,5 @@ +SET check_query_single_value_result = 1; + DROP TABLE IF EXISTS test.hits_log; DROP TABLE IF EXISTS test.hits_tinylog; DROP TABLE IF EXISTS test.hits_stripelog; diff --git a/dbms/tests/server-test.xml b/dbms/tests/server-test.xml index c20d34cce3f..d68cbca53c1 100644 --- a/dbms/tests/server-test.xml +++ b/dbms/tests/server-test.xml @@ -117,4 +117,10 @@ /clickhouse/task_queue/ddl /tmp/clickhouse/data/format_schemas/ + + + TOPSECRET.TOPSECRET + [hidden] + + diff --git a/debian/changelog b/debian/changelog index 52ac759a110..6abe0effc06 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,5 @@ -clickhouse (19.11.0) unstable; urgency=low +clickhouse (19.12.1.1) unstable; urgency=low * Modified source code - -- chertus Fri, 21 Jun 2019 18:47:02 +0300 + -- clickhouse-release Wed, 10 Jul 2019 22:57:50 +0300 diff --git a/debian/clickhouse-server.postinst b/debian/clickhouse-server.postinst index cd1590258ce..944b1e394f1 100644 --- a/debian/clickhouse-server.postinst +++ b/debian/clickhouse-server.postinst @@ -121,9 +121,9 @@ Please fix this and reinstall this package." >&2 TMPFILE=/tmp/test_setcap.sh command -v setcap >/dev/null \ - && echo > $TMPFILE && chmod a+x $TMPFILE && $TMPFILE && setcap "cap_net_admin,cap_ipc_lock+ep" $TMPFILE && $TMPFILE && rm $TMPFILE \ - && setcap "cap_net_admin,cap_ipc_lock+ep" "${CLICKHOUSE_BINDIR}/${CLICKHOUSE_GENERIC_PROGRAM}" \ - || echo "Cannot set 'net_admin' or 'ipc_lock' capability for clickhouse binary. This is optional. Taskstats accounting will be disabled. To enable taskstats accounting you may add the required capability later manually." + && echo > $TMPFILE && chmod a+x $TMPFILE && $TMPFILE && setcap "cap_net_admin,cap_ipc_lock,cap_sys_nice+ep" $TMPFILE && $TMPFILE && rm $TMPFILE \ + && setcap "cap_net_admin,cap_ipc_lock,cap_sys_nice+ep" "${CLICKHOUSE_BINDIR}/${CLICKHOUSE_GENERIC_PROGRAM}" \ + || echo "Cannot set 'net_admin' or 'ipc_lock' or 'sys_nice' capability for clickhouse binary. This is optional. Taskstats accounting will be disabled. To enable taskstats accounting you may add the required capability later manually." # Clean old dynamic compilation results if [ -d "${CLICKHOUSE_DATADIR_FROM_CONFIG}/build" ]; then diff --git a/debian/clickhouse-server.service b/debian/clickhouse-server.service index 7eaceeab986..4543b304197 100644 --- a/debian/clickhouse-server.service +++ b/debian/clickhouse-server.service @@ -11,7 +11,7 @@ RuntimeDirectory=clickhouse-server ExecStart=/usr/bin/clickhouse-server --config=/etc/clickhouse-server/config.xml --pid-file=/run/clickhouse-server/clickhouse-server.pid LimitCORE=infinity LimitNOFILE=500000 -CapabilityBoundingSet=CAP_NET_ADMIN CAP_IPC_LOCK +CapabilityBoundingSet=CAP_NET_ADMIN CAP_IPC_LOCK CAP_SYS_NICE [Install] WantedBy=multi-user.target diff --git a/debian/clickhouse-server.templates b/debian/clickhouse-server.templates index 3053c18c79f..fdab88cf877 100644 --- a/debian/clickhouse-server.templates +++ b/debian/clickhouse-server.templates @@ -1,3 +1,3 @@ Template: clickhouse-server/default-password Type: password -Description: Password for default user. +Description: Password for default user diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile index c6af86c5825..1afc097aafd 100644 --- a/docker/client/Dockerfile +++ b/docker/client/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=19.11.0 +ARG version=19.12.1.* RUN apt-get update \ && apt-get install --yes --no-install-recommends \ diff --git a/docker/images.json b/docker/images.json index 6ab12504a7d..e2282fcb653 100644 --- a/docker/images.json +++ b/docker/images.json @@ -7,7 +7,9 @@ "docker/test/performance": "yandex/clickhouse-performance-test", "docker/test/pvs": "yandex/clickhouse-pvs-test", "docker/test/stateful": "yandex/clickhouse-stateful-test", + "docker/test/stateful_with_coverage": "yandex/clickhouse-stateful-with-coverage-test", "docker/test/stateless": "yandex/clickhouse-stateless-test", + "docker/test/stateless_with_coverage": "yandex/clickhouse-stateless-with-coverage-test", "docker/test/unit": "yandex/clickhouse-unit-test", "docker/test/stress": "yandex/clickhouse-stress-test", "dbms/tests/integration/image": "yandex/clickhouse-integration-tests-runner" diff --git a/docker/packager/packager b/docker/packager/packager index 9dadb96b46d..0e8bf6ea98d 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -103,7 +103,7 @@ def run_vagrant_box_with_env(image_path, output_dir, ch_root): logging.info("Copying binary back") vagrant.copy_from_image("~/ClickHouse/dbms/programs/clickhouse", output_dir) -def parse_env_variables(build_type, compiler, sanitizer, package_type, cache, distcc_hosts, unbundled, split_binary, version, author, official, alien_pkgs): +def parse_env_variables(build_type, compiler, sanitizer, package_type, cache, distcc_hosts, unbundled, split_binary, version, author, official, alien_pkgs, with_coverage): result = [] cmake_flags = ['$CMAKE_FLAGS'] @@ -146,7 +146,10 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, cache, di cmake_flags.append('-DUNBUNDLED=1 -DENABLE_MYSQL=0 -DENABLE_POCO_ODBC=0 -DENABLE_ODBC=0') if split_binary: - cmake_flags.append('-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1') + cmake_flags.append('-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1 -DGLIBC_COMPATIBILITY=ON') + + if with_coverage: + cmake_flags.append('-DWITH_COVERAGE=1') if version: result.append("VERSION_STRING='{}'".format(version)) @@ -180,6 +183,7 @@ if __name__ == "__main__": parser.add_argument("--author", default="clickhouse") parser.add_argument("--official", action="store_true") parser.add_argument("--alien-pkgs", nargs='+', default=[]) + parser.add_argument("--with-coverage", action="store_true") args = parser.parse_args() if not os.path.isabs(args.output_dir): @@ -202,7 +206,7 @@ if __name__ == "__main__": env_prepared = parse_env_variables( args.build_type, args.compiler, args.sanitizer, args.package_type, args.cache, args.distcc_hosts, args.unbundled, args.split_binary, - args.version, args.author, args.official, args.alien_pkgs) + args.version, args.author, args.official, args.alien_pkgs, args.with_coverage) if args.package_type != "freebsd": run_docker_image_with_env(image_name, args.output_dir, env_prepared, ch_root, args.ccache_dir) else: diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index 879a9cd8c59..fed4295b76c 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=19.11.0 +ARG version=19.12.1.* ARG gosu_ver=1.10 RUN apt-get update \ diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile index 9743ec8624c..488a2554483 100644 --- a/docker/test/Dockerfile +++ b/docker/test/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=19.11.0 +ARG version=19.12.1.* RUN apt-get update && \ apt-get install -y apt-transport-https dirmngr && \ diff --git a/docker/test/coverage/Dockerfile b/docker/test/coverage/Dockerfile new file mode 100644 index 00000000000..c0c31a42571 --- /dev/null +++ b/docker/test/coverage/Dockerfile @@ -0,0 +1,36 @@ +# docker build -t yandex/clickhouse-coverage . +FROM yandex/clickhouse-deb-builder + +RUN apt-get --allow-unauthenticated update -y \ + && env DEBIAN_FRONTEND=noninteractive \ + apt-get --allow-unauthenticated install --yes --no-install-recommends \ + bash \ + fakeroot \ + cmake \ + ccache \ + curl \ + software-properties-common + + +RUN echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic main" >> /etc/apt/sources.list + +RUN apt-get --allow-unauthenticated update -y \ + && env DEBIAN_FRONTEND=noninteractive \ + apt-get --allow-unauthenticated install --yes --no-install-recommends \ + perl \ + lcov \ + llvm-9 \ + tzdata + + +ENV COVERAGE_DIR=/coverage_reports +ENV SOURCE_DIR=/build +ENV OUTPUT_DIR=/output +ENV IGNORE='.*contrib.*' + + +CMD mkdir -p /build/obj-x86_64-linux-gnu && cd /build/obj-x86_64-linux-gnu && CC=clang-7 CXX=clang++-7 cmake .. && cd /; \ + dpkg -i /package_folder/clickhouse-common-static_*.deb; \ + llvm-profdata-9 merge -sparse ${COVERAGE_DIR}/* -o clickhouse.profdata && \ + llvm-cov-9 export /usr/bin/clickhouse -instr-profile=clickhouse.profdata -j=16 -format=lcov -skip-functions -ignore-filename-regex $IGNORE > output.lcov && \ + genhtml output.lcov --ignore-errors source --output-directory ${OUTPUT_DIR} diff --git a/docker/test/integration/Dockerfile b/docker/test/integration/Dockerfile index d582247db74..c5f4629ba72 100644 --- a/docker/test/integration/Dockerfile +++ b/docker/test/integration/Dockerfile @@ -1,8 +1,10 @@ # docker build -t yandex/clickhouse-integration-test . FROM ubuntu:18.04 +RUN echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main" >> /etc/apt/sources.list + RUN apt-get update \ - && env DEBIAN_FRONTEND=noninteractive apt-get -y install tzdata python llvm-6.0 llvm-6.0-dev libreadline-dev libicu-dev bsdutils \ + && env DEBIAN_FRONTEND=noninteractive apt-get -y install tzdata python llvm-6.0 llvm-6.0-dev libreadline-dev libicu-dev bsdutils llvm-8 \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/debconf \ @@ -15,5 +17,6 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone RUN echo "TSAN_OPTIONS='halt_on_error=1 history_size=7'" >> /etc/environment; \ echo "UBSAN_OPTIONS='print_stacktrace=1'" >> /etc/environment; \ echo "ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-6.0/bin/llvm-symbolizer" >> /etc/environment; \ + echo "TSAN_SYMBOLIZER_PATH=/usr/lib/llvm-8/bin/llvm-symbolizer" >> /etc/environment; \ echo "UBSAN_SYMBOLIZER_PATH=/usr/lib/llvm-6.0/bin/llvm-symbolizer" >> /etc/environment; \ echo "LLVM_SYMBOLIZER_PATH=/usr/lib/llvm-6.0/bin/llvm-symbolizer" >> /etc/environment; diff --git a/docker/test/stateful/Dockerfile b/docker/test/stateful/Dockerfile index 65943bf6955..466578c8b98 100644 --- a/docker/test/stateful/Dockerfile +++ b/docker/test/stateful/Dockerfile @@ -16,8 +16,16 @@ CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \ dpkg -i package_folder/clickhouse-server_*.deb; \ dpkg -i package_folder/clickhouse-client_*.deb; \ dpkg -i package_folder/clickhouse-test_*.deb; \ + ln -s /usr/share/clickhouse-test/config/zookeeper.xml /etc/clickhouse-server/config.d/; \ + ln -s /usr/share/clickhouse-test/config/listen.xml /etc/clickhouse-server/config.d/; \ + ln -s /usr/share/clickhouse-test/config/part_log.xml /etc/clickhouse-server/config.d/; \ + ln -s /usr/share/clickhouse-test/config/log_queries.xml /etc/clickhouse-server/users.d/; \ + ln -s /usr/share/clickhouse-test/config/readonly.xml /etc/clickhouse-server/users.d/; \ + ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/; \ + ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/; \ + ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/; \ ln -s /usr/lib/llvm-8/bin/llvm-symbolizer /usr/bin/llvm-symbolizer; \ - echo "TSAN_OPTIONS='halt_on_error=1 history_size=7'" >> /etc/environment; \ + echo "TSAN_OPTIONS='verbosity=1000 halt_on_error=1 history_size=7'" >> /etc/environment; \ echo "TSAN_SYMBOLIZER_PATH=/usr/lib/llvm-8/bin/llvm-symbolizer" >> /etc/environment; \ echo "UBSAN_OPTIONS='print_stacktrace=1'" >> /etc/environment; \ echo "ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-6.0/bin/llvm-symbolizer" >> /etc/environment; \ diff --git a/docker/test/stateful_with_coverage/Dockerfile b/docker/test/stateful_with_coverage/Dockerfile new file mode 100644 index 00000000000..2a566bdcf01 --- /dev/null +++ b/docker/test/stateful_with_coverage/Dockerfile @@ -0,0 +1,19 @@ +# docker build -t yandex/clickhouse-stateful-test . +FROM yandex/clickhouse-stateless-test + +RUN echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic main" >> /etc/apt/sources.list + +RUN apt-get update -y \ + && env DEBIAN_FRONTEND=noninteractive \ + apt-get install --yes --no-install-recommends \ + python-requests \ + llvm-8 \ + llvm-9 + +COPY s3downloader /s3downloader +COPY run.sh /run.sh + +ENV DATASETS="hits visits" + +CMD ["/bin/bash", "/run.sh"] + diff --git a/docker/test/stateful_with_coverage/run.sh b/docker/test/stateful_with_coverage/run.sh new file mode 100755 index 00000000000..6ec0bfa155a --- /dev/null +++ b/docker/test/stateful_with_coverage/run.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +kill_clickhouse () { + while kill -0 `pgrep -u clickhouse`; + do + kill `pgrep -u clickhouse` 2>/dev/null + echo "Process" `pgrep -u clickhouse` "still alive" + sleep 10 + done +} + +start_clickhouse () { + LLVM_PROFILE_FILE='server_%h_%p_%m.profraw' sudo -Eu clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml & +} + +wait_llvm_profdata () { + while kill -0 `pgrep llvm-profdata-9`; + do + echo "Waiting for profdata " `pgrep llvm-profdata-9` "still alive" + sleep 3 + done +} + +merge_client_files_in_background () { + client_files=`ls /client_*profraw 2>/dev/null` + if [ ! -z "$client_files" ] + then + llvm-profdata-9 merge -sparse $client_files -o merged_client_`date +%s`.profraw + rm $client_files + fi +} + +chmod 777 / + +dpkg -i package_folder/clickhouse-common-static_*.deb; \ + dpkg -i package_folder/clickhouse-common-static-dbg_*.deb; \ + dpkg -i package_folder/clickhouse-server_*.deb; \ + dpkg -i package_folder/clickhouse-client_*.deb; \ + dpkg -i package_folder/clickhouse-test_*.deb + +mkdir -p /var/lib/clickhouse +mkdir -p /var/log/clickhouse-server +chmod 777 -R /var/log/clickhouse-server/ + +ln -s /usr/share/clickhouse-test/config/zookeeper.xml /etc/clickhouse-server/config.d/; \ + ln -s /usr/share/clickhouse-test/config/listen.xml /etc/clickhouse-server/config.d/; \ + ln -s /usr/share/clickhouse-test/config/part_log.xml /etc/clickhouse-server/config.d/; \ + ln -s /usr/share/clickhouse-test/config/log_queries.xml /etc/clickhouse-server/users.d/; \ + ln -s /usr/share/clickhouse-test/config/readonly.xml /etc/clickhouse-server/users.d/; \ + ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/; \ + ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/; \ + ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/; \ + ln -s /usr/lib/llvm-8/bin/llvm-symbolizer /usr/bin/llvm-symbolizer + + +service zookeeper start + +sleep 5 + +start_clickhouse + +sleep 5 + +/s3downloader --dataset-names $DATASETS + +chmod 777 -R /var/lib/clickhouse + +while /bin/true; do + merge_client_files_in_background + sleep 2 +done & + +LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-client --query "SHOW DATABASES" +LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-client --query "CREATE DATABASE datasets" +LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-client --query "CREATE DATABASE test" + +kill_clickhouse +start_clickhouse + +sleep 10 + +LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-client --query "SHOW TABLES FROM datasets" +LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-client --query "SHOW TABLES FROM test" +LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits" +LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits" +LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-client --query "SHOW TABLES FROM test" +LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-test --shard --zookeeper --no-stateless $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt + +kill_clickhouse + +wait_llvm_profdata + +sleep 3 + +wait_llvm_profdata # 100% merged all parts + + +cp /*.profraw /profraw ||: diff --git a/docker/test/stateful_with_coverage/s3downloader b/docker/test/stateful_with_coverage/s3downloader new file mode 100755 index 00000000000..f8e2bf3cbe4 --- /dev/null +++ b/docker/test/stateful_with_coverage/s3downloader @@ -0,0 +1,86 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import os +import sys +import tarfile +import logging +import argparse +import requests +import tempfile + + +DEFAULT_URL = 'https://clickhouse-datasets.s3.yandex.net' + +AVAILABLE_DATASETS = { + 'hits': 'hits_v1.tar', + 'visits': 'visits_v1.tar', +} + +def _get_temp_file_name(): + return os.path.join(tempfile._get_default_tempdir(), next(tempfile._get_candidate_names())) + +def build_url(base_url, dataset): + return os.path.join(base_url, dataset, 'partitions', AVAILABLE_DATASETS[dataset]) + +def dowload_with_progress(url, path): + logging.info("Downloading from %s to temp path %s", url, path) + with open(path, 'w') as f: + response = requests.get(url, stream=True) + response.raise_for_status() + total_length = response.headers.get('content-length') + if total_length is None or int(total_length) == 0: + logging.info("No content-length, will download file without progress") + f.write(response.content) + else: + dl = 0 + total_length = int(total_length) + logging.info("Content length is %ld bytes", total_length) + for data in response.iter_content(chunk_size=4096): + dl += len(data) + f.write(data) + if sys.stdout.isatty(): + done = int(50 * dl / total_length) + percent = int(100 * float(dl) / total_length) + sys.stdout.write("\r[{}{}] {}%".format('=' * done, ' ' * (50-done), percent)) + sys.stdout.flush() + sys.stdout.write("\n") + logging.info("Downloading finished") + +def unpack_to_clickhouse_directory(tar_path, clickhouse_path): + logging.info("Will unpack data from temp path %s to clickhouse db %s", tar_path, clickhouse_path) + with tarfile.open(tar_path, 'r') as comp_file: + comp_file.extractall(path=clickhouse_path) + logging.info("Unpack finished") + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser( + description="Simple tool for dowloading datasets for clickhouse from S3") + + parser.add_argument('--dataset-names', required=True, nargs='+', choices=AVAILABLE_DATASETS.keys()) + parser.add_argument('--url-prefix', default=DEFAULT_URL) + parser.add_argument('--clickhouse-data-path', default='/var/lib/clickhouse/') + + args = parser.parse_args() + datasets = args.dataset_names + logging.info("Will fetch following datasets: %s", ', '.join(datasets)) + for dataset in datasets: + logging.info("Processing %s", dataset) + temp_archive_path = _get_temp_file_name() + try: + download_url_for_dataset = build_url(args.url_prefix, dataset) + dowload_with_progress(download_url_for_dataset, temp_archive_path) + unpack_to_clickhouse_directory(temp_archive_path, args.clickhouse_data_path) + except Exception as ex: + logging.info("Some exception occured %s", str(ex)) + raise + finally: + logging.info("Will remove dowloaded file %s from filesystem if it exists", temp_archive_path) + if os.path.exists(temp_archive_path): + os.remove(temp_archive_path) + logging.info("Processing of %s finished", dataset) + logging.info("Fetch finished, enjoy your tables!") + + diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile index 16f8fc437bc..e2cd5eee933 100644 --- a/docker/test/stateless/Dockerfile +++ b/docker/test/stateless/Dockerfile @@ -39,13 +39,14 @@ CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \ ln -s /usr/share/clickhouse-test/config/zookeeper.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/listen.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/part_log.xml /etc/clickhouse-server/config.d/; \ + ln -s /usr/share/clickhouse-test/config/query_masking_rules.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/log_queries.xml /etc/clickhouse-server/users.d/; \ ln -s /usr/share/clickhouse-test/config/readonly.xml /etc/clickhouse-server/users.d/; \ ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/; \ ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/; \ ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/; \ ln -s /usr/lib/llvm-8/bin/llvm-symbolizer /usr/bin/llvm-symbolizer; \ - echo "TSAN_OPTIONS='halt_on_error=1 history_size=7'" >> /etc/environment; \ + echo "TSAN_OPTIONS='verbosity=1000 halt_on_error=1 history_size=7'" >> /etc/environment; \ echo "UBSAN_OPTIONS='print_stacktrace=1'" >> /etc/environment; \ echo "ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-6.0/bin/llvm-symbolizer" >> /etc/environment; \ echo "UBSAN_SYMBOLIZER_PATH=/usr/lib/llvm-6.0/bin/llvm-symbolizer" >> /etc/environment; \ diff --git a/docker/test/stateless_with_coverage/Dockerfile b/docker/test/stateless_with_coverage/Dockerfile new file mode 100644 index 00000000000..b9da18223ab --- /dev/null +++ b/docker/test/stateless_with_coverage/Dockerfile @@ -0,0 +1,36 @@ +# docker build -t yandex/clickhouse-stateless-with-coverage-test . +FROM yandex/clickhouse-deb-builder + +RUN echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic main" >> /etc/apt/sources.list + +RUN apt-get update -y \ + && env DEBIAN_FRONTEND=noninteractive \ + apt-get install --yes --no-install-recommends \ + bash \ + tzdata \ + fakeroot \ + debhelper \ + zookeeper \ + zookeeperd \ + expect \ + python \ + python-lxml \ + python-termcolor \ + python-requests \ + curl \ + sudo \ + openssl \ + netcat-openbsd \ + telnet \ + moreutils \ + brotli \ + gdb \ + lsof \ + llvm-9 + + +ENV TZ=Europe/Moscow +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +COPY run.sh /run.sh + +CMD ["/bin/bash", "/run.sh"] diff --git a/docker/test/stateless_with_coverage/run.sh b/docker/test/stateless_with_coverage/run.sh new file mode 100755 index 00000000000..50082337757 --- /dev/null +++ b/docker/test/stateless_with_coverage/run.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +kill_clickhouse () { + while kill -0 `pgrep -u clickhouse`; + do + kill `pgrep -u clickhouse` 2>/dev/null + echo "Process" `pgrep -u clickhouse` "still alive" + sleep 10 + done +} + +wait_llvm_profdata () { + while kill -0 `pgrep llvm-profdata-9`; + do + echo "Waiting for profdata " `pgrep llvm-profdata-9` "still alive" + sleep 3 + done +} + +start_clickhouse () { + LLVM_PROFILE_FILE='server_%h_%p_%m.profraw' sudo -Eu clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml & +} + +merge_client_files_in_background () { + client_files=`ls /client_*profraw 2>/dev/null` + if [ ! -z "$client_files" ] + then + llvm-profdata-9 merge -sparse $client_files -o merged_client_`date +%s`.profraw + rm $client_files + fi +} + +chmod 777 / + +dpkg -i package_folder/clickhouse-common-static_*.deb; \ + dpkg -i package_folder/clickhouse-common-static-dbg_*.deb; \ + dpkg -i package_folder/clickhouse-server_*.deb; \ + dpkg -i package_folder/clickhouse-client_*.deb; \ + dpkg -i package_folder/clickhouse-test_*.deb + + +mkdir -p /var/lib/clickhouse +mkdir -p /var/log/clickhouse-server +chmod 777 -R /var/lib/clickhouse +chmod 777 -R /var/log/clickhouse-server/ + +ln -s /usr/share/clickhouse-test/config/zookeeper.xml /etc/clickhouse-server/config.d/; \ + ln -s /usr/share/clickhouse-test/config/listen.xml /etc/clickhouse-server/config.d/; \ + ln -s /usr/share/clickhouse-test/config/part_log.xml /etc/clickhouse-server/config.d/; \ + ln -s /usr/share/clickhouse-test/config/log_queries.xml /etc/clickhouse-server/users.d/; \ + ln -s /usr/share/clickhouse-test/config/readonly.xml /etc/clickhouse-server/users.d/; \ + ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/; \ + ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/; \ + ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/; \ + ln -s /usr/lib/llvm-8/bin/llvm-symbolizer /usr/bin/llvm-symbolizer + +service zookeeper start +sleep 5 + +start_clickhouse + +sleep 10 + +while /bin/true; do + merge_client_files_in_background + sleep 2 +done & + +LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-test --shard --zookeeper $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt + +kill_clickhouse + +wait_llvm_profdata + +sleep 3 + +wait_llvm_profdata # 100% merged all parts + +cp /*.profraw /profraw ||: diff --git a/docker/test/stress/Dockerfile b/docker/test/stress/Dockerfile index 0ee49d64d51..b0b94ccc579 100644 --- a/docker/test/stress/Dockerfile +++ b/docker/test/stress/Dockerfile @@ -34,13 +34,8 @@ CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \ ln -s /usr/share/clickhouse-test/config/log_queries.xml /etc/clickhouse-server/users.d/; \ ln -s /usr/share/clickhouse-test/config/part_log.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/lib/llvm-8/bin/llvm-symbolizer /usr/bin/llvm-symbolizer; \ - echo "TSAN_OPTIONS='halt_on_error=1 history_size=7'" >> /etc/environment; \ - echo "TSAN_SYMBOLIZER_PATH=/usr/lib/llvm-8/bin/llvm-symbolizer" >> /etc/environment; \ + echo "TSAN_OPTIONS='halt_on_error=1 history_size=7 ignore_noninstrumented_modules=1 verbosity=1'" >> /etc/environment; \ echo "UBSAN_OPTIONS='print_stacktrace=1'" >> /etc/environment; \ - echo "ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-6.0/bin/llvm-symbolizer" >> /etc/environment; \ - echo "UBSAN_SYMBOLIZER_PATH=/usr/lib/llvm-6.0/bin/llvm-symbolizer" >> /etc/environment; \ - echo "TSAN_SYMBOLIZER_PATH=/usr/lib/llvm-6.0/bin/llvm-symbolizer" >> /etc/environment; \ - echo "LLVM_SYMBOLIZER_PATH=/usr/lib/llvm-6.0/bin/llvm-symbolizer" >> /etc/environment; \ service clickhouse-server start && sleep 5 \ && /s3downloader --dataset-names $DATASETS \ && chmod 777 -R /var/lib/clickhouse \ diff --git a/docs/en/database_engines/index.md b/docs/en/database_engines/index.md new file mode 100644 index 00000000000..3190a79017f --- /dev/null +++ b/docs/en/database_engines/index.md @@ -0,0 +1,9 @@ +# Database Engines + +Database engines provide working with tables. + +By default, ClickHouse uses its native database engine which provides configurable [table engines](../operations/table_engines/index.md) and [SQL dialect](../query_language/syntax.md). + +Also you can use the following database engines: + +- [MySQL](mysql.md) diff --git a/docs/en/database_engines/mysql.md b/docs/en/database_engines/mysql.md new file mode 100644 index 00000000000..fd5417ee27d --- /dev/null +++ b/docs/en/database_engines/mysql.md @@ -0,0 +1,122 @@ +# MySQL + +Allows to connect to some database on remote MySQL server and perform `INSERT` and `SELECT` queries with tables to exchange data between ClickHouse and MySQL. + +The `MySQL` database engine translate queries to the MySQL server, so you can perform operations such as `SHOW TABLES` or `SHOW CREATE TABLE`. + +You cannot perform with tables the following queries: + +- `ATTACH`/`DETACH` +- `DROP` +- `RENAME` +- `CREATE TABLE` +- `ALTER` + + +## Creating a Database + +``` sql +CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster] +ENGINE = MySQL('host:port', 'database', 'user', 'password') +``` + +**Engine Parameters** + +- `host:port` — MySQL server address. +- `database` — Remote database name. +- `user` — MySQL user. +- `password` — User password. + + +## Data Types Support + +MySQL | ClickHouse +------|------------ +UNSIGNED TINYINT | [UInt8](../data_types/int_uint.md) +TINYINT | [Int8](../data_types/int_uint.md) +UNSIGNED SMALLINT | [UInt16](../data_types/int_uint.md) +SMALLINT | [Int16](../data_types/int_uint.md) +UNSIGNED INT, UNSIGNED MEDIUMINT | [UInt32](../data_types/int_uint.md) +INT, MEDIUMINT | [Int32](../data_types/int_uint.md) +UNSIGNED BIGINT | [UInt64](../data_types/int_uint.md) +BIGINT | [Int64](../data_types/int_uint.md) +FLOAT | [Float32](../data_types/float.md) +DOUBLE | [Float64](../data_types/float.md) +DATE | [Date](../data_types/date.md) +DATETIME, TIMESTAMP | [DateTime](../data_types/datetime.md) +BINARY | [FixedString](../data_types/fixedstring.md) + +All other MySQL data types are converted into [String](../data_types/string.md). + +[Nullable](../data_types/nullable.md) data type is supported. + + +## Examples of Use + +Table in MySQL: + +``` +mysql> USE test; +Database changed + +mysql> CREATE TABLE `mysql_table` ( + -> `int_id` INT NOT NULL AUTO_INCREMENT, + -> `float` FLOAT NOT NULL, + -> PRIMARY KEY (`int_id`)); +Query OK, 0 rows affected (0,09 sec) + +mysql> insert into mysql_table (`int_id`, `float`) VALUES (1,2); +Query OK, 1 row affected (0,00 sec) + +mysql> select * from mysql_table; ++--------+-------+ +| int_id | value | ++--------+-------+ +| 1 | 2 | ++--------+-------+ +1 row in set (0,00 sec) +``` + +Database in ClickHouse, exchanging data with the MySQL server: + +```sql +CREATE DATABASE mysql_db ENGINE = MySQL('localhost:3306', 'test', 'my_user', 'user_password') +``` +```sql +SHOW DATABASES +``` +```text +┌─name─────┐ +│ default │ +│ mysql_db │ +│ system │ +└──────────┘ +``` +```sql +SHOW TABLES FROM mysql_db +``` +```text +┌─name─────────┐ +│ mysql_table │ +└──────────────┘ +``` +```sql +SELECT * FROM mysql_db.mysql_table +``` +```text +┌─int_id─┬─value─┐ +│ 1 │ 2 │ +└────────┴───────┘ +``` +```sql +INSERT INTO mysql_db.mysql_table VALUES (3,4) +``` +```sql +SELECT * FROM mysql_db.mysql_table +``` +```text +┌─int_id─┬─value─┐ +│ 1 │ 2 │ +│ 3 │ 4 │ +└────────┴───────┘ +``` diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index 29055aac09e..71f28263270 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -1,10 +1,14 @@ # Formats for input and output data {#formats} -ClickHouse can accept (`INSERT`) and return (`SELECT`) data in various formats. +ClickHouse can accept and return data in various formats. A format supported +for input can be used to parse the data provided to `INSERT`s, to perform +`SELECT`s from a file-backed table such as File, URL or HDFS, or to read an +external dictionary. A format supported for output can be used to arrange the +results of a `SELECT`, and to perform `INSERT`s into a file-backed table. -The table below lists supported formats and how they can be used in `INSERT` and `SELECT` queries. +The supported formats are: -| Format | INSERT | SELECT | +| Format | Input | Output | | ------- | -------- | -------- | | [TabSeparated](#tabseparated) | ✔ | ✔ | | [TabSeparatedRaw](#tabseparatedraw) | ✗ | ✔ | @@ -165,8 +169,7 @@ clickhouse-client --format_csv_delimiter="|" --query="INSERT INTO test.csv FORMA When parsing, all values can be parsed either with or without quotes. Both double and single quotes are supported. Rows can also be arranged without quotes. In this case, they are parsed up to the delimiter character or line feed (CR or LF). In violation of the RFC, when parsing rows without quotes, the leading and trailing spaces and tabs are ignored. For the line feed, Unix (LF), Windows (CR LF) and Mac OS Classic (CR LF) types are all supported. -Empty unquoted input values are replaced with default values for the respective -columns, if +Empty unquoted input values are replaced with default values for the respective columns, if [input_format_defaults_for_omitted_fields](../operations/settings/settings.md#session_settings-input_format_defaults_for_omitted_fields) is enabled. @@ -320,7 +323,7 @@ When using this format, ClickHouse outputs rows as separated, newline-delimited ```json {"SearchPhrase":"curtain designs","count()":"1064"} {"SearchPhrase":"baku","count()":"1000"} -{"SearchPhrase":"","count":"8267016"} +{"SearchPhrase":"","count()":"8267016"} ``` When inserting the data, you should provide a separate JSON object for each row. @@ -383,6 +386,60 @@ Unlike the [JSON](#json) format, there is no substitution of invalid UTF-8 seque !!! note "Note" Any set of bytes can be output in the strings. Use the `JSONEachRow` format if you are sure that the data in the table can be formatted as JSON without losing any information. +### Usage of Nested Structures {#jsoneachrow-nested} + +If you have a table with the [Nested](../data_types/nested_data_structures/nested.md) data type columns, you can insert JSON data having the same structure. Enable this functionality with the [input_format_import_nested_json](../operations/settings/settings.md#settings-input_format_import_nested_json) setting. + +For example, consider the following table: + +```sql +CREATE TABLE json_each_row_nested (n Nested (s String, i Int32) ) ENGINE = Memory +``` + +As you can find in the `Nested` data type description, ClickHouse treats each component of the nested structure as a separate column, `n.s` and `n.i` for our table. So you can insert the data the following way: + +```sql +INSERT INTO json_each_row_nested FORMAT JSONEachRow {"n.s": ["abc", "def"], "n.i": [1, 23]} +``` + +To insert data as hierarchical JSON object set [input_format_import_nested_json=1](../operations/settings/settings.md#settings-input_format_import_nested_json). + +```json +{ + "n": { + "s": ["abc", "def"], + "i": [1, 23] + } +} +``` + +Without this setting ClickHouse throws the exception. + +```sql +SELECT name, value FROM system.settings WHERE name = 'input_format_import_nested_json' +``` +```text +┌─name────────────────────────────┬─value─┐ +│ input_format_import_nested_json │ 0 │ +└─────────────────────────────────┴───────┘ +``` +```sql +INSERT INTO json_each_row_nested FORMAT JSONEachRow {"n": {"s": ["abc", "def"], "i": [1, 23]}} +``` +```text +Code: 117. DB::Exception: Unknown field found while parsing JSONEachRow format: n: (at row 1) +``` +```sql +SET input_format_import_nested_json=1 +INSERT INTO json_each_row_nested FORMAT JSONEachRow {"n": {"s": ["abc", "def"], "i": [1, 23]}} +SELECT * FROM json_each_row_nested +``` +```text +┌─n.s───────────┬─n.i────┐ +│ ['abc','def'] │ [1,23] │ +└───────────────┴────────┘ +``` + ## Native {#native} The most efficient format. Data is written and read by blocks in binary format. For each block, the number of rows, number of columns, column names and types, and parts of columns in this block are recorded one after another. In other words, this format is "columnar" – it doesn't convert columns to rows. This is the format used in the native interface for interaction between servers, for using the command-line client, and for C++ clients. diff --git a/docs/en/interfaces/third-party/integrations.md b/docs/en/interfaces/third-party/integrations.md index 7c302884631..f96507320a5 100644 --- a/docs/en/interfaces/third-party/integrations.md +++ b/docs/en/interfaces/third-party/integrations.md @@ -14,6 +14,7 @@ - [clickhousedb_fdw](https://github.com/Percona-Lab/clickhousedb_fdw) - [infi.clickhouse_fdw](https://github.com/Infinidat/infi.clickhouse_fdw) (uses [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm)) - [pg2ch](https://github.com/mkabilov/pg2ch) + - [clickhouse_fdw](https://github.com/adjust/clickhouse_fdw) - [MSSQL](https://en.wikipedia.org/wiki/Microsoft_SQL_Server) - [ClickHouseMigrator](https://github.com/zlzforever/ClickHouseMigrator) - Message queues diff --git a/docs/en/operations/server_settings/settings.md b/docs/en/operations/server_settings/settings.md index 493a558c8e2..2f87233e6a3 100644 --- a/docs/en/operations/server_settings/settings.md +++ b/docs/en/operations/server_settings/settings.md @@ -514,7 +514,7 @@ Use the following parameters to configure logging: ``` -## path +## path {#server_settings-path} The path to the directory containing data. diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index b7b12e31c06..c68643d3877 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -199,15 +199,10 @@ Ok. ## input_format_defaults_for_omitted_fields {#session_settings-input_format_defaults_for_omitted_fields} -When performing `INSERT` queries, replace omitted input column values with -default values of the respective columns. This option only applies to -[JSONEachRow](../../interfaces/formats.md#jsoneachrow) and -[CSV](../../interfaces/formats.md#csv) formats. +When performing `INSERT` queries, replace omitted input column values with default values of the respective columns. This option only applies to [JSONEachRow](../../interfaces/formats.md#jsoneachrow) and [CSV](../../interfaces/formats.md#csv) formats. !!! note "Note" - When this option is enabled, extended table metadata are sent -from server to client. It consumes additional computing resources on the server -and can reduce performance. + When this option is enabled, extended table metadata are sent from server to client. It consumes additional computing resources on the server and can reduce performance. Possible values: @@ -236,6 +231,25 @@ Possible values: Default value: 0. +## input_format_import_nested_json {#settings-input_format_import_nested_json} + +Enables or disables inserting of JSON data with nested objects. + +Supported formats: + +- [JSONEachRow](../../interfaces/formats.md#jsoneachrow) + +Possible values: + +- 0 — Disabled. +- 1 — Enabled. + +Default value: 0. + +**See Also** + +- [Usage of Nested Structures](../../interfaces/formats.md#jsoneachrow-nested) with the `JSONEachRow` format. + ## input_format_with_names_use_header {#settings-input_format_with_names_use_header} Enables or disables checking the column order when inserting data. @@ -254,6 +268,27 @@ Possible values: Default value: 1. +## date_time_input_format {#settings-date_time_input_format} + +Enables or disables extended parsing of date and time formatted strings. + +The setting doesn't apply to [date and time functions](../../query_language/functions/date_time_functions.md). + +Possible values: + +- `'best_effort'` — Enables extended parsing. + + ClickHouse can parse the basic format `YYYY-MM-DD HH:MM:SS` and all the [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601) date and time formats. For example, `'2018-06-08T01:02:03.000Z'`. + +- `'basic'` — Use basic parser. + + ClickHouse can parse only the basic format. + +**See Also** + +- [DateTime data type.](../../data_types/datetime.md) +- [Functions for working with dates and times.](../../query_language/functions/date_time_functions.md) + ## join_default_strictness {#settings-join_default_strictness} Sets default strictness for [JOIN clauses](../../query_language/select.md#select-join). diff --git a/docs/en/operations/table_engines/collapsingmergetree.md b/docs/en/operations/table_engines/collapsingmergetree.md index cf521ac4cd1..b4f7bd8e6cf 100644 --- a/docs/en/operations/table_engines/collapsingmergetree.md +++ b/docs/en/operations/table_engines/collapsingmergetree.md @@ -2,7 +2,7 @@ The engine inherits from [MergeTree](mergetree.md) and adds the logic of rows collapsing to data parts merge algorithm. -`CollapsingMergeTree` asynchronously deletes (collapses) pairs of rows if all of the fields in a row are equivalent excepting the particular field `Sign` which can have `1` and `-1` values. Rows without a pair are kept. For more details see the [Collapsing](#table_engine-collapsingmergetree-collapsing) section of the document. +`CollapsingMergeTree` asynchronously deletes (collapses) pairs of rows if all of the fields in a sorting key (`ORDER BY`) are equivalent excepting the particular field `Sign` which can have `1` and `-1` values. Rows without a pair are kept. For more details see the [Collapsing](#table_engine-collapsingmergetree-collapsing) section of the document. The engine may significantly reduce the volume of storage and increase efficiency of `SELECT` query as a consequence. @@ -80,7 +80,7 @@ At some moment later we register the change of user activity and write it with t └─────────────────────┴───────────┴──────────┴──────┘ ``` -The first row cancels the previous state of the object (user). It should copy all of the fields of the canceled state excepting `Sign`. +The first row cancels the previous state of the object (user). It should copy the sorting key fields of the canceled state excepting `Sign`. The second row contains the current state. @@ -99,13 +99,13 @@ Why we need 2 rows for each change read in the [Algorithm](#table_engine-collaps **Peculiar properties of such approach** -1. The program that writes the data should remember the state of an object to be able to cancel it. "Cancel" string should be the copy of "state" string with the opposite `Sign`. It increases the initial size of storage but allows to write the data quickly. +1. The program that writes the data should remember the state of an object to be able to cancel it. "Cancel" string should contain copies of the sorting key fields of the "state" string and the opposite `Sign`. It increases the initial size of storage but allows to write the data quickly. 2. Long growing arrays in columns reduce the efficiency of the engine due to load for writing. The more straightforward data, the higher efficiency. 3. The `SELECT` results depend strongly on the consistency of object changes history. Be accurate when preparing data for inserting. You can get unpredictable results in inconsistent data, for example, negative values for non-negative metrics such as session depth. ### Algorithm {#table_engine-collapsingmergetree-collapsing-algorithm} -When ClickHouse merges data parts, each group of consecutive rows with the same primary key is reduced to not more than two rows, one with `Sign = 1` ("state" row) and another with `Sign = -1` ("cancel" row). In other words, entries collapse. +When ClickHouse merges data parts, each group of consecutive rows with the same sorting key (`ORDER BY`) is reduced to not more than two rows, one with `Sign = 1` ("state" row) and another with `Sign = -1` ("cancel" row). In other words, entries collapse. For each resulting data part ClickHouse saves: @@ -119,9 +119,9 @@ For each resulting data part ClickHouse saves: Thus, collapsing should not change the results of calculating statistics. Changes gradually collapsed so that in the end only the last state of almost every object left. -The `Sign` is required because the merging algorithm doesn't guarantee that all of the rows with the same primary key will be in the same resulting data part and even on the same physical server. ClickHouse process `SELECT` queries with multiple threads, and it can not predict the order of rows in the result. The aggregation is required if there is a need to get completely "collapsed" data from `CollapsingMergeTree` table. +The `Sign` is required because the merging algorithm doesn't guarantee that all of the rows with the same sorting key will be in the same resulting data part and even on the same physical server. ClickHouse process `SELECT` queries with multiple threads, and it can not predict the order of rows in the result. The aggregation is required if there is a need to get completely "collapsed" data from `CollapsingMergeTree` table. -To finalize collapsing write a query with `GROUP BY` clause and aggregate functions that account for the sign. For example, to calculate quantity, use `sum(Sign)` instead of `count()`. To calculate the sum of something, use `sum(Sign * x)` instead of `sum(x)`, and so on, and also add `HAVING sum(Sign) > 0`. +To finalize collapsing, write a query with `GROUP BY` clause and aggregate functions that account for the sign. For example, to calculate quantity, use `sum(Sign)` instead of `count()`. To calculate the sum of something, use `sum(Sign * x)` instead of `sum(x)`, and so on, and also add `HAVING sum(Sign) > 0`. The aggregates `count`, `sum` and `avg` could be calculated this way. The aggregate `uniq` could be calculated if an object has at least one state not collapsed. The aggregates `min` and `max` could not be calculated because `CollapsingMergeTree` does not save values history of the collapsed states. diff --git a/docs/en/operations/table_engines/file.md b/docs/en/operations/table_engines/file.md index 284f8deff58..9e3c5b3400b 100644 --- a/docs/en/operations/table_engines/file.md +++ b/docs/en/operations/table_engines/file.md @@ -1,6 +1,7 @@ -# File(InputFormat) {#table_engines-file} +# File {#table_engines-file} -The data source is a file that stores data in one of the supported input formats (TabSeparated, Native, etc.). +The File table engine keeps the data in a file in one of the supported [file +formats](../../interfaces/formats.md#formats) (TabSeparated, Native, etc.). Usage examples: @@ -14,7 +15,10 @@ Usage examples: File(Format) ``` -`Format` should be supported for either `INSERT` and `SELECT`. For the full list of supported formats see [Formats](../../interfaces/formats.md#formats). +The `Format` parameter specifies one of the available file formats. To perform +`SELECT` queries, the format must be supported for input, and to perform +`INSERT` queries -- for output. The available formats are listed in the +[Formats](../../interfaces/formats.md#formats) section. ClickHouse does not allow to specify filesystem path for`File`. It will use folder defined by [path](../server_settings/settings.md) setting in server configuration. @@ -67,7 +71,7 @@ $ echo -e "1,2\n3,4" | clickhouse-local -q "CREATE TABLE table (a Int64, b Int64 ## Details of Implementation -- Multiple SELECT queries can be performed concurrently, but INSERT queries will wait each other. +- Multiple `SELECT` queries can be performed concurrently, but `INSERT` queries will wait each other. - Not supported: - `ALTER` - `SELECT ... SAMPLE` diff --git a/docs/en/operations/table_engines/mergetree.md b/docs/en/operations/table_engines/mergetree.md index 86734031f3f..523b442319a 100644 --- a/docs/en/operations/table_engines/mergetree.md +++ b/docs/en/operations/table_engines/mergetree.md @@ -78,6 +78,7 @@ For a description of request parameters, see [request description](../../query_l - `index_granularity` — The granularity of an index. The number of data rows between the "marks" of an index. By default, 8192. The list of all available parameters you can see in [MergeTreeSettings.h](https://github.com/yandex/ClickHouse/blob/master/dbms/src/Storages/MergeTree/MergeTreeSettings.h). - `use_minimalistic_part_header_in_zookeeper` — Storage method of the data parts headers in ZooKeeper. If `use_minimalistic_part_header_in_zookeeper=1`, then ZooKeeper stores less data. For more information refer the [setting description](../server_settings/settings.md#server-settings-use_minimalistic_part_header_in_zookeeper) in the "Server configuration parameters" chapter. - `min_merge_bytes_to_use_direct_io` — The minimum data volume for merge operation required for using of the direct I/O access to the storage disk. During the merging of the data parts, ClickHouse calculates summary storage volume of all the data to be merged. If the volume exceeds `min_merge_bytes_to_use_direct_io` bytes, then ClickHouse reads and writes the data using direct I/O interface (`O_DIRECT` option) to the storage disk. If `min_merge_bytes_to_use_direct_io = 0`, then the direct I/O is disabled. Default value: `10 * 1024 * 1024 * 1024` bytes. + - `merge_with_ttl_timeout` — Minimal time in seconds, when merge with TTL can be repeated. Default value: 86400 (1 day). **Example of sections setting** @@ -315,12 +316,42 @@ For concurrent table access, we use multi-versioning. In other words, when a tab Reading from a table is automatically parallelized. -## TTL for columns and tables +## TTL for columns and tables {#table_engine-mergetree-ttl} -Data with expired TTL is removed while executing merges. +Determines the lifetime of values. -If TTL is set for column, when it expires, value will be replaced by default. If all values in columns were zeroed in part, data for this column will be deleted from disk for part. You are not allowed to set TTL for all key columns. If TTL is set for table, when it expires, row will be deleted. +The `TTL` clause can be set for the whole table and for each individual column. If `TTL` is set for the whole table, individual `TTL` for columns are ignored. -When TTL expires on some value or row in part, extra merge will be executed. To control frequency of merges with TTL you can set `merge_with_ttl_timeout`. If it is too low, many extra merges and lack of regular merges can reduce the perfomance. + +The table must have the column of the [Date](../../data_types/date.md) or [DateTime](../../data_types/datetime.md) data type. This date column should be used in the `TTL` clause. You can only set lifetime of the data as an interval from the date column value. + +``` +TTL date_time + interval +``` + +You can set the `interval` by any expression, returning the value of the `DateTime` data type. For example, you can use [time interval](../../query_language/operators.md#operators-datetime) operators. + +``` +TTL date_time + INTERVAL 1 MONTH +TTL date_time + INTERVAL 15 HOUR +``` + +**Column TTL** + +When the values in the column expire, ClickHouse replace them with the default values for the column data type. If all the column values in the data part become expired, ClickHouse deletes this column from the data part in a filesystem. + +The `TTL` clause cannot be used for key columns. + +**Table TTL** + +When some data in table expires, ClickHouse deletes all the corresponding rows. + +**Cleaning up of Data** + +Data with expired TTL is removed, when ClickHouse merges data parts. + +When ClickHouse see that some data is expired, it performs off-schedule merge. To control frequency of such merges, you can set [merge_with_ttl_timeout](#mergetree_setting-merge_with_ttl_timeout). If it is too low, many off-schedule merges consume much resources. + +If you perform the `SELECT` query between merges you can get the expired data. To avoid it, use the [OPTIMIZE](../../query_language/misc.md#misc_operations-optimize) query before `SELECT`. [Original article](https://clickhouse.yandex/docs/en/operations/table_engines/mergetree/) diff --git a/docs/en/query_language/agg_functions/combinators.md b/docs/en/query_language/agg_functions/combinators.md index 852b396332c..133b4d489d7 100644 --- a/docs/en/query_language/agg_functions/combinators.md +++ b/docs/en/query_language/agg_functions/combinators.md @@ -22,13 +22,21 @@ Example 2: `uniqArray(arr)` – Count the number of unique elements in all 'arr' ## -State -If you apply this combinator, the aggregate function doesn't return the resulting value (such as the number of unique values for the `uniq` function), but an intermediate state of the aggregation (for `uniq`, this is the hash table for calculating the number of unique values). This is an AggregateFunction(...) that can be used for further processing or stored in a table to finish aggregating later. To work with these states, use the [AggregatingMergeTree](../../operations/table_engines/aggregatingmergetree.md) table engine, the functions [`finalizeAggregation`](../functions/other_functions.md#finalizeaggregation) and [`runningAccumulate`](../functions/other_functions.md#function-runningaccumulate), and the combinators -Merge and -MergeState described below. +If you apply this combinator, the aggregate function doesn't return the resulting value (such as the number of unique values for the [uniq](reference.md#agg_function-uniq) function), but an intermediate state of the aggregation (for `uniq`, this is the hash table for calculating the number of unique values). This is an `AggregateFunction(...)` that can be used for further processing or stored in a table to finish aggregating later. -## -Merge +To work with these states, use: + +- [AggregatingMergeTree](../../operations/table_engines/aggregatingmergetree.md) table engine. +- [finalizeAggregation](../functions/other_functions.md#function-finalizeaggregation) function. +- [runningAccumulate](../functions/other_functions.md#function-runningaccumulate) function. +- [-Merge](#aggregate_functions_combinators_merge) combinator. +- [-MergeState](#aggregate_functions_combinators_mergestate) combinator. + +## -Merge {#aggregate_functions_combinators_merge} If you apply this combinator, the aggregate function takes the intermediate aggregation state as an argument, combines the states to finish aggregation, and returns the resulting value. -## -MergeState. +## -MergeState {#aggregate_functions_combinators_mergestate} Merges the intermediate aggregation states in the same way as the -Merge combinator. However, it doesn't return the resulting value, but an intermediate aggregation state, similar to the -State combinator. diff --git a/docs/en/query_language/agg_functions/reference.md b/docs/en/query_language/agg_functions/reference.md index 71bd1c2f198..76fb90d4fc4 100644 --- a/docs/en/query_language/agg_functions/reference.md +++ b/docs/en/query_language/agg_functions/reference.md @@ -360,7 +360,7 @@ GROUP BY timeslot ## skewPop -Computes the [skewness](https://en.wikipedia.org/wiki/Skewness) for sequence. +Computes the [skewness](https://en.wikipedia.org/wiki/Skewness) of a sequence. ``` skewPop(expr) @@ -372,9 +372,9 @@ skewPop(expr) **Returned value** -The skewness of given distribution. Type — [Float64](../../data_types/float.md) +The skewness of the given distribution. Type — [Float64](../../data_types/float.md) -**Example of Use** +**Example** ```sql SELECT skewPop(value) FROM series_with_value_column @@ -382,9 +382,9 @@ SELECT skewPop(value) FROM series_with_value_column ## skewSamp -Computes the [sample skewness](https://en.wikipedia.org/wiki/Skewness) for sequence. +Computes the [sample skewness](https://en.wikipedia.org/wiki/Skewness) of a sequence. -It represents an unbiased estimate of the skewness of a random variable, if passed values form it's sample. +It represents an unbiased estimate of the skewness of a random variable if passed values form its sample. ``` skewSamp(expr) @@ -396,9 +396,9 @@ skewSamp(expr) **Returned value** -The skewness of given distribution. Type — [Float64](../../data_types/float.md). If `n <= 1` (`n` is a size of the sample), then the function returns `nan`. +The skewness of the given distribution. Type — [Float64](../../data_types/float.md). If `n <= 1` (`n` is the size of the sample), then the function returns `nan`. -**Example of Use** +**Example** ```sql SELECT skewSamp(value) FROM series_with_value_column @@ -406,7 +406,7 @@ SELECT skewSamp(value) FROM series_with_value_column ## kurtPop -Computes the [kurtosis](https://en.wikipedia.org/wiki/Kurtosis) for sequence. +Computes the [kurtosis](https://en.wikipedia.org/wiki/Kurtosis) of a sequence. ``` kurtPop(expr) @@ -418,9 +418,9 @@ kurtPop(expr) **Returned value** -The kurtosis of given distribution. Type — [Float64](../../data_types/float.md) +The kurtosis of the given distribution. Type — [Float64](../../data_types/float.md) -**Example of Use** +**Example** ```sql SELECT kurtPop(value) FROM series_with_value_column @@ -428,9 +428,9 @@ SELECT kurtPop(value) FROM series_with_value_column ## kurtSamp -Computes the [sample kurtosis](https://en.wikipedia.org/wiki/Kurtosis) for sequence. +Computes the [sample kurtosis](https://en.wikipedia.org/wiki/Kurtosis) of a sequence. -It represents an unbiased estimate of the kurtosis of a random variable, if passed values form it's sample. +It represents an unbiased estimate of the kurtosis of a random variable if passed values form its sample. ``` kurtSamp(expr) @@ -442,9 +442,9 @@ kurtSamp(expr) **Returned value** -The kurtosis of given distribution. Type — [Float64](../../data_types/float.md). If `n <= 1` (`n` is a size of the sample), then the function returns `nan`. +The kurtosis of the given distribution. Type — [Float64](../../data_types/float.md). If `n <= 1` (`n` is a size of the sample), then the function returns `nan`. -**Example of Use** +**Example** ```sql SELECT kurtSamp(value) FROM series_with_value_column @@ -763,7 +763,7 @@ All the quantile functions also have corresponding quantiles functions: `quantil Calculates the amount `Σ((x - x̅)^2) / (n - 1)`, where `n` is the sample size and `x̅`is the average value of `x`. -It represents an unbiased estimate of the variance of a random variable, if passed values form it's sample. +It represents an unbiased estimate of the variance of a random variable if passed values form its sample. Returns `Float64`. When `n <= 1`, returns `+∞`. diff --git a/docs/en/query_language/create.md b/docs/en/query_language/create.md index 926ba95173f..94d32cc49f5 100644 --- a/docs/en/query_language/create.md +++ b/docs/en/query_language/create.md @@ -1,14 +1,31 @@ -## CREATE DATABASE +## CREATE DATABASE {#query_language-create-database} -Creating db_name databases +Creates database. ``` sql -CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster] +CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster] [ENGINE = engine(...)] ``` -`A database` is just a directory for tables. -If `IF NOT EXISTS` is included, the query won't return an error if the database already exists. +### Clauses +- `IF NOT EXISTS` + + If the `db_name` database already exists then: + + - If clause is specified, ClickHouse doesn't create a new database and doesn't throw an exception. + - If clause is not specified, then ClickHouse doesn't create a new database and throw and exception. + +- `ON CLUSTER` + + ClickHouse creates the `db_name` database on all the servers of a specified cluster. + +- `ENGINE` + + - [MySQL](../database_engines/mysql.md) + + Allows to retrieve data from the remote MySQL server. + + By default, ClickHouse uses its own [database engine](../database_engines/index.md). ## CREATE TABLE {#create-table-query} @@ -82,10 +99,7 @@ It is not possible to set default values for elements in nested data structures. ### TTL expression -Can be specified only for MergeTree-family tables. An expression for setting storage time for values. It must depends on `Date` or `DateTime` column and has one `Date` or `DateTime` column as a result. Example: - `TTL date + INTERVAL 1 DAY` - -You are not allowed to set TTL for key columns. For more details, see [TTL for columns and tables](../operations/table_engines/mergetree.md) +Defines storage time for values. Can be specified only for MergeTree-family tables. For the detailed description, see [TTL for columns and tables](../operations/table_engines/mergetree.md#table_engine-mergetree-ttl). ## Column Compression Codecs @@ -93,22 +107,26 @@ Besides default data compression, defined in [server settings](../operations/ser Supported compression algorithms: -- `NONE` - no compression for data applied -- `LZ4` -- `LZ4HC(level)` - (level) - LZ4\_HC compression algorithm with defined level. -Possible `level` range: \[3, 12\]. Default value: 9. Greater values stands for better compression and higher CPU usage. Recommended value range: [4,9]. -- `ZSTD(level)` - ZSTD compression algorithm with defined `level`. Possible `level` value range: \[1, 22\]. Default value: 1. -Greater values stands for better compression and higher CPU usage. -- `Delta(delta_bytes)` - compression approach when raw values are replace with difference of two neighbour values. Up to `delta_bytes` are used for storing delta value. -Possible `delta_bytes` values: 1, 2, 4, 8. Default value for delta bytes is `sizeof(type)`, if it is equals to 1, 2, 4, 8 and equals to 1 otherwise. -- `DoubleDelta` - stores delta of deltas in compact binary form, compressing values down to 1 bit (in the best case). Best compression rates are achieved on monotonic sequences with constant stride, e.g. time samples. Can be used against any fixed-width type. Implementation is based on [Gorilla paper](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf), and extended to support 64bit types. The drawback is 1 extra bit for 32-byte wide deltas: 5-bit prefix instead of 4-bit prefix. -- `Gorilla` - stores (parts of) xored values in compact binary form, compressing values down to 1 bit (in the best case). Best compression rate is achieved when neighbouring values are binary equal. Basic use case - floating point data that do not change rapidly. Implementation is based on [Gorilla paper](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf), and extended to support 64bit types. +- `NONE` — No compression. +- `LZ4` — Lossless [data compression algorithm](https://github.com/lz4/lz4) used by default. Applies LZ4 fast compression. +- `LZ4HC[(level)]` — LZ4 CH (high compression) algorithm with configurable level. Default level: 9. If you set `level <= 0`, the default level is applied. Possible levels: [1, 12]. Recommended levels are in range: [4, 9]. +- `ZSTD[(level)]` — [ZSTD compression algorithm](https://en.wikipedia.org/wiki/Zstandard) with configurable `level`. Possible levels: [1, 22]. Default value: 1. +- `Delta(delta_bytes)` — compression approach, when raw values are replaced with the difference of two neighbour values. Up to `delta_bytes` are used for storing delta value, so `delta_bytes` is a maximum size of raw values. +Possible `delta_bytes` values: 1, 2, 4, 8. Default value for `delta_bytes` is `sizeof(type)`, if it is equals to 1, 2, 4, 8. Otherwise it equals 1. +- `DoubleDelta` — Compresses values down to 1 bit (in the best case), using deltas calculation. Best compression rates are achieved on monotonic sequences with constant stride, for example, time series data. Can be used with any fixed-width type. Implements the algorithm used in Gorilla TSDB, extending it to support 64 bit types. Uses 1 extra bit for 32 byte deltas: 5 bit prefix instead of 4 bit prefix. For additional information, see the "Compressing time stamps" section of the [Gorilla: A Fast, Scalable, In-Memory Time Series Database](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf) document. +- `Gorilla` — Compresses values down to 1 bit (in the best case). The codec is efficient when storing series of floating point values that change slowly, because the best compression rate is achieved when neighbouring values are binary equal. Implements the algorithm used in Gorilla TSDB, extending it to support 64 bit types. For additional information, see the "Compressing values" section of the [Gorilla: A Fast, Scalable, In-Memory Time Series Database](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf) document. + +High compression levels useful for asymmetric scenarios, like compress once, decompress a lot of times. Greater levels stands for better compression and higher CPU usage. + +!!!warning + You cannot decompress ClickHouse database files with external utilities, for example, `lz4`. Use the special utility [clickhouse-compressor](https://github.com/yandex/ClickHouse/tree/master/dbms/programs/compressor). Syntax example: + ``` CREATE TABLE codec_example ( - dt Date CODEC(ZSTD), /* используется уровень сжатия по-умолчанию */ + dt Date CODEC(ZSTD), ts DateTime CODEC(LZ4HC), float_value Float32 CODEC(NONE), double_value Float64 CODEC(LZ4HC(9)) @@ -120,6 +138,7 @@ ORDER BY dt Codecs can be combined in a pipeline. Default table codec is not included into pipeline (if it should be applied to a column, you have to specify it explicitly in pipeline). Example below shows an optimization approach for storing timeseries metrics. Usually, values for particular metric, stored in `path` does not differ significantly from point to point. Using delta-encoding allows to reduce disk space usage significantly. + ``` CREATE TABLE timeseries_example ( diff --git a/docs/en/query_language/functions/array_functions.md b/docs/en/query_language/functions/array_functions.md index 938a10ee8d0..94d79a1898b 100644 --- a/docs/en/query_language/functions/array_functions.md +++ b/docs/en/query_language/functions/array_functions.md @@ -58,11 +58,10 @@ arrayConcat(arrays) - `arrays` – Arbitrary number of arguments of [Array](../../data_types/array.md) type. **Example** -``` sql +```sql SELECT arrayConcat([1, 2], [3, 4], [5, 6]) AS res ``` - -``` +```text ┌─res───────────┐ │ [1,2,3,4,5,6] │ └───────────────┘ @@ -204,7 +203,7 @@ Returns the array \[1, 2, 3, ..., length (arr) \] This function is normally used with ARRAY JOIN. It allows counting something just once for each array after applying ARRAY JOIN. Example: -``` sql +```sql SELECT count() AS Reaches, countIf(num = 1) AS Hits @@ -215,8 +214,7 @@ ARRAY JOIN WHERE CounterID = 160656 LIMIT 10 ``` - -``` +```text ┌─Reaches─┬──Hits─┐ │ 95606 │ 31406 │ └─────────┴───────┘ @@ -224,15 +222,14 @@ LIMIT 10 In this example, Reaches is the number of conversions (the strings received after applying ARRAY JOIN), and Hits is the number of pageviews (strings before ARRAY JOIN). In this particular case, you can get the same result in an easier way: -``` sql +```sql SELECT sum(length(GoalsReached)) AS Reaches, count() AS Hits FROM test.hits WHERE (CounterID = 160656) AND notEmpty(GoalsReached) ``` - -``` +```text ┌─Reaches─┬──Hits─┐ │ 95606 │ 31406 │ └─────────┴───────┘ @@ -248,7 +245,7 @@ For example: arrayEnumerateUniq(\[10, 20, 10, 30\]) = \[1, 1, 2, 1\]. This function is useful when using ARRAY JOIN and aggregation of array elements. Example: -``` sql +```sql SELECT Goals.ID AS GoalID, sum(Sign) AS Reaches, @@ -262,8 +259,7 @@ GROUP BY GoalID ORDER BY Reaches DESC LIMIT 10 ``` - -``` +```text ┌──GoalID─┬─Reaches─┬─Visits─┐ │ 53225 │ 3214 │ 1097 │ │ 2825062 │ 3188 │ 1097 │ @@ -282,11 +278,10 @@ In this example, each goal ID has a calculation of the number of conversions (ea The arrayEnumerateUniq function can take multiple arrays of the same size as arguments. In this case, uniqueness is considered for tuples of elements in the same positions in all the arrays. -``` sql +```sql SELECT arrayEnumerateUniq([1, 1, 1, 2, 2, 2], [1, 1, 2, 1, 1, 2]) AS res ``` - -``` +```text ┌─res───────────┐ │ [1,2,1,1,2,1] │ └───────────────┘ @@ -308,11 +303,10 @@ arrayPopBack(array) **Example** -``` sql +```sql SELECT arrayPopBack([1, 2, 3]) AS res ``` - -``` +```text ┌─res───┐ │ [1,2] │ └───────┘ @@ -332,11 +326,10 @@ arrayPopFront(array) **Example** -``` sql +```sql SELECT arrayPopFront([1, 2, 3]) AS res ``` - -``` +```text ┌─res───┐ │ [2,3] │ └───────┘ @@ -357,11 +350,10 @@ arrayPushBack(array, single_value) **Example** -``` sql +```sql SELECT arrayPushBack(['a'], 'b') AS res ``` - -``` +```text ┌─res───────┐ │ ['a','b'] │ └───────────┘ @@ -382,11 +374,10 @@ arrayPushFront(array, single_value) **Example** -``` sql +```sql SELECT arrayPushBack(['b'], 'a') AS res ``` - -``` +```text ┌─res───────┐ │ ['a','b'] │ └───────────┘ @@ -446,11 +437,10 @@ arraySlice(array, offset[, length]) **Example** -``` sql +```sql SELECT arraySlice([1, 2, NULL, 4, 5], 2, 3) AS res ``` - -``` +```text ┌─res────────┐ │ [2,NULL,4] │ └────────────┘ @@ -464,10 +454,10 @@ Sorts the elements of the `arr` array in ascending order. If the `func` function Example of integer values sorting: -``` sql +```sql SELECT arraySort([1, 3, 3, 0]); ``` -``` +```text ┌─arraySort([1, 3, 3, 0])─┐ │ [0,1,3,3] │ └─────────────────────────┘ @@ -475,10 +465,10 @@ SELECT arraySort([1, 3, 3, 0]); Example of string values sorting: -``` sql +```sql SELECT arraySort(['hello', 'world', '!']); ``` -``` +```text ┌─arraySort(['hello', 'world', '!'])─┐ │ ['!','hello','world'] │ └────────────────────────────────────┘ @@ -486,10 +476,10 @@ SELECT arraySort(['hello', 'world', '!']); Consider the following sorting order for the `NULL`, `NaN` and `Inf` values: -``` sql +```sql SELECT arraySort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf]); ``` -``` +```text ┌─arraySort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf])─┐ │ [-inf,-4,1,2,3,inf,nan,nan,NULL,NULL] │ └───────────────────────────────────────────────────────────┘ @@ -504,10 +494,10 @@ Note that `arraySort` is a [higher-order function](higher_order_functions.md). Y Let's consider the following example: -``` sql +```sql SELECT arraySort((x) -> -x, [1, 2, 3]) as res; ``` -``` +```text ┌─res─────┐ │ [3,2,1] │ └─────────┘ @@ -517,11 +507,10 @@ For each element of the source array, the lambda function returns the sorting ke The lambda function can accept multiple arguments. In this case, you need to pass the `arraySort` function several arrays of identical length that the arguments of lambda function will correspond to. The resulting array will consist of elements from the first input array; elements from the next input array(s) specify the sorting keys. For example: -``` sql +```sql SELECT arraySort((x, y) -> y, ['hello', 'world'], [2, 1]) as res; ``` - -``` +```text ┌─res────────────────┐ │ ['world', 'hello'] │ └────────────────────┘ @@ -531,19 +520,19 @@ Here, the elements that are passed in the second array ([2, 1]) define a sorting Other examples are shown below. -``` sql +```sql SELECT arraySort((x, y) -> y, [0, 1, 2], ['c', 'b', 'a']) as res; ``` -``` sql +```text ┌─res─────┐ │ [2,1,0] │ └─────────┘ ``` -``` sql +```sql SELECT arraySort((x, y) -> -y, [0, 1, 2], [1, 2, 3]) as res; ``` -``` sql +```text ┌─res─────┐ │ [2,1,0] │ └─────────┘ @@ -558,10 +547,10 @@ Sorts the elements of the `arr` array in descending order. If the `func` functio Example of integer values sorting: -``` sql +```sql SELECT arrayReverseSort([1, 3, 3, 0]); ``` -``` +```text ┌─arrayReverseSort([1, 3, 3, 0])─┐ │ [3,3,1,0] │ └────────────────────────────────┘ @@ -569,10 +558,10 @@ SELECT arrayReverseSort([1, 3, 3, 0]); Example of string values sorting: -``` sql +```sql SELECT arrayReverseSort(['hello', 'world', '!']); ``` -``` +```text ┌─arrayReverseSort(['hello', 'world', '!'])─┐ │ ['world','hello','!'] │ └───────────────────────────────────────────┘ @@ -580,10 +569,10 @@ SELECT arrayReverseSort(['hello', 'world', '!']); Consider the following sorting order for the `NULL`, `NaN` and `Inf` values: -``` sql +```sql SELECT arrayReverseSort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf]) as res; ``` -``` sql +```text ┌─res───────────────────────────────────┐ │ [inf,3,2,1,-4,-inf,nan,nan,NULL,NULL] │ └───────────────────────────────────────┘ @@ -596,10 +585,10 @@ SELECT arrayReverseSort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf]) as res; Note that the `arrayReverseSort` is a [higher-order function](higher_order_functions.md). You can pass a lambda function to it as the first argument. Example is shown below. -``` sql +```sql SELECT arrayReverseSort((x) -> -x, [1, 2, 3]) as res; ``` -``` +```text ┌─res─────┐ │ [1,2,3] │ └─────────┘ @@ -612,10 +601,10 @@ The array is sorted in the following way: The lambda function can accept multiple arguments. In this case, you need to pass the `arrayReverseSort` function several arrays of identical length that the arguments of lambda function will correspond to. The resulting array will consist of elements from the first input array; elements from the next input array(s) specify the sorting keys. For example: -``` sql +```sql SELECT arrayReverseSort((x, y) -> y, ['hello', 'world'], [2, 1]) as res; ``` -``` sql +```text ┌─res───────────────┐ │ ['hello','world'] │ └───────────────────┘ @@ -628,18 +617,18 @@ In this example, the array is sorted in the following way: Other examples are shown below. -``` sql +```sql SELECT arrayReverseSort((x, y) -> y, [4, 3, 5], ['a', 'b', 'c']) AS res; ``` -``` sql +```text ┌─res─────┐ │ [5,3,4] │ └─────────┘ ``` -``` sql +```sql SELECT arrayReverseSort((x, y) -> -y, [4, 3, 5], [1, 2, 3]) AS res; ``` -``` sql +```text ┌─res─────┐ │ [4,3,5] │ └─────────┘ diff --git a/docs/en/query_language/functions/date_time_functions.md b/docs/en/query_language/functions/date_time_functions.md index 8ae05caf935..e8716d8c542 100644 --- a/docs/en/query_language/functions/date_time_functions.md +++ b/docs/en/query_language/functions/date_time_functions.md @@ -95,6 +95,12 @@ Returns the date. Rounds down a date or date with time to the nearest Monday. Returns the date. +## toStartOfWeek(t[,mode]) + +Rounds down a date or date with time to the nearest Sunday or Monday by mode. +Returns the date. +The mode argument works exactly like the mode argument to toWeek(). For the single-argument syntax, a mode value of 0 is used. + ## toStartOfDay Rounds down a date with time to the start of the day. @@ -169,6 +175,72 @@ Converts a date or date with time to a UInt16 number containing the ISO Year num Converts a date or date with time to a UInt8 number containing the ISO Week number. +## toWeek(date[,mode]) +This function returns the week number for date or datetime. The two-argument form of toWeek() enables you to specify whether the week starts on Sunday or Monday and whether the return value should be in the range from 0 to 53 or from 1 to 53. If the mode argument is omitted, the default mode is 0. +`toISOWeek() `is a compatibility function that is equivalent to `toWeek(date,3)`. +The following table describes how the mode argument works. + +| Mode | First day of week | Range | Week 1 is the first week … | +| ----------- | -------- | -------- | ------------------ | +|0|Sunday|0-53|with a Sunday in this year +|1|Monday|0-53|with 4 or more days this year +|2|Sunday|1-53|with a Sunday in this year +|3|Monday|1-53|with 4 or more days this year +|4|Sunday|0-53|with 4 or more days this year +|5|Monday|0-53|with a Monday in this year +|6|Sunday|1-53|with 4 or more days this year +|7|Monday|1-53|with a Monday in this year +|8|Sunday|1-53|contains January 1 +|9|Monday|1-53|contains January 1 + +For mode values with a meaning of “with 4 or more days this year,” weeks are numbered according to ISO 8601:1988: + +- If the week containing January 1 has 4 or more days in the new year, it is week 1. + +- Otherwise, it is the last week of the previous year, and the next week is week 1. + +For mode values with a meaning of “contains January 1”, the week contains January 1 is week 1. It doesn't matter how many days in the new year the week contained, even if it contained only one day. + +``` +toWeek(date, [, mode][, Timezone]) +``` +**Parameters** + +- `date` – Date or DateTime. +- `mode` – Optional parameter, Range of values is [0,9], default is 0. +- `Timezone` – Optional parameter, it behaves like any other conversion function. + +**Example** + +``` sql +SELECT toDate('2016-12-27') AS date, toWeek(date) AS week0, toWeek(date,1) AS week1, toWeek(date,9) AS week9; +``` + +``` +┌───────date─┬─week0─┬─week1─┬─week9─┐ +│ 2016-12-27 │ 52 │ 52 │ 1 │ +└────────────┴───────┴───────┴───────┘ +``` + +## toYearWeek(date[,mode]) +Returns year and week for a date. The year in the result may be different from the year in the date argument for the first and the last week of the year. + +The mode argument works exactly like the mode argument to toWeek(). For the single-argument syntax, a mode value of 0 is used. + +`toISOYear() `is a compatibility function that is equivalent to `intDiv(toYearWeek(date,3),100)`. + +**Example** + +``` sql +SELECT toDate('2016-12-27') AS date, toYearWeek(date) AS yearWeek0, toYearWeek(date,1) AS yearWeek1, toYearWeek(date,9) AS yearWeek9; +``` + +``` +┌───────date─┬─yearWeek0─┬─yearWeek1─┬─yearWeek9─┐ +│ 2016-12-27 │ 201652 │ 201652 │ 201701 │ +└────────────┴───────────┴───────────┴───────────┘ +``` + ## now Accepts zero arguments and returns the current time at one of the moments of request execution. diff --git a/docs/en/query_language/functions/ext_dict_functions.md b/docs/en/query_language/functions/ext_dict_functions.md index 017d941b9f6..6494a2b643e 100644 --- a/docs/en/query_language/functions/ext_dict_functions.md +++ b/docs/en/query_language/functions/ext_dict_functions.md @@ -96,7 +96,7 @@ LIMIT 3 Checks whether the dictionary has the key. ``` -dictHas('dict_name', id) +dictHas('dict_name', id_expr) ``` **Parameters** @@ -116,7 +116,7 @@ Type: `UInt8`. For the hierarchical dictionary, returns an array of dictionary keys starting from passed `id_expr` and continuing along the chain of parent elements. ``` -dictGetHierarchy('dict_name', id) +dictGetHierarchy('dict_name', id_expr) ``` **Parameters** diff --git a/docs/en/query_language/functions/geo.md b/docs/en/query_language/functions/geo.md index 97c4d1b5b4f..2c84a4516ba 100644 --- a/docs/en/query_language/functions/geo.md +++ b/docs/en/query_language/functions/geo.md @@ -151,4 +151,36 @@ SELECT geohashDecode('ezs42') AS res └─────────────────────────────────┘ ``` +## geoToH3 + +Calculates [H3](https://uber.github.io/h3/#/documentation/overview/introduction) point index `(lon, lat)` with specified resolution. + +``` +geoToH3(lon, lat, resolution) +``` + +**Input values** + +- `lon` — Longitude. Type: [Float64](../../data_types/float.md). +- `lat` — Latitude. Type: [Float64](../../data_types/float.md). +- `resolution` — Index resolution. Range: `[0, 15]`. Type: [UInt8](../../data_types/int_uint.md). + +**Returned values** + +- Hexagon index number. +- 0 in case of error. + +Type: [UInt64](../../data_types/int_uint.md). + +**Example** + +``` sql +SELECT geoToH3(37.79506683, 55.71290588, 15) as h3Index +``` +``` +┌────────────h3Index─┐ +│ 644325524701193974 │ +└────────────────────┘ +``` + [Original article](https://clickhouse.yandex/docs/en/query_language/functions/geo/) diff --git a/docs/en/query_language/functions/hash_functions.md b/docs/en/query_language/functions/hash_functions.md index 14577a6afe4..368cfa1622b 100644 --- a/docs/en/query_language/functions/hash_functions.md +++ b/docs/en/query_language/functions/hash_functions.md @@ -212,7 +212,7 @@ SELECT metroHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00: ## jumpConsistentHash Calculates JumpConsistentHash form a UInt64. -Accepts a UInt64-type argument. Returns Int32. +Accepts two arguments: a UInt64-type key and the number of buckets. Returns Int32. For more information, see the link: [JumpConsistentHash](https://arxiv.org/pdf/1406.2294.pdf) ## murmurHash2_32, murmurHash2_64 diff --git a/docs/en/query_language/functions/ip_address_functions.md b/docs/en/query_language/functions/ip_address_functions.md index 3fb97e60c1c..ce58a187853 100644 --- a/docs/en/query_language/functions/ip_address_functions.md +++ b/docs/en/query_language/functions/ip_address_functions.md @@ -155,7 +155,6 @@ Accepts an IPv4 and an UInt8 value containing the [CIDR](https://en.wikipedia.or ```sql SELECT IPv4CIDRToRange(toIPv4('192.168.5.2'), 16) ``` - ``` ┌─IPv4CIDRToRange(toIPv4('192.168.5.2'), 16)─┐ │ ('192.168.0.0','192.168.255.255') │ diff --git a/docs/en/query_language/functions/other_functions.md b/docs/en/query_language/functions/other_functions.md index d3b7b2082d4..007f1352775 100644 --- a/docs/en/query_language/functions/other_functions.md +++ b/docs/en/query_language/functions/other_functions.md @@ -627,15 +627,36 @@ SELECT replicate(1, ['a', 'b', 'c']) └───────────────────────────────┘ ``` -## filesystemAvailable +## filesystemAvailable {#function-filesystemavailable} -Returns the remaining space information of the disk, in bytes. This information is evaluated using the configured by path. +Returns the amount of remaining space in the filesystem where the files of the databases located. See the [path](../../operations/server_settings/settings.md#server_settings-path) server setting description. + +``` +filesystemAvailable() +``` + +**Returned values** + +- Amount of remaining space in bytes. + +Type: [UInt64](../../data_types/int_uint.md). + +**Example** + +```sql +SELECT filesystemAvailable() AS "Free space", toTypeName(filesystemAvailable()) AS "Type" +``` +```text +┌──Free space─┬─Type───┐ +│ 18152624128 │ UInt64 │ +└─────────────┴────────┘ +``` ## filesystemCapacity Returns the capacity information of the disk, in bytes. This information is evaluated using the configured by path. -## finalizeAggregation +## finalizeAggregation {#function-finalizeaggregation} Takes state of aggregate function. Returns result of aggregation (finalized state). diff --git a/docs/en/query_language/operators.md b/docs/en/query_language/operators.md index 00a1cc5fc9e..4e79c9e805f 100644 --- a/docs/en/query_language/operators.md +++ b/docs/en/query_language/operators.md @@ -65,7 +65,7 @@ Groups of operators are listed in order of priority (the higher it is in the lis `a GLOBAL NOT IN ...` – The `globalNotIn(a, b) function.` -## Operator for Working With Dates and Times +## Operator for Working With Dates and Times {#operators-datetime} ``` sql EXTRACT(part FROM date); @@ -94,13 +94,13 @@ SELECT EXTRACT(MONTH FROM toDate('2017-06-15')); SELECT EXTRACT(YEAR FROM toDate('2017-06-15')); ``` -In the following example we create a table and insert into it a value with the `DateTime` type. +In the following example we create a table and insert into it a value with the `DateTime` type. ``` sql CREATE TABLE test.Orders ( - OrderId UInt64, - OrderName String, + OrderId UInt64, + OrderName String, OrderDate DateTime ) ENGINE = Log; @@ -110,11 +110,11 @@ ENGINE = Log; INSERT INTO test.Orders VALUES (1, 'Jarlsberg Cheese', toDateTime('2008-10-11 13:23:44')); ``` ``` sql -SELECT - toYear(OrderDate) AS OrderYear, - toMonth(OrderDate) AS OrderMonth, - toDayOfMonth(OrderDate) AS OrderDay, - toHour(OrderDate) AS OrderHour, +SELECT + toYear(OrderDate) AS OrderYear, + toMonth(OrderDate) AS OrderMonth, + toDayOfMonth(OrderDate) AS OrderDay, + toHour(OrderDate) AS OrderHour, toMinute(OrderDate) AS OrderMinute, toSecond(OrderDate) AS OrderSecond FROM test.Orders; diff --git a/docs/en/query_language/system.md b/docs/en/query_language/system.md new file mode 100644 index 00000000000..c5f4830723f --- /dev/null +++ b/docs/en/query_language/system.md @@ -0,0 +1,34 @@ +# SYSTEM Queries {#query_language-system} + +- [STOP DISTRIBUTED SENDS](#query_language-system-stop-distributed-sends) +- [FLUSH DISTRIBUTED](#query_language-system-flush-distributed) +- [START DISTRIBUTED SENDS](#query_language-system-start-distributed-sends) + +## Managing Distributed Tables {#query_language-system-distributed} + +ClickHouse can manage [distributed](../operations/table_engines/distributed.md) tables. When a user inserts data into such table, ClickHouse creates a queue of the data which should be sent to servers of the cluster, then asynchronously sends them. You can control the processing of queue by using the requests [STOP DISTRIBUTED SENDS](#query_language-system-stop-distributed-sends), [FLUSH DISTRIBUTED](#query_language-system-flush-distributed) and [START DISTRIBUTED SENDS](#query_language-system-start-distributed-sends). Also, you can synchronously insert distributed data with the `insert_distributed_sync` setting. + + +### STOP DISTRIBUTED SENDS {#query_language-system-stop-distributed-sends} + +Disables background data distributing, when inserting data into the distributed tables. + +``` +SYSTEM STOP DISTRIBUTED SENDS [db.] +``` + +### FLUSH DISTRIBUTED {#query_language-system-flush-distributed} + +Forces ClickHouse to send data to the servers of the cluster in synchronous mode. If some of the servers are not available, ClickHouse throws an exception and stops query processing. When servers are back into operation, you should repeat the query. + +``` +SYSTEM FLUSH DISTRIBUTED [db.] +``` + +### START DISTRIBUTED SENDS {#query_language-system-start-distributed-sends} + +Enables background data distributing, when inserting data into the distributed tables. + +``` +SYSTEM START DISTRIBUTED SENDS [db.] +``` diff --git a/docs/fa/database_engines/index.md b/docs/fa/database_engines/index.md new file mode 120000 index 00000000000..bbdb762a4ad --- /dev/null +++ b/docs/fa/database_engines/index.md @@ -0,0 +1 @@ +../../en/database_engines/index.md \ No newline at end of file diff --git a/docs/fa/database_engines/mysql.md b/docs/fa/database_engines/mysql.md new file mode 120000 index 00000000000..51ac4126e2d --- /dev/null +++ b/docs/fa/database_engines/mysql.md @@ -0,0 +1 @@ +../../en/database_engines/mysql.md \ No newline at end of file diff --git a/docs/fa/interfaces/formats.md b/docs/fa/interfaces/formats.md index 49286b0bdd9..a8c91c73b8a 100644 --- a/docs/fa/interfaces/formats.md +++ b/docs/fa/interfaces/formats.md @@ -328,6 +328,60 @@ JSON با جاوااسکریپت سازگار است. برای اطمینان ا برای پارس کردن، هر ترتیبی برای مقادیر ستون های مختلف پشتیبانی می شود. حذف شدن بعضی مقادیر قابل قبول است، آنها با مقادیر پیش فرض خود برابر هستند. در این مورد، صفر و سطر های خالی به عنوان مقادیر پیش فرض قرار می گیرند. مقادیر پیچیده که می توانند در جدول مشخص شوند، به عنوان مقادیر پیش فرض پشتیبانی نمی شوند. Whitespace بین element ها نادیده گرفته می شوند. اگر کاما بعد از object ها قرار گیرند، نادیده گرفته می شوند. object ها نیازی به جداسازی با استفاده از new line را ندارند. +### Usage of Nested Structures {#jsoneachrow-nested} + +If you have a table with the [Nested](../data_types/nested_data_structures/nested.md) data type columns, you can insert JSON data having the same structure. Enable this functionality with the [input_format_import_nested_json](../operations/settings/settings.md#settings-input_format_import_nested_json) setting. + +For example, consider the following table: + +```sql +CREATE TABLE json_each_row_nested (n Nested (s String, i Int32) ) ENGINE = Memory +``` + +As you can find in the `Nested` data type description, ClickHouse treats each component of the nested structure as a separate column, `n.s` and `n.i` for our table. So you can insert the data the following way: + +```sql +INSERT INTO json_each_row_nested FORMAT JSONEachRow {"n.s": ["abc", "def"], "n.i": [1, 23]} +``` + +To insert data as hierarchical JSON object set [input_format_import_nested_json=1](../operations/settings/settings.md#settings-input_format_import_nested_json). + +```json +{ + "n": { + "s": ["abc", "def"], + "i": [1, 23] + } +} +``` + +Without this setting ClickHouse throws the exception. + +```sql +SELECT name, value FROM system.settings WHERE name = 'input_format_import_nested_json' +``` +```text +┌─name────────────────────────────┬─value─┐ +│ input_format_import_nested_json │ 0 │ +└─────────────────────────────────┴───────┘ +``` +```sql +INSERT INTO json_each_row_nested FORMAT JSONEachRow {"n": {"s": ["abc", "def"], "i": [1, 23]}} +``` +```text +Code: 117. DB::Exception: Unknown field found while parsing JSONEachRow format: n: (at row 1) +``` +```sql +SET input_format_import_nested_json=1 +INSERT INTO json_each_row_nested FORMAT JSONEachRow {"n": {"s": ["abc", "def"], "i": [1, 23]}} +SELECT * FROM json_each_row_nested +``` +```text +┌─n.s───────────┬─n.i────┐ +│ ['abc','def'] │ [1,23] │ +└───────────────┴────────┘ +``` + ## Native کارآمدترین فرمت. داده ها توسط بلاک ها و در فرمت باینری نوشته و خوانده می شوند. برای هر بلاک، تعداد سطرها، تعداد ستون ها، نام ستون ها و type آنها، و بخش هایی از ستون ها در این بلاک یکی پس از دیگری ثبت می شوند. به عبارت دیگر، این فرمت "columnar" است - این فرمت ستون ها را به سطر تبدیل نمی کند. این فرمت در حالت native interface و بین سرور و محیط ترمینال و همچنین کلاینت C++ استفاده می شود. diff --git a/docs/fa/query_language/system.md b/docs/fa/query_language/system.md new file mode 120000 index 00000000000..6061858c3f2 --- /dev/null +++ b/docs/fa/query_language/system.md @@ -0,0 +1 @@ +../../en/query_language/system.md \ No newline at end of file diff --git a/docs/ru/database_engines/index.md b/docs/ru/database_engines/index.md new file mode 100644 index 00000000000..3190a79017f --- /dev/null +++ b/docs/ru/database_engines/index.md @@ -0,0 +1,9 @@ +# Database Engines + +Database engines provide working with tables. + +By default, ClickHouse uses its native database engine which provides configurable [table engines](../operations/table_engines/index.md) and [SQL dialect](../query_language/syntax.md). + +Also you can use the following database engines: + +- [MySQL](mysql.md) diff --git a/docs/ru/database_engines/mysql.md b/docs/ru/database_engines/mysql.md new file mode 100644 index 00000000000..fd5417ee27d --- /dev/null +++ b/docs/ru/database_engines/mysql.md @@ -0,0 +1,122 @@ +# MySQL + +Allows to connect to some database on remote MySQL server and perform `INSERT` and `SELECT` queries with tables to exchange data between ClickHouse and MySQL. + +The `MySQL` database engine translate queries to the MySQL server, so you can perform operations such as `SHOW TABLES` or `SHOW CREATE TABLE`. + +You cannot perform with tables the following queries: + +- `ATTACH`/`DETACH` +- `DROP` +- `RENAME` +- `CREATE TABLE` +- `ALTER` + + +## Creating a Database + +``` sql +CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster] +ENGINE = MySQL('host:port', 'database', 'user', 'password') +``` + +**Engine Parameters** + +- `host:port` — MySQL server address. +- `database` — Remote database name. +- `user` — MySQL user. +- `password` — User password. + + +## Data Types Support + +MySQL | ClickHouse +------|------------ +UNSIGNED TINYINT | [UInt8](../data_types/int_uint.md) +TINYINT | [Int8](../data_types/int_uint.md) +UNSIGNED SMALLINT | [UInt16](../data_types/int_uint.md) +SMALLINT | [Int16](../data_types/int_uint.md) +UNSIGNED INT, UNSIGNED MEDIUMINT | [UInt32](../data_types/int_uint.md) +INT, MEDIUMINT | [Int32](../data_types/int_uint.md) +UNSIGNED BIGINT | [UInt64](../data_types/int_uint.md) +BIGINT | [Int64](../data_types/int_uint.md) +FLOAT | [Float32](../data_types/float.md) +DOUBLE | [Float64](../data_types/float.md) +DATE | [Date](../data_types/date.md) +DATETIME, TIMESTAMP | [DateTime](../data_types/datetime.md) +BINARY | [FixedString](../data_types/fixedstring.md) + +All other MySQL data types are converted into [String](../data_types/string.md). + +[Nullable](../data_types/nullable.md) data type is supported. + + +## Examples of Use + +Table in MySQL: + +``` +mysql> USE test; +Database changed + +mysql> CREATE TABLE `mysql_table` ( + -> `int_id` INT NOT NULL AUTO_INCREMENT, + -> `float` FLOAT NOT NULL, + -> PRIMARY KEY (`int_id`)); +Query OK, 0 rows affected (0,09 sec) + +mysql> insert into mysql_table (`int_id`, `float`) VALUES (1,2); +Query OK, 1 row affected (0,00 sec) + +mysql> select * from mysql_table; ++--------+-------+ +| int_id | value | ++--------+-------+ +| 1 | 2 | ++--------+-------+ +1 row in set (0,00 sec) +``` + +Database in ClickHouse, exchanging data with the MySQL server: + +```sql +CREATE DATABASE mysql_db ENGINE = MySQL('localhost:3306', 'test', 'my_user', 'user_password') +``` +```sql +SHOW DATABASES +``` +```text +┌─name─────┐ +│ default │ +│ mysql_db │ +│ system │ +└──────────┘ +``` +```sql +SHOW TABLES FROM mysql_db +``` +```text +┌─name─────────┐ +│ mysql_table │ +└──────────────┘ +``` +```sql +SELECT * FROM mysql_db.mysql_table +``` +```text +┌─int_id─┬─value─┐ +│ 1 │ 2 │ +└────────┴───────┘ +``` +```sql +INSERT INTO mysql_db.mysql_table VALUES (3,4) +``` +```sql +SELECT * FROM mysql_db.mysql_table +``` +```text +┌─int_id─┬─value─┐ +│ 1 │ 2 │ +│ 3 │ 4 │ +└────────┴───────┘ +``` diff --git a/docs/ru/getting_started/example_datasets/star_schema.md b/docs/ru/getting_started/example_datasets/star_schema.md index 1d8af3b29a5..545eaeea6a6 100644 --- a/docs/ru/getting_started/example_datasets/star_schema.md +++ b/docs/ru/getting_started/example_datasets/star_schema.md @@ -101,11 +101,11 @@ CREATE TABLE lineorder_flat ENGINE = MergeTree PARTITION BY toYear(LO_ORDERDATE) ORDER BY (LO_ORDERDATE, LO_ORDERKEY) AS -SELECT * -FROM lineorder -ANY INNER JOIN customer ON LO_CUSTKEY = C_CUSTKEY -ANY INNER JOIN supplier ON LO_SUPPKEY = S_SUPPKEY -ANY INNER JOIN part ON LO_PARTKEY = P_PARTKEY; +SELECT l.*, c.*, s.*, p.* +FROM lineorder l + ANY INNER JOIN customer c ON (c.C_CUSTKEY = l.LO_CUSTKEY) + ANY INNER JOIN supplier s ON (s.S_SUPPKEY = l.LO_SUPPKEY) + ANY INNER JOIN part p ON (p.P_PARTKEY = l.LO_PARTKEY); ALTER TABLE lineorder_flat DROP COLUMN C_CUSTKEY, DROP COLUMN S_SUPPKEY, DROP COLUMN P_PARTKEY; ``` diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index c02ed3d4993..bc685443b0d 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -165,6 +165,8 @@ clickhouse-client --format_csv_delimiter="|" --query="INSERT INTO test.csv FORMA При парсинге, все значения могут парситься как в кавычках, так и без кавычек. Поддерживаются как двойные, так и одинарные кавычки. Строки также могут быть без кавычек. В этом случае они парсятся до символа-разделителя или перевода строки (CR или LF). В нарушение RFC, в случае парсинга строк не в кавычках, начальные и конечные пробелы и табы игнорируются. В качестве перевода строки, поддерживаются как Unix (LF), так и Windows (CR LF) и Mac OS Classic (LF CR) варианты. +Если установлена настройка [input_format_defaults_for_omitted_fields = 1](../operations/settings/settings.md#session_settings-input_format_defaults_for_omitted_fields), то пустые значения без кавычек заменяются значениями по умолчанию для типа данных столбца. + `NULL` форматируется в виде `\N`. Формат CSV поддерживает вывод totals и extremes аналогично `TabSeparated`. diff --git a/docs/ru/operations/server_settings/settings.md b/docs/ru/operations/server_settings/settings.md index 8f1bbcb7488..c1444668bb2 100644 --- a/docs/ru/operations/server_settings/settings.md +++ b/docs/ru/operations/server_settings/settings.md @@ -514,7 +514,7 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat ``` -## path +## path {#server_settings-path} Путь к каталогу с данными. diff --git a/docs/ru/operations/settings/query_complexity.md b/docs/ru/operations/settings/query_complexity.md index ae7232d8ae8..24286999685 100644 --- a/docs/ru/operations/settings/query_complexity.md +++ b/docs/ru/operations/settings/query_complexity.md @@ -157,7 +157,7 @@ ## max_ast_elements Максимальное количество элементов синтаксического дерева запроса. Если превышено - кидается исключение. -Аналогично, проверяется уже после парсинга запроса. По умолчанию: 10 000. +Аналогично, проверяется уже после парсинга запроса. По умолчанию: 50 000. ## max_rows_in_set diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index e320231d397..fa3c67c687c 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -181,20 +181,15 @@ Ok. ## input_format_defaults_for_omitted_fields {#session_settings-input_format_defaults_for_omitted_fields} -Включает/выключает расширенный обмен данными между клиентом ClickHouse и сервером ClickHouse. Параметр применяется для запросов `INSERT`. +При вставке данных запросом `INSERT`, заменяет пропущенные поля значениям по умолчанию для типа данных столбца. -При выполнении запроса`INSERT`, клиент ClickHouse подготавливает данные и отправляет их на сервер для записи. При подготовке данных клиент получает структуру таблицы от сервера. В некоторых случаях клиенту требуется больше информации, чем сервер отправляет по умолчанию. Включите расширенный обмен данными с помощью настройки `input_format_defaults_for_omitted_fields = 1`. +Поддерживаемые форматы вставки: -Если расширенный обмен данными включен, сервер отправляет дополнительные метаданные вместе со структурой таблицы. Состав метаданных зависит от операции. - -Операции, для которых может потребоваться включить расширенный обмен данными: - -- Вставка данных в формате [JSONEachRow](../../interfaces/formats.md#jsoneachrow). - -Для всех остальных операций ClickHouse не применяет этот параметр. +- [JSONEachRow](../../interfaces/formats.md#jsoneachrow) +- [CSV](../../interfaces/formats.md#csv) !!! note "Примечание" - Функциональность расширенного обмена данными потребляет дополнительные вычислительные ресурсы на сервере и может снизить производительность. + Когда опция включена, сервер отправляет клиенту расширенные метаданные. Это требует дополнительных вычислительных ресурсов на сервере и может снизить производительность. Возможные значения: diff --git a/docs/ru/operations/table_engines/file.md b/docs/ru/operations/table_engines/file.md index 731204f928a..b67823b988a 100644 --- a/docs/ru/operations/table_engines/file.md +++ b/docs/ru/operations/table_engines/file.md @@ -68,7 +68,7 @@ $ echo -e "1,2\n3,4" | clickhouse-local -q "CREATE TABLE table (a Int64, b Int64 ## Детали реализации -- Поддерживается многопоточное чтение и однопоточная запись. +- Поддерживается одновременное выполнение множества запросов `SELECT`, запросы `INSERT` могут выполняться только последовательно. - Не поддерживается: - использование операций `ALTER` и `SELECT...SAMPLE`; - индексы; diff --git a/docs/ru/operations/table_engines/kafka.md b/docs/ru/operations/table_engines/kafka.md index 3fe2e4d5cba..086d4fb4f08 100644 --- a/docs/ru/operations/table_engines/kafka.md +++ b/docs/ru/operations/table_engines/kafka.md @@ -25,7 +25,7 @@ SETTINGS [kafka_row_delimiter = 'delimiter_symbol',] [kafka_schema = '',] [kafka_num_consumers = N,] - [kafka_skip_broken_messages = <0|1>] + [kafka_skip_broken_messages = N] ``` Обязательные параметры: @@ -40,7 +40,7 @@ SETTINGS - `kafka_row_delimiter` – символ-разделитель записей (строк), которым завершается сообщение. - `kafka_schema` – опциональный параметр, необходимый, если используется формат, требующий определения схемы. Например, [Cap'n Proto](https://capnproto.org/) требует путь к файлу со схемой и название корневого объекта `schema.capnp:Message`. - `kafka_num_consumers` – количество потребителей (consumer) на таблицу. По умолчанию: `1`. Укажите больше потребителей, если пропускная способность одного потребителя недостаточна. Общее число потребителей не должно превышать количество партиций в топике, так как на одну партицию может быть назначено не более одного потребителя. -- `kafka_skip_broken_messages` – режим обработки сообщений Kafka. Если `kafka_skip_broken_messages = 1`, то движок отбрасывает сообщения Кафки, которые не получилось обработать. Одно сообщение в точности соответствует одной записи (строке). +- `kafka_skip_broken_messages` – максимальное количество некорректных сообщений в блоке. Если `kafka_skip_broken_messages = N`, то движок отбрасывает `N` сообщений Кафки, которые не получилось обработать. Одно сообщение в точности соответствует одной записи (строке). Значение по умолчанию – 0. Примеры diff --git a/docs/ru/query_language/agg_functions/combinators.md b/docs/ru/query_language/agg_functions/combinators.md index dfee76bb79d..1fcdb111e17 100644 --- a/docs/ru/query_language/agg_functions/combinators.md +++ b/docs/ru/query_language/agg_functions/combinators.md @@ -23,13 +23,22 @@ ## -State -В случае применения этого комбинатора, агрегатная функция возвращает не готовое значение (например, в случае функции `uniq` — количество уникальных значений), а промежуточное состояние агрегации (например, в случае функции `uniq` — хэш-таблицу для расчёта количества уникальных значений), которое имеет тип AggregateFunction(...) и может использоваться для дальнейшей обработки или может быть сохранено в таблицу для последующей доагрегации - смотрите разделы «AggregatingMergeTree» и «функции для работы с промежуточными состояниями агрегации». +В случае применения этого комбинатора, агрегатная функция возвращает не готовое значение (например, в случае функции [uniq](reference.md#agg_function-uniq) — количество уникальных значений), а промежуточное состояние агрегации (например, в случае функции `uniq` — хэш-таблицу для расчёта количества уникальных значений), которое имеет тип `AggregateFunction(...)` и может использоваться для дальнейшей обработки или может быть сохранено в таблицу для последующей доагрегации. -## -Merge +Для работы с промежуточными состояниями предназначены: + +- Движок таблиц [AggregatingMergeTree](../../operations/table_engines/aggregatingmergetree.md). +- Функция [finalizeAggregation](../functions/other_functions.md#function-finalizeaggregation). +- Функция [runningAccumulate](../functions/other_functions.md#function-runningaccumulate). +- Комбинатор [-Merge](#aggregate_functions_combinators_merge). +- Комбинатор [-MergeState](#aggregate_functions_combinators_mergestate). + + +## -Merge {#aggregate_functions_combinators_merge} В случае применения этого комбинатора, агрегатная функция будет принимать в качестве аргумента промежуточное состояние агрегации, доагрегировать (объединять вместе) эти состояния, и возвращать готовое значение. -## -MergeState. +## -MergeState {#aggregate_functions_combinators_mergestate} Выполняет слияние промежуточных состояний агрегации, аналогично комбинатору -Merge, но возвращает не готовое значение, а промежуточное состояние агрегации, аналогично комбинатору -State. diff --git a/docs/ru/query_language/agg_functions/reference.md b/docs/ru/query_language/agg_functions/reference.md index 0fe7a1c10ea..6247b74f657 100644 --- a/docs/ru/query_language/agg_functions/reference.md +++ b/docs/ru/query_language/agg_functions/reference.md @@ -83,7 +83,7 @@ binary decimal SELECT groupBitAnd(num) FROM t ``` -Где `num` – столбец с тестовыми данными. +Где `num` — столбец с тестовыми данными. Результат: @@ -126,7 +126,7 @@ binary decimal SELECT groupBitOr(num) FROM t ``` -Где `num` – столбец с тестовыми данными. +Где `num` — столбец с тестовыми данными. Результат: @@ -169,7 +169,7 @@ binary decimal SELECT groupBitXor(num) FROM t ``` -Где `num` – столбец с тестовыми данными. +Где `num` — столбец с тестовыми данными. Результат: @@ -300,6 +300,98 @@ GROUP BY timeslot └─────────────────────┴──────────────────────────────────────────────┘ ``` +## skewPop + +Вычисляет [коэффициент асимметрии](https://ru.wikipedia.org/wiki/Коэффициент_асимметрии) для последовательности. + +``` +skewPop(expr) +``` + +**Параметры** + +`expr` — [Выражение](../syntax.md#syntax-expressions), возвращающее число. + +**Возвращаемое значение** + +Коэффициент асимметрии заданного распределения. Тип — [Float64](../../data_types/float.md) + +**Пример** + +```sql +SELECT skewPop(value) FROM series_with_value_column +``` + +## skewSamp + +Вычисляет [выборочный коэффициент асимметрии](https://ru.wikipedia.org/wiki/Статистика_(функция_выборки)) для последовательности. + +Он представляет собой несмещенную оценку асимметрии случайной величины, если переданные значения образуют ее выборку. + +``` +skewSamp(expr) +``` + +**Параметры** + +`expr` — [Выражение](../syntax.md#syntax-expressions), возвращающее число. + +**Возвращаемое значение** + +Коэффициент асимметрии заданного распределения. Тип — [Float64](../../data_types/float.md). Если `n <= 1` (`n` — размер выборки), тогда функция возвращает `nan`. + +**Пример** + +```sql +SELECT skewSamp(value) FROM series_with_value_column +``` + +## kurtPop + +Вычисляет [коэффициент эксцесса](https://ru.wikipedia.org/wiki/Коэффициент_эксцесса) последовательности. + +``` +kurtPop(expr) +``` + +**Параметры** + +`expr` — [Выражение](../syntax.md#syntax-expressions), возвращающее число. + +**Возвращаемое значение** + +Коэффициент эксцесса заданного распределения. Тип — [Float64](../../data_types/float.md) + +**Пример** + +```sql +SELECT kurtPop(value) FROM series_with_value_column +``` + +## kurtSamp + +Вычисляет [выборочный коэффициент эксцесса](https://ru.wikipedia.org/wiki/Статистика_(функция_выборки)) для последовательности. + +Он представляет собой несмещенную оценку эксцесса случайной величины, если переданные значения образуют ее выборку. + +``` +kurtSamp(expr) +``` + +**Параметры** + +`expr` — [Выражение](../syntax.md#syntax-expressions), возвращающее число. + +**Возвращаемое значение** + +Коэффициент эксцесса заданного распределения. Тип — [Float64](../../data_types/float.md). Если `n <= 1` (`n` — размер выборки), тогда функция возвращает `nan`. + +**Пример** + +```sql +SELECT kurtSamp(value) FROM series_with_value_column +``` + ## timeSeriesGroupSum(uid, timestamp, value) {#agg_function-timeseriesgroupsum} `timeSeriesGroupSum` агрегирует временные ряды в которых не совпадают моменты. diff --git a/docs/ru/query_language/dicts/external_dicts_dict_structure.md b/docs/ru/query_language/dicts/external_dicts_dict_structure.md index 50a2469358b..482f47d4a8f 100644 --- a/docs/ru/query_language/dicts/external_dicts_dict_structure.md +++ b/docs/ru/query_language/dicts/external_dicts_dict_structure.md @@ -84,7 +84,7 @@ ClickHouse поддерживает следующие виды ключей: При запросе в функции `dictGet*` в качестве ключа передаётся кортеж. Пример: `dictGetString('dict_name', 'attr_name', tuple('string for field1', num_for_field2))`. -## Атрибуты +## Атрибуты {#ext_dict_structure-attributes} Пример конфигурации: diff --git a/docs/ru/query_language/functions/ext_dict_functions.md b/docs/ru/query_language/functions/ext_dict_functions.md index 8901292aeb2..3fb4a110e88 100644 --- a/docs/ru/query_language/functions/ext_dict_functions.md +++ b/docs/ru/query_language/functions/ext_dict_functions.md @@ -1,40 +1,192 @@ # Функции для работы с внешними словарями {#ext_dict_functions} -Информация о подключении и настройке внешних словарей смотрите в разделе [Внешние словари](../dicts/external_dicts.md). +Для получения информации о подключении и настройке, читайте раздел про [внешние словари](../dicts/external_dicts.md). -## dictGetUInt8, dictGetUInt16, dictGetUInt32, dictGetUInt64 +## dictGet -## dictGetInt8, dictGetInt16, dictGetInt32, dictGetInt64 +Получение значения из внешнего словаря. -## dictGetFloat32, dictGetFloat64 +``` +dictGet('dict_name', 'attr_name', id_expr) +dictGetOrDefault('dict_name', 'attr_name', id_expr, default_value_expr) +``` -## dictGetDate, dictGetDateTime +**Параметры** -## dictGetUUID +- `dict_name` — Название словаря. [Строковый литерал](../syntax.md#syntax-string-literal). +- `attr_name` — Название колонки словаря. [Строковый литерал](../syntax.md#syntax-string-literal). +- `id_expr` — Значение ключа. [Выражение](../syntax.md#syntax-expressions) возвращает значение типа [UInt64](../../data_types/int_uint.md) или [Tuple](../../data_types/tuple.md) в зависимости от конфигурации словаря. +- `default_value_expr` — Значение которое возвращается, если словарь не содержит колонку с ключом `id_expr`. [Выражение](../syntax.md#syntax-expressions) возвращает значение такого же типа, что и у атрибута `attr_name`. -## dictGetString -`dictGetT('dict_name', 'attr_name', id)` -- получить из словаря dict_name значение атрибута attr_name по ключу id. -`dict_name` и `attr_name` - константные строки. -`id` должен иметь тип UInt64. -Если ключа `id` нет в словаре - вернуть значение по умолчанию, заданное в описании словаря. +**Возвращаемое значение** -## dictGetTOrDefault +- Если ClickHouse успешно обрабатывает атрибут в соотвествии с указаным [типом данных](../dicts/external_dicts_dict_structure.md#ext_dict_structure-attributes), то функция возвращает значение для заданного ключа `id_expr`. +- Если запрашиваемого `id_expr` не оказалось в словаре: -`dictGetT('dict_name', 'attr_name', id, default)` + - `dictGet` возвратит содержимое элемента `` определенного в настройках словаря. + - `dictGetOrDefault` вернет значение переданного `default_value_expr` параметра. -Аналогично функциям `dictGetT`, но значение по умолчанию берётся из последнего аргумента функции. +ClickHouse бросает исключение, если не может обработать значение атрибута или значение несопоставимо с типом атрибута. -## dictIsIn -`dictIsIn('dict_name', child_id, ancestor_id)` -- для иерархического словаря dict_name - узнать, находится ли ключ child_id внутри ancestor_id (или совпадает с ancestor_id). Возвращает UInt8. +**Пример использования** + +Создайте файл `ext-dict-text.csv` со следующим содержимым: + +```text +1,1 +2,2 +``` + +Первая колонка - это `id`, вторая - `c1` + +Конфигурация внешнего словаря: + +```xml + + + ext-dict-test + + + /path-to/ext-dict-test.csv + CSV + + + + + + + + id + + + c1 + UInt32 + + + + 0 + + +``` + +Выполните запрос: + +```sql +SELECT + dictGetOrDefault('ext-dict-test', 'c1', number + 1, toUInt32(number * 10)) AS val, + toТипName(val) AS Type +FROM system.numbers +LIMIT 3 +``` +```text +┌─val─┬─type───┐ +│ 1 │ UInt32 │ +│ 2 │ UInt32 │ +│ 20 │ UInt32 │ +└─────┴────────┘ +``` + +**Смотрите также** + +- [Внешние словари](../dicts/external_dicts.md) -## dictGetHierarchy -`dictGetHierarchy('dict_name', id)` -- для иерархического словаря dict_name - вернуть массив ключей словаря, начиная с id и продолжая цепочкой родительских элементов. Возвращает Array(UInt64). ## dictHas -`dictHas('dict_name', id)` -- проверить наличие ключа в словаре. Возвращает значение типа UInt8, равное 0, если ключа нет и 1, если ключ есть. + +Проверяет наличие записи с заданным ключом в словаре. + +``` +dictHas('dict_name', id_expr) +``` + +**Параметры** + +- `dict_name` — Название словаря. [Строковый литерал](../syntax.md#syntax-string-literal). +- `id_expr` — Значение ключа. [Выражение](../syntax.md#syntax-expressions) возвращает значение типа [UInt64](../../data_types/int_uint.md). + +**Возвращаемое значение** + +- 0, если ключ не был обнаружен +- 1, если ключ присутствует в словаре + +Тип: `UInt8`. + +## dictGetHierarchy + +Для иерархических словарей возвращает массив ключей, содержащий ключ `id_expr` и все ключи родительских элементов по цепочке. + +``` +dictGetHierarchy('dict_name', id_expr) +``` + +**Параметры** + +- `dict_name` — Название словаря. [Строковый литерал](../syntax.md#syntax-string-literal). +- `id_expr` — Значение ключа. [Выражение](../syntax.md#syntax-expressions) возвращает значение типа [UInt64](../../data_types/int_uint.md). + +**Возвращаемое значение** + +Иерархию ключей словаря. + +Тип: [Array(UInt64)](../../data_types/array.md). + +## dictIsIn + +Осуществляет проверку - является ли ключ родительским во всей иерархической цепочке словаря. + +`dictIsIn ('dict_name', child_id_expr, ancestor_id_expr)` + +**Параметры** + +- `dict_name` — Название словаря. [Строковый литерал](../syntax.md#syntax-string-literal). +- `child_id_expr` — Ключ который должен быть проверен. [Выражение](../syntax.md#syntax-expressions) возвращает значение типа [UInt64](../../data_types/int_uint.md). +- `ancestor_id_expr` — Родительский ключ для ключа `child_id_expr`. [Выражение](../syntax.md#syntax-expressions) возвращает значение типа [UInt64](../../data_types/int_uint.md). + +**Возвращаемое значение** + +- 0, если `child_id_expr` не является потомком для `ancestor_id_expr`. +- 1, если `child_id_expr` является потомком для `ancestor_id_expr` или если `child_id_expr` равен `ancestor_id_expr`. + +Тип: `UInt8`. + +## Другие функции {#ext_dict_functions-other} + +ClickHouse поддерживает специализированные функции для конвертации значений атрибутов словаря к определенному типу, независимо от настроек словаря. + +Функции: + +- `dictGetInt8`, `dictGetInt16`, `dictGetInt32`, `dictGetInt64` +- `dictGetUInt8`, `dictGetUInt16`, `dictGetUInt32`, `dictGetUInt64` +- `dictGetFloat32`, `dictGetFloat64` +- `dictGetDate` +- `dictGetDateTime` +- `dictGetUUID` +- `dictGetString` + +Все эти функции имеют так же `OrDefault` версию. Например, `dictGetDateOrDefault`. + +Синтаксис: + +``` +dictGet[Тип]('dict_name', 'attr_name', id_expr) +dictGet[Тип]OrDefault('dict_name', 'attr_name', id_expr, default_value_expr) +``` + +**Параметры** + +- `dict_name` — Название словаря. [Строковый литерал](../syntax.md#syntax-string-literal). +- `attr_name` — Название колонки словаря. [Строковый литерал](../syntax.md#syntax-string-literal). +- `id_expr` — Значение ключа. [Выражение](../syntax.md#syntax-expressions) возвращает значение типа [UInt64](../../data_types/int_uint.md). +- `default_value_expr` — Значение которое возвращается, если словарь не содержит строку с ключом `id_expr`. [Выражение](../syntax.md#syntax-expressions) возвращает значение с таким же типом, что и тип атрибута `attr_name`. + +**Возвращаемое значение** + +- Если ClickHouse успешно обрабатывает атрибут в соотвествии с указаным [типом данных](../dicts/external_dicts_dict_structure.md#ext_dict_structure-attributes),то функция возвращает значение для заданного ключа `id_expr`. +- Если запрашиваемого `id_expr` не оказалось в словаре: + + - `dictGet[Тип]` возвратит содержимое элемента `` определенного в настройках словаря. + - `dictGet[Тип]OrDefault` вернет значение переданного `default_value_expr` параметра. + +ClickHouse бросает исключение, если не может обработать значение атрибута или значение несопоставимо с типом атрибута [Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/ext_dict_functions/) diff --git a/docs/ru/query_language/functions/geo.md b/docs/ru/query_language/functions/geo.md index 33092cf804b..b8e37c15aca 100644 --- a/docs/ru/query_language/functions/geo.md +++ b/docs/ru/query_language/functions/geo.md @@ -132,13 +132,17 @@ SELECT geohashEncode(-5.60302734375, 42.593994140625, 0) AS res Декодирует любую строку, закодированную в geohash, на долготу и широту. +``` +geohashDecode(geohash_string) +``` + **Входные значения** -- encoded string — строка, содержащая geohash. +- `geohash_string` — строка, содержащая geohash. **Возвращаемые значения** -- (longitude, latitude) — широта и долгота. Кортеж из двух значений типа `Float64`. +- `(longitude, latitude)` — широта и долгота. Кортеж из двух значений типа `Float64`. **Пример** @@ -154,7 +158,7 @@ SELECT geohashDecode('ezs42') AS res ## geoToH3 -Получает H3 индекс точки (lon, lat) с заданным разрешением +Получает H3 индекс точки `(lon, lat)` с заданным разрешением ``` geoToH3(lon, lat, resolution) @@ -162,15 +166,16 @@ geoToH3(lon, lat, resolution) **Входные значения** -- `lon` - географическая долгота. Тип данных — [Float64](../../data_types/float.md). -- `lat` - географическая широта. Тип данных — [Float64](../../data_types/float.md). -- `resolution` - требуемое разрешение индекса. Тип данных — [UInt8](../../data_types/int_uint.md). Диапазон возможных значение — `[0, 15]`. +- `lon` — географическая долгота. Тип данных — [Float64](../../data_types/float.md). +- `lat` — географическая широта. Тип данных — [Float64](../../data_types/float.md). +- `resolution` — требуемое разрешение индекса. Тип данных — [UInt8](../../data_types/int_uint.md). Диапазон возможных значений — `[0, 15]`. **Возвращаемые значения** -Возвращает значение с типом [UInt64] (../../data_types/int_uint.md). -`0` в случае ошибки. -Иначе возвращается индексный номер шестиугольника. +- Порядковый номер шестиугольника. +- 0 в случае ошибки. + +Тип — [UInt64](../../data_types/int_uint.md). **Пример** diff --git a/docs/ru/query_language/functions/ip_address_functions.md b/docs/ru/query_language/functions/ip_address_functions.md index 647dcb5fdb2..57c11b46d81 100644 --- a/docs/ru/query_language/functions/ip_address_functions.md +++ b/docs/ru/query_language/functions/ip_address_functions.md @@ -146,32 +146,30 @@ SELECT └─────────────────────────────────────┴─────────────────────┘ ``` -## IPv4CIDRtoIPv4Range(ipv4, cidr), +## IPv4CIDRToRange(ipv4, cidr), Принимает на вход IPv4 и значение `UInt8`, содержащее [CIDR](https://ru.wikipedia.org/wiki/Бесклассовая_адресация). Возвращает кортеж с двумя IPv4, содержащими нижний и более высокий диапазон подсети. ```sql -SELECT IPv4CIDRtoIPv4Range(toIPv4('192.168.5.2'), 16) +SELECT IPv4CIDRToRange(toIPv4('192.168.5.2'), 16) +``` +```text +┌─IPv4CIDRToRange(toIPv4('192.168.5.2'), 16)─┐ +│ ('192.168.0.0','192.168.255.255') │ +└────────────────────────────────────────────┘ ``` -``` -┌─IPv4CIDRtoIPv4Range(toIPv4('192.168.5.2'), 16)─┐ -│ ('192.168.0.0','192.168.255.255') │ -└────────────────────────────────────────────────┘ -``` - -## IPv6CIDRtoIPv6Range(ipv6, cidr), +## IPv6CIDRToRange(ipv6, cidr), Принимает на вход IPv6 и значение `UInt8`, содержащее CIDR. Возвращает кортеж с двумя IPv6, содержащими нижний и более высокий диапазон подсети. ```sql -SELECT IPv6CIDRtoIPv6Range(toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001'), 32); +SELECT IPv6CIDRToRange(toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001'), 32) ``` - -``` -┌─IPv6CIDRtoIPv6Range(toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001'), 32)─┐ -│ ('2001:db8::','2001:db8:ffff:ffff:ffff:ffff:ffff:ffff') │ -└────────────────────────────────────────────────────────────────────────────┘ +```text +┌─IPv6CIDRToRange(toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001'), 32)─┐ +│ ('2001:db8::','2001:db8:ffff:ffff:ffff:ffff:ffff:ffff') │ +└────────────────────────────────────────────────────────────────────────┘ ``` ## toIPv4(string) diff --git a/docs/ru/query_language/functions/other_functions.md b/docs/ru/query_language/functions/other_functions.md index 62af103e02d..6e4913638ee 100644 --- a/docs/ru/query_language/functions/other_functions.md +++ b/docs/ru/query_language/functions/other_functions.md @@ -600,6 +600,39 @@ SELECT replicate(1, ['a', 'b', 'c']) └───────────────────────────────┘ ``` +## filesystemAvailable {#function-filesystemavailable} + +Возвращает объем оставшегося места в файловой системе, в которой расположены файлы баз данных. Смотрите описание конфигурационного параметра сервера [path](../../operations/server_settings/settings.md#server_settings-path). + +``` +filesystemAvailable() +``` + +**Возвращаемое значение** + +- Объем свободного места. + +Тип — [UInt64](../../data_types/int_uint.md). + +**Пример** + +```sql +SELECT filesystemAvailable() AS "Free space", toTypeName(filesystemAvailable()) AS "Type" +``` +```text +┌──Free space─┬─Type───┐ +│ 18152624128 │ UInt64 │ +└─────────────┴────────┘ +``` + +## filesystemCapacity + +Возвращает данные о ёмкости диска. + +## finalizeAggregation {#function-finalizeaggregation} + +Принимает состояние агрегатной функции. Возвращает результат агрегирования. + ## runningAccumulate {#function-runningaccumulate} Принимает на вход состояния агрегатной функции и возвращает столбец со значениями, которые представляют собой результат мёржа этих состояний для выборки строк из блока от первой до текущей строки. Например, принимает состояние агрегатной функции (например, `runningAccumulate(uniqState(UserID))`), и для каждой строки блока возвращает результат агрегатной функции после мёржа состояний функции для всех предыдущих строк и текущей. Таким образом, результат зависит от разбиения данных по блокам и от порядка данных в блоке. diff --git a/docs/ru/query_language/syntax.md b/docs/ru/query_language/syntax.md index e1eca7e3ff7..f48c8d236e4 100644 --- a/docs/ru/query_language/syntax.md +++ b/docs/ru/query_language/syntax.md @@ -65,7 +65,7 @@ INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def') Примеры: `1`, `18446744073709551615`, `0xDEADBEEF`, `01`, `0.1`, `1e100`, `-1e-100`, `inf`, `nan`. -### Строковые +### Строковые {#syntax-string-literal} Поддерживаются только строковые литералы в одинарных кавычках. Символы внутри могут быть экранированы с помощью обратного слеша. Следующие escape-последовательности имеют соответствующее специальное значение: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\a`, `\v`, `\xHH`. Во всех остальных случаях, последовательности вида `\c`, где `c` — любой символ, преобразуется в `c` . Таким образом, могут быть использованы последовательности `\'` и `\\`. Значение будет иметь тип [String](../data_types/string.md). diff --git a/docs/ru/query_language/system.md b/docs/ru/query_language/system.md new file mode 100644 index 00000000000..fcf6fb8eced --- /dev/null +++ b/docs/ru/query_language/system.md @@ -0,0 +1,33 @@ +# SYSTEM Queries {#query_language-system} + +- [STOP DISTRIBUTED SENDS](#query_language-system-stop-distributed-sends) +- [FLUSH DISTRIBUTED](#query_language-system-flush-distributed) +- [START DISTRIBUTED SENDS](#query_language-system-start-distributed-sends) + +## Managing Distributed Tables {#query_language-system-distributed} + +ClickHouse can manage [distributed](../operations/table_engines/distributed.md) tables. When a user inserts data into such table, ClickHouse creates a queue of the data which should be sent to servers of the cluster, then asynchronously sends them. You can control the processing of queue by using the requests [STOP DISTRIBUTED SENDS](#query_language-system-stop-distributed-sends), [FLUSH DISTRIBUTED](#query_language-system-flush-distributed) and [START DISTRIBUTED SENDS](#query_language-system-start-distributed-sends). + +### STOP DISTRIBUTED SENDS {#query_language-system-stop-distributed-sends} + +Disables asynchronous distribution of data between servers of the cluster. + +``` +SYSTEM STOP DISTRIBUTED SENDS [db.] +``` + +### FLUSH DISTRIBUTED {#query_language-system-flush-distributed} + +Forces ClickHouse to send data to the servers of the cluster in synchronous mode. If some of the servers are not available, ClickHouse throws an exception and stops query processing. When servers back into operation, you should repeat the query. + +``` +SYSTEM FLUSH DISTRIBUTED [db.] +``` + +### START DISTRIBUTED SENDS {#query_language-system-start-distributed-sends} + +Enables asynchronous distribution of data between servers of cluster. + +``` +SYSTEM START DISTRIBUTED SENDS [db.] +``` diff --git a/docs/toc_en.yml b/docs/toc_en.yml index 7ca367a0cec..d75f81a556c 100644 --- a/docs/toc_en.yml +++ b/docs/toc_en.yml @@ -61,6 +61,10 @@ nav: - 'IPv4': 'data_types/domains/ipv4.md' - 'IPv6': 'data_types/domains/ipv6.md' +- 'Database Engines': + - 'Introduction': 'database_engines/index.md' + - 'MySQL': 'database_engines/mysql.md' + - 'Table Engines': - 'Introduction': 'operations/table_engines/index.md' - 'MergeTree Family': @@ -104,6 +108,7 @@ nav: - 'INSERT INTO': 'query_language/insert_into.md' - 'CREATE': 'query_language/create.md' - 'ALTER': 'query_language/alter.md' + - 'SYSTEM': 'query_language/system.md' - 'Other Kinds of Queries': 'query_language/misc.md' - 'Functions': - 'Introduction': 'query_language/functions/index.md' diff --git a/docs/toc_fa.yml b/docs/toc_fa.yml index ffc9e547423..70a0f222d4d 100644 --- a/docs/toc_fa.yml +++ b/docs/toc_fa.yml @@ -61,6 +61,10 @@ nav: - 'IPv4': 'data_types/domains/ipv4.md' - 'IPv6': 'data_types/domains/ipv6.md' +- 'Database Engines': + - 'Introduction': 'database_engines/index.md' + - 'MySQL': 'database_engines/mysql.md' + - 'Table Engines': - 'Introduction': 'operations/table_engines/index.md' - 'MergeTree Family': @@ -104,6 +108,7 @@ nav: - 'INSERT INTO': 'query_language/insert_into.md' - 'CREATE': 'query_language/create.md' - 'ALTER': 'query_language/alter.md' + - 'SYSTEM': 'query_language/system.md' - 'Other Kinds of Queries': 'query_language/misc.md' - 'Functions': - 'Introduction': 'query_language/functions/index.md' diff --git a/docs/toc_ru.yml b/docs/toc_ru.yml index 67c3b530d70..d2f463fcc85 100644 --- a/docs/toc_ru.yml +++ b/docs/toc_ru.yml @@ -62,6 +62,10 @@ nav: - 'IPv4': 'data_types/domains/ipv4.md' - 'IPv6': 'data_types/domains/ipv6.md' +- 'Движки баз данных': + - 'Введение': 'database_engines/index.md' + - 'MySQL': 'database_engines/mysql.md' + - 'Движки таблиц': - 'Введение': 'operations/table_engines/index.md' - 'Семейство MergeTree': @@ -105,6 +109,7 @@ nav: - 'INSERT INTO': 'query_language/insert_into.md' - 'CREATE': 'query_language/create.md' - 'ALTER': 'query_language/alter.md' + - 'SYSTEM': 'query_language/system.md' - 'Прочие виды запросов': 'query_language/misc.md' - 'Функции': - 'Введение': 'query_language/functions/index.md' diff --git a/docs/toc_zh.yml b/docs/toc_zh.yml index d1388ab3dd7..40b1c97d2a8 100644 --- a/docs/toc_zh.yml +++ b/docs/toc_zh.yml @@ -60,6 +60,10 @@ nav: - 'IPv4': 'data_types/domains/ipv4.md' - 'IPv6': 'data_types/domains/ipv6.md' +- 'Database Engines': + - 'Introduction': 'database_engines/index.md' + - 'MySQL': 'database_engines/mysql.md' + - 'Table Engines': - 'Introduction': 'operations/table_engines/index.md' - 'MergeTree Family': @@ -103,6 +107,7 @@ nav: - 'INSERT INTO': 'query_language/insert_into.md' - 'CREATE': 'query_language/create.md' - 'ALTER': 'query_language/alter.md' + - 'SYSTEM': 'query_language/system.md' - 'Other kinds of queries': 'query_language/misc.md' - 'Functions': - '介绍': 'query_language/functions/index.md' diff --git a/docs/zh/database_engines/index.md b/docs/zh/database_engines/index.md new file mode 120000 index 00000000000..bbdb762a4ad --- /dev/null +++ b/docs/zh/database_engines/index.md @@ -0,0 +1 @@ +../../en/database_engines/index.md \ No newline at end of file diff --git a/docs/zh/database_engines/mysql.md b/docs/zh/database_engines/mysql.md new file mode 120000 index 00000000000..51ac4126e2d --- /dev/null +++ b/docs/zh/database_engines/mysql.md @@ -0,0 +1 @@ +../../en/database_engines/mysql.md \ No newline at end of file diff --git a/docs/zh/development/architecture.md b/docs/zh/development/architecture.md deleted file mode 120000 index abda4dd48a8..00000000000 --- a/docs/zh/development/architecture.md +++ /dev/null @@ -1 +0,0 @@ -../../en/development/architecture.md \ No newline at end of file diff --git a/docs/zh/development/architecture.md b/docs/zh/development/architecture.md new file mode 100644 index 00000000000..ad00ce932d1 --- /dev/null +++ b/docs/zh/development/architecture.md @@ -0,0 +1,194 @@ +# ClickHouse 架构概述 + +ClickHouse 是一个真正的列式数据库管理系统（DBMS)。在 ClickHouse 中，数据始终是按列存储的，包括失量（向量或列块）执行的过程。只要有可能，操作都是基于失量进行分派的，而不是单个的值，这被称为“矢量化查询执行”，它有利于降低实际的数据处理开销。 + +> 这个想法并不新鲜，其可以追溯到 `APL` 编程语言及其后代：`A +`、`J`、`K` 和 `Q`。失量编程被大量用于科学数据处理中。即使在关系型数据库中，这个想法也不是什么新的东西：比如，失量编程也被大量用于 `Vectorwise` 系统中。 + +通常有两种不同的加速查询处理的方法：矢量化查询执行和运行时代码生成。在后者中，动态地为每一类查询生成代码，消除了间接分派和动态分派。这两种方法中，并没有哪一种严格地比另一种好。运行时代码生成可以更好地将多个操作融合在一起，从而充分利用 CPU 执行单元和流水线。矢量化查询执行不是特别实用，因为它涉及必须写到缓存并读回的临时向量。如果 L2 缓存容纳不下临时数据，那么这将成为一个问题。但矢量化查询执行更容易利用 CPU 的 SIMD 功能。朋友写的一篇[研究论文](http://15721.courses.cs.cmu.edu/spring2016/papers/p5-sompolski.pdf)表明，将两种方法结合起来是更好的选择。ClickHouse 使用了矢量化查询执行，同时初步提供了有限的运行时动态代码生成。 + +## 列（Columns） + +要表示内存中的列（实际上是列块），需使用 `IColumn` 接口。该接口提供了用于实现各种关系操作符的辅助方法。几乎所有的操作都是不可变的：这些操作不会更改原始列，但是会创建一个新的修改后的列。比如，`IColumn::filter` 方法接受过滤字节掩码，用于 `WHERE` 和 `HAVING` 关系操作符中。另外的例子：`IColumn::permute` 方法支持 `ORDER BY` 实现，`IColumn::cut` 方法支持 `LIMIT` 实现等等。 + +不同的 `IColumn` 实现（`ColumnUInt8`、`ColumnString` 等）负责不同的列内存布局。内存布局通常是一个连续的数组。对于数据类型为整型的列，只是一个连续的数组，比如 `std::vector`。对于 `String` 列和 `Array` 列，则由两个向量组成：其中一个向量连续存储所有的 `String` 或数组元素，另一个存储每一个 `String` 或 `Array` 的起始元素在第一个向量中的偏移。而 `ColumnConst` 则仅在内存中存储一个值，但是看起来像一个列。 + +## Field + +尽管如此，有时候也可能需要处理单个值。表示单个值，可以使用 `Field`。`Field` 是 `UInt64`、`Int64`、`Float64`、`String` 和 `Array` 组成的联合。`IColumn` 拥有 `operator[]` 方法来获取第 `n` 个值成为一个 `Field`，同时也拥有 `insert` 方法将一个 `Field` 追加到一个列的末尾。这些方法并不高效，因为它们需要处理表示单一值的临时 `Field` 对象，但是有更高效的方法比如 `insertFrom` 和 `insertRangeFrom` 等。 + +`Field` 中并没有足够的关于一个表（table）的特定数据类型的信息。比如，`UInt8`、`UInt16`、`UInt32` 和 `UInt64` 在 `Field` 中均表示为 `UInt64`。 + +## 抽象漏洞 + +`IColumn` 具有用于数据的常见关系转换的方法，但这些方法并不能够满足所有需求。比如，`ColumnUInt64` 没有用于计算两列和的方法，`ColumnString` 没有用于进行子串搜索的方法。这些无法计算的例程在 `Icolumn` 之外实现。 + +列（Columns)上的各种函数可以通过使用 `Icolumn` 的方法来提取 `Field` 值，或根据特定的 `Icolumn` 实现的数据内存布局的知识，以一种通用但不高效的方式实现。为此，函数将会转换为特定的 `IColumn` 类型并直接处理内部表示。比如，`ColumnUInt64` 具有 `getData` 方法，该方法返回一个指向列的内部数组的引用，然后一个单独的例程可以直接读写或填充该数组。实际上，“抽象漏洞（leaky abstractions）”允许我们以更高效的方式来实现各种特定的例程。 + +## 数据类型 + +`IDataType` 负责序列化和反序列化：读写二进制或文本形式的列或单个值构成的块。`IDataType` 直接与表的数据类型相对应。比如，有 `DataTypeUInt32`、`DataTypeDateTime`、`DataTypeString` 等数据类型。 + +`IDataType` 与 `IColumn` 之间的关联并不大。不同的数据类型在内存中能够用相同的 `IColumn` 实现来表示。比如，`DataTypeUInt32` 和 `DataTypeDateTime` 都是用 `ColumnUInt32` 或 `ColumnConstUInt32` 来表示的。另外，相同的数据类型也可以用不同的 `IColumn` 实现来表示。比如，`DataTypeUInt8` 既可以使用 `ColumnUInt8` 来表示，也可以使用过 `ColumnConstUInt8` 来表示。 + +`IDataType` 仅存储元数据。比如，`DataTypeUInt8` 不存储任何东西（除了 vptr）；`DataTypeFixedString` 仅存储 `N`（固定长度字符串的串长度）。 + +`IDataType` 具有针对各种数据格式的辅助函数。比如如下一些辅助函数：序列化一个值并加上可能的引号；序列化一个值用于 JSON 格式；序列化一个值作为 XML 格式的一部分。辅助函数与数据格式并没有直接的对应。比如，两种不同的数据格式 `Pretty` 和 `TabSeparated` 均可以使用 `IDataType` 接口提供的 `serializeTextEscaped` 这一辅助函数。 + +## 块（Block） + +`Block` 是表示内存中表的子集（chunk）的容器，是由三元组：`(IColumn, IDataType, 列名)` 构成的集合。在查询执行期间，数据是按 `Block` 进行处理的。如果我们有一个 `Block`，那么就有了数据（在 `IColumn` 对象中），有了数据的类型信息告诉我们如何处理该列，同时也有了列名（来自表的原始列名，或人为指定的用于临时计算结果的名字）。 + +当我们遍历一个块中的列进行某些函数计算时，会把结果列加入到块中，但不会更改函数参数中的列，因为操作是不可变的。之后，不需要的列可以从块中删除，但不是修改。这对于消除公共子表达式非常方便。 + +`Block` 用于处理数据块。注意，对于相同类型的计算，列名和类型对不同的块保持相同，仅列数据不同。最好把块数据（block data）和块头（block header）分离开来，因为小块大小会因复制共享指针和列名而带来很高的临时字符串开销。 + +## 块流（Block Streams） + +块流用于处理数据。我们可以使用块流从某个地方读取数据，执行数据转换，或将数据写到某个地方。`IBlockInputStream` 具有 `read` 方法，其能够在数据可用时获取下一个块。`IBlockOutputStream` 具有 `write` 方法，其能够将块写到某处。 + +块流负责： + +1. 读或写一个表。表仅返回一个流用于读写块。 +2. 完成数据格式化。比如，如果你打算将数据以 `Pretty` 格式输出到终端，你可以创建一个块输出流，将块写入该流中，然后进行格式化。 +3. 执行数据转换。假设你现在有 `IBlockInputStream` 并且打算创建一个过滤流，那么你可以创建一个 `FilterBlockInputStream` 并用 `IBlockInputStream` 进行初始化。之后，当你从 `FilterBlockInputStream` 中拉取块时，会从你的流中提取一个块，对其进行过滤，然后将过滤后的块返回给你。查询执行流水线就是以这种方式表示的。 + +还有一些更复杂的转换。比如，当你从 `AggregatingBlockInputStream` 拉取数据时，会从数据源读取全部数据进行聚集，然后将聚集后的数据流返回给你。另一个例子：`UnionBlockInputStream` 的构造函数接受多个输入源和多个线程，其能够启动多线程从多个输入源并行读取数据。 + +> 块流使用“pull”方法来控制流：当你从第一个流中拉取块时，它会接着从嵌套的流中拉取所需的块，然后整个执行流水线开始工作。”pull“和“push”都不是最好的方案，因为控制流不是明确的，这限制了各种功能的实现，比如多个查询同步执行（多个流水线合并到一起）。这个限制可以通过协程或直接运行互相等待的线程来解决。如果控制流明确，那么我们会有更多的可能性：如果我们定位了数据从一个计算单元传递到那些外部的计算单元中其中一个计算单元的逻辑。阅读这篇[文章](http://journal.stuffwithstuff.com/2013/01/13/iteration-inside-and-out/)来获取更多的想法。 + +我们需要注意，查询执行流水线在每一步都会创建临时数据。我们要尽量使块的大小足够小，从而 CPU 缓存能够容纳下临时数据。在这个假设下，与其他计算相比，读写临时数据几乎是没有任何开销的。我们也可以考虑一种替代方案：将流水线中的多个操作融合在一起，使流水线尽可能短，并删除大量临时数据。这可能是一个优点，但同时也有缺点。比如，拆分流水线使得中间数据缓存、获取同时运行的类似查询的中间数据以及相似查询的流水线合并等功能很容易实现。 + +## 格式（Formats） + +数据格式同块流一起实现。既有仅用于向客户端输出数据的”展示“格式，如 `IBlockOutputStream` 提供的 `Pretty` 格式，也有其它输入输出格式，比如 `TabSeparated` 或 `JSONEachRow`。 + +此外还有行流：`IRowInputStream` 和 `IRowOutputStream`。它们允许你按行 pull/push 数据，而不是按块。行流只需要简单地面向行格式实现。包装器 `BlockInputStreamFromRowInputStream` 和 `BlockOutputStreamFromRowOutputStream` 允许你将面向行的流转换为正常的面向块的流。 + +## I/O + +对于面向字节的输入输出，有 `ReadBuffer` 和 `WriteBuffer` 这两个抽象类。它们用来替代 C++ 的 `iostream`。不用担心：每个成熟的 C++ 项目都会有充分的理由使用某些东西来代替 `iostream`。 + +`ReadBuffer` 和 `WriteBuffer` 由一个连续的缓冲区和指向缓冲区中某个位置的一个指针组成。实现中，缓冲区可能拥有内存，也可能不拥有内存。有一个虚方法会使用随后的数据来填充缓冲区（针对 `ReadBuffer`）或刷新缓冲区（针对 `WriteBuffer`），该虚方法很少被调用。 + +`ReadBuffer` 和 `WriteBuffer` 的实现用于处理文件、文件描述符和网络套接字（socket），也用于实现压缩（`CompressedWriteBuffer` 在写入数据前需要先用一个 `WriteBuffer` 进行初始化并进行压缩）和其它用途。`ConcatReadBuffer`、`LimitReadBuffer` 和 `HashingWriteBuffer` 的用途正如其名字所描述的一样。 + +`ReadBuffer` 和 `WriteBuffer` 仅处理字节。为了实现格式化输入和输出（比如以十进制格式写一个数字），`ReadHelpers` 和 `WriteHelpers` 头文件中有一些辅助函数可用。 + +让我们来看一下，当你把一个结果集以 `JSON` 格式写到标准输出（stdout）时会发生什么。你已经准备好从 `IBlockInputStream` 获取结果集，然后创建 `WriteBufferFromFileDescriptor(STDOUT_FILENO)` 用于写字节到标准输出，创建 `JSONRowOutputStream` 并用 `WriteBuffer` 初始化，用于将行以 `JSON` 格式写到标准输出，你还可以在其上创建 `BlockOutputStreamFromRowOutputStream`，将其表示为 `IBlockOutputStream`。然后调用 `copyData` 将数据从 `IBlockInputStream` 传输到 `IBlockOutputStream`，一切工作正常。在内部，`JSONRowOutputStream` 会写入 JSON 分隔符，并以指向 `IColumn` 的引用和行数作为参数调用 `IDataType::serializeTextJSON` 函数。随后，`IDataType::serializeTextJSON` 将会调用 `WriteHelpers.h` 中的一个方法：比如，`writeText` 用于数值类型，`writeJSONString` 用于 `DataTypeString` 。 + +## 表（Tables） + +表由 `IStorage` 接口表示。该接口的不同实现对应不同的表引擎。比如 `StorageMergeTree`、`StorageMemory` 等。这些类的实例就是表。 + +`IStorage` 中最重要的方法是 `read` 和 `write`，除此之外还有 `alter`、`rename` 和 `drop` 等方法。`read` 方法接受如下参数：需要从表中读取的列集，需要执行的 `AST` 查询，以及所需返回的流的数量。`read` 方法的返回值是一个或多个 `IBlockInputStream` 对象，以及在查询执行期间在一个表引擎内完成的关于数据处理阶段的信息。 + +在大多数情况下，`read` 方法仅负责从表中读取指定的列，而不会进行进一步的数据处理。进一步的数据处理均由查询解释器完成，不由 `IStorage` 负责。 + +但是也有值得注意的例外： + +- AST 查询被传递给 `read` 方法，表引擎可以使用它来判断是否能够使用索引，从而从表中读取更少的数据。 +- 有时候，表引擎能够将数据处理到一个特定阶段。比如，`StorageDistributed` 可以向远程服务器发送查询，要求它们将来自不同的远程服务器能够合并的数据处理到某个阶段，并返回预处理后的数据，然后查询解释器完成后续的数据处理。 + +表的 `read` 方法能够返回多个 `IBlockInputStream` 对象以允许并行处理数据。多个块输入流能够从一个表中并行读取。然后你可以通过不同的转换对这些流进行装饰（比如表达式求值或过滤），转换过程能够独立计算，并在其上创建一个 `UnionBlockInputStream`，以并行读取多个流。 + +另外也有 `TableFunction`。`TableFunction` 能够在查询的 `FROM` 字句中返回一个临时的 `IStorage` 以供使用。 + +要快速了解如何实现自己的表引擎，可以查看一些简单的表引擎，比如 `StorageMemory` 或 `StorageTinyLog`。 + +> 作为 `read` 方法的结果，`IStorage` 返回 `QueryProcessingStage` - 关于 storage 里哪部分查询已经被计算的信息。当前我们仅有非常粗粒度的信息。Storage 无法告诉我们“对于这个范围的数据，我已经处理完了 WHERE 字句里的这部分表达式”。我们需要在这个地方继续努力。 + +## 解析器（Parsers） + +查询由一个手写递归下降解析器解析。比如， `ParserSelectQuery` 只是针对查询的不同部分递归地调用下层解析器。解析器创建 `AST`。`AST` 由节点表示，节点是 `IAST` 的实例。 + +> 由于历史原因，未使用解析器生成器。 + +## 解释器（Interpreters） + +解释器负责从 `AST` 创建查询执行流水线。既有一些简单的解释器，如 `InterpreterExistsQuery` 和 `InterpreterDropQuery`，也有更复杂的解释器，如 `InterpreterSelectQuery`。查询执行流水线由块输入或输出流组成。比如，`SELECT` 查询的解释结果是从 `FROM` 字句的结果集中读取数据的 `IBlockInputStream`；`INSERT` 查询的结果是写入需要插入的数据的 `IBlockOutputStream`；`SELECT INSERT` 查询的解释结果是 `IBlockInputStream`，它在第一次读取时返回一个空结果集，同时将数据从 `SELECT` 复制到 `INSERT`。 + +`InterpreterSelectQuery` 使用 `ExpressionAnalyzer` 和 `ExpressionActions` 机制来进行查询分析和转换。这是大多数基于规则的查询优化完成的地方。`ExpressionAnalyzer` 非常混乱，应该进行重写：不同的查询转换和优化应该被提取出来并划分成不同的类，从而允许模块化转换或查询。 + +## 函数（Functions） + +函数既有普通函数，也有聚合函数。对于聚合函数，请看下一节。 + +普通函数不会改变行数 - 它们的执行看起来就像是独立地处理每一行数据。实际上，函数不会作用于一个单独的行上，而是作用在以 `Block` 为单位的数据上，以实现向量查询执行。 + +还有一些杂项函数，比如 [blockSize](../query_language/functions/other_functions.md#function-blocksize)、[rowNumberInBlock](../query_language/functions/other_functions.md#function-rownumberinblock)，以及 [runningAccumulate](../query_language/functions/other_functions.md#function-runningaccumulate)，它们对块进行处理，并且不遵从行的独立性。 + +ClickHouse 具有强类型，因此隐式类型转换不会发生。如果函数不支持某个特定的类型组合，则会抛出异常。但函数可以通过重载以支持许多不同的类型组合。比如，`plus` 函数（用于实现 `+` 运算符）支持任意数字类型的组合：`UInt8` + `Float32`，`UInt16` + `Int8` 等。同时，一些可变参数的函数能够级接收任意数目的参数，比如 `concat` 函数。 + +实现函数可能有些不方便，因为函数的实现需要包含所有支持该操作的数据类型和 `IColumn` 类型。比如，`plus` 函数能够利用 C++ 模板针对不同的数字类型组合、常量以及非常量的左值和右值进行代码生成。 + +> 这是一个实现动态代码生成的好地方，从而能够避免模板代码膨胀。同样，运行时代码生成也使得实现融合函数成为可能，比如融合“乘-加”，或者在单层循环迭代中进行多重比较。 + +由于向量查询执行，函数不会“短路”。比如，如果你写 `WHERE f(x) AND g(y)`，两边都会进行计算，即使是对于 `f(x)` 为 0 的行（除非 `f(x)` 是零常量表达式）。但是如果 `f(x)` 的选择条件很高，并且计算 `f(x)` 比计算 `g(y)` 要划算得多，那么最好进行多遍计算：首先计算 `f(x)`，根据计算结果对列数据进行过滤，然后计算 `g(y)`，之后只需对较小数量的数据进行过滤。 + +## 聚合函数 + +聚合函数是状态函数。它们将传入的值激活到某个状态，并允许你从该状态获取结果。聚合函数使用 `IAggregateFunction` 接口进行管理。状态可以非常简单（`AggregateFunctionCount` 的状态只是一个单一的`UInt64` 值），也可以非常复杂（`AggregateFunctionUniqCombined` 的状态是由一个线性数组、一个散列表和一个 `HyperLogLog` 概率数据结构组合而成的）。 + +为了能够在执行一个基数很大的 `GROUP BY` 查询时处理多个聚合状态，需要在 `Arena`（一个内存池）或任何合适的内存块中分配状态。状态可以有一个非平凡的构造器和析构器：比如，复杂的聚合状态能够自己分配额外的内存。这需要注意状态的创建和销毁并恰当地传递状态的所有权，以跟踪谁将何时销毁状态。 + +聚合状态可以被序列化和反序列化，以在分布式查询执行期间通过网络传递或者在内存不够的时候将其写到硬盘。聚合状态甚至可以通过 `DataTypeAggregateFunction` 存储到一个表中，以允许数据的增量聚合。 + +> 聚合函数状态的序列化数据格式目前尚未版本化。如果只是临时存储聚合状态，这样是可以的。但是我们有 `AggregatingMergeTree` 表引擎用于增量聚合，并且人们已经在生产中使用它。这就是为什么在未来当我们更改任何聚合函数的序列化格式时需要增加向后兼容的支持。 + +## 服务器（Server） + +服务器实现了多个不同的接口： + +- 一个用于任何外部客户端的 HTTP 接口。 +- 一个用于本机 ClickHouse 客户端以及在分布式查询执行中跨服务器通信的 TCP 接口。 +- 一个用于传输数据以进行拷贝的接口。 + +在内部，它只是一个没有协程、纤程等的基础多线程服务器。服务器不是为处理高速率的简单查询设计的，而是为处理相对低速率的复杂查询设计的，每一个复杂查询能够对大量的数据进行处理分析。 + +服务器使用必要的查询执行需要的环境初始化 `Context` 类：可用数据库列表、用户和访问权限、设置、集群、进程列表和查询日志等。这些环境被解释器使用。 + +我们维护了服务器 TCP 协议的完全向后向前兼容性：旧客户端可以和新服务器通信，新客户端也可以和旧服务器通信。但是我们并不想永久维护它，我们将在大约一年后删除对旧版本的支持。 + +> 对于所有的外部应用，我们推荐使用 HTTP 接口，因为该接口很简单，容易使用。TCP 接口与内部数据结构的联系更加紧密：它使用内部格式传递数据块，并使用自定义帧来压缩数据。我们没有发布该协议的 C 库，因为它需要链接大部分的 ClickHouse 代码库，这是不切实际的。 + +## 分布式查询执行 + +集群设置中的服务器大多是独立的。你可以在一个集群中的一个或多个服务器上创建一个 `Distributed` 表。`Distributed` 表本身并不存储数据，它只为集群的多个节点上的所有本地表提供一个“视图（view）”。当从 `Distributed` 表中进行 SELECT 时，它会重写该查询，根据负载平衡设置来选择远程节点，并将查询发送给节点。`Distributed` 表请求远程服务器处理查询，直到可以合并来自不同服务器的中间结果的阶段。然后它接收中间结果并进行合并。分布式表会尝试将尽可能多的工作分配给远程服务器，并且不会通过网络发送太多的中间数据。 + +> 当 `IN` 或 `JOIN` 子句中包含子查询并且每个子查询都使用分布式表时，事情会变得更加复杂。我们有不同的策略来执行这些查询。 + +分布式查询执行没有全局查询计划。每个节点都有针对自己的工作部分的本地查询计划。我们仅有简单的一次性分布式查询执行：将查询发送给远程节点，然后合并结果。但是对于具有高基数的 `GROUP BY` 或具有大量临时数据的 `JOIN` 这样困难的查询的来说，这是不可行的：在这种情况下，我们需要在服务器之间“改组”数据，这需要额外的协调。ClickHouse 不支持这类查询执行，我们需要在这方面进行努力。 + +## Merge Tree + +`MergeTree` 是一系列支持按主键索引的存储引擎。主键可以是一个任意的列或表达式的元组。`MergeTree` 表中的数据存储于“分块”中。每一个分块以主键序存储数据（数据按主键元组的字典序排序）。表的所有列都存储在这些“分块”中分离的 `column.bin` 文件中。`column.bin` 文件由压缩块组成，每一个块通常是 64 KB 到 1 MB 大小的未压缩数据，具体取决于平均值大小。这些块由一个接一个连续放置的列值组成。每一列的列值顺序相同（顺序由主键定义），因此当你按多列进行迭代时，你能够得到相应列的值。 + +主键本身是“稀疏”的。它并不是索引单一的行，而是索引某个范围内的数据。一个单独的 `primary.idx` 文件具有每个第 N 行的主键值，其中 N 称为 `index_granularity`（通常，N = 8192）。同时，对于每一列，都有带有标记的 `column.mrk` 文件，该文件记录的是每个第 N 行在数据文件中的偏移量。每个标记是一个 pair：文件中的偏移量到压缩块的起始，以及解压缩块中的偏移量到数据的起始。通常，压缩块根据标记对齐，并且解压缩块中的偏移量为 0。`primary.idx` 的数据始终驻留在内存，同时 `column.mrk` 的数据被缓存。 + +当我们要从 `MergeTree` 的一个分块中读取部分内容时，我们会查看 `primary.idx` 数据并查找可能包含所请求数据的范围，然后查看 `column.mrk` 并计算偏移量从而得知从哪里开始读取些范围的数据。由于稀疏性，可能会读取额外的数据。ClickHouse 不适用于高负载的简单点查询，因为对于每一个键，整个 `index_granularity` 范围的行的数据都需要读取，并且对于每一列需要解压缩整个压缩块。我们使索引稀疏，是因为每一个单一的服务器需要在索引没有明显内存消耗的情况下，维护数万亿行的数据。另外，由于主键是稀疏的，导致其不是唯一的：无法在 INSERT 时检查一个键在表中是否存在。你可以在一个表中使用同一个键创建多个行。 + +当你向 `MergeTree` 中插入一堆数据时，数据按主键排序并形成一个新的分块。为了保证分块的数量相对较少，有后台线程定期选择一些分块并将它们合并成一个有序的分块，这就是 `MergeTree` 的名称来源。当然，合并会导致“写入放大”。所有的分块都是不可变的：它们仅会被创建和删除，不会被修改。当运行 `SELECT` 查询时，`MergeTree` 会保存一个表的快照（分块集合）。合并之后，还会保留旧的分块一段时间，以便发生故障后更容易恢复，因此如果我们发现某些合并后的分块可能已损坏，我们可以将其替换为原分块。 + +`MergeTree` 不是 LSM 树，因为它不包含”memtable“和”log“：插入的数据直接写入文件系统。这使得它仅适用于批量插入数据，而不适用于非常频繁地一行一行插入 - 大约每秒一次是没问题的，但是每秒一千次就会有问题。我们这样做是为了简单起见，因为我们已经在我们的应用中批量插入数据。 + +> `MergeTree` 表只能有一个（主）索引：没有任何辅助索引。在一个逻辑表下，允许有多个物理表示，比如，可以以多个物理顺序存储数据，或者同时表示预聚合数据和原始数据。 + +有些 `MergeTree` 引擎会在后台合并期间做一些额外工作，比如 `CollapsingMergeTree` 和 `AggregatingMergeTree`。这可以视为对更新的特殊支持。请记住这些不是真正的更新，因为用户通常无法控制后台合并将会执行的时间，并且 `MergeTree` 中的数据几乎总是存储在多个分块中，而不是完全合并的形式。 + +## 复制（Replication） + +ClickHouse 中的复制是基于表实现的。你可以在同一个服务器上有一些可复制的表和不可复制的表。你也可以以不同的方式进行表的复制，比如一个表进行双因子复制，另一个进行三因子复制。 + +复制是在 `ReplicatedMergeTree` 存储引擎中实现的。`ZooKeeper` 中的路径被指定为存储引擎的参数。`ZooKeeper` 中所有具有相同路径的表互为副本：它们同步数据并保持一致性。只需创建或删除表，就可以实现动态添加或删除副本。 + +复制使用异步多主机方案。你可以将数据插入到与 `ZooKeeper` 进行会话的任意副本中，并将数据复制到所有其它副本中。由于 ClickHouse 不支持 UPDATEs，因此复制是无冲突的。由于没有对插入的仲裁确认，如果一个节点发生故障，刚刚插入的数据可能会丢失。 + +用于复制的元数据存储在 ZooKeeper 中。其中一个复制日志列出了要执行的操作。操作包括：获取分块、合并分块和删除分区等。每一个副本将复制日志复制到其队列中，然后执行队列中的操作。比如，在插入时，在复制日志中创建“获取分块”这一操作，然后每一个副本都会去下载该分块。所有副本之间会协调进行合并以获得相同字节的结果。所有的分块在所有的副本上以相同的方式合并。为实现该目的，其中一个副本被选为领导者，该副本首先进行合并，并把“合并分块”操作写到日志中。 + +复制是物理的：只有压缩的分块会在节点之间传输，查询则不会。为了降低网络成本（避免网络放大），大多数情况下，会在每一个副本上独立地处理合并。只有在存在显著的合并延迟的情况下，才会通过网络发送大块的合并分块。 + +另外，每一个副本将其状态作为分块和校验和组成的集合存储在 ZooKeeper 中。当本地文件系统中的状态与 ZooKeeper 中引用的状态不同时，该副本会通过从其它副本下载缺失和损坏的分块来恢复其一致性。当本地文件系统中出现一些意外或损坏的数据时，ClickHouse 不会将其删除，而是将其移动到一个单独的目录下并忘记它。 + +> ClickHouse 集群由独立的分片组成，每一个分片由多个副本组成。集群不是弹性的，因此在添加新的分片后，数据不会自动在分片之间重新平衡。相反，集群负载将变得不均衡。该实现为你提供了更多控制，对于相对较小的集群，例如只有数十个节点的集群来说是很好的。但是对于我们在生产中使用的具有数百个节点的集群来说，这种方法成为一个重大缺陷。我们应该实现一个表引擎，使得该引擎能够跨集群扩展数据，同时具有动态复制的区域，这些区域能够在集群之间自动拆分和平衡。 + +[来源文章](https://clickhouse.yandex/docs/en/development/architecture/) diff --git a/docs/zh/interfaces/formats.md b/docs/zh/interfaces/formats.md index d9ce4f3c1da..65358115295 100644 --- a/docs/zh/interfaces/formats.md +++ b/docs/zh/interfaces/formats.md @@ -327,6 +327,60 @@ ClickHouse 支持 [NULL](../query_language/syntax.md), 在 JSON 格式中以 `nu 对于解析，任何顺序都支持不同列的值。可以省略某些值 - 它们被视为等于它们的默认值。在这种情况下，零和空行被用作默认值。 作为默认值，不支持表中指定的复杂值。元素之间的空白字符被忽略。如果在对象之后放置逗号，它将被忽略。对象不一定必须用新行分隔。 +### Usage of Nested Structures {#jsoneachrow-nested} + +If you have a table with the [Nested](../data_types/nested_data_structures/nested.md) data type columns, you can insert JSON data having the same structure. Enable this functionality with the [input_format_import_nested_json](../operations/settings/settings.md#settings-input_format_import_nested_json) setting. + +For example, consider the following table: + +```sql +CREATE TABLE json_each_row_nested (n Nested (s String, i Int32) ) ENGINE = Memory +``` + +As you can find in the `Nested` data type description, ClickHouse treats each component of the nested structure as a separate column, `n.s` and `n.i` for our table. So you can insert the data the following way: + +```sql +INSERT INTO json_each_row_nested FORMAT JSONEachRow {"n.s": ["abc", "def"], "n.i": [1, 23]} +``` + +To insert data as hierarchical JSON object set [input_format_import_nested_json=1](../operations/settings/settings.md#settings-input_format_import_nested_json). + +```json +{ + "n": { + "s": ["abc", "def"], + "i": [1, 23] + } +} +``` + +Without this setting ClickHouse throws the exception. + +```sql +SELECT name, value FROM system.settings WHERE name = 'input_format_import_nested_json' +``` +```text +┌─name────────────────────────────┬─value─┐ +│ input_format_import_nested_json │ 0 │ +└─────────────────────────────────┴───────┘ +``` +```sql +INSERT INTO json_each_row_nested FORMAT JSONEachRow {"n": {"s": ["abc", "def"], "i": [1, 23]}} +``` +```text +Code: 117. DB::Exception: Unknown field found while parsing JSONEachRow format: n: (at row 1) +``` +```sql +SET input_format_import_nested_json=1 +INSERT INTO json_each_row_nested FORMAT JSONEachRow {"n": {"s": ["abc", "def"], "i": [1, 23]}} +SELECT * FROM json_each_row_nested +``` +```text +┌─n.s───────────┬─n.i────┐ +│ ['abc','def'] │ [1,23] │ +└───────────────┴────────┘ +``` + ## Native {#native} 最高性能的格式。 据通过二进制格式的块进行写入和读取。对于每个块，该块中的行数，列数，列名称和类型以及列的部分将被相继记录。 换句话说，这种格式是 “列式”的 - 它不会将列转换为行。 这是用于在服务器之间进行交互的本地界面中使用的格式，用于使用命令行客户端和 C++ 客户端。 diff --git a/docs/zh/query_language/functions/other_functions.md b/docs/zh/query_language/functions/other_functions.md index 85804c0a75d..84fbdaeb3ca 100644 --- a/docs/zh/query_language/functions/other_functions.md +++ b/docs/zh/query_language/functions/other_functions.md @@ -637,7 +637,7 @@ SELECT replicate(1, ['a', 'b', 'c']) 返回磁盘的容量信息，以字节为单位。使用配置文件中的path配置评估此信息。 -## finalizeAggregation +## finalizeAggregation {#function-finalizeaggregation} 获取聚合函数的状态。返回聚合结果（最终状态）。 diff --git a/docs/zh/query_language/system.md b/docs/zh/query_language/system.md new file mode 120000 index 00000000000..6061858c3f2 --- /dev/null +++ b/docs/zh/query_language/system.md @@ -0,0 +1 @@ +../../en/query_language/system.md \ No newline at end of file diff --git a/libs/libcommon/CMakeLists.txt b/libs/libcommon/CMakeLists.txt index e3fac95694a..1e3be7f61d6 100644 --- a/libs/libcommon/CMakeLists.txt +++ b/libs/libcommon/CMakeLists.txt @@ -67,7 +67,7 @@ if (USE_UNWIND) target_include_directories (common BEFORE PRIVATE ${UNWIND_INCLUDE_DIR}) if (NOT USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING) target_link_libraries (common PRIVATE ${UNWIND_LIBRARY}) - endif () + endif () endif () # When testing for memory leaks with Valgrind, dont link tcmalloc or jemalloc. diff --git a/libs/libcommon/include/common/DateLUTImpl.h b/libs/libcommon/include/common/DateLUTImpl.h index 28d536fc93d..344d363b0d7 100644 --- a/libs/libcommon/include/common/DateLUTImpl.h +++ b/libs/libcommon/include/common/DateLUTImpl.h @@ -20,6 +20,16 @@ #endif #endif +/// Flags for toYearWeek() function. +enum class WeekModeFlag : UInt8 +{ + MONDAY_FIRST = 1, + YEAR = 2, + FIRST_WEEKDAY = 4, + NEWYEAR_DAY = 8 +}; +typedef std::pair YearWeek; + /** Lookup table to conversion of time to date, and to month / year / day of week / day of month and so on. * First time was implemented for OLAPServer, that needed to do billions of such transformations. */ @@ -379,6 +389,167 @@ public: return toISOWeek(toDayNum(t)); } + /* + The bits in week_mode has the following meaning: + WeekModeFlag::MONDAY_FIRST (0) If not set Sunday is first day of week + If set Monday is first day of week + WeekModeFlag::YEAR (1) If not set Week is in range 0-53 + + Week 0 is returned for the the last week of the previous year (for + a date at start of january) In this case one can get 53 for the + first week of next year. This flag ensures that the week is + relevant for the given year. Note that this flag is only + releveant if WeekModeFlag::JANUARY is not set. + + If set Week is in range 1-53. + + In this case one may get week 53 for a date in January (when + the week is that last week of previous year) and week 1 for a + date in December. + + WeekModeFlag::FIRST_WEEKDAY (2) If not set Weeks are numbered according + to ISO 8601:1988 + If set The week that contains the first + 'first-day-of-week' is week 1. + + WeekModeFlag::NEWYEAR_DAY (3) If not set no meaning + If set The week that contains the January 1 is week 1. + Week is in range 1-53. + And ignore WeekModeFlag::YEAR, WeekModeFlag::FIRST_WEEKDAY + + ISO 8601:1988 means that if the week containing January 1 has + four or more days in the new year, then it is week 1; + Otherwise it is the last week of the previous year, and the + next week is week 1. + */ + inline YearWeek toYearWeek(DayNum d, UInt8 week_mode) const + { + bool newyear_day_mode = week_mode & static_cast(WeekModeFlag::NEWYEAR_DAY); + week_mode = check_week_mode(week_mode); + bool monday_first_mode = week_mode & static_cast(WeekModeFlag::MONDAY_FIRST); + bool week_year_mode = week_mode & static_cast(WeekModeFlag::YEAR); + bool first_weekday_mode = week_mode & static_cast(WeekModeFlag::FIRST_WEEKDAY); + + // Calculate week number of WeekModeFlag::NEWYEAR_DAY mode + if (newyear_day_mode) + { + return toYearWeekOfNewyearMode(d, monday_first_mode); + } + + YearWeek yw(toYear(d), 0); + UInt16 days = 0; + UInt16 daynr = makeDayNum(yw.first, toMonth(d), toDayOfMonth(d)); + UInt16 first_daynr = makeDayNum(yw.first, 1, 1); + + // 0 for monday, 1 for tuesday ... + // get weekday from first day in year. + UInt16 weekday = calc_weekday(DayNum(first_daynr), !monday_first_mode); + + if (toMonth(d) == 1 && toDayOfMonth(d) <= static_cast(7 - weekday)) + { + if (!week_year_mode && ((first_weekday_mode && weekday != 0) || (!first_weekday_mode && weekday >= 4))) + return yw; + week_year_mode = 1; + (yw.first)--; + first_daynr -= (days = calc_days_in_year(yw.first)); + weekday = (weekday + 53 * 7 - days) % 7; + } + + if ((first_weekday_mode && weekday != 0) || (!first_weekday_mode && weekday >= 4)) + days = daynr - (first_daynr + (7 - weekday)); + else + days = daynr - (first_daynr - weekday); + + if (week_year_mode && days >= 52 * 7) + { + weekday = (weekday + calc_days_in_year(yw.first)) % 7; + if ((!first_weekday_mode && weekday < 4) || (first_weekday_mode && weekday == 0)) + { + (yw.first)++; + yw.second = 1; + return yw; + } + } + yw.second = days / 7 + 1; + return yw; + } + + /// Calculate week number of WeekModeFlag::NEWYEAR_DAY mode + /// The week number 1 is the first week in year that contains January 1, + inline YearWeek toYearWeekOfNewyearMode(DayNum d, bool monday_first_mode) const + { + YearWeek yw(0, 0); + UInt16 offset_day = monday_first_mode ? 0U : 1U; + + // Checking the week across the year + yw.first = toYear(DayNum(d + 7 - toDayOfWeek(DayNum(d + offset_day)))); + + DayNum first_day = makeDayNum(yw.first, 1, 1); + DayNum this_day = d; + + if (monday_first_mode) + { + // Rounds down a date to the nearest Monday. + first_day = toFirstDayNumOfWeek(first_day); + this_day = toFirstDayNumOfWeek(d); + } + else + { + // Rounds down a date to the nearest Sunday. + if (toDayOfWeek(first_day) != 7) + first_day = DayNum(first_day - toDayOfWeek(first_day)); + if (toDayOfWeek(d) != 7) + this_day = DayNum(d - toDayOfWeek(d)); + } + yw.second = (this_day - first_day) / 7 + 1; + return yw; + } + + /** + * get first day of week with week_mode, return Sunday or Monday + */ + inline DayNum toFirstDayNumOfWeek(DayNum d, UInt8 week_mode) const + { + bool monday_first_mode = week_mode & static_cast(WeekModeFlag::MONDAY_FIRST); + if (monday_first_mode) + { + return toFirstDayNumOfWeek(d); + } + else + { + return (toDayOfWeek(d) != 7) ? DayNum(d - toDayOfWeek(d)) : d; + } + } + + /* + * check and change mode to effective + */ + inline UInt8 check_week_mode(UInt8 mode) const + { + UInt8 week_format = (mode & 7); + if (!(week_format & static_cast(WeekModeFlag::MONDAY_FIRST))) + week_format ^= static_cast(WeekModeFlag::FIRST_WEEKDAY); + return week_format; + } + + /* + * Calc weekday from d + * Returns 0 for monday, 1 for tuesday ... + */ + inline unsigned calc_weekday(DayNum d, bool sunday_first_day_of_week) const + { + if (!sunday_first_day_of_week) + return toDayOfWeek(d) - 1; + else + return toDayOfWeek(DayNum(d + 1)) - 1; + } + + /* Calc days in one year. */ + inline unsigned calc_days_in_year(UInt16 year) const + { + return ((year & 3) == 0 && (year % 100 || (year % 400 == 0 && year)) ? 366 : 365); + } + /// Number of month from some fixed moment in the past (year * 12 + month) inline unsigned toRelativeMonthNum(DayNum d) const { diff --git a/libs/libcommon/include/common/JSON.h b/libs/libcommon/include/common/JSON.h index 9bb109a840f..5f3d9325626 100644 --- a/libs/libcommon/include/common/JSON.h +++ b/libs/libcommon/include/common/JSON.h @@ -60,7 +60,18 @@ public: checkInit(); } - JSON(const JSON & rhs) : ptr_begin(rhs.ptr_begin), ptr_end(rhs.ptr_end), level(rhs.level) {} + JSON(const JSON & rhs) + { + *this = rhs; + } + + JSON & operator=(const JSON & rhs) + { + ptr_begin = rhs.ptr_begin; + ptr_end = rhs.ptr_end; + level = rhs.level; + return *this; + } const char * data() const { return ptr_begin; } const char * dataEnd() const { return ptr_end; } diff --git a/libs/libcommon/include/common/config_common.h.in b/libs/libcommon/include/common/config_common.h.in index 0cc0950efba..247afd87aea 100644 --- a/libs/libcommon/include/common/config_common.h.in +++ b/libs/libcommon/include/common/config_common.h.in @@ -7,3 +7,4 @@ #cmakedefine01 USE_READLINE #cmakedefine01 USE_LIBEDIT #cmakedefine01 HAVE_READLINE_HISTORY +#cmakedefine01 NOT_UNBUNDLED diff --git a/libs/libcommon/include/common/logger_useful.h b/libs/libcommon/include/common/logger_useful.h index 245a79c7982..c1c39047540 100644 --- a/libs/libcommon/include/common/logger_useful.h +++ b/libs/libcommon/include/common/logger_useful.h @@ -4,41 +4,39 @@ #include #include +#include +#include +#include +#include #ifndef QUERY_PREVIEW_LENGTH #define QUERY_PREVIEW_LENGTH 160 #endif using Poco::Logger; +using Poco::Message; +using DB::LogsLevel; +using DB::CurrentThread; /// Logs a message to a specified logger with that level. -#define LOG_TRACE(logger, message) do { \ - if ((logger)->trace()) {\ - std::stringstream oss_internal_rare; \ - oss_internal_rare << message; \ - (logger)->trace(oss_internal_rare.str());}} while(false) +#define LOG_SIMPLE(logger, message, priority, PRIORITY) do \ +{ \ + const bool is_clients_log = (CurrentThread::getGroup() != nullptr) && \ + (CurrentThread::getGroup()->client_logs_level >= (priority)); \ + if ((logger)->is((PRIORITY)) || is_clients_log) { \ + std::stringstream oss_internal_rare; \ + oss_internal_rare << message; \ + if (auto channel = (logger)->getChannel()) { \ + channel->log(Message((logger)->name(), oss_internal_rare.str(), (PRIORITY))); \ + } \ + } \ +} while (false) -#define LOG_DEBUG(logger, message) do { \ - if ((logger)->debug()) {\ - std::stringstream oss_internal_rare; \ - oss_internal_rare << message; \ - (logger)->debug(oss_internal_rare.str());}} while(false) -#define LOG_INFO(logger, message) do { \ - if ((logger)->information()) {\ - std::stringstream oss_internal_rare; \ - oss_internal_rare << message; \ - (logger)->information(oss_internal_rare.str());}} while(false) +#define LOG_TRACE(logger, message) LOG_SIMPLE(logger, message, LogsLevel::trace, Message::PRIO_TRACE) +#define LOG_DEBUG(logger, message) LOG_SIMPLE(logger, message, LogsLevel::debug, Message::PRIO_DEBUG) +#define LOG_INFO(logger, message) LOG_SIMPLE(logger, message, LogsLevel::information, Message::PRIO_INFORMATION) +#define LOG_WARNING(logger, message) LOG_SIMPLE(logger, message, LogsLevel::warning, Message::PRIO_WARNING) +#define LOG_ERROR(logger, message) LOG_SIMPLE(logger, message, LogsLevel::error, Message::PRIO_ERROR) -#define LOG_WARNING(logger, message) do { \ - if ((logger)->warning()) {\ - std::stringstream oss_internal_rare; \ - oss_internal_rare << message; \ - (logger)->warning(oss_internal_rare.str());}} while(false) - -#define LOG_ERROR(logger, message) do { \ - if ((logger)->error()) {\ - std::stringstream oss_internal_rare; \ - oss_internal_rare << message; \ - (logger)->error(oss_internal_rare.str());}} while(false) diff --git a/libs/libcommon/include/common/memory.h b/libs/libcommon/include/common/memory.h new file mode 100644 index 00000000000..d8dced79cfb --- /dev/null +++ b/libs/libcommon/include/common/memory.h @@ -0,0 +1,65 @@ +#pragma once + +#include +#include + +#if __has_include() +#include +#endif + +#if USE_JEMALLOC +#include + +#if JEMALLOC_VERSION_MAJOR < 4 + #undef USE_JEMALLOC + #define USE_JEMALLOC 0 + #include +#endif +#endif + +#define ALWAYS_INLINE inline __attribute__((__always_inline__)) +#define NO_INLINE __attribute__((__noinline__)) + +namespace Memory +{ + +ALWAYS_INLINE void * newImpl(std::size_t size) +{ + auto * ptr = malloc(size); + if (likely(ptr != nullptr)) + return ptr; + + /// @note no std::get_new_handler logic implemented + throw std::bad_alloc{}; +} + +ALWAYS_INLINE void * newNoExept(std::size_t size) noexcept +{ + return malloc(size); +} + +ALWAYS_INLINE void deleteImpl(void * ptr) noexcept +{ + free(ptr); +} + +#if USE_JEMALLOC + +ALWAYS_INLINE void deleteSized(void * ptr, std::size_t size) noexcept +{ + if (unlikely(ptr == nullptr)) + return; + + sdallocx(ptr, size, 0); +} + +#else + +ALWAYS_INLINE void deleteSized(void * ptr, std::size_t size [[maybe_unused]]) noexcept +{ + free(ptr); +} + +#endif + +} diff --git a/libs/libcommon/src/DateLUT.cpp b/libs/libcommon/src/DateLUT.cpp index c7dcf5689ff..ce3e7e32a26 100644 --- a/libs/libcommon/src/DateLUT.cpp +++ b/libs/libcommon/src/DateLUT.cpp @@ -64,7 +64,12 @@ std::string determineDefaultTimeZone() /// /usr/share/zoneinfo//UTC -> UCT /// But the preferred time zone name is pointed by the first link (UTC), and the second link is just an internal detail. if (fs::is_symlink(tz_file_path)) + { tz_file_path = fs::read_symlink(tz_file_path); + /// If it's relative - make it absolute. + if (tz_file_path.is_relative()) + tz_file_path = (fs::path("/etc/") / tz_file_path).lexically_normal(); + } } try diff --git a/libs/libcommon/src/demangle.cpp b/libs/libcommon/src/demangle.cpp index eab8a55abe0..e65ead7524b 100644 --- a/libs/libcommon/src/demangle.cpp +++ b/libs/libcommon/src/demangle.cpp @@ -1,6 +1,14 @@ #include -#if _MSC_VER +#if defined(__has_feature) + #if __has_feature(memory_sanitizer) + #define MEMORY_SANITIZER 1 + #endif +#elif defined(__MEMORY_SANITIZER__) + #define MEMORY_SANITIZER 1 +#endif + +#if _MSC_VER || MEMORY_SANITIZER std::string demangle(const char * name, int & status) { diff --git a/libs/libloggers/loggers/ExtendedLogChannel.cpp b/libs/libloggers/loggers/ExtendedLogChannel.cpp index 857765a94de..8040a094a15 100644 --- a/libs/libloggers/loggers/ExtendedLogChannel.cpp +++ b/libs/libloggers/loggers/ExtendedLogChannel.cpp @@ -25,7 +25,11 @@ ExtendedLogMessage ExtendedLogMessage::getFrom(const Poco::Message & base) msg_ext.time_microseconds = static_cast(tv.tv_usec); if (current_thread) - msg_ext.query_id = CurrentThread::getQueryId(); + { + auto query_id_ref = CurrentThread::getQueryId(); + if (query_id_ref.size) + msg_ext.query_id.assign(query_id_ref.data, query_id_ref.size); + } msg_ext.thread_number = getThreadNumber(); diff --git a/libs/libmysqlxx/CMakeLists.txt b/libs/libmysqlxx/CMakeLists.txt index 166c10e69f7..263a031d7b0 100644 --- a/libs/libmysqlxx/CMakeLists.txt +++ b/libs/libmysqlxx/CMakeLists.txt @@ -12,7 +12,6 @@ add_library (mysqlxx include/mysqlxx/Connection.h include/mysqlxx/Exception.h - include/mysqlxx/Manip.h include/mysqlxx/mysqlxx.h include/mysqlxx/Null.h include/mysqlxx/Pool.h diff --git a/libs/libmysqlxx/include/mysqlxx/Manip.h b/libs/libmysqlxx/include/mysqlxx/Manip.h deleted file mode 100644 index 4b777b68f21..00000000000 --- a/libs/libmysqlxx/include/mysqlxx/Manip.h +++ /dev/null @@ -1,483 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -#include -#include -#include - - -namespace mysqlxx -{ - -/** @brief Манипулятор ostream, который escape-ит строки для записи в tab delimited файл. - * Использование: tab_separated_ostr << mysqlxx::escape << x; - */ -enum escape_enum -{ - escape -}; - - -/** @brief Манипулятор ostream, который quote-ит строки для записи в MySQL запрос. - * Внимание! Не использует функции MySQL API, а использует свой метод quote-инга, - * который может быть некорректным при использовании некоторых кодировок - * (multi-byte attack), а также может оказаться некорректным при изменении libmysqlclient. - * Это сделано для увеличения производительности и это имеет значение. - * - * Использование: query << mysqlxx::quote << x; - */ -enum quote_enum -{ - quote -}; - - -struct EscapeManipResult -{ - std::ostream & ostr; - - EscapeManipResult(std::ostream & ostr_) : ostr(ostr_) {} - - std::ostream & operator<< (bool value) { return ostr << static_cast(value); } - std::ostream & operator<< (char value) { return ostr << static_cast(value); } - std::ostream & operator<< (unsigned char value) { return ostr << static_cast(value); } - std::ostream & operator<< (signed char value) { return ostr << static_cast(value); } - - template - typename std::enable_if::value, std::ostream &>::type - operator<< (T value) { return ostr << value; } - - std::ostream & operator<< (const LocalDate & value) { return ostr << value; } - std::ostream & operator<< (const LocalDateTime & value) { return ostr << value; } - - std::ostream & operator<< (const std::string & value) - { - writeEscapedData(value.data(), value.length()); - return ostr; - } - - - std::ostream & operator<< (const char * value) - { - while (const char * it = std::strpbrk(value, "\t\n\\")) - { - ostr.write(value, it - value); - switch (*it) - { - case '\t': - ostr.write("\\t", 2); - break; - case '\n': - ostr.write("\\n", 2); - break; - case '\\': - ostr.write("\\\\", 2); - break; - default: - ; - } - value = it + 1; - } - return ostr << value; - } - - - std::ostream & operator<< (const Value & string) - { - writeEscapedData(string.data(), string.size()); - return ostr; - } - - - std::ostream & operator<< (const Row & row) - { - for (size_t i = 0; i < row.size(); ++i) - { - if (i != 0) - ostr << '\t'; - - if (row[i].isNull()) - { - ostr << "\\N"; - continue; - } - - (*this) << row[i]; - } - - return ostr; - } - - - template - std::ostream & operator<< (const Null & value) - { - if(value.is_null) - ostr << "\\N"; - else - *this << value.data; - - return ostr ; - } - - - template - std::ostream & operator<< (const std::vector & value) - { - throw Poco::Exception(std::string(__PRETTY_FUNCTION__) + " is not implemented"); - } - -private: - - void writeEscapedData(const char * data, size_t length) - { - size_t i = 0; - - while (true) - { - size_t remaining_length = std::strlen(data); - (*this) << data; - if (i + remaining_length == length) - break; - - ostr.write("\\0", 2); - i += remaining_length + 1; - data += remaining_length + 1; - } - } -}; - -inline EscapeManipResult operator<< (std::ostream & ostr, escape_enum manip) -{ - return EscapeManipResult(ostr); -} - - -struct QuoteManipResult -{ -public: - std::ostream & ostr; - - QuoteManipResult(std::ostream & ostr_) : ostr(ostr_) {} - - std::ostream & operator<< (bool value) { return ostr << static_cast(value); } - std::ostream & operator<< (char value) { return ostr << static_cast(value); } - std::ostream & operator<< (unsigned char value) { return ostr << static_cast(value); } - std::ostream & operator<< (signed char value) { return ostr << static_cast(value); } - - template - typename std::enable_if::value, std::ostream &>::type - operator<< (T value) { return ostr << value; } - - std::ostream & operator<< (const LocalDate & value) { return ostr << '\'' << value << '\''; } - std::ostream & operator<< (const LocalDateTime & value) { return ostr << '\'' << value << '\''; } - - std::ostream & operator<< (const std::string & value) - { - ostr.put('\''); - writeEscapedData(value.data(), value.length()); - ostr.put('\''); - - return ostr; - } - - - std::ostream & operator<< (const char * value) - { - ostr.put('\''); - writeEscapedCString(value); - ostr.put('\''); - return ostr; - } - - template - std::ostream & operator<< (const Null & value) - { - if(value.is_null) - { - ostr << "\\N"; - } - else - { - *this << value.data; - } - return ostr ; - } - - template - std::ostream & operator<< (const std::vector & value) - { - throw Poco::Exception(std::string(__PRETTY_FUNCTION__) + " is not implemented"); - } - -private: - - void writeEscapedCString(const char * value) - { - while (const char * it = std::strpbrk(value, "'\\\"")) - { - ostr.write(value, it - value); - switch (*it) - { - case '"': - ostr.write("\\\"", 2); - break; - case '\'': - ostr.write("\\'", 2); - break; - case '\\': - ostr.write("\\\\", 2); - break; - default: - ; - } - value = it + 1; - } - ostr << value; - } - - - void writeEscapedData(const char * data, size_t length) - { - size_t i = 0; - - while (true) - { - size_t remaining_length = std::strlen(data); - writeEscapedCString(data); - if (i + remaining_length == length) - break; - - ostr.write("\\0", 2); - i += remaining_length + 1; - data += remaining_length + 1; - } - } -}; - -inline QuoteManipResult operator<< (std::ostream & ostr, quote_enum manip) -{ - return QuoteManipResult(ostr); -} - - -/** Манипулятор istream, позволяет считывать значения из tab delimited файла. - */ -enum unescape_enum -{ - unescape -}; - - -/** Манипулятор istream, который позволяет читать значения в кавычках или без. - */ -enum unquote_enum -{ - unquote -}; - - -inline void parseEscapeSequence(std::istream & istr, std::string & value) -{ - char c = istr.get(); - if (!istr.good()) - throw Poco::Exception("Cannot parse string: unexpected end of input."); - - switch(c) - { - case 'b': - value.push_back('\b'); - break; - case 'f': - value.push_back('\f'); - break; - case 'n': - value.push_back('\n'); - break; - case 'r': - value.push_back('\r'); - break; - case 't': - value.push_back('\t'); - break; - default: - value.push_back(c); - break; - } -} - - -struct UnEscapeManipResult -{ - std::istream & istr; - - UnEscapeManipResult(std::istream & istr_) : istr(istr_) {} - - std::istream & operator>> (bool & value) { int tmp = 0; istr >> tmp; value = tmp; return istr; } - std::istream & operator>> (char & value) { int tmp = 0; istr >> tmp; value = tmp; return istr; } - std::istream & operator>> (unsigned char & value) { int tmp = 0; istr >> tmp; value = tmp; return istr; } - std::istream & operator>> (signed char & value) { int tmp = 0; istr >> tmp; value = tmp; return istr; } - - template - typename std::enable_if::value, std::istream &>::type - operator>> (T & value) { return istr >> value; } - - std::istream & operator>> (std::string & value) - { - value.clear(); - - char c; - while (1) - { - istr.get(c); - if (!istr.good()) - break; - - switch (c) - { - case '\\': - parseEscapeSequence(istr, value); - break; - - case '\t': - istr.unget(); - return istr; - break; - - case '\n': - istr.unget(); - return istr; - break; - - default: - value.push_back(c); - break; - } - } - return istr; - } - - /// Чтение NULL-able типа. - template - std::istream & operator>> (Null & value) - { - char c; - istr.get(c); - if (c == '\\' && istr.peek() == 'N') - { - value.is_null = true; - istr.ignore(); - } - else - { - istr.unget(); - value.is_null = false; - *this >> value.data; - } - return istr; - } - - std::istream & operator>> (LocalDate & value) - { - std::string s; - (*this) >> s; - value = LocalDate(s); - return istr; - } - - std::istream & operator>> (LocalDateTime & value) - { - std::string s; - (*this) >> s; - value = LocalDateTime(s); - return istr; - } - - template - std::istream & operator>> (std::vector & value) - { - throw Poco::Exception(std::string(__PRETTY_FUNCTION__) + " is not implemented"); - } -}; - -inline UnEscapeManipResult operator>> (std::istream & istr, unescape_enum manip) -{ - return UnEscapeManipResult(istr); -} - - -struct UnQuoteManipResult -{ -public: - std::istream & istr; - - UnQuoteManipResult(std::istream & istr_) : istr(istr_) {} - - std::istream & operator>> (bool & value) { int tmp = 0; istr >> tmp; value = tmp; return istr; } - std::istream & operator>> (char & value) { int tmp = 0; istr >> tmp; value = tmp; return istr; } - std::istream & operator>> (unsigned char & value) { int tmp = 0; istr >> tmp; value = tmp; return istr; } - std::istream & operator>> (signed char & value) { int tmp = 0; istr >> tmp; value = tmp; return istr; } - - template - typename std::enable_if::value, std::istream &>::type - operator>> (T & value) { return istr >> value; } - - std::istream & operator>> (std::string & value) - { - value.clear(); - readQuote(); - - char c; - while (1) - { - istr.get(c); - if (!istr.good()) - break; - - switch (c) - { - case '\\': - parseEscapeSequence(istr, value); - break; - - case '\'': - return istr; - break; - - default: - value.push_back(c); - break; - } - } - throw Poco::Exception("Cannot parse string: unexpected end of input."); - } - - template - std::istream & operator>> (std::vector & value) - { - throw Poco::Exception(std::string(__PRETTY_FUNCTION__) + " is not implemented"); - } - -private: - - void readQuote() - { - char c = istr.get(); - if (!istr.good()) - throw Poco::Exception("Cannot parse string: unexpected end of input."); - if (c != '\'') - throw Poco::Exception("Cannot parse string: missing opening single quote."); - } -}; - -inline UnQuoteManipResult operator>> (std::istream & istr, unquote_enum manip) -{ - return UnQuoteManipResult(istr); -} - - -} diff --git a/libs/libmysqlxx/include/mysqlxx/mysqlxx.h b/libs/libmysqlxx/include/mysqlxx/mysqlxx.h index f9da8ec5c04..179d550519e 100644 --- a/libs/libmysqlxx/include/mysqlxx/mysqlxx.h +++ b/libs/libmysqlxx/include/mysqlxx/mysqlxx.h @@ -2,7 +2,6 @@ #include #include -#include #include #include #include diff --git a/libs/libmysqlxx/src/tests/mysqlxx_test.cpp b/libs/libmysqlxx/src/tests/mysqlxx_test.cpp index d8c15eecaba..d99900b1a39 100644 --- a/libs/libmysqlxx/src/tests/mysqlxx_test.cpp +++ b/libs/libmysqlxx/src/tests/mysqlxx_test.cpp @@ -2,7 +2,7 @@ #include -int main(int argc, char ** argv) +int main(int, char **) { try { @@ -25,10 +25,10 @@ int main(int argc, char ** argv) << ", " << row[1].getDate() << ", " << row[1].getDateTime() << std::endl - << mysqlxx::escape << row[1].getDate() << ", " << mysqlxx::escape << row[1].getDateTime() << std::endl - << mysqlxx::quote << row[1].getDate() << ", " << mysqlxx::quote << row[1].getDateTime() << std::endl - << mysqlxx::escape << row[1].getDate() << ", " << mysqlxx::escape << row[1].getDateTime() << std::endl - << mysqlxx::quote << row[1].getDate() << ", " << mysqlxx::quote << row[1].getDateTime() << std::endl + << row[1].getDate() << ", " << row[1].getDateTime() << std::endl + << row[1].getDate() << ", " << row[1].getDateTime() << std::endl + << row[1].getDate() << ", " << row[1].getDateTime() << std::endl + << row[1].getDate() << ", " << row[1].getDateTime() << std::endl ; time_t t1 = row[0]; @@ -51,7 +51,7 @@ int main(int argc, char ** argv) mysqlxx::UseQueryResult result = connection.query("SELECT 'abc\\\\def' x").use(); mysqlxx::Row row = result.fetch(); std::cerr << row << std::endl; - std::cerr << mysqlxx::escape << row << std::endl; + std::cerr << row << std::endl; } { @@ -71,7 +71,7 @@ int main(int argc, char ** argv) for (Queries::iterator it = queries.begin(); it != queries.end(); ++it) { std::cerr << it->str() << std::endl; - std::cerr << mysqlxx::escape << it->store().at(0) << std::endl; + std::cerr << it->store().at(0) << std::endl; } } @@ -95,7 +95,7 @@ int main(int argc, char ** argv) for (Queries::iterator it = queries.begin(); it != queries.end(); ++it) { std::cerr << it->str() << std::endl; - std::cerr << mysqlxx::escape << it->store().at(0) << std::endl; + std::cerr << it->store().at(0) << std::endl; } } diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 3b523822451..b3df25d13e6 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -24,6 +24,7 @@ if (NOT DEFINED ENABLE_UTILS OR ENABLE_UTILS) add_subdirectory (zookeeper-copy-tree) add_subdirectory (zookeeper-remove-by-list) add_subdirectory (zookeeper-create-entry-to-download-part) + add_subdirectory (zookeeper-adjust-block-numbers-to-parts) add_subdirectory (wikistat-loader) add_subdirectory (fill-factor) add_subdirectory (check-marks) diff --git a/utils/check-marks/CMakeLists.txt b/utils/check-marks/CMakeLists.txt index 86cff8fb233..bfb200b8d28 100644 --- a/utils/check-marks/CMakeLists.txt +++ b/utils/check-marks/CMakeLists.txt @@ -1,2 +1,2 @@ add_executable (check-marks main.cpp) -target_link_libraries(check-marks PRIVATE clickhouse_compression clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY}) +target_link_libraries(check-marks PRIVATE dbms ${Boost_PROGRAM_OPTIONS_LIBRARY}) diff --git a/utils/compressor/CMakeLists.txt b/utils/compressor/CMakeLists.txt index e4f99c4b73a..3fdf8aa5eaf 100644 --- a/utils/compressor/CMakeLists.txt +++ b/utils/compressor/CMakeLists.txt @@ -10,7 +10,7 @@ add_executable (mutator mutator.cpp) target_link_libraries(mutator PRIVATE clickhouse_common_io) add_executable (decompress_perf decompress_perf.cpp) -target_link_libraries(decompress_perf PRIVATE clickhouse_common_io clickhouse_compression ${LZ4_LIBRARY}) +target_link_libraries(decompress_perf PRIVATE dbms ${LZ4_LIBRARY}) if (NOT USE_INTERNAL_ZSTD_LIBRARY AND ZSTD_INCLUDE_DIR) target_include_directories (zstd_test BEFORE PRIVATE ${ZSTD_INCLUDE_DIR}) diff --git a/utils/release/push_packages b/utils/release/push_packages index a32ceb0777f..d432136a5bd 100755 --- a/utils/release/push_packages +++ b/utils/release/push_packages @@ -6,6 +6,7 @@ import os import logging import shutil import base64 +import pexpect # Do nothing if keys are not provided @@ -116,6 +117,20 @@ def debsign(path, gpg_passphrase, gpg_sec_key_path, gpg_pub_key_path, gpg_user): logging.error("Cannot debsign packages on path %s, with user key", path) raise ex +def rpmsign(path, gpg_passphrase, gpg_sec_key_path, gpg_pub_key_path, gpg_user): + try: + with GpgKey(gpg_sec_key_path, gpg_pub_key_path): + for package in os.listdir(path): + package_path = os.path.join(path, package) + logging.info("Signing %s", package_path) + proc = pexpect.spawn('rpm --resign -D "_signature gpg" -D "_gpg_name {username}" {package}'.format(username=gpg_user, package=package_path)) + proc.expect_exact("Enter pass phrase: ") + proc.sendline(gpg_passphrase) + proc.expect(pexpect.EOF) + logging.info("Signed successfully") + except Exception as ex: + logging.error("Cannot rpmsign packages on path %s, with user key", path) + raise ex def transfer_packages_scp(ssh_key, path, repo_user, repo_url, incoming_directory): logging.info("Transferring packages via scp to %s", repo_url) @@ -212,6 +227,7 @@ if __name__ == "__main__": if args.rpm_directory: if not is_open_source: raise Exception("Cannot upload .rpm package to {}".format(args.repo_url)) + rpmsign(args.rpm_directory, args.gpg_passphrase, args.gpg_sec_key_path, args.gpg_pub_key_path, args.gpg_key_user) packages.append((args.rpm_directory, 'rpm')) if args.tgz_directory: diff --git a/utils/release/release_lib.sh b/utils/release/release_lib.sh index 75307dfe0b0..4eaa4d4ebbd 100644 --- a/utils/release/release_lib.sh +++ b/utils/release/release_lib.sh @@ -226,12 +226,22 @@ function make_rpm { PACKAGE=clickhouse-server ARCH=all TARGET=noarch - unpack_pack + deb_unpack + mv ${PACKAGE}-$VERSION_FULL-2.spec ${PACKAGE}-$VERSION_FULL-2.spec_tmp + echo "Requires: clickhouse-common-static = $VERSION_FULL-2" >> ${PACKAGE}-$VERSION_FULL-2.spec + echo "Requires: tzdata" >> ${PACKAGE}-$VERSION_FULL-2.spec + echo "Requires: initscripts" >> ${PACKAGE}-$VERSION_FULL-2.spec + cat ${PACKAGE}-$VERSION_FULL-2.spec_tmp >> ${PACKAGE}-$VERSION_FULL-2.spec + rpm_pack PACKAGE=clickhouse-client ARCH=all TARGET=noarch - unpack_pack + deb_unpack + mv ${PACKAGE}-$VERSION_FULL-2.spec ${PACKAGE}-$VERSION_FULL-2.spec_tmp + echo "Requires: clickhouse-common-static = $VERSION_FULL-2" >> ${PACKAGE}-$VERSION_FULL-2.spec + cat ${PACKAGE}-$VERSION_FULL-2.spec_tmp >> ${PACKAGE}-$VERSION_FULL-2.spec + rpm_pack PACKAGE=clickhouse-test ARCH=all diff --git a/utils/zookeeper-adjust-block-numbers-to-parts/main.cpp b/utils/zookeeper-adjust-block-numbers-to-parts/main.cpp index dda1677f3a4..3e449043adc 100644 --- a/utils/zookeeper-adjust-block-numbers-to-parts/main.cpp +++ b/utils/zookeeper-adjust-block-numbers-to-parts/main.cpp @@ -1,20 +1,62 @@ #include #include #include +#include #include #include #include #include -size_t getMaxBlockSizeForPartition(zkutil::ZooKeeper & zk, + +std::vector getAllShards(zkutil::ZooKeeper & zk, const std::string & root) +{ + return zk.getChildren(root); +} + + +std::vector removeNotExistingShards(zkutil::ZooKeeper & zk, const std::string & root, const std::vector & shards) +{ + auto existing_shards = getAllShards(zk, root); + std::vector filtered_shards; + filtered_shards.reserve(shards.size()); + for (const auto & shard : shards) + if (std::find(existing_shards.begin(), existing_shards.end(), shard) == existing_shards.end()) + std::cerr << "Shard " << shard << " not found." << std::endl; + else + filtered_shards.emplace_back(shard); + return filtered_shards; +} + + +std::vector getAllTables(zkutil::ZooKeeper & zk, const std::string & root, const std::string & shard) +{ + return zk.getChildren(root + "/" + shard); +} + + +std::vector removeNotExistingTables(zkutil::ZooKeeper & zk, const std::string & root, const std::string & shard, const std::vector & tables) +{ + auto existing_tables = getAllTables(zk, root, shard); + std::vector filtered_tables; + filtered_tables.reserve(tables.size()); + for (const auto & table : tables) + if (std::find(existing_tables.begin(), existing_tables.end(), table) == existing_tables.end()) + std::cerr << "\tTable " << table << " not found on shard " << shard << "." << std::endl; + else + filtered_tables.emplace_back(table); + return filtered_tables; +} + + +Int64 getMaxBlockNumberForPartition(zkutil::ZooKeeper & zk, const std::string & replica_path, const std::string & partition_name, const DB::MergeTreeDataFormatVersion & format_version) { auto replicas_path = replica_path + "/replicas"; auto replica_hosts = zk.getChildren(replicas_path); - size_t max_block_num = 0; + Int64 max_block_num = 0; for (const auto & replica_host : replica_hosts) { auto parts = zk.getChildren(replicas_path + "/" + replica_host + "/parts"); @@ -24,40 +66,78 @@ size_t getMaxBlockSizeForPartition(zkutil::ZooKeeper & zk, { auto info = DB::MergeTreePartInfo::fromPartName(part, format_version); if (info.partition_id == partition_name) - max_block_num = std::max(info.max_block, max_block_num); + max_block_num = std::max(info.max_block, max_block_num); } catch (const DB::Exception & ex) { - std::cerr << "Exception on: " << ex.displayText() << " will skip part: " << part << std::endl; + std::cerr << ex.displayText() << ", Part " << part << "skipped." << std::endl; } } } return max_block_num; } -std::unordered_map getAllTablesBlockPaths(zkutil::ZooKeeper & zk, const std::string & root) + +Int64 getCurrentBlockNumberForPartition(zkutil::ZooKeeper & zk, const std::string & part_path) { - std::unordered_map result; - auto shards = zk.getChildren(root); - for (const auto & shard : shards) + Coordination::Stat stat; + zk.get(part_path, &stat); + + /// References: + /// https://stackoverflow.com/a/10347910 + /// https://bowenli86.github.io/2016/07/07/distributed%20system/zookeeper/How-does-ZooKeeper-s-persistent-sequential-id-work/ + return (stat.cversion + stat.numChildren) / 2; +} + + +std::unordered_map getPartitionsNeedAdjustingBlockNumbers( + zkutil::ZooKeeper & zk, const std::string & root, const std::vector & shards, const std::vector & tables) +{ + std::unordered_map result; + + std::vector use_shards = shards.empty() ? getAllShards(zk, root) : removeNotExistingShards(zk, root, shards); + + for (const auto & shard : use_shards) { - std::string shard_path = root + "/" + shard; - auto tables = zk.getChildren(shard_path); - for (auto table : tables) + std::cout << "Shard: " << shard << std::endl; + std::vector use_tables = tables.empty() ? getAllTables(zk, root, shard) : removeNotExistingTables(zk, root, shard, tables); + + for (auto table : use_tables) { - std::cerr << "Searching for nodes in: " << table << std::endl; - std::string table_path = shard_path + "/" + table; - auto format_version = DB::ReplicatedMergeTreeTableMetadata::parse(zk.get(table_path + "/metadata")).data_format_version; + std::cout << "\tTable: " << table << std::endl; + std::string table_path = root + "/" + shard + "/" + table; std::string blocks_path = table_path + "/block_numbers"; - auto partitions = zk.getChildren(blocks_path); - if (!partitions.empty()) + + std::vector partitions; + DB::MergeTreeDataFormatVersion format_version; + try { - for (auto partition : partitions) + format_version = DB::ReplicatedMergeTreeTableMetadata::parse(zk.get(table_path + "/metadata")).data_format_version; + partitions = zk.getChildren(blocks_path); + } + catch (const DB::Exception & ex) + { + std::cerr << ex.displayText() << ", table " << table << " skipped." << std::endl; + continue; + } + + for (auto partition : partitions) + { + try { std::string part_path = blocks_path + "/" + partition; - size_t partition_max_block = getMaxBlockSizeForPartition(zk, table_path, partition, format_version); - std::cerr << "\tFound max block number: " << partition_max_block << " for part: " << partition << std::endl; - result.emplace(part_path, partition_max_block); + Int64 partition_max_block = getMaxBlockNumberForPartition(zk, table_path, partition, format_version); + Int64 current_block_number = getCurrentBlockNumberForPartition(zk, part_path); + if (current_block_number < partition_max_block + 1) + { + std::cout << "\t\tPartition: " << partition << ": current block_number: " << current_block_number + << ", max block number: " << partition_max_block << ". Adjusting is required." << std::endl; + result.emplace(part_path, partition_max_block); + } + } + catch (const DB::Exception & ex) + { + std::cerr << ex.displayText() << ", partition " << partition << " skipped." << std::endl; } } } @@ -66,67 +146,137 @@ std::unordered_map getAllTablesBlockPaths(zkutil::ZooKeeper } -void rotateNodes(zkutil::ZooKeeper & zk, const std::string & path, size_t max_block_num) +void setCurrentBlockNumber(zkutil::ZooKeeper & zk, const std::string & path, Int64 new_current_block_number) { - Coordination::Requests requests; - std::string block_prefix = path + "/block-"; - std::string current = zk.create(block_prefix, "", zkutil::CreateMode::EphemeralSequential); - size_t current_block_num = DB::parse(current.c_str() + block_prefix.size(), current.size() - block_prefix.size()); - if (current_block_num >= max_block_num) - { - std::cerr << "Nothing to rotate, current block num: " << current_block_num << " max_block_num:" << max_block_num << std::endl; - return; - } + Int64 current_block_number = getCurrentBlockNumberForPartition(zk, path); - size_t need_to_rotate = max_block_num - current_block_num; - std::cerr << "Will rotate: " << need_to_rotate << " block numbers from " << current_block_num << " to " << max_block_num << std::endl; - - for (size_t i = 0; i < need_to_rotate; ++i) + auto create_ephemeral_nodes = [&](size_t count) { - if (requests.size() == 50) + std::string block_prefix = path + "/block-"; + Coordination::Requests requests; + requests.reserve(count); + for (size_t i = 0; i != count; ++i) + requests.emplace_back(zkutil::makeCreateRequest(block_prefix, "", zkutil::CreateMode::EphemeralSequential)); + auto responses = zk.multi(requests); + + std::vector paths_created; + paths_created.reserve(responses.size()); + for (const auto & response : responses) { - std::cerr << "Rotating: " << i << " block numbers" << std::endl; - zk.multi(requests); - requests.clear(); + const auto * create_response = dynamic_cast(response.get()); + if (!create_response) + { + std::cerr << "\tCould not create ephemeral node " << block_prefix << std::endl; + return false; + } + paths_created.emplace_back(create_response->path_created); } - requests.emplace_back(zkutil::makeCreateRequest(path + "/block-", "", zkutil::CreateMode::EphemeralSequential)); - } - if (!requests.empty()) - { - zk.multi(requests); - } + + std::sort(paths_created.begin(), paths_created.end()); + for (const auto & path_created : paths_created) + { + Int64 number = DB::parse(path_created.c_str() + block_prefix.size(), path_created.size() - block_prefix.size()); + if (number != current_block_number) + { + char suffix[11] = ""; + sprintf(suffix, "%010ld", current_block_number); + std::string expected_path = block_prefix + suffix; + std::cerr << "\t" << path_created << ": Ephemeral node has been created with an unexpected path (expected something like " + << expected_path << ")." << std::endl; + return false; + } + std::cout << "\t" << path_created << std::endl; + ++current_block_number; + } + + return true; + }; + + if (current_block_number >= new_current_block_number) + return; + + std::cout << "Creating ephemeral sequential nodes:" << std::endl; + create_ephemeral_nodes(1); /// Firstly try to create just a single node. + + /// Create other nodes in batches of 50 nodes. + while (current_block_number + 50 <= new_current_block_number) + create_ephemeral_nodes(50); + + create_ephemeral_nodes(new_current_block_number - current_block_number); } + int main(int argc, char ** argv) try { - boost::program_options::options_description desc("Allowed options"); + /// Parse the command line. + namespace po = boost::program_options; + po::options_description desc("Allowed options"); desc.add_options() - ("help,h", "produce help message") - ("address,a", boost::program_options::value()->required(), "addresses of ZooKeeper instances, comma separated. Example: example01e.yandex.ru:2181") - ("path,p", boost::program_options::value()->required(), "path of replica queue to insert node (without trailing slash)"); + ("help,h", "show help") + ("zookeeper,z", po::value(), "Addresses of ZooKeeper instances, comma-separated. Example: example01e.yandex.ru:2181") + ("path,p", po::value(), "[optional] Path of replica queue to insert node (without trailing slash). By default it's /clickhouse/tables") + ("shard,s", po::value(), "[optional] Shards to process, comma-separated. If not specified then the utility will process all the shards.") + ("table,t", po::value(), "[optional] Tables to process, comma-separated. If not specified then the utility will process all the tables.") + ("dry-run", "[optional] Specify if you want this utility just to analyze block numbers without any changes."); - boost::program_options::variables_map options; - boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), options); + po::variables_map options; + po::store(po::parse_command_line(argc, argv, desc), options); - if (options.count("help")) + auto show_usage = [&] { - std::cout << "Util for /block_numbers node adjust with max block number in partition" << std::endl; - std::cout << "Usage: " << argv[0] << " [options]" << std::endl; + std::cout << "Usage: " << std::endl; + std::cout << " " << argv[0] << " [options]" << std::endl; std::cout << desc << std::endl; + }; + + if (options.count("help") || (argc == 1)) + { + std::cout << "This utility adjusts the /block_numbers zookeeper nodes to the correct block number in partition." << std::endl; + std::cout << "It might be useful when incorrect block numbers stored in zookeeper don't allow you to insert data into a table or drop/detach a partition." << std::endl; + show_usage(); + return 0; + } + + if (!options.count("zookeeper")) + { + std::cerr << "Option --zookeeper should be set." << std::endl; + show_usage(); return 1; } - std::string global_path = options.at("path").as(); + std::string root = options.count("path") ? options.at("path").as() : "/clickhouse/tables"; - zkutil::ZooKeeper zookeeper(options.at("address").as()); + std::vector shards, tables; + if (options.count("shard")) + boost::split(shards, options.at("shard").as(), boost::algorithm::is_any_of(",")); + if (options.count("table")) + boost::split(tables, options.at("table").as(), boost::algorithm::is_any_of(",")); - auto all_path = getAllTablesBlockPaths(zookeeper, global_path); - for (const auto & [path, max_block_num] : all_path) + /// Check if the adjusting of the block numbers is required. + std::cout << "Checking if adjusting of the block numbers is required:" << std::endl; + zkutil::ZooKeeper zookeeper(options.at("zookeeper").as()); + auto part_paths_with_max_block_numbers = getPartitionsNeedAdjustingBlockNumbers(zookeeper, root, shards, tables); + + if (part_paths_with_max_block_numbers.empty()) { - std::cerr << "Rotating on: " << path << std::endl; - rotateNodes(zookeeper, path, max_block_num); + std::cout << "No adjusting required." << std::endl; + return 0; } + + std::cout << "Required adjusting of " << part_paths_with_max_block_numbers.size() << " block numbers." << std::endl; + + /// Adjust the block numbers. + if (options.count("dry-run")) + { + std::cout << "This is a dry-run, exiting." << std::endl; + return 0; + } + + std::cout << std::endl << "Adjusting the block numbers:" << std::endl; + for (const auto & [part_path, max_block_number] : part_paths_with_max_block_numbers) + setCurrentBlockNumber(zookeeper, part_path, max_block_number + 1); + return 0; } catch (const Poco::Exception & e) diff --git a/website/deprecated/reference_en.html b/website/deprecated/reference_en.html index a789aeb238d..2e6fe0ac30b 100644 --- a/website/deprecated/reference_en.html +++ b/website/deprecated/reference_en.html @@ -7014,7 +7014,7 @@ Maximum nesting depth of a query syntactic tree. If exceeded, an exception is th ===max_ast_elements=== Maximum number of elements in a query syntactic tree. If exceeded, an exception is thrown. -In the same way as the previous setting, it is checked only after parsing the query. By default, 10,000. +In the same way as the previous setting, it is checked only after parsing the query. By default, 50,000. ===max_rows_in_set=== diff --git a/website/deprecated/reference_ru.html b/website/deprecated/reference_ru.html index e18966ae322..89c91d7d1c1 100644 --- a/website/deprecated/reference_ru.html +++ b/website/deprecated/reference_ru.html @@ -7260,7 +7260,7 @@ any (только для group_by_overflow_mode) - продолжить агре ===max_ast_elements=== Максимальное количество элементов синтаксического дерева запроса. Если превышено - кидается исключение. -Аналогично, проверяется уже после парсинга запроса. По умолчанию: 10 000. +Аналогично, проверяется уже после парсинга запроса. По умолчанию: 50 000. ===max_rows_in_set=== diff --git a/website/index.html b/website/index.html index e87223555fd..9b799638c26 100644 --- a/website/index.html +++ b/website/index.html @@ -94,7 +94,7 @@
- Upcoming Meetups: Minsk on July 11 and Shenzhen on October 20 + Upcoming Meetups: Saint Petersburg on July 27 and Shenzhen on October 20