diff --git a/.gitmodules b/.gitmodules index 847abf7d931..e5be5438cc7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -97,9 +97,9 @@ [submodule "contrib/rapidjson"] path = contrib/rapidjson url = https://github.com/Tencent/rapidjson -[submodule "contrib/mimalloc"] - path = contrib/mimalloc - url = https://github.com/ClickHouse-Extras/mimalloc [submodule "contrib/fastops"] path = contrib/fastops url = https://github.com/ClickHouse-Extras/fastops +[submodule "contrib/orc"] + path = contrib/orc + url = https://github.com/apache/orc diff --git a/CHANGELOG.md b/CHANGELOG.md index 607f650deeb..7d6714b6474 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +## ClickHouse release 19.13.3.26, 2019-08-22 + +### Bug Fix +* Fix `ALTER TABLE ... UPDATE` query for tables with `enable_mixed_granularity_parts=1`. [#6543](https://github.com/yandex/ClickHouse/pull/6543) ([alesapin](https://github.com/alesapin)) +* Fix NPE when using IN clause with a subquery with a tuple. [#6125](https://github.com/yandex/ClickHouse/issues/6125) [#6550](https://github.com/yandex/ClickHouse/pull/6550) ([tavplubix](https://github.com/tavplubix)) +* Fixed an issue that if a stale replica becomes alive, it may still have data parts that were removed by DROP PARTITION. [#6522](https://github.com/yandex/ClickHouse/issues/6522) [#6523](https://github.com/yandex/ClickHouse/pull/6523) ([tavplubix](https://github.com/tavplubix)) +* Fixed issue with parsing CSV [#6426](https://github.com/yandex/ClickHouse/issues/6426) [#6559](https://github.com/yandex/ClickHouse/pull/6559) ([tavplubix](https://github.com/tavplubix)) +* Fixed data race in system.parts table and ALTER query. This fixes [#6245](https://github.com/yandex/ClickHouse/issues/6245). [#6513](https://github.com/yandex/ClickHouse/pull/6513) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fixed wrong code in mutations that may lead to memory corruption. Fixed segfault with read of address `0x14c0` that may happed due to concurrent `DROP TABLE` and `SELECT` from `system.parts` or `system.parts_columns`. Fixed race condition in preparation of mutation queries. Fixed deadlock caused by `OPTIMIZE` of Replicated tables and concurrent modification operations like ALTERs. [#6514](https://github.com/yandex/ClickHouse/pull/6514) ([alexey-milovidov](https://github.com/alexey-milovidov)) + ## ClickHouse release 19.13.2.19, 2019-08-14 ### New Feature @@ -31,6 +41,16 @@ * Fix build with external `libcxx` [#6010](https://github.com/yandex/ClickHouse/pull/6010) ([Ivan](https://github.com/abyss7)) * Fix shared build with `rdkafka` library [#6101](https://github.com/yandex/ClickHouse/pull/6101) ([Ivan](https://github.com/abyss7)) +## ClickHouse release 19.11.8.46, 2019-08-22 + +### Bug Fix +* Fix `ALTER TABLE ... UPDATE` query for tables with `enable_mixed_granularity_parts=1`. [#6543](https://github.com/yandex/ClickHouse/pull/6543) ([alesapin](https://github.com/alesapin)) +* Fix NPE when using IN clause with a subquery with a tuple. [#6125](https://github.com/yandex/ClickHouse/issues/6125) [#6550](https://github.com/yandex/ClickHouse/pull/6550) ([tavplubix](https://github.com/tavplubix)) +* Fixed an issue that if a stale replica becomes alive, it may still have data parts that were removed by DROP PARTITION. [#6522](https://github.com/yandex/ClickHouse/issues/6522) [#6523](https://github.com/yandex/ClickHouse/pull/6523) ([tavplubix](https://github.com/tavplubix)) +* Fixed issue with parsing CSV [#6426](https://github.com/yandex/ClickHouse/issues/6426) [#6559](https://github.com/yandex/ClickHouse/pull/6559) ([tavplubix](https://github.com/tavplubix)) +* Fixed data race in system.parts table and ALTER query. This fixes [#6245](https://github.com/yandex/ClickHouse/issues/6245). [#6513](https://github.com/yandex/ClickHouse/pull/6513) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fixed wrong code in mutations that may lead to memory corruption. Fixed segfault with read of address `0x14c0` that may happed due to concurrent `DROP TABLE` and `SELECT` from `system.parts` or `system.parts_columns`. Fixed race condition in preparation of mutation queries. Fixed deadlock caused by `OPTIMIZE` of Replicated tables and concurrent modification operations like ALTERs. [#6514](https://github.com/yandex/ClickHouse/pull/6514) ([alexey-milovidov](https://github.com/alexey-milovidov)) + ## ClickHouse release 19.11.7.40, 2019-08-14 ### Bug fix diff --git a/CMakeLists.txt b/CMakeLists.txt index 8466fa5d33d..5330c8daeb5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,14 +1,22 @@ +foreach(policy + CMP0023 + CMP0048 # CMake 3.0 + CMP0074 # CMake 3.12 + CMP0077 + CMP0079 + ) + if(POLICY ${policy}) + cmake_policy(SET ${policy} NEW) + endif() +endforeach() + project(ClickHouse) cmake_minimum_required(VERSION 3.3) -foreach(policy - CMP0023 - CMP0074 # CMake 3.12 - ) - if(POLICY ${policy}) - cmake_policy(SET ${policy} NEW) - endif() -endforeach() +# Ignore export() since we don't use it, +# but it gets broken with a global targets via link_libraries() +macro (export) +endmacro () set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/") set(CMAKE_EXPORT_COMPILE_COMMANDS 1) # Write compile_commands.json @@ -128,12 +136,6 @@ if (CMAKE_SYSTEM_PROCESSOR MATCHES "amd64|x86_64") endif () endif () -if (GLIBC_COMPATIBILITY) - set (USE_INTERNAL_MEMCPY ON) -else () - message (WARNING "Option GLIBC_COMPATIBILITY must be turned on for production builds.") -endif () - string(REGEX MATCH "-?[0-9]+(.[0-9]+)?$" COMPILER_POSTFIX ${CMAKE_CXX_COMPILER}) find_program (LLD_PATH NAMES "lld${COMPILER_POSTFIX}" "lld") @@ -172,20 +174,15 @@ if (ARCH_NATIVE) set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=native") endif () -# Special options for better optimized code with clang -#if (COMPILER_CLANG) -# set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -Wno-unused-command-line-argument -mllvm -inline-threshold=10000") -#endif () - if (CMAKE_VERSION VERSION_LESS "3.8.0") if (NOT MSVC) - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++1z") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") endif () else () set (CMAKE_CXX_STANDARD 17) set (CMAKE_CXX_EXTENSIONS 0) # https://cmake.org/cmake/help/latest/prop_tgt/CXX_EXTENSIONS.html#prop_tgt:CXX_EXTENSIONS set (CMAKE_CXX_STANDARD_REQUIRED ON) - set (CXX_FLAGS_INTERNAL_COMPILER "-std=c++1z") + set (CXX_FLAGS_INTERNAL_COMPILER "-std=c++17") endif () if (COMPILER_GCC OR COMPILER_CLANG) @@ -207,17 +204,13 @@ endif() set (CMAKE_BUILD_COLOR_MAKEFILE ON) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS} ${PLATFORM_EXTRA_CXX_FLAG} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${CXX_WARNING_FLAGS}") -#set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${CMAKE_CXX_FLAGS_ADD}") set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_CXX_FLAGS_ADD}") set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_CXX_FLAGS_ADD}") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${CMAKE_C_FLAGS_ADD}") -#set (CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${CMAKE_C_FLAGS_ADD}") set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_C_FLAGS_ADD}") set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_C_FLAGS_ADD}") -# Uses MAKE_STATIC_LIBRARIES - option (UNBUNDLED "Try find all libraries in system. We recommend to avoid this mode for production builds, because we cannot guarantee exact versions and variants of libraries your system has installed. This mode exists for enthusiastic developers who search for trouble. Also it is useful for maintainers of OS packages." OFF) if (UNBUNDLED) @@ -225,150 +218,28 @@ if (UNBUNDLED) else () set(NOT_UNBUNDLED 1) endif () + # Using system libs can cause lot of warnings in includes. if (UNBUNDLED OR NOT (OS_LINUX OR APPLE) OR ARCH_32) option (NO_WERROR "Disable -Werror compiler option" ON) endif () - -set(THREADS_PREFER_PTHREAD_FLAG ON) -find_package (Threads) - -include (cmake/find_cxx.cmake) - -include (cmake/test_compiler.cmake) - -if (OS_LINUX AND COMPILER_CLANG AND USE_STATIC_LIBRARIES) - option (USE_LIBCXX "Use libc++ and libc++abi instead of libstdc++ (only make sense on Linux)" ${HAVE_LIBCXX}) - - if (USE_LIBCXX) - set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_LIBCPP_DEBUG=0") # More checks in debug build. - endif () +# Make this extra-checks for correct library dependencies. +if (NOT SANITIZE) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-undefined") + set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-undefined") endif () -if (USE_LIBCXX) - set (STATIC_STDLIB_FLAGS "") -else () - set (STATIC_STDLIB_FLAGS "-static-libgcc -static-libstdc++") -endif () +include(cmake/dbms_glob_sources.cmake) +include(cmake/default_libs.cmake) -if (MAKE_STATIC_LIBRARIES AND NOT APPLE AND NOT (COMPILER_CLANG AND OS_FREEBSD)) - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${STATIC_STDLIB_FLAGS}") - - # Along with executables, we also build example of shared library for "library dictionary source"; and it also should be self-contained. - set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${STATIC_STDLIB_FLAGS}") -endif () - -if (USE_STATIC_LIBRARIES AND HAVE_NO_PIE) - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FLAG_NO_PIE}") - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FLAG_NO_PIE}") -endif () - -# TODO: only make this extra-checks in CI builds, since a lot of contrib libs won't link - -# CI works around this problem by explicitly adding GLIBC_COMPATIBILITY flag. -if (NOT SANITIZE AND YANDEX_OFFICIAL_BUILD) - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-undefined") - set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-undefined") -endif () - -include (cmake/find_unwind.cmake) - -if (USE_INTERNAL_UNWIND_LIBRARY) - option (USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING "Use internal unwind library for exception handling" ${USE_STATIC_LIBRARIES}) -endif () - - -# Set standard, system and compiler libraries explicitly. -# This is intended for more control of what we are linking. +###################################### +### Add targets below this comment ### +###################################### string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC) set (CMAKE_POSTFIX_VARIABLE "CMAKE_${CMAKE_BUILD_TYPE_UC}_POSTFIX") -set (DEFAULT_LIBS "") -if (OS_LINUX AND NOT UNBUNDLED AND (GLIBC_COMPATIBILITY OR USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING OR USE_LIBCXX)) - # Note: this probably has no effect, but I'm not an expert in CMake. - set (CMAKE_C_IMPLICIT_LINK_LIBRARIES "") - set (CMAKE_CXX_IMPLICIT_LINK_LIBRARIES "") - - # Disable default linked libraries. - set (DEFAULT_LIBS "-nodefaultlibs") - - # We need builtins from Clang's RT even without libcxx - for ubsan+int128. See https://bugs.llvm.org/show_bug.cgi?id=16404 - set (BUILTINS_LIB_PATH "") - if (COMPILER_CLANG) - execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-${CMAKE_SYSTEM_PROCESSOR}.a OUTPUT_VARIABLE BUILTINS_LIB_PATH OUTPUT_STRIP_TRAILING_WHITESPACE) - else () - set (BUILTINS_LIB_PATH "-lgcc") - endif () - - string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC) - set (CMAKE_POSTFIX_VARIABLE "CMAKE_${CMAKE_BUILD_TYPE_UC}_POSTFIX") - - # Add C++ libraries. - # - # This consist of: - # - C++ standard library (like implementation of std::string); - # - C++ ABI implementation (functions for exceptions like __cxa_throw, RTTI, etc); - # - functions for internal implementation of exception handling (stack unwinding based on DWARF info; TODO replace with bundled libunwind); - # - compiler builtins (example: functions for implementation of __int128 operations); - # - # There are two variants of C++ library: libc++ (from LLVM compiler infrastructure) and libstdc++ (from GCC). - - if (USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING) - if (USE_STATIC_LIBRARIES) - set (EXCEPTION_HANDLING_LIBRARY "${ClickHouse_BINARY_DIR}/contrib/libunwind-cmake/libunwind_static${${CMAKE_POSTFIX_VARIABLE}}.a") - else () - set (EXCEPTION_HANDLING_LIBRARY "${ClickHouse_BINARY_DIR}/contrib/libunwind-cmake/libunwind_shared${${CMAKE_POSTFIX_VARIABLE}}.so") - endif () - else () - set (EXCEPTION_HANDLING_LIBRARY "-lgcc_eh") - endif () - - message (STATUS "Using exception handling library: ${EXCEPTION_HANDLING_LIBRARY}") - - if (USE_LIBCXX) - if (USE_INTERNAL_LIBCXX_LIBRARY) - set (LIBCXX_LIBS "${ClickHouse_BINARY_DIR}/contrib/libcxx-cmake/libcxx_static${${CMAKE_POSTFIX_VARIABLE}}.a ${ClickHouse_BINARY_DIR}/contrib/libcxxabi-cmake/libcxxabi_static${${CMAKE_POSTFIX_VARIABLE}}.a") - else () - set (LIBCXX_LIBS "-lc++ -lc++abi -lc++fs") - endif () - - set (DEFAULT_LIBS "${DEFAULT_LIBS} -Wl,-Bstatic ${LIBCXX_LIBS} ${EXCEPTION_HANDLING_LIBRARY} ${BUILTINS_LIB_PATH} -Wl,-Bdynamic") - else () - set (DEFAULT_LIBS "${DEFAULT_LIBS} -Wl,-Bstatic -lstdc++ -lstdc++fs ${EXCEPTION_HANDLING_LIBRARY} ${COVERAGE_OPTION} ${BUILTINS_LIB_PATH} -Wl,-Bdynamic") - endif () - - # Linking with GLIBC prevents portability of binaries to older systems. - # We overcome this behaviour by statically linking with our own implementation of all new symbols (that don't exist in older Libc or have infamous "symbol versioning"). - # The order of linking is important: 'glibc-compatibility' must be before libc but after all other libraries. - if (GLIBC_COMPATIBILITY) - message (STATUS "Some symbols from glibc will be replaced for compatibility") - - string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC) - set (CMAKE_POSTFIX_VARIABLE "CMAKE_${CMAKE_BUILD_TYPE_UC}_POSTFIX") - - # FIXME: glibc-compatibility may be non-static in some builds! - set (DEFAULT_LIBS "${DEFAULT_LIBS} ${ClickHouse_BINARY_DIR}/libs/libglibc-compatibility/libglibc-compatibility${${CMAKE_POSTFIX_VARIABLE}}.a") - endif () - - # Add Libc. GLIBC is actually a collection of interdependent libraries. - set (DEFAULT_LIBS "${DEFAULT_LIBS} -lrt -ldl -lpthread -lm -lc") - - # Note: we'd rather use Musl libc library, but it's little bit more difficult to use. - - message(STATUS "Default libraries: ${DEFAULT_LIBS}") -endif () - -if (NOT GLIBC_COMPATIBILITY) - set (M_LIBRARY m) -endif () - -if (DEFAULT_LIBS) - # Add default libs to all targets as the last dependency. - set(CMAKE_CXX_STANDARD_LIBRARIES ${DEFAULT_LIBS}) - set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS}) -endif () - if (NOT MAKE_STATIC_LIBRARIES) set(CMAKE_POSITION_INDEPENDENT_CODE ON) endif () @@ -421,20 +292,12 @@ if (UNBUNDLED) else () set(NOT_UNBUNDLED 1) endif () + # Using system libs can cause lot of warnings in includes. if (UNBUNDLED OR NOT (OS_LINUX OR APPLE) OR ARCH_32) option (NO_WERROR "Disable -Werror compiler option" ON) endif () -if (USE_LIBCXX) - set (HAVE_LIBCXX 1) - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") -endif() - -if (USE_LIBCXX AND USE_INTERNAL_LIBCXX_LIBRARY) - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -nostdinc++ -isystem ${LIBCXX_INCLUDE_DIR} -isystem ${LIBCXXABI_INCLUDE_DIR}") -endif () - message (STATUS "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE} ; USE_STATIC_LIBRARIES=${USE_STATIC_LIBRARIES} MAKE_STATIC_LIBRARIES=${MAKE_STATIC_LIBRARIES} SPLIT_SHARED=${SPLIT_SHARED_LIBRARIES} UNBUNDLED=${UNBUNDLED} CCACHE=${CCACHE_FOUND} ${CCACHE_VERSION}") include(GNUInstallDirs) @@ -477,10 +340,10 @@ include (cmake/find_consistent-hashing.cmake) include (cmake/find_base64.cmake) include (cmake/find_parquet.cmake) include (cmake/find_hyperscan.cmake) -include (cmake/find_mimalloc.cmake) include (cmake/find_simdjson.cmake) include (cmake/find_rapidjson.cmake) include (cmake/find_fastops.cmake) +include (cmake/find_orc.cmake) find_contrib_lib(cityhash) find_contrib_lib(farmhash) @@ -499,79 +362,11 @@ include (libs/libmysqlxx/cmake/find_mysqlclient.cmake) include (cmake/print_flags.cmake) +install (EXPORT global DESTINATION cmake) + add_subdirectory (contrib EXCLUDE_FROM_ALL) add_subdirectory (libs) add_subdirectory (utils) add_subdirectory (dbms) include (cmake/print_include_directories.cmake) - -if (GLIBC_COMPATIBILITY OR USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING) - # FIXME: actually glibc-compatibility should always be built first, - # because it's unconditionally linked via $DEFAULT_LIBS, - # and these looks like the first places that get linked. - function (add_default_dependencies target_name) - if (TARGET ${target_name}) - if (GLIBC_COMPATIBILITY) - add_dependencies(${target_name} glibc-compatibility) - endif () - - if (USE_LIBCXX AND USE_INTERNAL_LIBCXX_LIBRARY) - add_dependencies(${target_name} cxx_static cxxabi_static) - endif () - - if (USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING) - add_dependencies(${target_name} unwind_static) - endif () - endif () - endfunction () - - add_default_dependencies(ltdl) - add_default_dependencies(zlibstatic) - add_default_dependencies(jemalloc) - add_default_dependencies(memcpy) - add_default_dependencies(Foundation) - add_default_dependencies(common) - add_default_dependencies(gtest) - add_default_dependencies(lz4) - add_default_dependencies(zstd) - add_default_dependencies(snappy) - add_default_dependencies(arrow) - add_default_dependencies(protoc) - add_default_dependencies(thrift_static) - add_default_dependencies(cityhash) - add_default_dependencies(farmhash) - add_default_dependencies(murmurhash) - add_default_dependencies(metrohash) - add_default_dependencies(metrohash128) - add_default_dependencies(consistent-hashing) - add_default_dependencies(double-conversion) - add_default_dependencies(cctz) - add_default_dependencies(kj) - add_default_dependencies(simdjson) - add_default_dependencies(apple_rt) - add_default_dependencies(h3) - add_default_dependencies(re2) - add_default_dependencies(re2_st) - add_default_dependencies(hs_compile_shared) - add_default_dependencies(hs_exec_shared) - add_default_dependencies(hs_shared) - add_default_dependencies(widechar_width) - add_default_dependencies(string_utils) - add_default_dependencies(consistent-hashing-sumbur) - add_default_dependencies(boost_program_options_internal) - add_default_dependencies(boost_system_internal) - add_default_dependencies(boost_regex_internal) - add_default_dependencies(roaring) - add_default_dependencies(btrie) - add_default_dependencies(cpuid) - add_default_dependencies(mysqlclient) - add_default_dependencies(zlib) - add_default_dependencies(thrift) - add_default_dependencies(brotli) - add_default_dependencies(libprotobuf) - add_default_dependencies(base64) - add_default_dependencies(readpassphrase) - add_default_dependencies(unwind_static) - add_default_dependencies(fastops) -endif () diff --git a/README.md b/README.md index ac579d3f2a3..d5b0bf63165 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ ClickHouse is an open-source column-oriented database management system that all ## Upcoming Events * [ClickHouse Meetup in Moscow](https://yandex.ru/promo/clickhouse/moscow-2019) on September 5. +* [ClickHouse Meetup in Munich](https://www.meetup.com/ClickHouse-Meetup-Munich/events/264185199/) on September 17. * [ClickHouse Meetup in Paris](https://www.eventbrite.com/e/clickhouse-paris-meetup-2019-registration-68493270215) on October 3. * [ClickHouse Meetup in Hong Kong](https://www.meetup.com/Hong-Kong-Machine-Learning-Meetup/events/263580542/) on October 17. * [ClickHouse Meetup in Shenzhen](https://www.huodongxing.com/event/3483759917300) on October 20. diff --git a/cmake/default_libs.cmake b/cmake/default_libs.cmake new file mode 100644 index 00000000000..54a01042558 --- /dev/null +++ b/cmake/default_libs.cmake @@ -0,0 +1,48 @@ +# Set standard, system and compiler libraries explicitly. +# This is intended for more control of what we are linking. + +set (DEFAULT_LIBS "-nodefaultlibs") + +if (OS_LINUX) + # We need builtins from Clang's RT even without libcxx - for ubsan+int128. + # See https://bugs.llvm.org/show_bug.cgi?id=16404 + if (COMPILER_CLANG) + execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-${CMAKE_SYSTEM_PROCESSOR}.a OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE) + else () + set (BUILTINS_LIBRARY "-lgcc") + endif () + + set (DEFAULT_LIBS "${DEFAULT_LIBS} ${BUILTINS_LIBRARY} ${COVERAGE_OPTION} -lc -lm -lrt -lpthread -ldl") + + message(STATUS "Default libraries: ${DEFAULT_LIBS}") +endif () + +set(CMAKE_CXX_STANDARD_LIBRARIES ${DEFAULT_LIBS}) +set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS}) + +# Global libraries + +add_library(global-libs INTERFACE) + +# Unfortunately '-pthread' doesn't work with '-nodefaultlibs'. +# Just make sure we have pthreads at all. +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_package(Threads REQUIRED) + +add_subdirectory(libs/libglibc-compatibility) +include (cmake/find_unwind.cmake) +include (cmake/find_cxx.cmake) + +add_library(global-group INTERFACE) +target_link_libraries(global-group INTERFACE + -Wl,--start-group + $ + -Wl,--end-group +) + +link_libraries(global-group) + +install( + TARGETS global-group global-libs + EXPORT global +) diff --git a/cmake/find_capnp.cmake b/cmake/find_capnp.cmake index ec591afdc38..572fc1b3341 100644 --- a/cmake/find_capnp.cmake +++ b/cmake/find_capnp.cmake @@ -1,50 +1,20 @@ -option (ENABLE_CAPNP "Enable Cap'n Proto" ON) - -if (ENABLE_CAPNP) - # cmake 3.5.1 bug: - # capnproto uses this cmake feature: - # target_compile_features(kj PUBLIC cxx_constexpr) - # old cmake adds -std=gnu++11 to end of all compile commands (even if -std=gnu++17 already present in compile string) - # cmake 3.9.1 (ubuntu artful) have no this bug (c++17 support added to cmake 3.8.2) - if (CMAKE_VERSION VERSION_LESS "3.8.0") - set (USE_INTERNAL_CAPNP_LIBRARY_DEFAULT 0) - set (MISSING_INTERNAL_CAPNP_LIBRARY 1) - else () - set (USE_INTERNAL_CAPNP_LIBRARY_DEFAULT ${NOT_UNBUNDLED}) - endif () - - option (USE_INTERNAL_CAPNP_LIBRARY "Set to FALSE to use system capnproto library instead of bundled" ${USE_INTERNAL_CAPNP_LIBRARY_DEFAULT}) - - if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/capnproto/c++/CMakeLists.txt") - if (USE_INTERNAL_CAPNP_LIBRARY) - message (WARNING "submodule contrib/capnproto is missing. to fix try run: \n git submodule update --init --recursive") - endif () - set (USE_INTERNAL_CAPNP_LIBRARY 0) - set (MISSING_INTERNAL_CAPNP_LIBRARY 1) - endif () - - if (NOT USE_INTERNAL_CAPNP_LIBRARY) - set (CAPNP_PATHS "/usr/local/opt/capnp/lib") - set (CAPNP_INCLUDE_PATHS "/usr/local/opt/capnp/include") - find_library (CAPNP capnp PATHS ${CAPNP_PATHS}) - find_library (CAPNPC capnpc PATHS ${CAPNP_PATHS}) - find_library (KJ kj PATHS ${CAPNP_PATHS}) - set (CAPNP_LIBRARY ${CAPNPC} ${CAPNP} ${KJ}) - find_path (CAPNP_INCLUDE_DIR NAMES capnp/schema-parser.h PATHS ${CAPNP_INCLUDE_PATHS}) - endif () - - if (CAPNP_INCLUDE_DIR AND CAPNP_LIBRARY) - set(USE_CAPNP 1) - elseif (NOT MISSING_INTERNAL_CAPNP_LIBRARY) - set (USE_INTERNAL_CAPNP_LIBRARY 1) - set (CAPNP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/capnproto/c++/src") - set (CAPNP_LIBRARY capnpc) - set (USE_CAPNP 1) - endif () -endif () +option (USE_CAPNP "Enable Cap'n Proto" ON) if (USE_CAPNP) - message (STATUS "Using capnp=${USE_CAPNP}: ${CAPNP_INCLUDE_DIR} : ${CAPNP_LIBRARY}") -else () - message (STATUS "Build without capnp (support for Cap'n Proto format will be disabled)") + option (USE_INTERNAL_CAPNP_LIBRARY "Set to FALSE to use system capnproto library instead of bundled" ${NOT_UNBUNDLED}) + + # FIXME: refactor to use `add_library(… IMPORTED)` if possible. + if (NOT USE_INTERNAL_CAPNP_LIBRARY) + find_library (KJ kj) + find_library (CAPNP capnp) + find_library (CAPNPC capnpc) + + set (CAPNP_LIBRARIES ${CAPNPC} ${CAPNP} ${KJ}) + else () + add_subdirectory(contrib/capnproto-cmake) + + set (CAPNP_LIBRARIES capnpc) + endif () + + message (STATUS "Using capnp: ${CAPNP_LIBRARIES}") endif () diff --git a/cmake/find_ccache.cmake b/cmake/find_ccache.cmake index c2d467ca6ba..95d6b208cfa 100644 --- a/cmake/find_ccache.cmake +++ b/cmake/find_ccache.cmake @@ -1,13 +1,14 @@ find_program (CCACHE_FOUND ccache) -if (CCACHE_FOUND AND NOT CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache" AND NOT CMAKE_CXX_COMPILER MATCHES "ccache") - execute_process(COMMAND ${CCACHE_FOUND} "-V" OUTPUT_VARIABLE CCACHE_VERSION) - string(REGEX REPLACE "ccache version ([0-9\\.]+).*" "\\1" CCACHE_VERSION ${CCACHE_VERSION}) - if (CCACHE_VERSION VERSION_GREATER "3.2.0" OR NOT CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - #message(STATUS "Using ${CCACHE_FOUND} ${CCACHE_VERSION}") - set_property (GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND}) - set_property (GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND}) - else () - message(STATUS "Not using ${CCACHE_FOUND} ${CCACHE_VERSION} bug: https://bugzilla.samba.org/show_bug.cgi?id=8118") - endif () +if (CCACHE_FOUND AND NOT CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache" AND NOT CMAKE_CXX_COMPILER MATCHES "ccache") + execute_process(COMMAND ${CCACHE_FOUND} "-V" OUTPUT_VARIABLE CCACHE_VERSION) + string(REGEX REPLACE "ccache version ([0-9\\.]+).*" "\\1" CCACHE_VERSION ${CCACHE_VERSION}) + + if (CCACHE_VERSION VERSION_GREATER "3.2.0" OR NOT CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + #message(STATUS "Using ${CCACHE_FOUND} ${CCACHE_VERSION}") + set_property (GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND}) + set_property (GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND}) + else () + message(STATUS "Not using ${CCACHE_FOUND} ${CCACHE_VERSION} bug: https://bugzilla.samba.org/show_bug.cgi?id=8118") + endif () endif () diff --git a/cmake/find_cxx.cmake b/cmake/find_cxx.cmake index 2b2952f6efd..f84a76183ec 100644 --- a/cmake/find_cxx.cmake +++ b/cmake/find_cxx.cmake @@ -1,26 +1,50 @@ -if (NOT APPLE) +if (OS_LINUX AND COMPILER_CLANG) + option (USE_LIBCXX "Use libc++ and libc++abi instead of libstdc++" ON) option (USE_INTERNAL_LIBCXX_LIBRARY "Set to FALSE to use system libcxx and libcxxabi libraries instead of bundled" ${NOT_UNBUNDLED}) +endif() + +if (USE_LIBCXX) + set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_LIBCPP_DEBUG=0") # More checks in debug build. endif () +# FIXME: make better check for submodule presence if (USE_INTERNAL_LIBCXX_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libcxx/include/vector") message (WARNING "submodule contrib/libcxx is missing. to fix try run: \n git submodule update --init --recursive") set (USE_INTERNAL_LIBCXX_LIBRARY 0) endif () +# FIXME: make better check for submodule presence if (USE_INTERNAL_LIBCXX_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libcxxabi/src") message (WARNING "submodule contrib/libcxxabi is missing. to fix try run: \n git submodule update --init --recursive") - set (USE_INTERNAL_LIBCXXABI_LIBRARY 0) + set (USE_INTERNAL_LIBCXX_LIBRARY 0) endif () -if (NOT USE_INTERNAL_LIBCXX_LIBRARY) - find_library (LIBCXX_LIBRARY c++) - find_library (LIBCXXABI_LIBRARY c++abi) +if (USE_LIBCXX) + if (NOT USE_INTERNAL_LIBCXX_LIBRARY) + find_library (LIBCXX_LIBRARY c++) + find_library (LIBCXXFS_LIBRARY c++fs) + find_library (LIBCXXABI_LIBRARY c++abi) + + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") + + target_link_libraries(global-libs INTERFACE ${EXCEPTION_HANDLING_LIBRARY}) + else () + set (LIBCXX_LIBRARY cxx) + set (LIBCXXABI_LIBRARY cxxabi) + add_subdirectory(contrib/libcxxabi-cmake) + add_subdirectory(contrib/libcxx-cmake) + + # Exception handling library is embedded into libcxxabi. + endif () + + target_link_libraries(global-libs INTERFACE ${LIBCXX_LIBRARY} ${LIBCXXABI_LIBRARY} ${LIBCXXFS_LIBRARY}) + + set (HAVE_LIBCXX 1) + + message (STATUS "Using libcxx: ${LIBCXX_LIBRARY}") + message (STATUS "Using libcxxfs: ${LIBCXXFS_LIBRARY}") + message (STATUS "Using libcxxabi: ${LIBCXXABI_LIBRARY}") else () - set (LIBCXX_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcxx/include) - set (LIBCXXABI_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcxxabi/include) - set (LIBCXX_LIBRARY cxx_static) - set (LIBCXXABI_LIBRARY cxxabi_static) + target_link_libraries(global-libs INTERFACE -l:libstdc++.a -l:libstdc++fs.a) # Always link these libraries as static + target_link_libraries(global-libs INTERFACE ${EXCEPTION_HANDLING_LIBRARY}) endif () - -message (STATUS "Using libcxx: ${LIBCXX_LIBRARY}") -message (STATUS "Using libcxxabi: ${LIBCXXABI_LIBRARY}") diff --git a/cmake/find_mimalloc.cmake b/cmake/find_mimalloc.cmake deleted file mode 100644 index 1820421379f..00000000000 --- a/cmake/find_mimalloc.cmake +++ /dev/null @@ -1,17 +0,0 @@ -if (OS_LINUX AND NOT SANITIZE AND NOT ARCH_ARM AND NOT ARCH_32 AND NOT ARCH_PPC64LE) - option (ENABLE_MIMALLOC "Set to FALSE to disable usage of mimalloc for internal ClickHouse caches" FALSE) -endif () - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/mimalloc/include/mimalloc.h") - message (WARNING "submodule contrib/mimalloc is missing. to fix try run: \n git submodule update --init --recursive") - return() -endif () - -if (ENABLE_MIMALLOC) - message (FATAL_ERROR "Mimalloc is not production ready. (Disable with cmake -D ENABLE_MIMALLOC=0). If you want to use mimalloc, you must manually remove this message.") - - set (MIMALLOC_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/mimalloc/include) - set (USE_MIMALLOC 1) - set (MIMALLOC_LIBRARY mimalloc-static) - message (STATUS "Using mimalloc: ${MIMALLOC_INCLUDE_DIR} : ${MIMALLOC_LIBRARY}") -endif () diff --git a/cmake/find_orc.cmake b/cmake/find_orc.cmake new file mode 100644 index 00000000000..3676bec1b6b --- /dev/null +++ b/cmake/find_orc.cmake @@ -0,0 +1,8 @@ +##TODO replace hardcode to find procedure + +set(USE_ORC 0) +set(USE_INTERNAL_ORC_LIBRARY ON) + +if (ARROW_LIBRARY) + set(USE_ORC 1) +endif() \ No newline at end of file diff --git a/cmake/find_unwind.cmake b/cmake/find_unwind.cmake index 25e088e8deb..ea6e1d4bacb 100644 --- a/cmake/find_unwind.cmake +++ b/cmake/find_unwind.cmake @@ -1,59 +1,17 @@ -include (CMakePushCheckState) -cmake_push_check_state () +option (USE_UNWIND "Enable libunwind (better stacktraces)" ON) -option (ENABLE_UNWIND "Enable libunwind (better stacktraces)" ON) +if (NOT CMAKE_SYSTEM MATCHES "Linux" OR ARCH_ARM OR ARCH_32) + set (USE_UNWIND OFF) +endif () -if (ENABLE_UNWIND) +if (USE_UNWIND) + add_subdirectory(contrib/libunwind-cmake) + set (UNWIND_LIBRARIES unwind) + set (EXCEPTION_HANDLING_LIBRARY ${UNWIND_LIBRARIES}) -if (CMAKE_SYSTEM MATCHES "Linux" AND NOT ARCH_ARM AND NOT ARCH_32) - option (USE_INTERNAL_UNWIND_LIBRARY "Set to FALSE to use system unwind library instead of bundled" ${NOT_UNBUNDLED}) + message (STATUS "Using libunwind: ${UNWIND_LIBRARIES}") else () - option (USE_INTERNAL_UNWIND_LIBRARY "Set to FALSE to use system unwind library instead of bundled" OFF) + set (EXCEPTION_HANDLING_LIBRARY gcc_eh) endif () -if (NOT USE_INTERNAL_UNWIND_LIBRARY) - find_library (UNWIND_LIBRARY unwind) - find_path (UNWIND_INCLUDE_DIR NAMES unwind.h PATHS ${UNWIND_INCLUDE_PATHS}) - - include (CheckCXXSourceCompiles) - set(CMAKE_REQUIRED_INCLUDES ${UNWIND_INCLUDE_DIR}) - set(CMAKE_REQUIRED_LIBRARIES ${UNWIND_LIBRARY}) - check_cxx_source_compiles(" - #include - #define UNW_LOCAL_ONLY - #include - int main () { - ucontext_t context; - unw_cursor_t cursor; - unw_init_local(&cursor, &context); - return 0; - } - " HAVE_UNW_INIT_LOCAL) - if (NOT HAVE_UNW_INIT_LOCAL) - set(UNWIND_LIBRARY "") - set(UNWIND_INCLUDE_DIR "") - endif () - -endif () - -if (UNWIND_LIBRARY AND UNWIND_INCLUDE_DIR) - set (USE_UNWIND 1) -elseif (CMAKE_SYSTEM MATCHES "Linux" AND NOT ARCH_ARM AND NOT ARCH_32 AND NOT UNBUNDLED) - set (USE_INTERNAL_UNWIND_LIBRARY 1) - - set (PACKAGE_VERSION "9.0.0svn" CACHE STRING "") - - set (UNWIND_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libunwind/include") - - set (LIBUNWIND_ENABLE_SHARED OFF CACHE BOOL "") - set (LIBUNWIND_ENABLE_STATIC ON CACHE BOOL "") - set (UNWIND_LIBRARY unwind_static) - - set (USE_UNWIND 1) -endif () - -endif () - -message (STATUS "Using unwind=${USE_UNWIND}: ${UNWIND_INCLUDE_DIR} : ${UNWIND_LIBRARY}") - -cmake_pop_check_state () +message (STATUS "Using exception handler: ${EXCEPTION_HANDLING_LIBRARY}") diff --git a/cmake/test_compiler.cmake b/cmake/test_compiler.cmake deleted file mode 100644 index 570c058b9f7..00000000000 --- a/cmake/test_compiler.cmake +++ /dev/null @@ -1,47 +0,0 @@ -include (CheckCXXSourceCompiles) -include (CMakePushCheckState) - -set(THREADS_PREFER_PTHREAD_FLAG ON) -find_package(Threads) - -cmake_push_check_state () - -if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") -# clang4 : -no-pie cause error -# clang6 : -no-pie cause warning - - if (MAKE_STATIC_LIBRARIES) - set (TEST_FLAG "-Wl,-Bstatic -stdlib=libc++ -lc++ -lc++abi -Wl,-Bdynamic") - else () - set (TEST_FLAG "-stdlib=libc++ -lc++ -lc++abi") - endif () - - set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG}") - set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} Threads::Threads) - - check_cxx_source_compiles(" - #include - int main() { - std::cerr << std::endl; - return 0; - } - " HAVE_LIBCXX) - -else () - - set (TEST_FLAG "-no-pie") - set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG}") - - check_cxx_source_compiles(" - int main() { - return 0; - } - " HAVE_NO_PIE) - - if (HAVE_NO_PIE) - set (FLAG_NO_PIE ${TEST_FLAG}) - endif () - -endif () - -cmake_pop_check_state () diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index c478311d77a..96462de0190 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -10,17 +10,19 @@ endif () set_property(DIRECTORY PROPERTY EXCLUDE_FROM_ALL 1) - -if (USE_INTERNAL_UNWIND_LIBRARY) - add_subdirectory (libunwind-cmake) -endif () - -if (USE_LIBCXX AND USE_INTERNAL_LIBCXX_LIBRARY) - add_subdirectory(libcxx-cmake) - add_subdirectory(libcxxabi-cmake) +if (USE_INTERNAL_ORC_LIBRARY) + set(BUILD_JAVA OFF) + set (ANALYZE_JAVA OFF) + set (BUILD_CPP_TESTS OFF) + set (BUILD_TOOLS OFF) + option(BUILD_JAVA OFF) + option (ANALYZE_JAVA OFF) + option (BUILD_CPP_TESTS OFF) + option (BUILD_TOOLS OFF) + set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/contrib/orc/cmake_modules") + add_subdirectory(orc) endif() - if (USE_INTERNAL_BOOST_LIBRARY) add_subdirectory (boost-cmake) endif () @@ -160,15 +162,6 @@ if (ENABLE_ODBC AND USE_INTERNAL_ODBC_LIBRARY) add_library(ODBC::ODBC ALIAS ${ODBC_LIBRARIES}) endif () -if (ENABLE_CAPNP AND USE_INTERNAL_CAPNP_LIBRARY) - set (BUILD_TESTING 0 CACHE INTERNAL "") - set (_save ${CMAKE_CXX_EXTENSIONS}) - set (CMAKE_CXX_EXTENSIONS) - add_subdirectory (capnproto/c++) - set (CMAKE_CXX_EXTENSIONS ${_save}) - target_include_directories(${CAPNP_LIBRARY} PUBLIC $) -endif () - if (USE_INTERNAL_PARQUET_LIBRARY) if (USE_INTERNAL_PARQUET_LIBRARY_NATIVE_CMAKE) # We dont use arrow's cmakefiles because they uses too many depends and download some libs in compile time diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index a7b6628ea4e..ba1ddc2414a 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -44,9 +44,73 @@ set( thriftcpp_threads_SOURCES add_library(${THRIFT_LIBRARY} ${thriftcpp_SOURCES} ${thriftcpp_threads_SOURCES}) set_target_properties(${THRIFT_LIBRARY} PROPERTIES CXX_STANDARD 14) # REMOVE after https://github.com/apache/thrift/pull/1641 target_include_directories(${THRIFT_LIBRARY} SYSTEM PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/thrift/lib/cpp/src PRIVATE ${Boost_INCLUDE_DIRS}) -target_link_libraries(${THRIFT_LIBRARY} PRIVATE Threads::Threads) +# === orc + +set(ORC_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/orc/c++) +set(ORC_INCLUDE_DIR ${ORC_SOURCE_DIR}/include) +set(ORC_SOURCE_SRC_DIR ${ORC_SOURCE_DIR}/src) +set(ORC_SOURCE_WRAP_DIR ${ORC_SOURCE_DIR}/wrap) + +set(ORC_BUILD_SRC_DIR ${CMAKE_CURRENT_BINARY_DIR}/../orc/c++/src) +set(ORC_BUILD_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/../orc/c++/include) + +set(GOOGLE_PROTOBUF_DIR ${ClickHouse_SOURCE_DIR}/contrib/protobuf/src/) +set(ORC_ADDITION_SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}) +set(ARROW_SRC_DIR ${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src) + +set(PROTOBUF_EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/../protobuf/cmake/protoc) +set(PROTO_DIR ${ORC_SOURCE_DIR}/../proto) + + +add_custom_command(OUTPUT orc_proto.pb.h orc_proto.pb.cc + COMMAND ${PROTOBUF_EXECUTABLE} + -I ${PROTO_DIR} + --cpp_out="${CMAKE_CURRENT_BINARY_DIR}" + "${PROTO_DIR}/orc_proto.proto") + +include_directories(SYSTEM ${ORC_INCLUDE_DIR}) +include_directories(SYSTEM ${ORC_SOURCE_SRC_DIR}) +include_directories(SYSTEM ${ORC_SOURCE_WRAP_DIR}) +include_directories(SYSTEM ${GOOGLE_PROTOBUF_DIR}) +include_directories(SYSTEM ${ORC_BUILD_SRC_DIR}) +include_directories(SYSTEM ${ORC_BUILD_INCLUDE_DIR}) +include_directories(SYSTEM ${ORC_ADDITION_SOURCE_DIR}) +include_directories(SYSTEM ${ARROW_SRC_DIR}) + + +set(ORC_SRCS + ${ARROW_SRC_DIR}/arrow/adapters/orc/adapter.cc + ${ORC_SOURCE_SRC_DIR}/Exceptions.cc + ${ORC_SOURCE_SRC_DIR}/OrcFile.cc + ${ORC_SOURCE_SRC_DIR}/Reader.cc + ${ORC_SOURCE_SRC_DIR}/ByteRLE.cc + ${ORC_SOURCE_SRC_DIR}/ColumnPrinter.cc + ${ORC_SOURCE_SRC_DIR}/ColumnReader.cc + ${ORC_SOURCE_SRC_DIR}/ColumnWriter.cc + ${ORC_SOURCE_SRC_DIR}/Common.cc + ${ORC_SOURCE_SRC_DIR}/Compression.cc + ${ORC_SOURCE_SRC_DIR}/Exceptions.cc + ${ORC_SOURCE_SRC_DIR}/Int128.cc + ${ORC_SOURCE_SRC_DIR}/LzoDecompressor.cc + ${ORC_SOURCE_SRC_DIR}/MemoryPool.cc + ${ORC_SOURCE_SRC_DIR}/OrcFile.cc + ${ORC_SOURCE_SRC_DIR}/Reader.cc + ${ORC_SOURCE_SRC_DIR}/RLE.cc + ${ORC_SOURCE_SRC_DIR}/RLEv1.cc + ${ORC_SOURCE_SRC_DIR}/RLEv2.cc + ${ORC_SOURCE_SRC_DIR}/Statistics.cc + ${ORC_SOURCE_SRC_DIR}/StripeStream.cc + ${ORC_SOURCE_SRC_DIR}/Timezone.cc + ${ORC_SOURCE_SRC_DIR}/TypeImpl.cc + ${ORC_SOURCE_SRC_DIR}/Vector.cc + ${ORC_SOURCE_SRC_DIR}/Writer.cc + ${ORC_SOURCE_SRC_DIR}/io/InputStream.cc + ${ORC_SOURCE_SRC_DIR}/io/OutputStream.cc + ${ORC_ADDITION_SOURCE_DIR}/orc_proto.pb.cc + ) + # === arrow @@ -103,6 +167,7 @@ set(ARROW_SRCS ${LIBRARY_DIR}/util/thread-pool.cc ${LIBRARY_DIR}/util/trie.cc ${LIBRARY_DIR}/util/utf8.cc + ${ORC_SRCS} ) set(ARROW_SRCS ${ARROW_SRCS} @@ -151,8 +216,9 @@ endif() add_library(${ARROW_LIBRARY} ${ARROW_SRCS}) +add_dependencies(${ARROW_LIBRARY} protoc) target_include_directories(${ARROW_LIBRARY} SYSTEM PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/cpp/src ${Boost_INCLUDE_DIRS}) -target_link_libraries(${ARROW_LIBRARY} PRIVATE ${DOUBLE_CONVERSION_LIBRARIES} Threads::Threads) +target_link_libraries(${ARROW_LIBRARY} PRIVATE ${DOUBLE_CONVERSION_LIBRARIES} ${Protobuf_LIBRARY}) if (ARROW_WITH_LZ4) target_link_libraries(${ARROW_LIBRARY} PRIVATE ${LZ4_LIBRARY}) endif() diff --git a/contrib/capnproto-cmake/CMakeLists.txt b/contrib/capnproto-cmake/CMakeLists.txt new file mode 100644 index 00000000000..d92a5a282ce --- /dev/null +++ b/contrib/capnproto-cmake/CMakeLists.txt @@ -0,0 +1,69 @@ +set (CAPNPROTO_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/capnproto/c++/src) + +set (KJ_SRCS + ${CAPNPROTO_SOURCE_DIR}/kj/array.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/common.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/debug.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/exception.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/io.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/memory.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/mutex.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/string.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/hash.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/table.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/thread.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/main.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/arena.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/test-helpers.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/units.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/encoding.c++ + + ${CAPNPROTO_SOURCE_DIR}/kj/refcount.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/string-tree.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/time.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/filesystem.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/filesystem-disk-unix.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/filesystem-disk-win32.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/parse/char.c++ +) + +add_library(kj ${KJ_SRCS}) +target_include_directories(kj PUBLIC ${CAPNPROTO_SOURCE_DIR}) +target_compile_options(kj PUBLIC -Wno-non-virtual-dtor) + +set (CAPNP_SRCS + ${CAPNPROTO_SOURCE_DIR}/capnp/c++.capnp.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/blob.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/arena.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/layout.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/list.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/any.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/message.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/schema.capnp.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/serialize.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/serialize-packed.c++ + + ${CAPNPROTO_SOURCE_DIR}/capnp/schema.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/schema-loader.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/dynamic.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/stringify.c++ +) + +add_library(capnp ${CAPNP_SRCS}) +target_link_libraries(capnp PUBLIC kj) + +set (CAPNPC_SRCS + ${CAPNPROTO_SOURCE_DIR}/capnp/compiler/type-id.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/compiler/error-reporter.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/compiler/lexer.capnp.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/compiler/lexer.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/compiler/grammar.capnp.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/compiler/parser.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/compiler/node-translator.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/compiler/compiler.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/schema-parser.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/serialize-text.c++ +) + +add_library(capnpc ${CAPNPC_SRCS}) +target_link_libraries(capnpc PUBLIC capnp) diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index 47f057c0559..e44c54d2b37 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -59,7 +59,6 @@ if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG") if (USE_UNWIND) target_compile_definitions (jemalloc PRIVATE -DJEMALLOC_PROF_LIBUNWIND=1) - target_include_directories (jemalloc BEFORE PRIVATE ${UNWIND_INCLUDE_DIR}) - target_link_libraries (jemalloc PRIVATE ${UNWIND_LIBRARY}) + target_link_libraries (jemalloc PRIVATE ${UNWIND_LIBRARIES}) endif () endif () diff --git a/contrib/libcxx-cmake/CMakeLists.txt b/contrib/libcxx-cmake/CMakeLists.txt index e9ca5e1e7cd..9609c7ca9e7 100644 --- a/contrib/libcxx-cmake/CMakeLists.txt +++ b/contrib/libcxx-cmake/CMakeLists.txt @@ -1,5 +1,4 @@ set(LIBCXX_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcxx) -#set(LIBCXX_BINARY_DIR ${ClickHouse_BINARY_DIR}/contrib/libcxx) set(SRCS ${LIBCXX_SOURCE_DIR}/src/optional.cpp @@ -16,10 +15,6 @@ ${LIBCXX_SOURCE_DIR}/src/condition_variable.cpp ${LIBCXX_SOURCE_DIR}/src/hash.cpp ${LIBCXX_SOURCE_DIR}/src/string.cpp ${LIBCXX_SOURCE_DIR}/src/debug.cpp -#${LIBCXX_SOURCE_DIR}/src/support/win32/support.cpp -#${LIBCXX_SOURCE_DIR}/src/support/win32/locale_win32.cpp -#${LIBCXX_SOURCE_DIR}/src/support/win32/thread_win32.cpp -#${LIBCXX_SOURCE_DIR}/src/support/solaris/xlocale.cpp ${LIBCXX_SOURCE_DIR}/src/stdexcept.cpp ${LIBCXX_SOURCE_DIR}/src/utility.cpp ${LIBCXX_SOURCE_DIR}/src/any.cpp @@ -43,9 +38,17 @@ ${LIBCXX_SOURCE_DIR}/src/system_error.cpp ${LIBCXX_SOURCE_DIR}/src/random.cpp ) -add_library(cxx_static ${SRCS}) +add_library(cxx ${SRCS}) -target_include_directories(cxx_static PUBLIC ${LIBCXX_SOURCE_DIR}/include) -target_compile_definitions(cxx_static PRIVATE -D_LIBCPP_BUILDING_LIBRARY -DLIBCXX_BUILDING_LIBCXXABI) -target_compile_options(cxx_static PRIVATE -nostdinc++) +target_include_directories(cxx SYSTEM BEFORE PUBLIC $) +target_compile_definitions(cxx PRIVATE -D_LIBCPP_BUILDING_LIBRARY -DLIBCXX_BUILDING_LIBCXXABI) +target_compile_options(cxx PUBLIC -nostdinc++ -Wno-reserved-id-macro) +target_link_libraries(cxx PUBLIC cxxabi) +install( + TARGETS cxx + EXPORT global + ARCHIVE DESTINATION lib + RUNTIME DESTINATION lib + LIBRARY DESTINATION lib +) diff --git a/contrib/libcxxabi-cmake/CMakeLists.txt b/contrib/libcxxabi-cmake/CMakeLists.txt index 2abece86691..68bb5606689 100644 --- a/contrib/libcxxabi-cmake/CMakeLists.txt +++ b/contrib/libcxxabi-cmake/CMakeLists.txt @@ -1,13 +1,10 @@ set(LIBCXXABI_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcxxabi) -set(LIBCXX_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcxx) -#set(LIBCXXABI_BINARY_DIR ${ClickHouse_BINARY_DIR}/contrib/libcxxabi) set(SRCS ${LIBCXXABI_SOURCE_DIR}/src/stdlib_stdexcept.cpp ${LIBCXXABI_SOURCE_DIR}/src/cxa_virtual.cpp ${LIBCXXABI_SOURCE_DIR}/src/cxa_thread_atexit.cpp ${LIBCXXABI_SOURCE_DIR}/src/fallback_malloc.cpp -#${LIBCXXABI_SOURCE_DIR}/src/cxa_noexception.cpp ${LIBCXXABI_SOURCE_DIR}/src/cxa_guard.cpp ${LIBCXXABI_SOURCE_DIR}/src/cxa_default_handlers.cpp ${LIBCXXABI_SOURCE_DIR}/src/cxa_personality.cpp @@ -25,10 +22,20 @@ ${LIBCXXABI_SOURCE_DIR}/src/cxa_vector.cpp ${LIBCXXABI_SOURCE_DIR}/src/stdlib_new_delete.cpp ) -add_library(cxxabi_static ${SRCS}) - -target_include_directories(cxxabi_static PUBLIC ${LIBCXXABI_SOURCE_DIR}/include ${LIBCXX_SOURCE_DIR}/include) -target_compile_definitions(cxxabi_static PRIVATE -D_LIBCPP_BUILDING_LIBRARY) -target_compile_options(cxxabi_static PRIVATE -nostdinc++ -fno-sanitize=undefined) # If we don't disable UBSan, infinite recursion happens in dynamic_cast. +add_library(cxxabi ${SRCS}) +target_include_directories(cxxabi SYSTEM BEFORE + PUBLIC $ + PRIVATE $ +) +target_compile_definitions(cxxabi PRIVATE -D_LIBCPP_BUILDING_LIBRARY) +target_compile_options(cxxabi PRIVATE -nostdinc++ -fno-sanitize=undefined -Wno-macro-redefined) # If we don't disable UBSan, infinite recursion happens in dynamic_cast. +target_link_libraries(cxxabi PUBLIC ${EXCEPTION_HANDLING_LIBRARY}) +install( + TARGETS cxxabi + EXPORT global + ARCHIVE DESTINATION lib + RUNTIME DESTINATION lib + LIBRARY DESTINATION lib +) diff --git a/contrib/librdkafka-cmake/CMakeLists.txt b/contrib/librdkafka-cmake/CMakeLists.txt index 75cd3968204..64dc83fa8b6 100644 --- a/contrib/librdkafka-cmake/CMakeLists.txt +++ b/contrib/librdkafka-cmake/CMakeLists.txt @@ -65,7 +65,7 @@ add_library(rdkafka ${SRCS}) target_include_directories(rdkafka SYSTEM PUBLIC include) target_include_directories(rdkafka SYSTEM PUBLIC ${RDKAFKA_SOURCE_DIR}) # Because weird logic with "include_next" is used. target_include_directories(rdkafka SYSTEM PRIVATE ${ZSTD_INCLUDE_DIR}/common) # Because wrong path to "zstd_errors.h" is used. -target_link_libraries(rdkafka PRIVATE ${ZLIB_LIBRARIES} ${ZSTD_LIBRARY} ${LZ4_LIBRARY} ${LIBGSASL_LIBRARY} Threads::Threads) +target_link_libraries(rdkafka PRIVATE ${ZLIB_LIBRARIES} ${ZSTD_LIBRARY} ${LZ4_LIBRARY} ${LIBGSASL_LIBRARY}) if(OPENSSL_SSL_LIBRARY AND OPENSSL_CRYPTO_LIBRARY) target_link_libraries(rdkafka PRIVATE ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY}) endif() diff --git a/contrib/libunwind-cmake/CMakeLists.txt b/contrib/libunwind-cmake/CMakeLists.txt index 4f24fe249f5..f09d0979692 100644 --- a/contrib/libunwind-cmake/CMakeLists.txt +++ b/contrib/libunwind-cmake/CMakeLists.txt @@ -24,9 +24,15 @@ set(LIBUNWIND_SOURCES ${LIBUNWIND_C_SOURCES} ${LIBUNWIND_ASM_SOURCES}) -add_library(unwind_static ${LIBUNWIND_SOURCES}) +add_library(unwind ${LIBUNWIND_SOURCES}) -target_include_directories(unwind_static SYSTEM BEFORE PUBLIC ${LIBUNWIND_SOURCE_DIR}/include) -target_compile_definitions(unwind_static PRIVATE -D_LIBUNWIND_NO_HEAP=1 -D_DEBUG -D_LIBUNWIND_IS_NATIVE_ONLY) -target_compile_options(unwind_static PRIVATE -fno-exceptions -funwind-tables -fno-sanitize=all -nostdinc++ -fno-rtti) -target_link_libraries(unwind_static PRIVATE Threads::Threads ${CMAKE_DL_LIBS}) +target_include_directories(unwind SYSTEM BEFORE PUBLIC $) +target_compile_definitions(unwind PRIVATE -D_LIBUNWIND_NO_HEAP=1 -D_DEBUG -D_LIBUNWIND_IS_NATIVE_ONLY) +target_compile_options(unwind PRIVATE -fno-exceptions -funwind-tables -fno-sanitize=all -nostdinc++ -fno-rtti) + +install( + TARGETS unwind + EXPORT global + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib +) diff --git a/contrib/mariadb-connector-c-cmake/CMakeLists.txt b/contrib/mariadb-connector-c-cmake/CMakeLists.txt index 1f453a7f6d1..2e80b0c325f 100644 --- a/contrib/mariadb-connector-c-cmake/CMakeLists.txt +++ b/contrib/mariadb-connector-c-cmake/CMakeLists.txt @@ -62,11 +62,6 @@ endif() add_library(mysqlclient ${SRCS}) -target_link_libraries(mysqlclient PRIVATE ${CMAKE_DL_LIBS} Threads::Threads) -if(M_LIBRARY) - target_link_libraries(mysqlclient PRIVATE ${M_LIBRARY}) -endif() - if(OPENSSL_LIBRARIES) target_link_libraries(mysqlclient PRIVATE ${OPENSSL_LIBRARIES}) target_compile_definitions(mysqlclient PRIVATE -D HAVE_OPENSSL -D HAVE_TLS) diff --git a/contrib/mimalloc b/contrib/mimalloc deleted file mode 160000 index a787bdebce9..00000000000 --- a/contrib/mimalloc +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a787bdebce94bf3776dc0d1ad597917f479ab8d5 diff --git a/contrib/orc b/contrib/orc new file mode 160000 index 00000000000..5981208e394 --- /dev/null +++ b/contrib/orc @@ -0,0 +1 @@ +Subproject commit 5981208e39447df84827f6a961d1da76bacb6078 diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index b589c398238..e7cc084237f 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -97,8 +97,6 @@ add_subdirectory (src) set(dbms_headers) set(dbms_sources) -include(../cmake/dbms_glob_sources.cmake) - add_headers_and_sources(clickhouse_common_io src/Common) add_headers_and_sources(clickhouse_common_io src/Common/HashTable) add_headers_and_sources(clickhouse_common_io src/IO) @@ -114,6 +112,7 @@ add_headers_and_sources(dbms src/Columns) add_headers_and_sources(dbms src/Storages) add_headers_and_sources(dbms src/Storages/Distributed) add_headers_and_sources(dbms src/Storages/MergeTree) +add_headers_and_sources(dbms src/Storages/LiveView) add_headers_and_sources(dbms src/Client) add_headers_and_sources(dbms src/Formats) add_headers_and_sources(dbms src/Processors) @@ -162,9 +161,7 @@ if (OS_FREEBSD) endif () if (USE_UNWIND) - if (NOT USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING) - target_link_libraries (clickhouse_common_io PRIVATE ${UNWIND_LIBRARY}) - endif () + target_link_libraries (clickhouse_common_io PRIVATE ${UNWIND_LIBRARIES}) endif () add_subdirectory(src/Common/ZooKeeper) @@ -240,15 +237,10 @@ target_link_libraries(clickhouse_common_io ${EXECINFO_LIBRARIES} PUBLIC ${Boost_SYSTEM_LIBRARY} + ${Boost_PROGRAM_OPTIONS_LIBRARY} PRIVATE apple_rt PUBLIC - Threads::Threads - PRIVATE - ${CMAKE_DL_LIBS} - PRIVATE - rt - PUBLIC roaring ) @@ -264,11 +256,6 @@ if(RE2_INCLUDE_DIR) target_include_directories(clickhouse_common_io SYSTEM BEFORE PUBLIC ${RE2_INCLUDE_DIR}) endif() -if (USE_MIMALLOC) - target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${MIMALLOC_INCLUDE_DIR}) - target_link_libraries (clickhouse_common_io PRIVATE ${MIMALLOC_LIBRARY}) -endif () - if(CPUID_LIBRARY) target_link_libraries(clickhouse_common_io PRIVATE ${CPUID_LIBRARY}) endif() @@ -296,7 +283,6 @@ target_link_libraries (dbms ${Boost_FILESYSTEM_LIBRARY} PUBLIC ${Boost_SYSTEM_LIBRARY} - Threads::Threads ) target_include_directories(dbms PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/src/Core/include) @@ -363,10 +349,7 @@ if (USE_ICU) endif () if (USE_CAPNP) - target_link_libraries (dbms PRIVATE ${CAPNP_LIBRARY}) - if (NOT USE_INTERNAL_CAPNP_LIBRARY) - target_include_directories (dbms SYSTEM BEFORE PRIVATE ${CAPNP_INCLUDE_DIR}) - endif () + target_link_libraries (dbms PRIVATE ${CAPNP_LIBRARIES}) endif () if (USE_PARQUET) @@ -379,7 +362,6 @@ endif () if(OPENSSL_CRYPTO_LIBRARY) target_link_libraries(dbms PRIVATE ${OPENSSL_CRYPTO_LIBRARY}) endif () -target_link_libraries(dbms PRIVATE Threads::Threads) target_include_directories (dbms SYSTEM BEFORE PRIVATE ${DIVIDE_INCLUDE_DIR}) target_include_directories (dbms SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) diff --git a/dbms/programs/CMakeLists.txt b/dbms/programs/CMakeLists.txt index 6626d90e5f5..0dcd4d7ab91 100644 --- a/dbms/programs/CMakeLists.txt +++ b/dbms/programs/CMakeLists.txt @@ -81,7 +81,6 @@ add_subdirectory (extract-from-config) add_subdirectory (compressor) add_subdirectory (copier) add_subdirectory (format) -add_subdirectory (clang) add_subdirectory (obfuscator) if (ENABLE_CLICKHOUSE_ODBC_BRIDGE) @@ -89,25 +88,21 @@ if (ENABLE_CLICKHOUSE_ODBC_BRIDGE) endif () if (CLICKHOUSE_ONE_SHARED) - add_library(clickhouse-lib SHARED ${CLICKHOUSE_SERVER_SOURCES} ${CLICKHOUSE_CLIENT_SOURCES} ${CLICKHOUSE_LOCAL_SOURCES} ${CLICKHOUSE_BENCHMARK_SOURCES} ${CLICKHOUSE_PERFORMANCE_TEST_SOURCES} ${CLICKHOUSE_COPIER_SOURCES} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES} ${CLICKHOUSE_COMPRESSOR_SOURCES} ${CLICKHOUSE_FORMAT_SOURCES} ${CLICKHOUSE_OBFUSCATOR_SOURCES} ${CLICKHOUSE_COMPILER_SOURCES} ${CLICKHOUSE_ODBC_BRIDGE_SOURCES}) - target_link_libraries(clickhouse-lib ${CLICKHOUSE_SERVER_LINK} ${CLICKHOUSE_CLIENT_LINK} ${CLICKHOUSE_LOCAL_LINK} ${CLICKHOUSE_BENCHMARK_LINK} ${CLICKHOUSE_PERFORMANCE_TEST_LINK} ${CLICKHOUSE_COPIER_LINK} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK} ${CLICKHOUSE_COMPRESSOR_LINK} ${CLICKHOUSE_FORMAT_LINK} ${CLICKHOUSE_OBFUSCATOR_LINK} ${CLICKHOUSE_COMPILER_LINK} ${CLICKHOUSE_ODBC_BRIDGE_LINK}) - target_include_directories(clickhouse-lib ${CLICKHOUSE_SERVER_INCLUDE} ${CLICKHOUSE_CLIENT_INCLUDE} ${CLICKHOUSE_LOCAL_INCLUDE} ${CLICKHOUSE_BENCHMARK_INCLUDE} ${CLICKHOUSE_PERFORMANCE_TEST_INCLUDE} ${CLICKHOUSE_COPIER_INCLUDE} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_INCLUDE} ${CLICKHOUSE_COMPRESSOR_INCLUDE} ${CLICKHOUSE_FORMAT_INCLUDE} ${CLICKHOUSE_OBFUSCATOR_INCLUDE} ${CLICKHOUSE_COMPILER_INCLUDE} ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE}) + add_library(clickhouse-lib SHARED ${CLICKHOUSE_SERVER_SOURCES} ${CLICKHOUSE_CLIENT_SOURCES} ${CLICKHOUSE_LOCAL_SOURCES} ${CLICKHOUSE_BENCHMARK_SOURCES} ${CLICKHOUSE_PERFORMANCE_TEST_SOURCES} ${CLICKHOUSE_COPIER_SOURCES} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES} ${CLICKHOUSE_COMPRESSOR_SOURCES} ${CLICKHOUSE_FORMAT_SOURCES} ${CLICKHOUSE_OBFUSCATOR_SOURCES} ${CLICKHOUSE_ODBC_BRIDGE_SOURCES}) + target_link_libraries(clickhouse-lib ${CLICKHOUSE_SERVER_LINK} ${CLICKHOUSE_CLIENT_LINK} ${CLICKHOUSE_LOCAL_LINK} ${CLICKHOUSE_BENCHMARK_LINK} ${CLICKHOUSE_PERFORMANCE_TEST_LINK} ${CLICKHOUSE_COPIER_LINK} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK} ${CLICKHOUSE_COMPRESSOR_LINK} ${CLICKHOUSE_FORMAT_LINK} ${CLICKHOUSE_OBFUSCATOR_LINK} ${CLICKHOUSE_ODBC_BRIDGE_LINK}) + target_include_directories(clickhouse-lib ${CLICKHOUSE_SERVER_INCLUDE} ${CLICKHOUSE_CLIENT_INCLUDE} ${CLICKHOUSE_LOCAL_INCLUDE} ${CLICKHOUSE_BENCHMARK_INCLUDE} ${CLICKHOUSE_PERFORMANCE_TEST_INCLUDE} ${CLICKHOUSE_COPIER_INCLUDE} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_INCLUDE} ${CLICKHOUSE_COMPRESSOR_INCLUDE} ${CLICKHOUSE_FORMAT_INCLUDE} ${CLICKHOUSE_OBFUSCATOR_INCLUDE} ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE}) set_target_properties(clickhouse-lib PROPERTIES SOVERSION ${VERSION_MAJOR}.${VERSION_MINOR} VERSION ${VERSION_SO} OUTPUT_NAME clickhouse DEBUG_POSTFIX "") install (TARGETS clickhouse-lib LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT clickhouse) endif() if (CLICKHOUSE_SPLIT_BINARY) set (CLICKHOUSE_ALL_TARGETS clickhouse-server clickhouse-client clickhouse-local clickhouse-benchmark clickhouse-performance-test - clickhouse-extract-from-config clickhouse-compressor clickhouse-format clickhouse-copier) + clickhouse-extract-from-config clickhouse-compressor clickhouse-format clickhouse-obfuscator clickhouse-copier) if (ENABLE_CLICKHOUSE_ODBC_BRIDGE) list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-odbc-bridge) endif () - if (USE_EMBEDDED_COMPILER) - list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-clang clickhouse-lld) - endif () - set_target_properties(${CLICKHOUSE_ALL_TARGETS} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..) add_custom_target (clickhouse-bundle ALL DEPENDS ${CLICKHOUSE_ALL_TARGETS}) @@ -115,10 +110,6 @@ if (CLICKHOUSE_SPLIT_BINARY) install(PROGRAMS clickhouse-split-helper DESTINATION ${CMAKE_INSTALL_BINDIR} RENAME clickhouse COMPONENT clickhouse) else () - if (USE_EMBEDDED_COMPILER) - # before add_executable ! - link_directories (${LLVM_LIBRARY_DIRS}) - endif () add_executable (clickhouse main.cpp) target_link_libraries (clickhouse PRIVATE clickhouse_common_io string_utils) target_include_directories (clickhouse BEFORE PRIVATE ${COMMON_INCLUDE_DIR}) @@ -154,9 +145,6 @@ else () if (ENABLE_CLICKHOUSE_OBFUSCATOR) clickhouse_target_link_split_lib(clickhouse obfuscator) endif () - if (USE_EMBEDDED_COMPILER) - target_link_libraries(clickhouse PRIVATE clickhouse-compiler-lib) - endif () set (CLICKHOUSE_BUNDLE) if (ENABLE_CLICKHOUSE_SERVER) @@ -213,18 +201,8 @@ else () list(APPEND CLICKHOUSE_BUNDLE clickhouse-odbc-bridge) endif() - # install always because depian package want this files: - add_custom_target (clickhouse-clang ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-clang DEPENDS clickhouse) - add_custom_target (clickhouse-lld ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-lld DEPENDS clickhouse) - list(APPEND CLICKHOUSE_BUNDLE clickhouse-clang clickhouse-lld) - install (TARGETS clickhouse RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) - install (FILES - ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-clang - ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-lld - DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) - add_custom_target (clickhouse-bundle ALL DEPENDS ${CLICKHOUSE_BUNDLE}) endif () diff --git a/dbms/programs/benchmark/Benchmark.cpp b/dbms/programs/benchmark/Benchmark.cpp index c69e9a54feb..e685425eefc 100644 --- a/dbms/programs/benchmark/Benchmark.cpp +++ b/dbms/programs/benchmark/Benchmark.cpp @@ -32,6 +32,8 @@ #include #include #include +#include +#include /** A tool for evaluating ClickHouse performance. @@ -41,6 +43,8 @@ namespace DB { +using Ports = std::vector; + namespace ErrorCodes { extern const int BAD_ARGUMENTS; @@ -50,17 +54,34 @@ namespace ErrorCodes class Benchmark : public Poco::Util::Application { public: - Benchmark(unsigned concurrency_, double delay_, - const String & host_, UInt16 port_, bool secure_, const String & default_database_, + Benchmark(unsigned concurrency_, double delay_, Strings && hosts_, Ports && ports_, + bool cumulative_, bool secure_, const String & default_database_, const String & user_, const String & password_, const String & stage, bool randomize_, size_t max_iterations_, double max_time_, - const String & json_path_, const Settings & settings_) + const String & json_path_, size_t confidence_, const Settings & settings_) : - concurrency(concurrency_), delay(delay_), queue(concurrency), - connections(concurrency, host_, port_, default_database_, user_, password_, "benchmark", Protocol::Compression::Enable, secure_ ? Protocol::Secure::Enable : Protocol::Secure::Disable), - randomize(randomize_), max_iterations(max_iterations_), max_time(max_time_), - json_path(json_path_), settings(settings_), global_context(Context::createGlobal()), pool(concurrency) + concurrency(concurrency_), delay(delay_), queue(concurrency), randomize(randomize_), + cumulative(cumulative_), max_iterations(max_iterations_), max_time(max_time_), + confidence(confidence_), json_path(json_path_), settings(settings_), + global_context(Context::createGlobal()), pool(concurrency) { + const auto secure = secure_ ? Protocol::Secure::Enable : Protocol::Secure::Disable; + size_t connections_cnt = std::max(ports_.size(), hosts_.size()); + + connections.reserve(connections_cnt); + comparison_info_total.reserve(connections_cnt); + comparison_info_per_interval.reserve(connections_cnt); + + for (size_t i = 0; i < connections_cnt; ++i) + { + UInt16 cur_port = i >= ports_.size() ? 9000 : ports_[i]; + std::string cur_host = i >= hosts_.size() ? "localhost" : hosts_[i]; + + connections.emplace_back(std::make_unique(concurrency, cur_host, cur_port, default_database_, user_, password_, "benchmark", Protocol::Compression::Enable, secure)); + comparison_info_per_interval.emplace_back(std::make_shared()); + comparison_info_total.emplace_back(std::make_shared()); + } + global_context.makeGlobalContext(); std::cerr << std::fixed << std::setprecision(3); @@ -101,21 +122,29 @@ public: } private: - using Query = std::string; + using Entry = ConnectionPool::Entry; + using EntryPtr = std::shared_ptr; + using EntryPtrs = std::vector; unsigned concurrency; double delay; + using Query = std::string; using Queries = std::vector; Queries queries; using Queue = ConcurrentBoundedQueue; Queue queue; - ConnectionPool connections; + using ConnectionPoolUniq = std::unique_ptr; + using ConnectionPoolUniqs = std::vector; + ConnectionPoolUniqs connections; + bool randomize; + bool cumulative; size_t max_iterations; double max_time; + size_t confidence; String json_path; Settings settings; Context global_context; @@ -128,12 +157,12 @@ private: struct Stats { - Stopwatch watch; std::atomic queries{0}; size_t read_rows = 0; size_t read_bytes = 0; size_t result_rows = 0; size_t result_bytes = 0; + double work_time = 0; using Sampler = ReservoirSampler; Sampler sampler {1 << 16}; @@ -141,6 +170,7 @@ private: void add(double seconds, size_t read_rows_inc, size_t read_bytes_inc, size_t result_rows_inc, size_t result_bytes_inc) { ++queries; + work_time += seconds; read_rows += read_rows_inc; read_bytes += read_bytes_inc; result_rows += result_rows_inc; @@ -150,8 +180,8 @@ private: void clear() { - watch.restart(); queries = 0; + work_time = 0; read_rows = 0; read_bytes = 0; result_rows = 0; @@ -160,15 +190,18 @@ private: } }; - Stats info_per_interval; - Stats info_total; + using MultiStats = std::vector>; + MultiStats comparison_info_per_interval; + MultiStats comparison_info_total; + StudentTTest t_test; + + Stopwatch total_watch; Stopwatch delay_watch; std::mutex mutex; ThreadPool pool; - void readQueries() { ReadBufferFromFileDescriptor in(STDIN_FILENO); @@ -213,7 +246,7 @@ private: return false; } - if (max_time > 0 && info_total.watch.elapsedSeconds() >= max_time) + if (max_time > 0 && total_watch.elapsedSeconds() >= max_time) { std::cout << "Stopping launch of queries. Requested time limit is exhausted.\n"; return false; @@ -227,8 +260,8 @@ private: if (delay > 0 && delay_watch.elapsedSeconds() > delay) { - printNumberOfQueriesExecuted(info_total.queries); - report(info_per_interval); + printNumberOfQueriesExecuted(queries_executed); + cumulative ? report(comparison_info_total) : report(comparison_info_per_interval); delay_watch.restart(); } } @@ -242,11 +275,17 @@ private: std::uniform_int_distribution distribution(0, queries.size() - 1); for (size_t i = 0; i < concurrency; ++i) - pool.schedule(std::bind(&Benchmark::thread, this, - connections.get(ConnectionTimeouts::getTCPTimeoutsWithoutFailover(settings)))); + { + EntryPtrs connection_entries; + connection_entries.reserve(connections.size()); + + for (const auto & connection : connections) + connection_entries.emplace_back(std::make_shared(connection->get(ConnectionTimeouts::getTCPTimeoutsWithoutFailover(settings)))); + + pool.schedule(std::bind(&Benchmark::thread, this, connection_entries)); + } InterruptListener interrupt_listener; - info_per_interval.watch.restart(); delay_watch.restart(); /// Push queries into queue @@ -262,20 +301,24 @@ private: } pool.wait(); - info_total.watch.stop(); + total_watch.stop(); if (!json_path.empty()) - reportJSON(info_total, json_path); + reportJSON(comparison_info_total, json_path); - printNumberOfQueriesExecuted(info_total.queries); - report(info_total); + printNumberOfQueriesExecuted(queries_executed); + report(comparison_info_total); } - void thread(ConnectionPool::Entry connection) + void thread(EntryPtrs & connection_entries) { Query query; + /// Randomly choosing connection index + pcg64 generator(randomSeed()); + std::uniform_int_distribution distribution(0, connection_entries.size() - 1); + try { /// In these threads we do not accept INT signal. @@ -296,8 +339,7 @@ private: if (shutdown || (max_iterations && queries_executed == max_iterations)) return; } - - execute(connection, query); + execute(connection_entries, query, distribution(generator)); ++queries_executed; } } @@ -309,20 +351,19 @@ private: } } - - void execute(ConnectionPool::Entry & connection, Query & query) + void execute(EntryPtrs & connection_entries, Query & query, size_t connection_index) { Stopwatch watch; RemoteBlockInputStream stream( - *connection, + *(*connection_entries[connection_index]), query, {}, global_context, &settings, nullptr, Tables(), query_processing_stage); Progress progress; stream.setProgressCallback([&progress](const Progress & value) { progress.incrementPiecewiseAtomically(value); }); stream.readPrefix(); - while (Block block = stream.read()) - ; + while (Block block = stream.read()); + stream.readSuffix(); const BlockStreamProfileInfo & info = stream.getProfileInfo(); @@ -330,33 +371,47 @@ private: double seconds = watch.elapsedSeconds(); std::lock_guard lock(mutex); - info_per_interval.add(seconds, progress.read_rows, progress.read_bytes, info.rows, info.bytes); - info_total.add(seconds, progress.read_rows, progress.read_bytes, info.rows, info.bytes); + + comparison_info_per_interval[connection_index]->add(seconds, progress.read_rows, progress.read_bytes, info.rows, info.bytes); + comparison_info_total[connection_index]->add(seconds, progress.read_rows, progress.read_bytes, info.rows, info.bytes); + t_test.add(connection_index, seconds); } - - void report(Stats & info) + void report(MultiStats & infos) { std::lock_guard lock(mutex); - /// Avoid zeros, nans or exceptions - if (0 == info.queries) - return; + std::cerr << "\n"; + for (size_t i = 0; i < infos.size(); ++i) + { + const auto & info = infos[i]; - double seconds = info.watch.elapsedSeconds(); + /// Avoid zeros, nans or exceptions + if (0 == info->queries) + return; - std::cerr - << "\n" - << "QPS: " << (info.queries / seconds) << ", " - << "RPS: " << (info.read_rows / seconds) << ", " - << "MiB/s: " << (info.read_bytes / seconds / 1048576) << ", " - << "result RPS: " << (info.result_rows / seconds) << ", " - << "result MiB/s: " << (info.result_bytes / seconds / 1048576) << "." - << "\n"; + double seconds = info->work_time / concurrency; + + std::cerr + << connections[i]->getDescription() << ", " + << "queries " << info->queries << ", " + << "QPS: " << (info->queries / seconds) << ", " + << "RPS: " << (info->read_rows / seconds) << ", " + << "MiB/s: " << (info->read_bytes / seconds / 1048576) << ", " + << "result RPS: " << (info->result_rows / seconds) << ", " + << "result MiB/s: " << (info->result_bytes / seconds / 1048576) << "." + << "\n"; + } + std::cerr << "\n"; auto print_percentile = [&](double percent) { - std::cerr << percent << "%\t" << info.sampler.quantileInterpolated(percent / 100.0) << " sec." << std::endl; + std::cerr << percent << "%\t\t"; + for (const auto & info : infos) + { + std::cerr << info->sampler.quantileInterpolated(percent / 100.0) << " sec." << "\t"; + } + std::cerr << "\n"; }; for (int percent = 0; percent <= 90; percent += 10) @@ -367,10 +422,16 @@ private: print_percentile(99.9); print_percentile(99.99); - info.clear(); + std::cerr << "\n" << t_test.compareAndReport(confidence).second << "\n"; + + if (!cumulative) + { + for (auto & info : infos) + info->clear(); + } } - void reportJSON(Stats & info, const std::string & filename) + void reportJSON(MultiStats & infos, const std::string & filename) { WriteBufferFromFile json_out(filename); @@ -381,36 +442,41 @@ private: json_out << double_quote << key << ": " << value << (with_comma ? ",\n" : "\n"); }; - auto print_percentile = [&json_out, &info](auto percent, bool with_comma = true) + auto print_percentile = [&json_out](Stats & info, auto percent, bool with_comma = true) { json_out << "\"" << percent << "\"" << ": " << info.sampler.quantileInterpolated(percent / 100.0) << (with_comma ? ",\n" : "\n"); }; json_out << "{\n"; - json_out << double_quote << "statistics" << ": {\n"; + for (size_t i = 0; i < infos.size(); ++i) + { + const auto & info = infos[i]; - double seconds = info.watch.elapsedSeconds(); - print_key_value("QPS", info.queries / seconds); - print_key_value("RPS", info.read_rows / seconds); - print_key_value("MiBPS", info.read_bytes / seconds); - print_key_value("RPS_result", info.result_rows / seconds); - print_key_value("MiBPS_result", info.result_bytes / seconds); - print_key_value("num_queries", info.queries.load(), false); + json_out << double_quote << connections[i]->getDescription() << ": {\n"; + json_out << double_quote << "statistics" << ": {\n"; - json_out << "},\n"; + print_key_value("QPS", info->queries / info->work_time); + print_key_value("RPS", info->read_rows / info->work_time); + print_key_value("MiBPS", info->read_bytes / info->work_time); + print_key_value("RPS_result", info->result_rows / info->work_time); + print_key_value("MiBPS_result", info->result_bytes / info->work_time); + print_key_value("num_queries", info->queries.load(), false); - json_out << double_quote << "query_time_percentiles" << ": {\n"; + json_out << "},\n"; + json_out << double_quote << "query_time_percentiles" << ": {\n"; - for (int percent = 0; percent <= 90; percent += 10) - print_percentile(percent); + for (int percent = 0; percent <= 90; percent += 10) + print_percentile(*info, percent); - print_percentile(95); - print_percentile(99); - print_percentile(99.9); - print_percentile(99.99, false); + print_percentile(*info, 95); + print_percentile(*info, 99); + print_percentile(*info, 99.9); + print_percentile(*info, 99.99, false); - json_out << "}\n"; + json_out << "}\n"; + json_out << (i == infos.size() - 1 ? "}\n" : "},\n"); + } json_out << "}\n"; } @@ -439,7 +505,7 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv) { using boost::program_options::value; - boost::program_options::options_description desc("Allowed options"); + boost::program_options::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth()); desc.add_options() ("help", "produce help message") ("concurrency,c", value()->default_value(1), "number of parallel queries") @@ -449,13 +515,15 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv) ("timelimit,t", value()->default_value(0.), "stop launch of queries after specified time limit") ("randomize,r", value()->default_value(false), "randomize order of execution") ("json", value()->default_value(""), "write final report to specified file in JSON format") - ("host,h", value()->default_value("localhost"), "") - ("port", value()->default_value(9000), "") + ("host,h", value()->multitoken(), "") + ("port,p", value()->multitoken(), "") + ("cumulative", "prints cumulative data instead of data per interval") ("secure,s", "Use TLS connection") ("user", value()->default_value("default"), "") ("password", value()->default_value(""), "") ("database", value()->default_value("default"), "") ("stacktrace", "print stack traces of exceptions") + ("confidence", value()->default_value(5), "set the level of confidence for T-test [0=80%, 1=90%, 2=95%, 3=98%, 4=99%, 5=99.5%(default)") ; Settings settings; @@ -475,12 +543,15 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv) print_stacktrace = options.count("stacktrace"); UseSSL use_ssl; + Ports ports = options.count("port") ? options["port"].as() : Ports({9000}); + Strings hosts = options.count("host") ? options["host"].as() : Strings({"localhost"}); Benchmark benchmark( options["concurrency"].as(), options["delay"].as(), - options["host"].as(), - options["port"].as(), + std::move(hosts), + std::move(ports), + options.count("cumulative"), options.count("secure"), options["database"].as(), options["user"].as(), @@ -490,6 +561,7 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv) options["iterations"].as(), options["timelimit"].as(), options["json"].as(), + options["confidence"].as(), settings); return benchmark.run(); } diff --git a/dbms/programs/clang/CMakeLists.txt b/dbms/programs/clang/CMakeLists.txt deleted file mode 100644 index 82f520614f4..00000000000 --- a/dbms/programs/clang/CMakeLists.txt +++ /dev/null @@ -1,38 +0,0 @@ -if (USE_EMBEDDED_COMPILER) - add_subdirectory ("Compiler-${LLVM_VERSION}") -endif () - -if (CLICKHOUSE_SPLIT_BINARY) - if (USE_EMBEDDED_COMPILER) - link_directories (${LLVM_LIBRARY_DIRS}) - add_executable (clickhouse-clang clickhouse-clang.cpp) - target_link_libraries (clickhouse-clang PRIVATE clickhouse-compiler-lib) - add_executable (clickhouse-lld clickhouse-lld.cpp) - target_link_libraries (clickhouse-lld PRIVATE clickhouse-compiler-lib) - install (TARGETS clickhouse-clang clickhouse-lld RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) - endif () -endif () - -set (TMP_HEADERS_DIR "${CMAKE_CURRENT_BINARY_DIR}/${INTERNAL_COMPILER_HEADERS_RELATIVE}") -# Make and install empty dir for debian package if compiler disabled -add_custom_target (make-headers-directory ALL COMMAND ${CMAKE_COMMAND} -E make_directory ${TMP_HEADERS_DIR}) -install (DIRECTORY ${TMP_HEADERS_DIR} DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/clickhouse/${INTERNAL_COMPILER_HEADERS_DIR} COMPONENT clickhouse) -# TODO: fix on macos copy_headers.sh: sed --posix - -if (USE_EMBEDDED_COMPILER) - set (COPY_HEADERS_COMPILER "${CMAKE_CURRENT_BINARY_DIR}/../${INTERNAL_COMPILER_EXECUTABLE}") - set (COPY_HEADERS_DEPENDS clickhouse-clang) -elseif (EXISTS ${INTERNAL_COMPILER_BIN_ROOT}${INTERNAL_COMPILER_EXECUTABLE}) - set (COPY_HEADERS_COMPILER "${INTERNAL_COMPILER_BIN_ROOT}${INTERNAL_COMPILER_EXECUTABLE}") -endif () - -if (COPY_HEADERS_COMPILER) - add_custom_target (copy-headers [ -f ${TMP_HEADERS_DIR}/dbms/src/Interpreters/SpecializedAggregator.h ] || env CLANG=${COPY_HEADERS_COMPILER} BUILD_PATH=${ClickHouse_BINARY_DIR} DESTDIR=${ClickHouse_SOURCE_DIR} CMAKE_CXX_COMPILER_VERSION=${CMAKE_CXX_COMPILER_VERSION} ${CMAKE_CURRENT_SOURCE_DIR}/copy_headers.sh ${ClickHouse_SOURCE_DIR} ${TMP_HEADERS_DIR} DEPENDS ${COPY_HEADERS_DEPENDS} WORKING_DIRECTORY ${ClickHouse_SOURCE_DIR} SOURCES copy_headers.sh) - - if (USE_INTERNAL_LLVM_LIBRARY) - set (CLANG_HEADERS_DIR "${ClickHouse_SOURCE_DIR}/contrib/llvm/clang/lib/Headers") - set (CLANG_HEADERS_DEST "${TMP_HEADERS_DIR}/usr/local/lib/clang/${LLVM_VERSION}/include") # original: ${LLVM_LIBRARY_OUTPUT_INTDIR}/clang/${CLANG_VERSION}/include - add_custom_target (copy-headers-clang ${CMAKE_COMMAND} -E make_directory ${CLANG_HEADERS_DEST} && ${CMAKE_COMMAND} -E copy_if_different ${CLANG_HEADERS_DIR}/* ${CLANG_HEADERS_DEST} ) - add_dependencies (copy-headers copy-headers-clang) - endif () -endif () diff --git a/dbms/programs/clang/Compiler-5.0.0/CMakeLists.txt b/dbms/programs/clang/Compiler-5.0.0/CMakeLists.txt deleted file mode 100644 index 83e38cea257..00000000000 --- a/dbms/programs/clang/Compiler-5.0.0/CMakeLists.txt +++ /dev/null @@ -1,53 +0,0 @@ -add_definitions(-Wno-error -Wno-unused-parameter -Wno-non-virtual-dtor -U_LIBCPP_DEBUG) - -link_directories(${LLVM_LIBRARY_DIRS}) - -add_library(clickhouse-compiler-lib - driver.cpp - cc1_main.cpp - cc1as_main.cpp - lld.cpp) - -target_compile_options(clickhouse-compiler-lib PRIVATE -fno-rtti -fno-exceptions -g0) - -string(REPLACE "${INCLUDE_DEBUG_HELPERS}" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) # cant compile with -fno-rtti - -llvm_libs_all(REQUIRED_LLVM_LIBRARIES) - -message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}") - -target_include_directories(clickhouse-compiler-lib SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS}) - -# This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory. - -target_link_libraries(clickhouse-compiler-lib PRIVATE - -clangBasic clangCodeGen clangDriver clangFrontend clangFrontendTool -clangRewriteFrontend clangARCMigrate clangStaticAnalyzerFrontend -clangParse clangSerialization clangSema clangEdit clangStaticAnalyzerCheckers -clangASTMatchers clangStaticAnalyzerCore clangAnalysis clangAST clangRewrite clangLex clangBasic - -lldCOFF -lldDriver -lldELF -#lldMinGW -lldMachO -lldReaderWriter -lldYAML -#lldCommon -lldCore -lldConfig - -${REQUIRED_LLVM_LIBRARIES} - -LLVMSupport - -#Polly -#PollyISL -#PollyPPCG - -PUBLIC ${ZLIB_LIBRARIES} ${EXECINFO_LIBRARIES} Threads::Threads -${MALLOC_LIBRARIES} -${GLIBC_COMPATIBILITY_LIBRARIES} -${MEMCPY_LIBRARIES} -) diff --git a/dbms/programs/clang/Compiler-5.0.0/LICENSE.TXT b/dbms/programs/clang/Compiler-5.0.0/LICENSE.TXT deleted file mode 100644 index b452ca2efd8..00000000000 --- a/dbms/programs/clang/Compiler-5.0.0/LICENSE.TXT +++ /dev/null @@ -1,63 +0,0 @@ -============================================================================== -LLVM Release License -============================================================================== -University of Illinois/NCSA -Open Source License - -Copyright (c) 2007-2016 University of Illinois at Urbana-Champaign. -All rights reserved. - -Developed by: - - LLVM Team - - University of Illinois at Urbana-Champaign - - http://llvm.org - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal with -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimers. - - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimers in the - documentation and/or other materials provided with the distribution. - - * Neither the names of the LLVM Team, University of Illinois at - Urbana-Champaign, nor the names of its contributors may be used to - endorse or promote products derived from this Software without specific - prior written permission. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE -SOFTWARE. - -============================================================================== -The LLVM software contains code written by third parties. Such software will -have its own individual LICENSE.TXT file in the directory in which it appears. -This file will describe the copyrights, license, and restrictions which apply -to that code. - -The disclaimer of warranty in the University of Illinois Open Source License -applies to all code in the LLVM Distribution, and nothing in any of the -other licenses gives permission to use the names of the LLVM Team or the -University of Illinois to endorse or promote products derived from this -Software. - -The following pieces of software have additional or alternate copyrights, -licenses, and/or restrictions: - -Program Directory -------- --------- - - diff --git a/dbms/programs/clang/Compiler-5.0.0/cc1_main.cpp b/dbms/programs/clang/Compiler-5.0.0/cc1_main.cpp deleted file mode 100644 index f6eabaf3387..00000000000 --- a/dbms/programs/clang/Compiler-5.0.0/cc1_main.cpp +++ /dev/null @@ -1,242 +0,0 @@ -//===-- cc1_main.cpp - Clang CC1 Compiler Frontend ------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This is the entry point to the clang -cc1 functionality, which implements the -// core compiler functionality along with a number of additional tools for -// demonstration and testing purposes. -// -//===----------------------------------------------------------------------===// - -#include "llvm/Option/Arg.h" -#include "clang/CodeGen/ObjectFilePCHContainerOperations.h" -#include "clang/Config/config.h" -#include "clang/Driver/DriverDiagnostic.h" -#include "clang/Driver/Options.h" -#include "clang/Frontend/CompilerInstance.h" -#include "clang/Frontend/CompilerInvocation.h" -#include "clang/Frontend/FrontendDiagnostic.h" -#include "clang/Frontend/TextDiagnosticBuffer.h" -#include "clang/Frontend/TextDiagnosticPrinter.h" -#include "clang/Frontend/Utils.h" -#include "clang/FrontendTool/Utils.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/LinkAllPasses.h" -#include "llvm/Option/ArgList.h" -#include "llvm/Option/OptTable.h" -#include "llvm/Support/Compiler.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/ManagedStatic.h" -#include "llvm/Support/Signals.h" -#include "llvm/Support/TargetSelect.h" -#include "llvm/Support/Timer.h" -#include "llvm/Support/raw_ostream.h" -#include - -#ifdef CLANG_HAVE_RLIMITS -#include -#endif - -// have no .a version in packages -#undef LINK_POLLY_INTO_TOOLS - -using namespace clang; -using namespace llvm::opt; - -//===----------------------------------------------------------------------===// -// Main driver -//===----------------------------------------------------------------------===// - -static void LLVMErrorHandler(void *UserData, const std::string &Message, - bool GenCrashDiag) { - DiagnosticsEngine &Diags = *static_cast(UserData); - - Diags.Report(diag::err_fe_error_backend) << Message; - - // Run the interrupt handlers to make sure any special cleanups get done, in - // particular that we remove files registered with RemoveFileOnSignal. - llvm::sys::RunInterruptHandlers(); - - // We cannot recover from llvm errors. When reporting a fatal error, exit - // with status 70 to generate crash diagnostics. For BSD systems this is - // defined as an internal software error. Otherwise, exit with status 1. - exit(GenCrashDiag ? 70 : 1); -} - -#ifdef LINK_POLLY_INTO_TOOLS -namespace polly { -void initializePollyPasses(llvm::PassRegistry &Registry); -} -#endif - -#ifdef CLANG_HAVE_RLIMITS -// The amount of stack we think is "sufficient". If less than this much is -// available, we may be unable to reach our template instantiation depth -// limit and other similar limits. -// FIXME: Unify this with the stack we request when spawning a thread to build -// a module. -static const int kSufficientStack = 8 << 20; - -#if defined(__linux__) && defined(__PIE__) -static size_t getCurrentStackAllocation() { - // If we can't compute the current stack usage, allow for 512K of command - // line arguments and environment. - size_t Usage = 512 * 1024; - if (FILE *StatFile = fopen("/proc/self/stat", "r")) { - // We assume that the stack extends from its current address to the end of - // the environment space. In reality, there is another string literal (the - // program name) after the environment, but this is close enough (we only - // need to be within 100K or so). - unsigned long StackPtr, EnvEnd; - // Disable silly GCC -Wformat warning that complains about length - // modifiers on ignored format specifiers. We want to retain these - // for documentation purposes even though they have no effect. -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wformat" -#endif - if (fscanf(StatFile, - "%*d %*s %*c %*d %*d %*d %*d %*d %*u %*lu %*lu %*lu %*lu %*lu " - "%*lu %*ld %*ld %*ld %*ld %*ld %*ld %*llu %*lu %*ld %*lu %*lu " - "%*lu %*lu %lu %*lu %*lu %*lu %*lu %*lu %*llu %*lu %*lu %*d %*d " - "%*u %*u %*llu %*lu %*ld %*lu %*lu %*lu %*lu %*lu %*lu %lu %*d", - &StackPtr, &EnvEnd) == 2) { -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif - Usage = StackPtr < EnvEnd ? EnvEnd - StackPtr : StackPtr - EnvEnd; - } - fclose(StatFile); - } - return Usage; -} - -#include - -LLVM_ATTRIBUTE_NOINLINE -static void ensureStackAddressSpace(int ExtraChunks = 0) { - // Linux kernels prior to 4.1 will sometimes locate the heap of a PIE binary - // relatively close to the stack (they are only guaranteed to be 128MiB - // apart). This results in crashes if we happen to heap-allocate more than - // 128MiB before we reach our stack high-water mark. - // - // To avoid these crashes, ensure that we have sufficient virtual memory - // pages allocated before we start running. - size_t Curr = getCurrentStackAllocation(); - const int kTargetStack = kSufficientStack - 256 * 1024; - if (Curr < kTargetStack) { - volatile char *volatile Alloc = - static_cast(alloca(kTargetStack - Curr)); - Alloc[0] = 0; - Alloc[kTargetStack - Curr - 1] = 0; - } -} -#else -static void ensureStackAddressSpace() {} -#endif - -/// Attempt to ensure that we have at least 8MiB of usable stack space. -static void ensureSufficientStack() { - struct rlimit rlim; - if (getrlimit(RLIMIT_STACK, &rlim) != 0) - return; - - // Increase the soft stack limit to our desired level, if necessary and - // possible. - if (rlim.rlim_cur != RLIM_INFINITY && rlim.rlim_cur < kSufficientStack) { - // Try to allocate sufficient stack. - if (rlim.rlim_max == RLIM_INFINITY || rlim.rlim_max >= kSufficientStack) - rlim.rlim_cur = kSufficientStack; - else if (rlim.rlim_cur == rlim.rlim_max) - return; - else - rlim.rlim_cur = rlim.rlim_max; - - if (setrlimit(RLIMIT_STACK, &rlim) != 0 || - rlim.rlim_cur != kSufficientStack) - return; - } - - // We should now have a stack of size at least kSufficientStack. Ensure - // that we can actually use that much, if necessary. - ensureStackAddressSpace(); -} -#else -static void ensureSufficientStack() {} -#endif - -int cc1_main(ArrayRef Argv, const char *Argv0, void *MainAddr) { - ensureSufficientStack(); - - std::unique_ptr Clang(new CompilerInstance()); - IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); - - // Register the support for object-file-wrapped Clang modules. - auto PCHOps = Clang->getPCHContainerOperations(); - PCHOps->registerWriter(llvm::make_unique()); - PCHOps->registerReader(llvm::make_unique()); - - // Initialize targets first, so that --version shows registered targets. - llvm::InitializeAllTargets(); - llvm::InitializeAllTargetMCs(); - llvm::InitializeAllAsmPrinters(); - llvm::InitializeAllAsmParsers(); - -#ifdef LINK_POLLY_INTO_TOOLS - llvm::PassRegistry &Registry = *llvm::PassRegistry::getPassRegistry(); - polly::initializePollyPasses(Registry); -#endif - - // Buffer diagnostics from argument parsing so that we can output them using a - // well formed diagnostic object. - IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); - TextDiagnosticBuffer *DiagsBuffer = new TextDiagnosticBuffer; - DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagsBuffer); - bool Success = CompilerInvocation::CreateFromArgs( - Clang->getInvocation(), Argv.begin(), Argv.end(), Diags); - - // Infer the builtin include path if unspecified. - if (Clang->getHeaderSearchOpts().UseBuiltinIncludes && - Clang->getHeaderSearchOpts().ResourceDir.empty()) - Clang->getHeaderSearchOpts().ResourceDir = - CompilerInvocation::GetResourcesPath(Argv0, MainAddr); - - // Create the actual diagnostics engine. - Clang->createDiagnostics(); - if (!Clang->hasDiagnostics()) - return 1; - - // Set an error handler, so that any LLVM backend diagnostics go through our - // error handler. - llvm::install_fatal_error_handler(LLVMErrorHandler, - static_cast(&Clang->getDiagnostics())); - - DiagsBuffer->FlushDiagnostics(Clang->getDiagnostics()); - if (!Success) - return 1; - - // Execute the frontend actions. - Success = ExecuteCompilerInvocation(Clang.get()); - - // If any timers were active but haven't been destroyed yet, print their - // results now. This happens in -disable-free mode. - llvm::TimerGroup::printAll(llvm::errs()); - - // Our error handler depends on the Diagnostics object, which we're - // potentially about to delete. Uninstall the handler now so that any - // later errors use the default handling behavior instead. - llvm::remove_fatal_error_handler(); - - // When running with -disable-free, don't do any destruction or shutdown. - if (Clang->getFrontendOpts().DisableFree) { - BuryPointer(std::move(Clang)); - return !Success; - } - - return !Success; -} diff --git a/dbms/programs/clang/Compiler-5.0.0/cc1as_main.cpp b/dbms/programs/clang/Compiler-5.0.0/cc1as_main.cpp deleted file mode 100644 index 2fc2b508ef2..00000000000 --- a/dbms/programs/clang/Compiler-5.0.0/cc1as_main.cpp +++ /dev/null @@ -1,540 +0,0 @@ -//===-- cc1as_main.cpp - Clang Assembler ---------------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This is the entry point to the clang -cc1as functionality, which implements -// the direct interface to the LLVM MC based assembler. -// -//===----------------------------------------------------------------------===// - -#include "clang/Basic/Diagnostic.h" -#include "clang/Basic/DiagnosticOptions.h" -#include "clang/Driver/DriverDiagnostic.h" -#include "clang/Driver/Options.h" -#include "clang/Frontend/FrontendDiagnostic.h" -#include "clang/Frontend/TextDiagnosticPrinter.h" -#include "clang/Frontend/Utils.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/StringSwitch.h" -#include "llvm/ADT/Triple.h" -#include "llvm/IR/DataLayout.h" -#include "llvm/MC/MCAsmBackend.h" -#include "llvm/MC/MCAsmInfo.h" -#include "llvm/MC/MCCodeEmitter.h" -#include "llvm/MC/MCContext.h" -#include "llvm/MC/MCInstrInfo.h" -#include "llvm/MC/MCObjectFileInfo.h" -#include "llvm/MC/MCParser/MCAsmParser.h" -#include "llvm/MC/MCParser/MCTargetAsmParser.h" -#include "llvm/MC/MCRegisterInfo.h" -#include "llvm/MC/MCStreamer.h" -#include "llvm/MC/MCSubtargetInfo.h" -#include "llvm/MC/MCTargetOptions.h" -#include "llvm/Option/Arg.h" -#include "llvm/Option/ArgList.h" -#include "llvm/Option/OptTable.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/FileSystem.h" -#include "llvm/Support/FormattedStream.h" -#include "llvm/Support/Host.h" -#include "llvm/Support/MemoryBuffer.h" -#include "llvm/Support/Path.h" -#include "llvm/Support/Signals.h" -#include "llvm/Support/SourceMgr.h" -#include "llvm/Support/TargetRegistry.h" -#include "llvm/Support/TargetSelect.h" -#include "llvm/Support/Timer.h" -#include "llvm/Support/raw_ostream.h" -#include -#include -using namespace clang; -using namespace clang::driver; -using namespace clang::driver::options; -using namespace llvm; -using namespace llvm::opt; - -namespace { - -/// \brief Helper class for representing a single invocation of the assembler. -struct AssemblerInvocation { - /// @name Target Options - /// @{ - - /// The name of the target triple to assemble for. - std::string Triple; - - /// If given, the name of the target CPU to determine which instructions - /// are legal. - std::string CPU; - - /// The list of target specific features to enable or disable -- this should - /// be a list of strings starting with '+' or '-'. - std::vector Features; - - /// The list of symbol definitions. - std::vector SymbolDefs; - - /// @} - /// @name Language Options - /// @{ - - std::vector IncludePaths; - unsigned NoInitialTextSection : 1; - unsigned SaveTemporaryLabels : 1; - unsigned GenDwarfForAssembly : 1; - unsigned RelaxELFRelocations : 1; - unsigned DwarfVersion; - std::string DwarfDebugFlags; - std::string DwarfDebugProducer; - std::string DebugCompilationDir; - llvm::DebugCompressionType CompressDebugSections = - llvm::DebugCompressionType::None; - std::string MainFileName; - - /// @} - /// @name Frontend Options - /// @{ - - std::string InputFile; - std::vector LLVMArgs; - std::string OutputPath; - enum FileType { - FT_Asm, ///< Assembly (.s) output, transliterate mode. - FT_Null, ///< No output, for timing purposes. - FT_Obj ///< Object file output. - }; - FileType OutputType; - unsigned ShowHelp : 1; - unsigned ShowVersion : 1; - - /// @} - /// @name Transliterate Options - /// @{ - - unsigned OutputAsmVariant; - unsigned ShowEncoding : 1; - unsigned ShowInst : 1; - - /// @} - /// @name Assembler Options - /// @{ - - unsigned RelaxAll : 1; - unsigned NoExecStack : 1; - unsigned FatalWarnings : 1; - unsigned IncrementalLinkerCompatible : 1; - - /// The name of the relocation model to use. - std::string RelocationModel; - - /// @} - -public: - AssemblerInvocation() { - Triple = ""; - NoInitialTextSection = 0; - InputFile = "-"; - OutputPath = "-"; - OutputType = FT_Asm; - OutputAsmVariant = 0; - ShowInst = 0; - ShowEncoding = 0; - RelaxAll = 0; - NoExecStack = 0; - FatalWarnings = 0; - IncrementalLinkerCompatible = 0; - DwarfVersion = 0; - } - - static bool CreateFromArgs(AssemblerInvocation &Res, - ArrayRef Argv, - DiagnosticsEngine &Diags); -}; - -} - -bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts, - ArrayRef Argv, - DiagnosticsEngine &Diags) { - bool Success = true; - - // Parse the arguments. - std::unique_ptr OptTbl(createDriverOptTable()); - - const unsigned IncludedFlagsBitmask = options::CC1AsOption; - unsigned MissingArgIndex, MissingArgCount; - InputArgList Args = OptTbl->ParseArgs(Argv, MissingArgIndex, MissingArgCount, - IncludedFlagsBitmask); - - // Check for missing argument error. - if (MissingArgCount) { - Diags.Report(diag::err_drv_missing_argument) - << Args.getArgString(MissingArgIndex) << MissingArgCount; - Success = false; - } - - // Issue errors on unknown arguments. - for (const Arg *A : Args.filtered(OPT_UNKNOWN)) { - Diags.Report(diag::err_drv_unknown_argument) << A->getAsString(Args); - Success = false; - } - - // Construct the invocation. - - // Target Options - Opts.Triple = llvm::Triple::normalize(Args.getLastArgValue(OPT_triple)); - Opts.CPU = Args.getLastArgValue(OPT_target_cpu); - Opts.Features = Args.getAllArgValues(OPT_target_feature); - - // Use the default target triple if unspecified. - if (Opts.Triple.empty()) - Opts.Triple = llvm::sys::getDefaultTargetTriple(); - - // Language Options - Opts.IncludePaths = Args.getAllArgValues(OPT_I); - Opts.NoInitialTextSection = Args.hasArg(OPT_n); - Opts.SaveTemporaryLabels = Args.hasArg(OPT_msave_temp_labels); - // Any DebugInfoKind implies GenDwarfForAssembly. - Opts.GenDwarfForAssembly = Args.hasArg(OPT_debug_info_kind_EQ); - - if (const Arg *A = Args.getLastArg(OPT_compress_debug_sections, - OPT_compress_debug_sections_EQ)) { - if (A->getOption().getID() == OPT_compress_debug_sections) { - // TODO: be more clever about the compression type auto-detection - Opts.CompressDebugSections = llvm::DebugCompressionType::GNU; - } else { - Opts.CompressDebugSections = - llvm::StringSwitch(A->getValue()) - .Case("none", llvm::DebugCompressionType::None) - .Case("zlib", llvm::DebugCompressionType::Z) - .Case("zlib-gnu", llvm::DebugCompressionType::GNU) - .Default(llvm::DebugCompressionType::None); - } - } - - Opts.RelaxELFRelocations = Args.hasArg(OPT_mrelax_relocations); - Opts.DwarfVersion = getLastArgIntValue(Args, OPT_dwarf_version_EQ, 2, Diags); - Opts.DwarfDebugFlags = Args.getLastArgValue(OPT_dwarf_debug_flags); - Opts.DwarfDebugProducer = Args.getLastArgValue(OPT_dwarf_debug_producer); - Opts.DebugCompilationDir = Args.getLastArgValue(OPT_fdebug_compilation_dir); - Opts.MainFileName = Args.getLastArgValue(OPT_main_file_name); - - // Frontend Options - if (Args.hasArg(OPT_INPUT)) { - bool First = true; - for (const Arg *A : Args.filtered(OPT_INPUT)) { - if (First) { - Opts.InputFile = A->getValue(); - First = false; - } else { - Diags.Report(diag::err_drv_unknown_argument) << A->getAsString(Args); - Success = false; - } - } - } - Opts.LLVMArgs = Args.getAllArgValues(OPT_mllvm); - Opts.OutputPath = Args.getLastArgValue(OPT_o); - if (Arg *A = Args.getLastArg(OPT_filetype)) { - StringRef Name = A->getValue(); - unsigned OutputType = StringSwitch(Name) - .Case("asm", FT_Asm) - .Case("null", FT_Null) - .Case("obj", FT_Obj) - .Default(~0U); - if (OutputType == ~0U) { - Diags.Report(diag::err_drv_invalid_value) << A->getAsString(Args) << Name; - Success = false; - } else - Opts.OutputType = FileType(OutputType); - } - Opts.ShowHelp = Args.hasArg(OPT_help); - Opts.ShowVersion = Args.hasArg(OPT_version); - - // Transliterate Options - Opts.OutputAsmVariant = - getLastArgIntValue(Args, OPT_output_asm_variant, 0, Diags); - Opts.ShowEncoding = Args.hasArg(OPT_show_encoding); - Opts.ShowInst = Args.hasArg(OPT_show_inst); - - // Assemble Options - Opts.RelaxAll = Args.hasArg(OPT_mrelax_all); - Opts.NoExecStack = Args.hasArg(OPT_mno_exec_stack); - Opts.FatalWarnings = Args.hasArg(OPT_massembler_fatal_warnings); - Opts.RelocationModel = Args.getLastArgValue(OPT_mrelocation_model, "pic"); - Opts.IncrementalLinkerCompatible = - Args.hasArg(OPT_mincremental_linker_compatible); - Opts.SymbolDefs = Args.getAllArgValues(OPT_defsym); - - return Success; -} - -static std::unique_ptr -getOutputStream(AssemblerInvocation &Opts, DiagnosticsEngine &Diags, - bool Binary) { - if (Opts.OutputPath.empty()) - Opts.OutputPath = "-"; - - // Make sure that the Out file gets unlinked from the disk if we get a - // SIGINT. - if (Opts.OutputPath != "-") - sys::RemoveFileOnSignal(Opts.OutputPath); - - std::error_code EC; - auto Out = llvm::make_unique( - Opts.OutputPath, EC, (Binary ? sys::fs::F_None : sys::fs::F_Text)); - if (EC) { - Diags.Report(diag::err_fe_unable_to_open_output) << Opts.OutputPath - << EC.message(); - return nullptr; - } - - return Out; -} - -static bool ExecuteAssembler(AssemblerInvocation &Opts, - DiagnosticsEngine &Diags) { - // Get the target specific parser. - std::string Error; - const Target *TheTarget = TargetRegistry::lookupTarget(Opts.Triple, Error); - if (!TheTarget) - return Diags.Report(diag::err_target_unknown_triple) << Opts.Triple; - - ErrorOr> Buffer = - MemoryBuffer::getFileOrSTDIN(Opts.InputFile); - - if (std::error_code EC = Buffer.getError()) { - Error = EC.message(); - return Diags.Report(diag::err_fe_error_reading) << Opts.InputFile; - } - - SourceMgr SrcMgr; - - // Tell SrcMgr about this buffer, which is what the parser will pick up. - SrcMgr.AddNewSourceBuffer(std::move(*Buffer), SMLoc()); - - // Record the location of the include directories so that the lexer can find - // it later. - SrcMgr.setIncludeDirs(Opts.IncludePaths); - - std::unique_ptr MRI(TheTarget->createMCRegInfo(Opts.Triple)); - assert(MRI && "Unable to create target register info!"); - - std::unique_ptr MAI(TheTarget->createMCAsmInfo(*MRI, Opts.Triple)); - assert(MAI && "Unable to create target asm info!"); - - // Ensure MCAsmInfo initialization occurs before any use, otherwise sections - // may be created with a combination of default and explicit settings. - MAI->setCompressDebugSections(Opts.CompressDebugSections); - - MAI->setRelaxELFRelocations(Opts.RelaxELFRelocations); - - bool IsBinary = Opts.OutputType == AssemblerInvocation::FT_Obj; - std::unique_ptr FDOS = getOutputStream(Opts, Diags, IsBinary); - if (!FDOS) - return true; - - // FIXME: This is not pretty. MCContext has a ptr to MCObjectFileInfo and - // MCObjectFileInfo needs a MCContext reference in order to initialize itself. - std::unique_ptr MOFI(new MCObjectFileInfo()); - - MCContext Ctx(MAI.get(), MRI.get(), MOFI.get(), &SrcMgr); - - bool PIC = false; - if (Opts.RelocationModel == "static") { - PIC = false; - } else if (Opts.RelocationModel == "pic") { - PIC = true; - } else { - assert(Opts.RelocationModel == "dynamic-no-pic" && - "Invalid PIC model!"); - PIC = false; - } - - MOFI->InitMCObjectFileInfo(Triple(Opts.Triple), PIC, CodeModel::Default, Ctx); - if (Opts.SaveTemporaryLabels) - Ctx.setAllowTemporaryLabels(false); - if (Opts.GenDwarfForAssembly) - Ctx.setGenDwarfForAssembly(true); - if (!Opts.DwarfDebugFlags.empty()) - Ctx.setDwarfDebugFlags(StringRef(Opts.DwarfDebugFlags)); - if (!Opts.DwarfDebugProducer.empty()) - Ctx.setDwarfDebugProducer(StringRef(Opts.DwarfDebugProducer)); - if (!Opts.DebugCompilationDir.empty()) - Ctx.setCompilationDir(Opts.DebugCompilationDir); - if (!Opts.MainFileName.empty()) - Ctx.setMainFileName(StringRef(Opts.MainFileName)); - Ctx.setDwarfVersion(Opts.DwarfVersion); - - // Build up the feature string from the target feature list. - std::string FS; - if (!Opts.Features.empty()) { - FS = Opts.Features[0]; - for (unsigned i = 1, e = Opts.Features.size(); i != e; ++i) - FS += "," + Opts.Features[i]; - } - - std::unique_ptr Str; - - std::unique_ptr MCII(TheTarget->createMCInstrInfo()); - std::unique_ptr STI( - TheTarget->createMCSubtargetInfo(Opts.Triple, Opts.CPU, FS)); - - raw_pwrite_stream *Out = FDOS.get(); - std::unique_ptr BOS; - - // FIXME: There is a bit of code duplication with addPassesToEmitFile. - if (Opts.OutputType == AssemblerInvocation::FT_Asm) { - MCInstPrinter *IP = TheTarget->createMCInstPrinter( - llvm::Triple(Opts.Triple), Opts.OutputAsmVariant, *MAI, *MCII, *MRI); - MCCodeEmitter *CE = nullptr; - MCAsmBackend *MAB = nullptr; - if (Opts.ShowEncoding) { - CE = TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx); - MCTargetOptions Options; - MAB = TheTarget->createMCAsmBackend(*MRI, Opts.Triple, Opts.CPU, Options); - } - auto FOut = llvm::make_unique(*Out); - Str.reset(TheTarget->createAsmStreamer( - Ctx, std::move(FOut), /*asmverbose*/ true, - /*useDwarfDirectory*/ true, IP, CE, MAB, Opts.ShowInst)); - } else if (Opts.OutputType == AssemblerInvocation::FT_Null) { - Str.reset(createNullStreamer(Ctx)); - } else { - assert(Opts.OutputType == AssemblerInvocation::FT_Obj && - "Invalid file type!"); - if (!FDOS->supportsSeeking()) { - BOS = make_unique(*FDOS); - Out = BOS.get(); - } - - MCCodeEmitter *CE = TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx); - MCTargetOptions Options; - MCAsmBackend *MAB = TheTarget->createMCAsmBackend(*MRI, Opts.Triple, - Opts.CPU, Options); - Triple T(Opts.Triple); - Str.reset(TheTarget->createMCObjectStreamer( - T, Ctx, *MAB, *Out, CE, *STI, Opts.RelaxAll, - Opts.IncrementalLinkerCompatible, - /*DWARFMustBeAtTheEnd*/ true)); - Str.get()->InitSections(Opts.NoExecStack); - } - - bool Failed = false; - - std::unique_ptr Parser( - createMCAsmParser(SrcMgr, Ctx, *Str.get(), *MAI)); - - // FIXME: init MCTargetOptions from sanitizer flags here. - MCTargetOptions Options; - std::unique_ptr TAP( - TheTarget->createMCAsmParser(*STI, *Parser, *MCII, Options)); - if (!TAP) - Failed = Diags.Report(diag::err_target_unknown_triple) << Opts.Triple; - - // Set values for symbols, if any. - for (auto &S : Opts.SymbolDefs) { - auto Pair = StringRef(S).split('='); - auto Sym = Pair.first; - auto Val = Pair.second; - int64_t Value; - // We have already error checked this in the driver. - Val.getAsInteger(0, Value); - Ctx.setSymbolValue(Parser->getStreamer(), Sym, Value); - } - - if (!Failed) { - Parser->setTargetParser(*TAP.get()); - Failed = Parser->Run(Opts.NoInitialTextSection); - } - - // Close Streamer first. - // It might have a reference to the output stream. - Str.reset(); - // Close the output stream early. - BOS.reset(); - FDOS.reset(); - - // Delete output file if there were errors. - if (Failed && Opts.OutputPath != "-") - sys::fs::remove(Opts.OutputPath); - - return Failed; -} - -static void LLVMErrorHandler(void *UserData, const std::string &Message, - bool GenCrashDiag) { - DiagnosticsEngine &Diags = *static_cast(UserData); - - Diags.Report(diag::err_fe_error_backend) << Message; - - // We cannot recover from llvm errors. - exit(1); -} - -int cc1as_main(ArrayRef Argv, const char *Argv0, void *MainAddr) { - // Initialize targets and assembly printers/parsers. - InitializeAllTargetInfos(); - InitializeAllTargetMCs(); - InitializeAllAsmParsers(); - - // Construct our diagnostic client. - IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); - TextDiagnosticPrinter *DiagClient - = new TextDiagnosticPrinter(errs(), &*DiagOpts); - DiagClient->setPrefix("clang -cc1as"); - IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); - DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient); - - // Set an error handler, so that any LLVM backend diagnostics go through our - // error handler. - ScopedFatalErrorHandler FatalErrorHandler - (LLVMErrorHandler, static_cast(&Diags)); - - // Parse the arguments. - AssemblerInvocation Asm; - if (!AssemblerInvocation::CreateFromArgs(Asm, Argv, Diags)) - return 1; - - if (Asm.ShowHelp) { - std::unique_ptr Opts(driver::createDriverOptTable()); - Opts->PrintHelp(llvm::outs(), "clang -cc1as", "Clang Integrated Assembler", - /*Include=*/driver::options::CC1AsOption, /*Exclude=*/0); - return 0; - } - - // Honor -version. - // - // FIXME: Use a better -version message? - if (Asm.ShowVersion) { - llvm::cl::PrintVersionMessage(); - return 0; - } - - // Honor -mllvm. - // - // FIXME: Remove this, one day. - if (!Asm.LLVMArgs.empty()) { - unsigned NumArgs = Asm.LLVMArgs.size(); - auto Args = llvm::make_unique(NumArgs + 2); - Args[0] = "clang (LLVM option parsing)"; - for (unsigned i = 0; i != NumArgs; ++i) - Args[i + 1] = Asm.LLVMArgs[i].c_str(); - Args[NumArgs + 1] = nullptr; - llvm::cl::ParseCommandLineOptions(NumArgs + 1, Args.get()); - } - - // Execute the invocation, unless there were parsing errors. - bool Failed = Diags.hasErrorOccurred() || ExecuteAssembler(Asm, Diags); - - // If any timers were active but haven't been destroyed yet, print their - // results now. - TimerGroup::printAll(errs()); - - return !!Failed; -} diff --git a/dbms/programs/clang/Compiler-5.0.0/driver.cpp b/dbms/programs/clang/Compiler-5.0.0/driver.cpp deleted file mode 100644 index 5aec2759f9e..00000000000 --- a/dbms/programs/clang/Compiler-5.0.0/driver.cpp +++ /dev/null @@ -1,519 +0,0 @@ -//===-- driver.cpp - Clang GCC-Compatible Driver --------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This is the entry point to the clang driver; it is a thin wrapper -// for functionality in the Driver clang library. -// -//===----------------------------------------------------------------------===// - -#include "clang/Basic/DiagnosticOptions.h" -#include "clang/Driver/Compilation.h" -#include "clang/Driver/Driver.h" -#include "clang/Driver/DriverDiagnostic.h" -#include "clang/Driver/Options.h" -#include "clang/Driver/ToolChain.h" -#include "clang/Frontend/ChainedDiagnosticConsumer.h" -#include "clang/Frontend/CompilerInvocation.h" -#include "clang/Frontend/SerializedDiagnosticPrinter.h" -#include "clang/Frontend/TextDiagnosticPrinter.h" -#include "clang/Frontend/Utils.h" -#include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/SmallString.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/Config/llvm-config.h" -#include "llvm/Option/ArgList.h" -#include "llvm/Option/OptTable.h" -#include "llvm/Option/Option.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/FileSystem.h" -#include "llvm/Support/Host.h" -#include "llvm/Support/ManagedStatic.h" -#include "llvm/Support/Path.h" -#include "llvm/Support/PrettyStackTrace.h" -#include "llvm/Support/Process.h" -#include "llvm/Support/Program.h" -#include "llvm/Support/Regex.h" -#include "llvm/Support/Signals.h" -#include "llvm/Support/StringSaver.h" -#include "llvm/Support/TargetSelect.h" -#include "llvm/Support/Timer.h" -#include "llvm/Support/raw_ostream.h" -#include -#include -#include -using namespace clang; -using namespace clang::driver; -using namespace llvm::opt; - -std::string GetExecutablePath(const char *Argv0, bool CanonicalPrefixes) { - if (!CanonicalPrefixes) { - SmallString<128> ExecutablePath(Argv0); - // Do a PATH lookup if Argv0 isn't a valid path. - if (!llvm::sys::fs::exists(ExecutablePath)) - if (llvm::ErrorOr P = - llvm::sys::findProgramByName(ExecutablePath)) - ExecutablePath = *P; - return ExecutablePath.str(); - } - - // This just needs to be some symbol in the binary; C++ doesn't - // allow taking the address of ::main however. - void *P = (void*) (intptr_t) GetExecutablePath; - return llvm::sys::fs::getMainExecutable(Argv0, P); -} - -static const char *GetStableCStr(std::set &SavedStrings, - StringRef S) { - return SavedStrings.insert(S).first->c_str(); -} - -/// ApplyQAOverride - Apply a list of edits to the input argument lists. -/// -/// The input string is a space separate list of edits to perform, -/// they are applied in order to the input argument lists. Edits -/// should be one of the following forms: -/// -/// '#': Silence information about the changes to the command line arguments. -/// -/// '^': Add FOO as a new argument at the beginning of the command line. -/// -/// '+': Add FOO as a new argument at the end of the command line. -/// -/// 's/XXX/YYY/': Substitute the regular expression XXX with YYY in the command -/// line. -/// -/// 'xOPTION': Removes all instances of the literal argument OPTION. -/// -/// 'XOPTION': Removes all instances of the literal argument OPTION, -/// and the following argument. -/// -/// 'Ox': Removes all flags matching 'O' or 'O[sz0-9]' and adds 'Ox' -/// at the end of the command line. -/// -/// \param OS - The stream to write edit information to. -/// \param Args - The vector of command line arguments. -/// \param Edit - The override command to perform. -/// \param SavedStrings - Set to use for storing string representations. -static void ApplyOneQAOverride(raw_ostream &OS, - SmallVectorImpl &Args, - StringRef Edit, - std::set &SavedStrings) { - // This does not need to be efficient. - - if (Edit[0] == '^') { - const char *Str = - GetStableCStr(SavedStrings, Edit.substr(1)); - OS << "### Adding argument " << Str << " at beginning\n"; - Args.insert(Args.begin() + 1, Str); - } else if (Edit[0] == '+') { - const char *Str = - GetStableCStr(SavedStrings, Edit.substr(1)); - OS << "### Adding argument " << Str << " at end\n"; - Args.push_back(Str); - } else if (Edit[0] == 's' && Edit[1] == '/' && Edit.endswith("/") && - Edit.slice(2, Edit.size()-1).find('/') != StringRef::npos) { - StringRef MatchPattern = Edit.substr(2).split('/').first; - StringRef ReplPattern = Edit.substr(2).split('/').second; - ReplPattern = ReplPattern.slice(0, ReplPattern.size()-1); - - for (unsigned i = 1, e = Args.size(); i != e; ++i) { - // Ignore end-of-line response file markers - if (Args[i] == nullptr) - continue; - std::string Repl = llvm::Regex(MatchPattern).sub(ReplPattern, Args[i]); - - if (Repl != Args[i]) { - OS << "### Replacing '" << Args[i] << "' with '" << Repl << "'\n"; - Args[i] = GetStableCStr(SavedStrings, Repl); - } - } - } else if (Edit[0] == 'x' || Edit[0] == 'X') { - auto Option = Edit.substr(1); - for (unsigned i = 1; i < Args.size();) { - if (Option == Args[i]) { - OS << "### Deleting argument " << Args[i] << '\n'; - Args.erase(Args.begin() + i); - if (Edit[0] == 'X') { - if (i < Args.size()) { - OS << "### Deleting argument " << Args[i] << '\n'; - Args.erase(Args.begin() + i); - } else - OS << "### Invalid X edit, end of command line!\n"; - } - } else - ++i; - } - } else if (Edit[0] == 'O') { - for (unsigned i = 1; i < Args.size();) { - const char *A = Args[i]; - // Ignore end-of-line response file markers - if (A == nullptr) - continue; - if (A[0] == '-' && A[1] == 'O' && - (A[2] == '\0' || - (A[3] == '\0' && (A[2] == 's' || A[2] == 'z' || - ('0' <= A[2] && A[2] <= '9'))))) { - OS << "### Deleting argument " << Args[i] << '\n'; - Args.erase(Args.begin() + i); - } else - ++i; - } - OS << "### Adding argument " << Edit << " at end\n"; - Args.push_back(GetStableCStr(SavedStrings, '-' + Edit.str())); - } else { - OS << "### Unrecognized edit: " << Edit << "\n"; - } -} - -/// ApplyQAOverride - Apply a comma separate list of edits to the -/// input argument lists. See ApplyOneQAOverride. -static void ApplyQAOverride(SmallVectorImpl &Args, - const char *OverrideStr, - std::set &SavedStrings) { - raw_ostream *OS = &llvm::errs(); - - if (OverrideStr[0] == '#') { - ++OverrideStr; - OS = &llvm::nulls(); - } - - *OS << "### CCC_OVERRIDE_OPTIONS: " << OverrideStr << "\n"; - - // This does not need to be efficient. - - const char *S = OverrideStr; - while (*S) { - const char *End = ::strchr(S, ' '); - if (!End) - End = S + strlen(S); - if (End != S) - ApplyOneQAOverride(*OS, Args, std::string(S, End), SavedStrings); - S = End; - if (*S != '\0') - ++S; - } -} - -extern int cc1_main(ArrayRef Argv, const char *Argv0, - void *MainAddr); -extern int cc1as_main(ArrayRef Argv, const char *Argv0, - void *MainAddr); - -static void insertTargetAndModeArgs(StringRef Target, StringRef Mode, - SmallVectorImpl &ArgVector, - std::set &SavedStrings) { - if (!Mode.empty()) { - // Add the mode flag to the arguments. - auto it = ArgVector.begin(); - if (it != ArgVector.end()) - ++it; - ArgVector.insert(it, GetStableCStr(SavedStrings, Mode)); - } - - if (!Target.empty()) { - auto it = ArgVector.begin(); - if (it != ArgVector.end()) - ++it; - const char *arr[] = {"-target", GetStableCStr(SavedStrings, Target)}; - ArgVector.insert(it, std::begin(arr), std::end(arr)); - } -} - -static void getCLEnvVarOptions(std::string &EnvValue, llvm::StringSaver &Saver, - SmallVectorImpl &Opts) { - llvm::cl::TokenizeWindowsCommandLine(EnvValue, Saver, Opts); - // The first instance of '#' should be replaced with '=' in each option. - for (const char *Opt : Opts) - if (char *NumberSignPtr = const_cast(::strchr(Opt, '#'))) - *NumberSignPtr = '='; -} - -static void SetBackdoorDriverOutputsFromEnvVars(Driver &TheDriver) { - // Handle CC_PRINT_OPTIONS and CC_PRINT_OPTIONS_FILE. - TheDriver.CCPrintOptions = !!::getenv("CC_PRINT_OPTIONS"); - if (TheDriver.CCPrintOptions) - TheDriver.CCPrintOptionsFilename = ::getenv("CC_PRINT_OPTIONS_FILE"); - - // Handle CC_PRINT_HEADERS and CC_PRINT_HEADERS_FILE. - TheDriver.CCPrintHeaders = !!::getenv("CC_PRINT_HEADERS"); - if (TheDriver.CCPrintHeaders) - TheDriver.CCPrintHeadersFilename = ::getenv("CC_PRINT_HEADERS_FILE"); - - // Handle CC_LOG_DIAGNOSTICS and CC_LOG_DIAGNOSTICS_FILE. - TheDriver.CCLogDiagnostics = !!::getenv("CC_LOG_DIAGNOSTICS"); - if (TheDriver.CCLogDiagnostics) - TheDriver.CCLogDiagnosticsFilename = ::getenv("CC_LOG_DIAGNOSTICS_FILE"); -} - -static void FixupDiagPrefixExeName(TextDiagnosticPrinter *DiagClient, - const std::string &Path) { - // If the clang binary happens to be named cl.exe for compatibility reasons, - // use clang-cl.exe as the prefix to avoid confusion between clang and MSVC. - StringRef ExeBasename(llvm::sys::path::filename(Path)); - if (ExeBasename.equals_lower("cl.exe")) - ExeBasename = "clang-cl.exe"; - DiagClient->setPrefix(ExeBasename); -} - -// This lets us create the DiagnosticsEngine with a properly-filled-out -// DiagnosticOptions instance. -static DiagnosticOptions * -CreateAndPopulateDiagOpts(ArrayRef argv) { - auto *DiagOpts = new DiagnosticOptions; - std::unique_ptr Opts(createDriverOptTable()); - unsigned MissingArgIndex, MissingArgCount; - InputArgList Args = - Opts->ParseArgs(argv.slice(1), MissingArgIndex, MissingArgCount); - // We ignore MissingArgCount and the return value of ParseDiagnosticArgs. - // Any errors that would be diagnosed here will also be diagnosed later, - // when the DiagnosticsEngine actually exists. - (void)ParseDiagnosticArgs(*DiagOpts, Args); - return DiagOpts; -} - -static void SetInstallDir(SmallVectorImpl &argv, - Driver &TheDriver, bool CanonicalPrefixes) { - // Attempt to find the original path used to invoke the driver, to determine - // the installed path. We do this manually, because we want to support that - // path being a symlink. - SmallString<128> InstalledPath(argv[0]); - - // Do a PATH lookup, if there are no directory components. - if (llvm::sys::path::filename(InstalledPath) == InstalledPath) - if (llvm::ErrorOr Tmp = llvm::sys::findProgramByName( - llvm::sys::path::filename(InstalledPath.str()))) - InstalledPath = *Tmp; - - // FIXME: We don't actually canonicalize this, we just make it absolute. - if (CanonicalPrefixes) - llvm::sys::fs::make_absolute(InstalledPath); - - StringRef InstalledPathParent(llvm::sys::path::parent_path(InstalledPath)); - if (llvm::sys::fs::exists(InstalledPathParent)) - TheDriver.setInstalledDir(InstalledPathParent); -} - -static int ExecuteCC1Tool(ArrayRef argv, StringRef Tool) { - void *GetExecutablePathVP = (void *)(intptr_t) GetExecutablePath; - if (Tool == "") - return cc1_main(argv.slice(2), argv[0], GetExecutablePathVP); - if (Tool == "as") - return cc1as_main(argv.slice(2), argv[0], GetExecutablePathVP); - - // Reject unknown tools. - llvm::errs() << "error: unknown integrated tool '" << Tool << "'\n"; - return 1; -} - -int mainEntryClickHouseClang(int argc_, char **argv_) { - llvm::sys::PrintStackTraceOnErrorSignal(argv_[0]); - llvm::PrettyStackTraceProgram X(argc_, argv_); - llvm::llvm_shutdown_obj Y; // Call llvm_shutdown() on exit. - - if (llvm::sys::Process::FixupStandardFileDescriptors()) - return 1; - - SmallVector argv; - llvm::SpecificBumpPtrAllocator ArgAllocator; - std::error_code EC = llvm::sys::Process::GetArgumentVector( - argv, llvm::makeArrayRef(argv_, argc_), ArgAllocator); - if (EC) { - llvm::errs() << "error: couldn't get arguments: " << EC.message() << '\n'; - return 1; - } - - llvm::InitializeAllTargets(); - std::string ProgName = argv[0]; - std::pair TargetAndMode = - ToolChain::getTargetAndModeFromProgramName(ProgName); - - llvm::BumpPtrAllocator A; - llvm::StringSaver Saver(A); - - // Parse response files using the GNU syntax, unless we're in CL mode. There - // are two ways to put clang in CL compatibility mode: argv[0] is either - // clang-cl or cl, or --driver-mode=cl is on the command line. The normal - // command line parsing can't happen until after response file parsing, so we - // have to manually search for a --driver-mode=cl argument the hard way. - // Finally, our -cc1 tools don't care which tokenization mode we use because - // response files written by clang will tokenize the same way in either mode. - bool ClangCLMode = false; - if (TargetAndMode.second == "--driver-mode=cl" || - std::find_if(argv.begin(), argv.end(), [](const char *F) { - return F && strcmp(F, "--driver-mode=cl") == 0; - }) != argv.end()) { - ClangCLMode = true; - } - enum { Default, POSIX, Windows } RSPQuoting = Default; - for (const char *F : argv) { - if (strcmp(F, "--rsp-quoting=posix") == 0) - RSPQuoting = POSIX; - else if (strcmp(F, "--rsp-quoting=windows") == 0) - RSPQuoting = Windows; - } - - // Determines whether we want nullptr markers in argv to indicate response - // files end-of-lines. We only use this for the /LINK driver argument with - // clang-cl.exe on Windows. - bool MarkEOLs = ClangCLMode; - - llvm::cl::TokenizerCallback Tokenizer; - if (RSPQuoting == Windows || (RSPQuoting == Default && ClangCLMode)) - Tokenizer = &llvm::cl::TokenizeWindowsCommandLine; - else - Tokenizer = &llvm::cl::TokenizeGNUCommandLine; - - if (MarkEOLs && argv.size() > 1 && StringRef(argv[1]).startswith("-cc1")) - MarkEOLs = false; - llvm::cl::ExpandResponseFiles(Saver, Tokenizer, argv, MarkEOLs); - - // Handle -cc1 integrated tools, even if -cc1 was expanded from a response - // file. - auto FirstArg = std::find_if(argv.begin() + 1, argv.end(), - [](const char *A) { return A != nullptr; }); - if (FirstArg != argv.end() && StringRef(*FirstArg).startswith("-cc1")) { - // If -cc1 came from a response file, remove the EOL sentinels. - if (MarkEOLs) { - auto newEnd = std::remove(argv.begin(), argv.end(), nullptr); - argv.resize(newEnd - argv.begin()); - } - return ExecuteCC1Tool(argv, argv[1] + 4); - } - - bool CanonicalPrefixes = true; - for (int i = 1, size = argv.size(); i < size; ++i) { - // Skip end-of-line response file markers - if (argv[i] == nullptr) - continue; - if (StringRef(argv[i]) == "-no-canonical-prefixes") { - CanonicalPrefixes = false; - break; - } - } - - // Handle CL and _CL_ which permits additional command line options to be - // prepended or appended. - if (ClangCLMode) { - // Arguments in "CL" are prepended. - llvm::Optional OptCL = llvm::sys::Process::GetEnv("CL"); - if (OptCL.hasValue()) { - SmallVector PrependedOpts; - getCLEnvVarOptions(OptCL.getValue(), Saver, PrependedOpts); - - // Insert right after the program name to prepend to the argument list. - argv.insert(argv.begin() + 1, PrependedOpts.begin(), PrependedOpts.end()); - } - // Arguments in "_CL_" are appended. - llvm::Optional Opt_CL_ = llvm::sys::Process::GetEnv("_CL_"); - if (Opt_CL_.hasValue()) { - SmallVector AppendedOpts; - getCLEnvVarOptions(Opt_CL_.getValue(), Saver, AppendedOpts); - - // Insert at the end of the argument list to append. - argv.append(AppendedOpts.begin(), AppendedOpts.end()); - } - } - - std::set SavedStrings; - // Handle CCC_OVERRIDE_OPTIONS, used for editing a command line behind the - // scenes. - if (const char *OverrideStr = ::getenv("CCC_OVERRIDE_OPTIONS")) { - // FIXME: Driver shouldn't take extra initial argument. - ApplyQAOverride(argv, OverrideStr, SavedStrings); - } - - std::string Path = GetExecutablePath(argv[0], CanonicalPrefixes); - - IntrusiveRefCntPtr DiagOpts = - CreateAndPopulateDiagOpts(argv); - - TextDiagnosticPrinter *DiagClient - = new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts); - FixupDiagPrefixExeName(DiagClient, Path); - - IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); - - DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient); - - if (!DiagOpts->DiagnosticSerializationFile.empty()) { - auto SerializedConsumer = - clang::serialized_diags::create(DiagOpts->DiagnosticSerializationFile, - &*DiagOpts, /*MergeChildRecords=*/true); - Diags.setClient(new ChainedDiagnosticConsumer( - Diags.takeClient(), std::move(SerializedConsumer))); - } - - ProcessWarningOptions(Diags, *DiagOpts, /*ReportDiags=*/false); - - Driver TheDriver(Path, llvm::sys::getDefaultTargetTriple(), Diags); - SetInstallDir(argv, TheDriver, CanonicalPrefixes); - - insertTargetAndModeArgs(TargetAndMode.first, TargetAndMode.second, argv, - SavedStrings); - - SetBackdoorDriverOutputsFromEnvVars(TheDriver); - - std::unique_ptr C(TheDriver.BuildCompilation(argv)); - int Res = 1; - if (C && !C->containsError()) { - SmallVector, 4> FailingCommands; - Res = TheDriver.ExecuteCompilation(*C, FailingCommands); - - // Force a crash to test the diagnostics. - if (TheDriver.GenReproducer) { - Diags.Report(diag::err_drv_force_crash) - << !::getenv("FORCE_CLANG_DIAGNOSTICS_CRASH"); - - // Pretend that every command failed. - FailingCommands.clear(); - for (const auto &J : C->getJobs()) - if (const Command *C = dyn_cast(&J)) - FailingCommands.push_back(std::make_pair(-1, C)); - } - - for (const auto &P : FailingCommands) { - int CommandRes = P.first; - const Command *FailingCommand = P.second; - if (!Res) - Res = CommandRes; - - // If result status is < 0, then the driver command signalled an error. - // If result status is 70, then the driver command reported a fatal error. - // On Windows, abort will return an exit code of 3. In these cases, - // generate additional diagnostic information if possible. - bool DiagnoseCrash = CommandRes < 0 || CommandRes == 70; -#ifdef LLVM_ON_WIN32 - DiagnoseCrash |= CommandRes == 3; -#endif - if (DiagnoseCrash) { - TheDriver.generateCompilationDiagnostics(*C, *FailingCommand); - break; - } - } - } - - Diags.getClient()->finish(); - - // If any timers were active but haven't been destroyed yet, print their - // results now. This happens in -disable-free mode. - llvm::TimerGroup::printAll(llvm::errs()); - -#ifdef LLVM_ON_WIN32 - // Exit status should not be negative on Win32, unless abnormal termination. - // Once abnormal termiation was caught, negative status should not be - // propagated. - if (Res < 0) - Res = 1; -#endif - - // If we have multiple failing commands, we return the result of the first - // failing command. - return Res; -} diff --git a/dbms/programs/clang/Compiler-5.0.0/lld.cpp b/dbms/programs/clang/Compiler-5.0.0/lld.cpp deleted file mode 100644 index 5af29868864..00000000000 --- a/dbms/programs/clang/Compiler-5.0.0/lld.cpp +++ /dev/null @@ -1,23 +0,0 @@ -#include "lld/Driver/Driver.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/StringSwitch.h" -#include "llvm/ADT/Twine.h" -#include "llvm/Support/ManagedStatic.h" -#include "llvm/Support/Path.h" -#include "llvm/Support/PrettyStackTrace.h" -#include "llvm/Support/Signals.h" - -using namespace lld; -using namespace llvm; -using namespace llvm::sys; - -int mainEntryClickHouseLLD(int Argc, char **Argv) -{ - // Standard set up, so program fails gracefully. - sys::PrintStackTraceOnErrorSignal(Argv[0]); - PrettyStackTraceProgram StackPrinter(Argc, Argv); - llvm_shutdown_obj Shutdown; - - std::vector Args(Argv, Argv + Argc); - return !elf::link(Args, true); -} diff --git a/dbms/programs/clang/Compiler-5.0.1 b/dbms/programs/clang/Compiler-5.0.1 deleted file mode 120000 index 7c8af57399f..00000000000 --- a/dbms/programs/clang/Compiler-5.0.1 +++ /dev/null @@ -1 +0,0 @@ -Compiler-5.0.0 \ No newline at end of file diff --git a/dbms/programs/clang/Compiler-5.0.2 b/dbms/programs/clang/Compiler-5.0.2 deleted file mode 120000 index 7c8af57399f..00000000000 --- a/dbms/programs/clang/Compiler-5.0.2 +++ /dev/null @@ -1 +0,0 @@ -Compiler-5.0.0 \ No newline at end of file diff --git a/dbms/programs/clang/Compiler-6.0.0/CMakeLists.txt b/dbms/programs/clang/Compiler-6.0.0/CMakeLists.txt deleted file mode 100644 index 4a046674afc..00000000000 --- a/dbms/programs/clang/Compiler-6.0.0/CMakeLists.txt +++ /dev/null @@ -1,54 +0,0 @@ - -add_definitions(-Wno-error -Wno-unused-parameter -Wno-non-virtual-dtor -U_LIBCPP_DEBUG) - -link_directories(${LLVM_LIBRARY_DIRS}) - -add_library(clickhouse-compiler-lib - driver.cpp - cc1_main.cpp - cc1as_main.cpp - lld.cpp) - -target_compile_options(clickhouse-compiler-lib PRIVATE -fno-rtti -fno-exceptions -g0) - -string(REPLACE "${INCLUDE_DEBUG_HELPERS}" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) # cant compile with -fno-rtti - -llvm_libs_all(REQUIRED_LLVM_LIBRARIES) - -message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}") - -target_include_directories(clickhouse-compiler-lib SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS}) - -# This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory. - -target_link_libraries(clickhouse-compiler-lib PRIVATE - -clangBasic clangCodeGen clangDriver -clangFrontend -clangFrontendTool -clangRewriteFrontend clangARCMigrate clangStaticAnalyzerFrontend -clangParse clangSerialization clangSema clangEdit clangStaticAnalyzerCheckers -clangASTMatchers clangStaticAnalyzerCore clangAnalysis clangAST clangRewrite clangLex clangBasic - -lldCOFF -lldDriver -lldELF -lldMinGW -lldMachO -lldReaderWriter -lldYAML -lldCommon -lldCore -#lldWasm - -${REQUIRED_LLVM_LIBRARIES} - -#Polly -#PollyISL -#PollyPPCG - -PUBLIC ${ZLIB_LIBRARIES} ${EXECINFO_LIBRARIES} Threads::Threads -${MALLOC_LIBRARIES} -${GLIBC_COMPATIBILITY_LIBRARIES} -${MEMCPY_LIBRARIES} -) diff --git a/dbms/programs/clang/Compiler-6.0.0/LICENSE.TXT b/dbms/programs/clang/Compiler-6.0.0/LICENSE.TXT deleted file mode 100644 index b452ca2efd8..00000000000 --- a/dbms/programs/clang/Compiler-6.0.0/LICENSE.TXT +++ /dev/null @@ -1,63 +0,0 @@ -============================================================================== -LLVM Release License -============================================================================== -University of Illinois/NCSA -Open Source License - -Copyright (c) 2007-2016 University of Illinois at Urbana-Champaign. -All rights reserved. - -Developed by: - - LLVM Team - - University of Illinois at Urbana-Champaign - - http://llvm.org - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal with -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimers. - - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimers in the - documentation and/or other materials provided with the distribution. - - * Neither the names of the LLVM Team, University of Illinois at - Urbana-Champaign, nor the names of its contributors may be used to - endorse or promote products derived from this Software without specific - prior written permission. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE -SOFTWARE. - -============================================================================== -The LLVM software contains code written by third parties. Such software will -have its own individual LICENSE.TXT file in the directory in which it appears. -This file will describe the copyrights, license, and restrictions which apply -to that code. - -The disclaimer of warranty in the University of Illinois Open Source License -applies to all code in the LLVM Distribution, and nothing in any of the -other licenses gives permission to use the names of the LLVM Team or the -University of Illinois to endorse or promote products derived from this -Software. - -The following pieces of software have additional or alternate copyrights, -licenses, and/or restrictions: - -Program Directory -------- --------- - - diff --git a/dbms/programs/clang/Compiler-6.0.0/cc1_main.cpp b/dbms/programs/clang/Compiler-6.0.0/cc1_main.cpp deleted file mode 100644 index f6eabaf3387..00000000000 --- a/dbms/programs/clang/Compiler-6.0.0/cc1_main.cpp +++ /dev/null @@ -1,242 +0,0 @@ -//===-- cc1_main.cpp - Clang CC1 Compiler Frontend ------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This is the entry point to the clang -cc1 functionality, which implements the -// core compiler functionality along with a number of additional tools for -// demonstration and testing purposes. -// -//===----------------------------------------------------------------------===// - -#include "llvm/Option/Arg.h" -#include "clang/CodeGen/ObjectFilePCHContainerOperations.h" -#include "clang/Config/config.h" -#include "clang/Driver/DriverDiagnostic.h" -#include "clang/Driver/Options.h" -#include "clang/Frontend/CompilerInstance.h" -#include "clang/Frontend/CompilerInvocation.h" -#include "clang/Frontend/FrontendDiagnostic.h" -#include "clang/Frontend/TextDiagnosticBuffer.h" -#include "clang/Frontend/TextDiagnosticPrinter.h" -#include "clang/Frontend/Utils.h" -#include "clang/FrontendTool/Utils.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/LinkAllPasses.h" -#include "llvm/Option/ArgList.h" -#include "llvm/Option/OptTable.h" -#include "llvm/Support/Compiler.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/ManagedStatic.h" -#include "llvm/Support/Signals.h" -#include "llvm/Support/TargetSelect.h" -#include "llvm/Support/Timer.h" -#include "llvm/Support/raw_ostream.h" -#include - -#ifdef CLANG_HAVE_RLIMITS -#include -#endif - -// have no .a version in packages -#undef LINK_POLLY_INTO_TOOLS - -using namespace clang; -using namespace llvm::opt; - -//===----------------------------------------------------------------------===// -// Main driver -//===----------------------------------------------------------------------===// - -static void LLVMErrorHandler(void *UserData, const std::string &Message, - bool GenCrashDiag) { - DiagnosticsEngine &Diags = *static_cast(UserData); - - Diags.Report(diag::err_fe_error_backend) << Message; - - // Run the interrupt handlers to make sure any special cleanups get done, in - // particular that we remove files registered with RemoveFileOnSignal. - llvm::sys::RunInterruptHandlers(); - - // We cannot recover from llvm errors. When reporting a fatal error, exit - // with status 70 to generate crash diagnostics. For BSD systems this is - // defined as an internal software error. Otherwise, exit with status 1. - exit(GenCrashDiag ? 70 : 1); -} - -#ifdef LINK_POLLY_INTO_TOOLS -namespace polly { -void initializePollyPasses(llvm::PassRegistry &Registry); -} -#endif - -#ifdef CLANG_HAVE_RLIMITS -// The amount of stack we think is "sufficient". If less than this much is -// available, we may be unable to reach our template instantiation depth -// limit and other similar limits. -// FIXME: Unify this with the stack we request when spawning a thread to build -// a module. -static const int kSufficientStack = 8 << 20; - -#if defined(__linux__) && defined(__PIE__) -static size_t getCurrentStackAllocation() { - // If we can't compute the current stack usage, allow for 512K of command - // line arguments and environment. - size_t Usage = 512 * 1024; - if (FILE *StatFile = fopen("/proc/self/stat", "r")) { - // We assume that the stack extends from its current address to the end of - // the environment space. In reality, there is another string literal (the - // program name) after the environment, but this is close enough (we only - // need to be within 100K or so). - unsigned long StackPtr, EnvEnd; - // Disable silly GCC -Wformat warning that complains about length - // modifiers on ignored format specifiers. We want to retain these - // for documentation purposes even though they have no effect. -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wformat" -#endif - if (fscanf(StatFile, - "%*d %*s %*c %*d %*d %*d %*d %*d %*u %*lu %*lu %*lu %*lu %*lu " - "%*lu %*ld %*ld %*ld %*ld %*ld %*ld %*llu %*lu %*ld %*lu %*lu " - "%*lu %*lu %lu %*lu %*lu %*lu %*lu %*lu %*llu %*lu %*lu %*d %*d " - "%*u %*u %*llu %*lu %*ld %*lu %*lu %*lu %*lu %*lu %*lu %lu %*d", - &StackPtr, &EnvEnd) == 2) { -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif - Usage = StackPtr < EnvEnd ? EnvEnd - StackPtr : StackPtr - EnvEnd; - } - fclose(StatFile); - } - return Usage; -} - -#include - -LLVM_ATTRIBUTE_NOINLINE -static void ensureStackAddressSpace(int ExtraChunks = 0) { - // Linux kernels prior to 4.1 will sometimes locate the heap of a PIE binary - // relatively close to the stack (they are only guaranteed to be 128MiB - // apart). This results in crashes if we happen to heap-allocate more than - // 128MiB before we reach our stack high-water mark. - // - // To avoid these crashes, ensure that we have sufficient virtual memory - // pages allocated before we start running. - size_t Curr = getCurrentStackAllocation(); - const int kTargetStack = kSufficientStack - 256 * 1024; - if (Curr < kTargetStack) { - volatile char *volatile Alloc = - static_cast(alloca(kTargetStack - Curr)); - Alloc[0] = 0; - Alloc[kTargetStack - Curr - 1] = 0; - } -} -#else -static void ensureStackAddressSpace() {} -#endif - -/// Attempt to ensure that we have at least 8MiB of usable stack space. -static void ensureSufficientStack() { - struct rlimit rlim; - if (getrlimit(RLIMIT_STACK, &rlim) != 0) - return; - - // Increase the soft stack limit to our desired level, if necessary and - // possible. - if (rlim.rlim_cur != RLIM_INFINITY && rlim.rlim_cur < kSufficientStack) { - // Try to allocate sufficient stack. - if (rlim.rlim_max == RLIM_INFINITY || rlim.rlim_max >= kSufficientStack) - rlim.rlim_cur = kSufficientStack; - else if (rlim.rlim_cur == rlim.rlim_max) - return; - else - rlim.rlim_cur = rlim.rlim_max; - - if (setrlimit(RLIMIT_STACK, &rlim) != 0 || - rlim.rlim_cur != kSufficientStack) - return; - } - - // We should now have a stack of size at least kSufficientStack. Ensure - // that we can actually use that much, if necessary. - ensureStackAddressSpace(); -} -#else -static void ensureSufficientStack() {} -#endif - -int cc1_main(ArrayRef Argv, const char *Argv0, void *MainAddr) { - ensureSufficientStack(); - - std::unique_ptr Clang(new CompilerInstance()); - IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); - - // Register the support for object-file-wrapped Clang modules. - auto PCHOps = Clang->getPCHContainerOperations(); - PCHOps->registerWriter(llvm::make_unique()); - PCHOps->registerReader(llvm::make_unique()); - - // Initialize targets first, so that --version shows registered targets. - llvm::InitializeAllTargets(); - llvm::InitializeAllTargetMCs(); - llvm::InitializeAllAsmPrinters(); - llvm::InitializeAllAsmParsers(); - -#ifdef LINK_POLLY_INTO_TOOLS - llvm::PassRegistry &Registry = *llvm::PassRegistry::getPassRegistry(); - polly::initializePollyPasses(Registry); -#endif - - // Buffer diagnostics from argument parsing so that we can output them using a - // well formed diagnostic object. - IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); - TextDiagnosticBuffer *DiagsBuffer = new TextDiagnosticBuffer; - DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagsBuffer); - bool Success = CompilerInvocation::CreateFromArgs( - Clang->getInvocation(), Argv.begin(), Argv.end(), Diags); - - // Infer the builtin include path if unspecified. - if (Clang->getHeaderSearchOpts().UseBuiltinIncludes && - Clang->getHeaderSearchOpts().ResourceDir.empty()) - Clang->getHeaderSearchOpts().ResourceDir = - CompilerInvocation::GetResourcesPath(Argv0, MainAddr); - - // Create the actual diagnostics engine. - Clang->createDiagnostics(); - if (!Clang->hasDiagnostics()) - return 1; - - // Set an error handler, so that any LLVM backend diagnostics go through our - // error handler. - llvm::install_fatal_error_handler(LLVMErrorHandler, - static_cast(&Clang->getDiagnostics())); - - DiagsBuffer->FlushDiagnostics(Clang->getDiagnostics()); - if (!Success) - return 1; - - // Execute the frontend actions. - Success = ExecuteCompilerInvocation(Clang.get()); - - // If any timers were active but haven't been destroyed yet, print their - // results now. This happens in -disable-free mode. - llvm::TimerGroup::printAll(llvm::errs()); - - // Our error handler depends on the Diagnostics object, which we're - // potentially about to delete. Uninstall the handler now so that any - // later errors use the default handling behavior instead. - llvm::remove_fatal_error_handler(); - - // When running with -disable-free, don't do any destruction or shutdown. - if (Clang->getFrontendOpts().DisableFree) { - BuryPointer(std::move(Clang)); - return !Success; - } - - return !Success; -} diff --git a/dbms/programs/clang/Compiler-6.0.0/cc1as_main.cpp b/dbms/programs/clang/Compiler-6.0.0/cc1as_main.cpp deleted file mode 100644 index caf8409054a..00000000000 --- a/dbms/programs/clang/Compiler-6.0.0/cc1as_main.cpp +++ /dev/null @@ -1,540 +0,0 @@ -//===-- cc1as_main.cpp - Clang Assembler ---------------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This is the entry point to the clang -cc1as functionality, which implements -// the direct interface to the LLVM MC based assembler. -// -//===----------------------------------------------------------------------===// - -#include "clang/Basic/Diagnostic.h" -#include "clang/Basic/DiagnosticOptions.h" -#include "clang/Driver/DriverDiagnostic.h" -#include "clang/Driver/Options.h" -#include "clang/Frontend/FrontendDiagnostic.h" -#include "clang/Frontend/TextDiagnosticPrinter.h" -#include "clang/Frontend/Utils.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/StringSwitch.h" -#include "llvm/ADT/Triple.h" -#include "llvm/IR/DataLayout.h" -#include "llvm/MC/MCAsmBackend.h" -#include "llvm/MC/MCAsmInfo.h" -#include "llvm/MC/MCCodeEmitter.h" -#include "llvm/MC/MCContext.h" -#include "llvm/MC/MCInstrInfo.h" -#include "llvm/MC/MCObjectFileInfo.h" -#include "llvm/MC/MCParser/MCAsmParser.h" -#include "llvm/MC/MCParser/MCTargetAsmParser.h" -#include "llvm/MC/MCRegisterInfo.h" -#include "llvm/MC/MCStreamer.h" -#include "llvm/MC/MCSubtargetInfo.h" -#include "llvm/MC/MCTargetOptions.h" -#include "llvm/Option/Arg.h" -#include "llvm/Option/ArgList.h" -#include "llvm/Option/OptTable.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/FileSystem.h" -#include "llvm/Support/FormattedStream.h" -#include "llvm/Support/Host.h" -#include "llvm/Support/MemoryBuffer.h" -#include "llvm/Support/Path.h" -#include "llvm/Support/Signals.h" -#include "llvm/Support/SourceMgr.h" -#include "llvm/Support/TargetRegistry.h" -#include "llvm/Support/TargetSelect.h" -#include "llvm/Support/Timer.h" -#include "llvm/Support/raw_ostream.h" -#include -#include -using namespace clang; -using namespace clang::driver; -using namespace clang::driver::options; -using namespace llvm; -using namespace llvm::opt; - -namespace { - -/// \brief Helper class for representing a single invocation of the assembler. -struct AssemblerInvocation { - /// @name Target Options - /// @{ - - /// The name of the target triple to assemble for. - std::string Triple; - - /// If given, the name of the target CPU to determine which instructions - /// are legal. - std::string CPU; - - /// The list of target specific features to enable or disable -- this should - /// be a list of strings starting with '+' or '-'. - std::vector Features; - - /// The list of symbol definitions. - std::vector SymbolDefs; - - /// @} - /// @name Language Options - /// @{ - - std::vector IncludePaths; - unsigned NoInitialTextSection : 1; - unsigned SaveTemporaryLabels : 1; - unsigned GenDwarfForAssembly : 1; - unsigned RelaxELFRelocations : 1; - unsigned DwarfVersion; - std::string DwarfDebugFlags; - std::string DwarfDebugProducer; - std::string DebugCompilationDir; - llvm::DebugCompressionType CompressDebugSections = - llvm::DebugCompressionType::None; - std::string MainFileName; - - /// @} - /// @name Frontend Options - /// @{ - - std::string InputFile; - std::vector LLVMArgs; - std::string OutputPath; - enum FileType { - FT_Asm, ///< Assembly (.s) output, transliterate mode. - FT_Null, ///< No output, for timing purposes. - FT_Obj ///< Object file output. - }; - FileType OutputType; - unsigned ShowHelp : 1; - unsigned ShowVersion : 1; - - /// @} - /// @name Transliterate Options - /// @{ - - unsigned OutputAsmVariant; - unsigned ShowEncoding : 1; - unsigned ShowInst : 1; - - /// @} - /// @name Assembler Options - /// @{ - - unsigned RelaxAll : 1; - unsigned NoExecStack : 1; - unsigned FatalWarnings : 1; - unsigned IncrementalLinkerCompatible : 1; - - /// The name of the relocation model to use. - std::string RelocationModel; - - /// @} - -public: - AssemblerInvocation() { - Triple = ""; - NoInitialTextSection = 0; - InputFile = "-"; - OutputPath = "-"; - OutputType = FT_Asm; - OutputAsmVariant = 0; - ShowInst = 0; - ShowEncoding = 0; - RelaxAll = 0; - NoExecStack = 0; - FatalWarnings = 0; - IncrementalLinkerCompatible = 0; - DwarfVersion = 0; - } - - static bool CreateFromArgs(AssemblerInvocation &Res, - ArrayRef Argv, - DiagnosticsEngine &Diags); -}; - -} - -bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts, - ArrayRef Argv, - DiagnosticsEngine &Diags) { - bool Success = true; - - // Parse the arguments. - std::unique_ptr OptTbl(createDriverOptTable()); - - const unsigned IncludedFlagsBitmask = options::CC1AsOption; - unsigned MissingArgIndex, MissingArgCount; - InputArgList Args = OptTbl->ParseArgs(Argv, MissingArgIndex, MissingArgCount, - IncludedFlagsBitmask); - - // Check for missing argument error. - if (MissingArgCount) { - Diags.Report(diag::err_drv_missing_argument) - << Args.getArgString(MissingArgIndex) << MissingArgCount; - Success = false; - } - - // Issue errors on unknown arguments. - for (const Arg *A : Args.filtered(OPT_UNKNOWN)) { - Diags.Report(diag::err_drv_unknown_argument) << A->getAsString(Args); - Success = false; - } - - // Construct the invocation. - - // Target Options - Opts.Triple = llvm::Triple::normalize(Args.getLastArgValue(OPT_triple)); - Opts.CPU = Args.getLastArgValue(OPT_target_cpu); - Opts.Features = Args.getAllArgValues(OPT_target_feature); - - // Use the default target triple if unspecified. - if (Opts.Triple.empty()) - Opts.Triple = llvm::sys::getDefaultTargetTriple(); - - // Language Options - Opts.IncludePaths = Args.getAllArgValues(OPT_I); - Opts.NoInitialTextSection = Args.hasArg(OPT_n); - Opts.SaveTemporaryLabels = Args.hasArg(OPT_msave_temp_labels); - // Any DebugInfoKind implies GenDwarfForAssembly. - Opts.GenDwarfForAssembly = Args.hasArg(OPT_debug_info_kind_EQ); - - if (const Arg *A = Args.getLastArg(OPT_compress_debug_sections, - OPT_compress_debug_sections_EQ)) { - if (A->getOption().getID() == OPT_compress_debug_sections) { - // TODO: be more clever about the compression type auto-detection - Opts.CompressDebugSections = llvm::DebugCompressionType::GNU; - } else { - Opts.CompressDebugSections = - llvm::StringSwitch(A->getValue()) - .Case("none", llvm::DebugCompressionType::None) - .Case("zlib", llvm::DebugCompressionType::Z) - .Case("zlib-gnu", llvm::DebugCompressionType::GNU) - .Default(llvm::DebugCompressionType::None); - } - } - - Opts.RelaxELFRelocations = Args.hasArg(OPT_mrelax_relocations); - Opts.DwarfVersion = getLastArgIntValue(Args, OPT_dwarf_version_EQ, 2, Diags); - Opts.DwarfDebugFlags = Args.getLastArgValue(OPT_dwarf_debug_flags); - Opts.DwarfDebugProducer = Args.getLastArgValue(OPT_dwarf_debug_producer); - Opts.DebugCompilationDir = Args.getLastArgValue(OPT_fdebug_compilation_dir); - Opts.MainFileName = Args.getLastArgValue(OPT_main_file_name); - - // Frontend Options - if (Args.hasArg(OPT_INPUT)) { - bool First = true; - for (const Arg *A : Args.filtered(OPT_INPUT)) { - if (First) { - Opts.InputFile = A->getValue(); - First = false; - } else { - Diags.Report(diag::err_drv_unknown_argument) << A->getAsString(Args); - Success = false; - } - } - } - Opts.LLVMArgs = Args.getAllArgValues(OPT_mllvm); - Opts.OutputPath = Args.getLastArgValue(OPT_o); - if (Arg *A = Args.getLastArg(OPT_filetype)) { - StringRef Name = A->getValue(); - unsigned OutputType = StringSwitch(Name) - .Case("asm", FT_Asm) - .Case("null", FT_Null) - .Case("obj", FT_Obj) - .Default(~0U); - if (OutputType == ~0U) { - Diags.Report(diag::err_drv_invalid_value) << A->getAsString(Args) << Name; - Success = false; - } else - Opts.OutputType = FileType(OutputType); - } - Opts.ShowHelp = Args.hasArg(OPT_help); - Opts.ShowVersion = Args.hasArg(OPT_version); - - // Transliterate Options - Opts.OutputAsmVariant = - getLastArgIntValue(Args, OPT_output_asm_variant, 0, Diags); - Opts.ShowEncoding = Args.hasArg(OPT_show_encoding); - Opts.ShowInst = Args.hasArg(OPT_show_inst); - - // Assemble Options - Opts.RelaxAll = Args.hasArg(OPT_mrelax_all); - Opts.NoExecStack = Args.hasArg(OPT_mno_exec_stack); - Opts.FatalWarnings = Args.hasArg(OPT_massembler_fatal_warnings); - Opts.RelocationModel = Args.getLastArgValue(OPT_mrelocation_model, "pic"); - Opts.IncrementalLinkerCompatible = - Args.hasArg(OPT_mincremental_linker_compatible); - Opts.SymbolDefs = Args.getAllArgValues(OPT_defsym); - - return Success; -} - -static std::unique_ptr -getOutputStream(AssemblerInvocation &Opts, DiagnosticsEngine &Diags, - bool Binary) { - if (Opts.OutputPath.empty()) - Opts.OutputPath = "-"; - - // Make sure that the Out file gets unlinked from the disk if we get a - // SIGINT. - if (Opts.OutputPath != "-") - sys::RemoveFileOnSignal(Opts.OutputPath); - - std::error_code EC; - auto Out = llvm::make_unique( - Opts.OutputPath, EC, (Binary ? sys::fs::F_None : sys::fs::F_Text)); - if (EC) { - Diags.Report(diag::err_fe_unable_to_open_output) << Opts.OutputPath - << EC.message(); - return nullptr; - } - - return Out; -} - -static bool ExecuteAssembler(AssemblerInvocation &Opts, - DiagnosticsEngine &Diags) { - // Get the target specific parser. - std::string Error; - const Target *TheTarget = TargetRegistry::lookupTarget(Opts.Triple, Error); - if (!TheTarget) - return Diags.Report(diag::err_target_unknown_triple) << Opts.Triple; - - ErrorOr> Buffer = - MemoryBuffer::getFileOrSTDIN(Opts.InputFile); - - if (std::error_code EC = Buffer.getError()) { - Error = EC.message(); - return Diags.Report(diag::err_fe_error_reading) << Opts.InputFile; - } - - SourceMgr SrcMgr; - - // Tell SrcMgr about this buffer, which is what the parser will pick up. - SrcMgr.AddNewSourceBuffer(std::move(*Buffer), SMLoc()); - - // Record the location of the include directories so that the lexer can find - // it later. - SrcMgr.setIncludeDirs(Opts.IncludePaths); - - std::unique_ptr MRI(TheTarget->createMCRegInfo(Opts.Triple)); - assert(MRI && "Unable to create target register info!"); - - std::unique_ptr MAI(TheTarget->createMCAsmInfo(*MRI, Opts.Triple)); - assert(MAI && "Unable to create target asm info!"); - - // Ensure MCAsmInfo initialization occurs before any use, otherwise sections - // may be created with a combination of default and explicit settings. - MAI->setCompressDebugSections(Opts.CompressDebugSections); - - MAI->setRelaxELFRelocations(Opts.RelaxELFRelocations); - - bool IsBinary = Opts.OutputType == AssemblerInvocation::FT_Obj; - std::unique_ptr FDOS = getOutputStream(Opts, Diags, IsBinary); - if (!FDOS) - return true; - - // FIXME: This is not pretty. MCContext has a ptr to MCObjectFileInfo and - // MCObjectFileInfo needs a MCContext reference in order to initialize itself. - std::unique_ptr MOFI(new MCObjectFileInfo()); - - MCContext Ctx(MAI.get(), MRI.get(), MOFI.get(), &SrcMgr); - - bool PIC = false; - if (Opts.RelocationModel == "static") { - PIC = false; - } else if (Opts.RelocationModel == "pic") { - PIC = true; - } else { - assert(Opts.RelocationModel == "dynamic-no-pic" && - "Invalid PIC model!"); - PIC = false; - } - - MOFI->InitMCObjectFileInfo(Triple(Opts.Triple), PIC, Ctx); - if (Opts.SaveTemporaryLabels) - Ctx.setAllowTemporaryLabels(false); - if (Opts.GenDwarfForAssembly) - Ctx.setGenDwarfForAssembly(true); - if (!Opts.DwarfDebugFlags.empty()) - Ctx.setDwarfDebugFlags(StringRef(Opts.DwarfDebugFlags)); - if (!Opts.DwarfDebugProducer.empty()) - Ctx.setDwarfDebugProducer(StringRef(Opts.DwarfDebugProducer)); - if (!Opts.DebugCompilationDir.empty()) - Ctx.setCompilationDir(Opts.DebugCompilationDir); - if (!Opts.MainFileName.empty()) - Ctx.setMainFileName(StringRef(Opts.MainFileName)); - Ctx.setDwarfVersion(Opts.DwarfVersion); - - // Build up the feature string from the target feature list. - std::string FS; - if (!Opts.Features.empty()) { - FS = Opts.Features[0]; - for (unsigned i = 1, e = Opts.Features.size(); i != e; ++i) - FS += "," + Opts.Features[i]; - } - - std::unique_ptr Str; - - std::unique_ptr MCII(TheTarget->createMCInstrInfo()); - std::unique_ptr STI( - TheTarget->createMCSubtargetInfo(Opts.Triple, Opts.CPU, FS)); - - raw_pwrite_stream *Out = FDOS.get(); - std::unique_ptr BOS; - - // FIXME: There is a bit of code duplication with addPassesToEmitFile. - if (Opts.OutputType == AssemblerInvocation::FT_Asm) { - MCInstPrinter *IP = TheTarget->createMCInstPrinter( - llvm::Triple(Opts.Triple), Opts.OutputAsmVariant, *MAI, *MCII, *MRI); - MCCodeEmitter *CE = nullptr; - MCAsmBackend *MAB = nullptr; - if (Opts.ShowEncoding) { - CE = TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx); - MCTargetOptions Options; - MAB = TheTarget->createMCAsmBackend(*STI, *MRI, Options); - } - auto FOut = llvm::make_unique(*Out); - Str.reset(TheTarget->createAsmStreamer( - Ctx, std::move(FOut), /*asmverbose*/ true, - /*useDwarfDirectory*/ true, IP, CE, MAB, Opts.ShowInst)); - } else if (Opts.OutputType == AssemblerInvocation::FT_Null) { - Str.reset(createNullStreamer(Ctx)); - } else { - assert(Opts.OutputType == AssemblerInvocation::FT_Obj && - "Invalid file type!"); - if (!FDOS->supportsSeeking()) { - BOS = make_unique(*FDOS); - Out = BOS.get(); - } - - MCCodeEmitter *CE = TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx); - MCTargetOptions Options; - MCAsmBackend *MAB = TheTarget->createMCAsmBackend(*STI, *MRI, Options); - Triple T(Opts.Triple); - Str.reset(TheTarget->createMCObjectStreamer( - T, Ctx, std::unique_ptr(MAB), *Out, std::unique_ptr(CE), *STI, - Opts.RelaxAll, Opts.IncrementalLinkerCompatible, - /*DWARFMustBeAtTheEnd*/ true)); - Str.get()->InitSections(Opts.NoExecStack); - } - - bool Failed = false; - - std::unique_ptr Parser( - createMCAsmParser(SrcMgr, Ctx, *Str.get(), *MAI)); - - // FIXME: init MCTargetOptions from sanitizer flags here. - MCTargetOptions Options; - std::unique_ptr TAP( - TheTarget->createMCAsmParser(*STI, *Parser, *MCII, Options)); - if (!TAP) - Failed = Diags.Report(diag::err_target_unknown_triple) << Opts.Triple; - - // Set values for symbols, if any. - for (auto &S : Opts.SymbolDefs) { - auto Pair = StringRef(S).split('='); - auto Sym = Pair.first; - auto Val = Pair.second; - int64_t Value = 0; - // We have already error checked this in the driver. - Val.getAsInteger(0, Value); - Ctx.setSymbolValue(Parser->getStreamer(), Sym, Value); - } - - if (!Failed) { - Parser->setTargetParser(*TAP.get()); - Failed = Parser->Run(Opts.NoInitialTextSection); - } - - // Close Streamer first. - // It might have a reference to the output stream. - Str.reset(); - // Close the output stream early. - BOS.reset(); - FDOS.reset(); - - // Delete output file if there were errors. - if (Failed && Opts.OutputPath != "-") - sys::fs::remove(Opts.OutputPath); - - return Failed; -} - -static void LLVMErrorHandler(void *UserData, const std::string &Message, - bool GenCrashDiag) { - DiagnosticsEngine &Diags = *static_cast(UserData); - - Diags.Report(diag::err_fe_error_backend) << Message; - - // We cannot recover from llvm errors. - exit(1); -} - -int cc1as_main(ArrayRef Argv, const char *Argv0, void *MainAddr) { - // Initialize targets and assembly printers/parsers. - InitializeAllTargetInfos(); - InitializeAllTargetMCs(); - InitializeAllAsmParsers(); - - // Construct our diagnostic client. - IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); - TextDiagnosticPrinter *DiagClient - = new TextDiagnosticPrinter(errs(), &*DiagOpts); - DiagClient->setPrefix("clang -cc1as"); - IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); - DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient); - - // Set an error handler, so that any LLVM backend diagnostics go through our - // error handler. - ScopedFatalErrorHandler FatalErrorHandler - (LLVMErrorHandler, static_cast(&Diags)); - - // Parse the arguments. - AssemblerInvocation Asm; - if (!AssemblerInvocation::CreateFromArgs(Asm, Argv, Diags)) - return 1; - - if (Asm.ShowHelp) { - std::unique_ptr Opts(driver::createDriverOptTable()); - Opts->PrintHelp(llvm::outs(), "clang -cc1as", "Clang Integrated Assembler", - /*Include=*/driver::options::CC1AsOption, /*Exclude=*/0, - /*ShowAllAliases=*/false); - return 0; - } - - // Honor -version. - // - // FIXME: Use a better -version message? - if (Asm.ShowVersion) { - llvm::cl::PrintVersionMessage(); - return 0; - } - - // Honor -mllvm. - // - // FIXME: Remove this, one day. - if (!Asm.LLVMArgs.empty()) { - unsigned NumArgs = Asm.LLVMArgs.size(); - auto Args = llvm::make_unique(NumArgs + 2); - Args[0] = "clang (LLVM option parsing)"; - for (unsigned i = 0; i != NumArgs; ++i) - Args[i + 1] = Asm.LLVMArgs[i].c_str(); - Args[NumArgs + 1] = nullptr; - llvm::cl::ParseCommandLineOptions(NumArgs + 1, Args.get()); - } - - // Execute the invocation, unless there were parsing errors. - bool Failed = Diags.hasErrorOccurred() || ExecuteAssembler(Asm, Diags); - - // If any timers were active but haven't been destroyed yet, print their - // results now. - TimerGroup::printAll(errs()); - - return !!Failed; -} diff --git a/dbms/programs/clang/Compiler-6.0.0/driver.cpp b/dbms/programs/clang/Compiler-6.0.0/driver.cpp deleted file mode 100644 index 30511b8253a..00000000000 --- a/dbms/programs/clang/Compiler-6.0.0/driver.cpp +++ /dev/null @@ -1,520 +0,0 @@ -//===-- driver.cpp - Clang GCC-Compatible Driver --------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This is the entry point to the clang driver; it is a thin wrapper -// for functionality in the Driver clang library. -// -//===----------------------------------------------------------------------===// - -#include "clang/Basic/DiagnosticOptions.h" -#include "clang/Driver/Compilation.h" -#include "clang/Driver/Driver.h" -#include "clang/Driver/DriverDiagnostic.h" -#include "clang/Driver/Options.h" -#include "clang/Driver/ToolChain.h" -#include "clang/Frontend/ChainedDiagnosticConsumer.h" -#include "clang/Frontend/CompilerInvocation.h" -#include "clang/Frontend/SerializedDiagnosticPrinter.h" -#include "clang/Frontend/TextDiagnosticPrinter.h" -#include "clang/Frontend/Utils.h" -#include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/SmallString.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/Config/llvm-config.h" -#include "llvm/Option/ArgList.h" -#include "llvm/Option/OptTable.h" -#include "llvm/Option/Option.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/FileSystem.h" -#include "llvm/Support/Host.h" -#include "llvm/Support/ManagedStatic.h" -#include "llvm/Support/Path.h" -#include "llvm/Support/PrettyStackTrace.h" -#include "llvm/Support/Process.h" -#include "llvm/Support/Program.h" -#include "llvm/Support/Regex.h" -#include "llvm/Support/Signals.h" -#include "llvm/Support/StringSaver.h" -#include "llvm/Support/TargetSelect.h" -#include "llvm/Support/Timer.h" -#include "llvm/Support/raw_ostream.h" -#include -#include -#include -using namespace clang; -using namespace clang::driver; -using namespace llvm::opt; - -std::string GetExecutablePath(const char *Argv0, bool CanonicalPrefixes) { - if (!CanonicalPrefixes) { - SmallString<128> ExecutablePath(Argv0); - // Do a PATH lookup if Argv0 isn't a valid path. - if (!llvm::sys::fs::exists(ExecutablePath)) - if (llvm::ErrorOr P = - llvm::sys::findProgramByName(ExecutablePath)) - ExecutablePath = *P; - return ExecutablePath.str(); - } - - // This just needs to be some symbol in the binary; C++ doesn't - // allow taking the address of ::main however. - void *P = (void*) (intptr_t) GetExecutablePath; - return llvm::sys::fs::getMainExecutable(Argv0, P); -} - -static const char *GetStableCStr(std::set &SavedStrings, - StringRef S) { - return SavedStrings.insert(S).first->c_str(); -} - -/// ApplyQAOverride - Apply a list of edits to the input argument lists. -/// -/// The input string is a space separate list of edits to perform, -/// they are applied in order to the input argument lists. Edits -/// should be one of the following forms: -/// -/// '#': Silence information about the changes to the command line arguments. -/// -/// '^': Add FOO as a new argument at the beginning of the command line. -/// -/// '+': Add FOO as a new argument at the end of the command line. -/// -/// 's/XXX/YYY/': Substitute the regular expression XXX with YYY in the command -/// line. -/// -/// 'xOPTION': Removes all instances of the literal argument OPTION. -/// -/// 'XOPTION': Removes all instances of the literal argument OPTION, -/// and the following argument. -/// -/// 'Ox': Removes all flags matching 'O' or 'O[sz0-9]' and adds 'Ox' -/// at the end of the command line. -/// -/// \param OS - The stream to write edit information to. -/// \param Args - The vector of command line arguments. -/// \param Edit - The override command to perform. -/// \param SavedStrings - Set to use for storing string representations. -static void ApplyOneQAOverride(raw_ostream &OS, - SmallVectorImpl &Args, - StringRef Edit, - std::set &SavedStrings) { - // This does not need to be efficient. - - if (Edit[0] == '^') { - const char *Str = - GetStableCStr(SavedStrings, Edit.substr(1)); - OS << "### Adding argument " << Str << " at beginning\n"; - Args.insert(Args.begin() + 1, Str); - } else if (Edit[0] == '+') { - const char *Str = - GetStableCStr(SavedStrings, Edit.substr(1)); - OS << "### Adding argument " << Str << " at end\n"; - Args.push_back(Str); - } else if (Edit[0] == 's' && Edit[1] == '/' && Edit.endswith("/") && - Edit.slice(2, Edit.size()-1).find('/') != StringRef::npos) { - StringRef MatchPattern = Edit.substr(2).split('/').first; - StringRef ReplPattern = Edit.substr(2).split('/').second; - ReplPattern = ReplPattern.slice(0, ReplPattern.size()-1); - - for (unsigned i = 1, e = Args.size(); i != e; ++i) { - // Ignore end-of-line response file markers - if (Args[i] == nullptr) - continue; - std::string Repl = llvm::Regex(MatchPattern).sub(ReplPattern, Args[i]); - - if (Repl != Args[i]) { - OS << "### Replacing '" << Args[i] << "' with '" << Repl << "'\n"; - Args[i] = GetStableCStr(SavedStrings, Repl); - } - } - } else if (Edit[0] == 'x' || Edit[0] == 'X') { - auto Option = Edit.substr(1); - for (unsigned i = 1; i < Args.size();) { - if (Option == Args[i]) { - OS << "### Deleting argument " << Args[i] << '\n'; - Args.erase(Args.begin() + i); - if (Edit[0] == 'X') { - if (i < Args.size()) { - OS << "### Deleting argument " << Args[i] << '\n'; - Args.erase(Args.begin() + i); - } else - OS << "### Invalid X edit, end of command line!\n"; - } - } else - ++i; - } - } else if (Edit[0] == 'O') { - for (unsigned i = 1; i < Args.size();) { - const char *A = Args[i]; - // Ignore end-of-line response file markers - if (A == nullptr) - continue; - if (A[0] == '-' && A[1] == 'O' && - (A[2] == '\0' || - (A[3] == '\0' && (A[2] == 's' || A[2] == 'z' || - ('0' <= A[2] && A[2] <= '9'))))) { - OS << "### Deleting argument " << Args[i] << '\n'; - Args.erase(Args.begin() + i); - } else - ++i; - } - OS << "### Adding argument " << Edit << " at end\n"; - Args.push_back(GetStableCStr(SavedStrings, '-' + Edit.str())); - } else { - OS << "### Unrecognized edit: " << Edit << "\n"; - } -} - -/// ApplyQAOverride - Apply a comma separate list of edits to the -/// input argument lists. See ApplyOneQAOverride. -static void ApplyQAOverride(SmallVectorImpl &Args, - const char *OverrideStr, - std::set &SavedStrings) { - raw_ostream *OS = &llvm::errs(); - - if (OverrideStr[0] == '#') { - ++OverrideStr; - OS = &llvm::nulls(); - } - - *OS << "### CCC_OVERRIDE_OPTIONS: " << OverrideStr << "\n"; - - // This does not need to be efficient. - - const char *S = OverrideStr; - while (*S) { - const char *End = ::strchr(S, ' '); - if (!End) - End = S + strlen(S); - if (End != S) - ApplyOneQAOverride(*OS, Args, std::string(S, End), SavedStrings); - S = End; - if (*S != '\0') - ++S; - } -} - -extern int cc1_main(ArrayRef Argv, const char *Argv0, - void *MainAddr); -extern int cc1as_main(ArrayRef Argv, const char *Argv0, - void *MainAddr); - -static void insertTargetAndModeArgs(const ParsedClangName &NameParts, - SmallVectorImpl &ArgVector, - std::set &SavedStrings) { - // Put target and mode arguments at the start of argument list so that - // arguments specified in command line could override them. Avoid putting - // them at index 0, as an option like '-cc1' must remain the first. - auto InsertionPoint = ArgVector.begin(); - if (InsertionPoint != ArgVector.end()) - ++InsertionPoint; - - if (NameParts.DriverMode) { - // Add the mode flag to the arguments. - ArgVector.insert(InsertionPoint, - GetStableCStr(SavedStrings, NameParts.DriverMode)); - } - - if (NameParts.TargetIsValid) { - const char *arr[] = {"-target", GetStableCStr(SavedStrings, - NameParts.TargetPrefix)}; - ArgVector.insert(InsertionPoint, std::begin(arr), std::end(arr)); - } -} - -static void getCLEnvVarOptions(std::string &EnvValue, llvm::StringSaver &Saver, - SmallVectorImpl &Opts) { - llvm::cl::TokenizeWindowsCommandLine(EnvValue, Saver, Opts); - // The first instance of '#' should be replaced with '=' in each option. - for (const char *Opt : Opts) - if (char *NumberSignPtr = const_cast(::strchr(Opt, '#'))) - *NumberSignPtr = '='; -} - -static void SetBackdoorDriverOutputsFromEnvVars(Driver &TheDriver) { - // Handle CC_PRINT_OPTIONS and CC_PRINT_OPTIONS_FILE. - TheDriver.CCPrintOptions = !!::getenv("CC_PRINT_OPTIONS"); - if (TheDriver.CCPrintOptions) - TheDriver.CCPrintOptionsFilename = ::getenv("CC_PRINT_OPTIONS_FILE"); - - // Handle CC_PRINT_HEADERS and CC_PRINT_HEADERS_FILE. - TheDriver.CCPrintHeaders = !!::getenv("CC_PRINT_HEADERS"); - if (TheDriver.CCPrintHeaders) - TheDriver.CCPrintHeadersFilename = ::getenv("CC_PRINT_HEADERS_FILE"); - - // Handle CC_LOG_DIAGNOSTICS and CC_LOG_DIAGNOSTICS_FILE. - TheDriver.CCLogDiagnostics = !!::getenv("CC_LOG_DIAGNOSTICS"); - if (TheDriver.CCLogDiagnostics) - TheDriver.CCLogDiagnosticsFilename = ::getenv("CC_LOG_DIAGNOSTICS_FILE"); -} - -static void FixupDiagPrefixExeName(TextDiagnosticPrinter *DiagClient, - const std::string &Path) { - // If the clang binary happens to be named cl.exe for compatibility reasons, - // use clang-cl.exe as the prefix to avoid confusion between clang and MSVC. - StringRef ExeBasename(llvm::sys::path::filename(Path)); - if (ExeBasename.equals_lower("cl.exe")) - ExeBasename = "clang-cl.exe"; - DiagClient->setPrefix(ExeBasename); -} - -// This lets us create the DiagnosticsEngine with a properly-filled-out -// DiagnosticOptions instance. -static DiagnosticOptions * -CreateAndPopulateDiagOpts(ArrayRef argv) { - auto *DiagOpts = new DiagnosticOptions; - std::unique_ptr Opts(createDriverOptTable()); - unsigned MissingArgIndex, MissingArgCount; - InputArgList Args = - Opts->ParseArgs(argv.slice(1), MissingArgIndex, MissingArgCount); - // We ignore MissingArgCount and the return value of ParseDiagnosticArgs. - // Any errors that would be diagnosed here will also be diagnosed later, - // when the DiagnosticsEngine actually exists. - (void)ParseDiagnosticArgs(*DiagOpts, Args); - return DiagOpts; -} - -static void SetInstallDir(SmallVectorImpl &argv, - Driver &TheDriver, bool CanonicalPrefixes) { - // Attempt to find the original path used to invoke the driver, to determine - // the installed path. We do this manually, because we want to support that - // path being a symlink. - SmallString<128> InstalledPath(argv[0]); - - // Do a PATH lookup, if there are no directory components. - if (llvm::sys::path::filename(InstalledPath) == InstalledPath) - if (llvm::ErrorOr Tmp = llvm::sys::findProgramByName( - llvm::sys::path::filename(InstalledPath.str()))) - InstalledPath = *Tmp; - - // FIXME: We don't actually canonicalize this, we just make it absolute. - if (CanonicalPrefixes) - llvm::sys::fs::make_absolute(InstalledPath); - - StringRef InstalledPathParent(llvm::sys::path::parent_path(InstalledPath)); - if (llvm::sys::fs::exists(InstalledPathParent)) - TheDriver.setInstalledDir(InstalledPathParent); -} - -static int ExecuteCC1Tool(ArrayRef argv, StringRef Tool) { - void *GetExecutablePathVP = (void *)(intptr_t) GetExecutablePath; - if (Tool == "") - return cc1_main(argv.slice(2), argv[0], GetExecutablePathVP); - if (Tool == "as") - return cc1as_main(argv.slice(2), argv[0], GetExecutablePathVP); - - // Reject unknown tools. - llvm::errs() << "error: unknown integrated tool '" << Tool << "'\n"; - return 1; -} - -int mainEntryClickHouseClang(int argc_, char **argv_) { - llvm::sys::PrintStackTraceOnErrorSignal(argv_[0]); - llvm::PrettyStackTraceProgram X(argc_, argv_); - llvm::llvm_shutdown_obj Y; // Call llvm_shutdown() on exit. - - if (llvm::sys::Process::FixupStandardFileDescriptors()) - return 1; - - SmallVector argv; - llvm::SpecificBumpPtrAllocator ArgAllocator; - std::error_code EC = llvm::sys::Process::GetArgumentVector( - argv, llvm::makeArrayRef(argv_, argc_), ArgAllocator); - if (EC) { - llvm::errs() << "error: couldn't get arguments: " << EC.message() << '\n'; - return 1; - } - - llvm::InitializeAllTargets(); - auto TargetAndMode = ToolChain::getTargetAndModeFromProgramName(argv[0]); - - llvm::BumpPtrAllocator A; - llvm::StringSaver Saver(A); - - // Parse response files using the GNU syntax, unless we're in CL mode. There - // are two ways to put clang in CL compatibility mode: argv[0] is either - // clang-cl or cl, or --driver-mode=cl is on the command line. The normal - // command line parsing can't happen until after response file parsing, so we - // have to manually search for a --driver-mode=cl argument the hard way. - // Finally, our -cc1 tools don't care which tokenization mode we use because - // response files written by clang will tokenize the same way in either mode. - bool ClangCLMode = false; - if (StringRef(TargetAndMode.DriverMode).equals("--driver-mode=cl") || - std::find_if(argv.begin(), argv.end(), [](const char *F) { - return F && strcmp(F, "--driver-mode=cl") == 0; - }) != argv.end()) { - ClangCLMode = true; - } - enum { Default, POSIX, Windows } RSPQuoting = Default; - for (const char *F : argv) { - if (strcmp(F, "--rsp-quoting=posix") == 0) - RSPQuoting = POSIX; - else if (strcmp(F, "--rsp-quoting=windows") == 0) - RSPQuoting = Windows; - } - - // Determines whether we want nullptr markers in argv to indicate response - // files end-of-lines. We only use this for the /LINK driver argument with - // clang-cl.exe on Windows. - bool MarkEOLs = ClangCLMode; - - llvm::cl::TokenizerCallback Tokenizer; - if (RSPQuoting == Windows || (RSPQuoting == Default && ClangCLMode)) - Tokenizer = &llvm::cl::TokenizeWindowsCommandLine; - else - Tokenizer = &llvm::cl::TokenizeGNUCommandLine; - - if (MarkEOLs && argv.size() > 1 && StringRef(argv[1]).startswith("-cc1")) - MarkEOLs = false; - llvm::cl::ExpandResponseFiles(Saver, Tokenizer, argv, MarkEOLs); - - // Handle -cc1 integrated tools, even if -cc1 was expanded from a response - // file. - auto FirstArg = std::find_if(argv.begin() + 1, argv.end(), - [](const char *A) { return A != nullptr; }); - if (FirstArg != argv.end() && StringRef(*FirstArg).startswith("-cc1")) { - // If -cc1 came from a response file, remove the EOL sentinels. - if (MarkEOLs) { - auto newEnd = std::remove(argv.begin(), argv.end(), nullptr); - argv.resize(newEnd - argv.begin()); - } - return ExecuteCC1Tool(argv, argv[1] + 4); - } - - bool CanonicalPrefixes = true; - for (int i = 1, size = argv.size(); i < size; ++i) { - // Skip end-of-line response file markers - if (argv[i] == nullptr) - continue; - if (StringRef(argv[i]) == "-no-canonical-prefixes") { - CanonicalPrefixes = false; - break; - } - } - - // Handle CL and _CL_ which permits additional command line options to be - // prepended or appended. - if (ClangCLMode) { - // Arguments in "CL" are prepended. - llvm::Optional OptCL = llvm::sys::Process::GetEnv("CL"); - if (OptCL.hasValue()) { - SmallVector PrependedOpts; - getCLEnvVarOptions(OptCL.getValue(), Saver, PrependedOpts); - - // Insert right after the program name to prepend to the argument list. - argv.insert(argv.begin() + 1, PrependedOpts.begin(), PrependedOpts.end()); - } - // Arguments in "_CL_" are appended. - llvm::Optional Opt_CL_ = llvm::sys::Process::GetEnv("_CL_"); - if (Opt_CL_.hasValue()) { - SmallVector AppendedOpts; - getCLEnvVarOptions(Opt_CL_.getValue(), Saver, AppendedOpts); - - // Insert at the end of the argument list to append. - argv.append(AppendedOpts.begin(), AppendedOpts.end()); - } - } - - std::set SavedStrings; - // Handle CCC_OVERRIDE_OPTIONS, used for editing a command line behind the - // scenes. - if (const char *OverrideStr = ::getenv("CCC_OVERRIDE_OPTIONS")) { - // FIXME: Driver shouldn't take extra initial argument. - ApplyQAOverride(argv, OverrideStr, SavedStrings); - } - - std::string Path = GetExecutablePath(argv[0], CanonicalPrefixes); - - IntrusiveRefCntPtr DiagOpts = - CreateAndPopulateDiagOpts(argv); - - TextDiagnosticPrinter *DiagClient - = new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts); - FixupDiagPrefixExeName(DiagClient, Path); - - IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); - - DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient); - - if (!DiagOpts->DiagnosticSerializationFile.empty()) { - auto SerializedConsumer = - clang::serialized_diags::create(DiagOpts->DiagnosticSerializationFile, - &*DiagOpts, /*MergeChildRecords=*/true); - Diags.setClient(new ChainedDiagnosticConsumer( - Diags.takeClient(), std::move(SerializedConsumer))); - } - - ProcessWarningOptions(Diags, *DiagOpts, /*ReportDiags=*/false); - - Driver TheDriver(Path, llvm::sys::getDefaultTargetTriple(), Diags); - SetInstallDir(argv, TheDriver, CanonicalPrefixes); - TheDriver.setTargetAndMode(TargetAndMode); - - insertTargetAndModeArgs(TargetAndMode, argv, SavedStrings); - - SetBackdoorDriverOutputsFromEnvVars(TheDriver); - - std::unique_ptr C(TheDriver.BuildCompilation(argv)); - int Res = 1; - if (C && !C->containsError()) { - SmallVector, 4> FailingCommands; - Res = TheDriver.ExecuteCompilation(*C, FailingCommands); - - // Force a crash to test the diagnostics. - if (TheDriver.GenReproducer) { - Diags.Report(diag::err_drv_force_crash) - << !::getenv("FORCE_CLANG_DIAGNOSTICS_CRASH"); - - // Pretend that every command failed. - FailingCommands.clear(); - for (const auto &J : C->getJobs()) - if (const Command *C = dyn_cast(&J)) - FailingCommands.push_back(std::make_pair(-1, C)); - } - - for (const auto &P : FailingCommands) { - int CommandRes = P.first; - const Command *FailingCommand = P.second; - if (!Res) - Res = CommandRes; - - // If result status is < 0, then the driver command signalled an error. - // If result status is 70, then the driver command reported a fatal error. - // On Windows, abort will return an exit code of 3. In these cases, - // generate additional diagnostic information if possible. - bool DiagnoseCrash = CommandRes < 0 || CommandRes == 70; -#ifdef LLVM_ON_WIN32 - DiagnoseCrash |= CommandRes == 3; -#endif - if (DiagnoseCrash) { - TheDriver.generateCompilationDiagnostics(*C, *FailingCommand); - break; - } - } - } - - Diags.getClient()->finish(); - - // If any timers were active but haven't been destroyed yet, print their - // results now. This happens in -disable-free mode. - llvm::TimerGroup::printAll(llvm::errs()); - -#ifdef LLVM_ON_WIN32 - // Exit status should not be negative on Win32, unless abnormal termination. - // Once abnormal termiation was caught, negative status should not be - // propagated. - if (Res < 0) - Res = 1; -#endif - - // If we have multiple failing commands, we return the result of the first - // failing command. - return Res; -} diff --git a/dbms/programs/clang/Compiler-6.0.0/lld.cpp b/dbms/programs/clang/Compiler-6.0.0/lld.cpp deleted file mode 100644 index 696ff84dfe6..00000000000 --- a/dbms/programs/clang/Compiler-6.0.0/lld.cpp +++ /dev/null @@ -1,23 +0,0 @@ -#include "lld/Common/Driver.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/StringSwitch.h" -#include "llvm/ADT/Twine.h" -#include "llvm/Support/ManagedStatic.h" -#include "llvm/Support/Path.h" -#include "llvm/Support/PrettyStackTrace.h" -#include "llvm/Support/Signals.h" - -using namespace lld; -using namespace llvm; -using namespace llvm::sys; - -int mainEntryClickHouseLLD(int Argc, char **Argv) -{ - // Standard set up, so program fails gracefully. - sys::PrintStackTraceOnErrorSignal(Argv[0]); - PrettyStackTraceProgram StackPrinter(Argc, Argv); - llvm_shutdown_obj Shutdown; - - std::vector Args(Argv, Argv + Argc); - return !elf::link(Args, true); -} diff --git a/dbms/programs/clang/Compiler-6.0.0svn b/dbms/programs/clang/Compiler-6.0.0svn deleted file mode 120000 index 7eba9cc37d0..00000000000 --- a/dbms/programs/clang/Compiler-6.0.0svn +++ /dev/null @@ -1 +0,0 @@ -Compiler-6.0.0 \ No newline at end of file diff --git a/dbms/programs/clang/Compiler-6.0.1 b/dbms/programs/clang/Compiler-6.0.1 deleted file mode 120000 index 7eba9cc37d0..00000000000 --- a/dbms/programs/clang/Compiler-6.0.1 +++ /dev/null @@ -1 +0,0 @@ -Compiler-6.0.0 \ No newline at end of file diff --git a/dbms/programs/clang/Compiler-7.0.0/CMakeLists.txt b/dbms/programs/clang/Compiler-7.0.0/CMakeLists.txt deleted file mode 100644 index a042c821ec4..00000000000 --- a/dbms/programs/clang/Compiler-7.0.0/CMakeLists.txt +++ /dev/null @@ -1,49 +0,0 @@ -add_definitions(-Wno-error -Wno-unused-parameter -Wno-non-virtual-dtor -U_LIBCPP_DEBUG) - -link_directories(${LLVM_LIBRARY_DIRS}) - -add_library(clickhouse-compiler-lib - driver.cpp - cc1_main.cpp - cc1gen_reproducer_main.cpp - cc1as_main.cpp - lld.cpp) - -target_compile_options(clickhouse-compiler-lib PRIVATE -fno-rtti -fno-exceptions -g0) - -string(REPLACE "${INCLUDE_DEBUG_HELPERS}" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) # cant compile with -fno-rtti - -llvm_libs_all(REQUIRED_LLVM_LIBRARIES) - -message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}") - -target_include_directories(clickhouse-compiler-lib SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS}) - -# This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory. - -target_link_libraries(clickhouse-compiler-lib PRIVATE -clangBasic clangCodeGen clangDriver -clangFrontend -clangFrontendTool -clangRewriteFrontend clangARCMigrate clangStaticAnalyzerFrontend -clangParse clangSerialization clangSema clangEdit clangStaticAnalyzerCheckers -clangASTMatchers clangStaticAnalyzerCore clangAnalysis clangAST clangRewrite clangLex clangBasic -clangCrossTU clangIndex - -lldCOFF -lldDriver -lldELF -lldMinGW -lldMachO -lldReaderWriter -lldYAML -lldCommon -lldCore - -${REQUIRED_LLVM_LIBRARIES} - -PUBLIC ${ZLIB_LIBRARIES} ${EXECINFO_LIBRARIES} Threads::Threads -${MALLOC_LIBRARIES} -${GLIBC_COMPATIBILITY_LIBRARIES} -${MEMCPY_LIBRARIES} -) diff --git a/dbms/programs/clang/Compiler-7.0.0/cc1_main.cpp b/dbms/programs/clang/Compiler-7.0.0/cc1_main.cpp deleted file mode 100644 index 214bfa72476..00000000000 --- a/dbms/programs/clang/Compiler-7.0.0/cc1_main.cpp +++ /dev/null @@ -1,239 +0,0 @@ -//===-- cc1_main.cpp - Clang CC1 Compiler Frontend ------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This is the entry point to the clang -cc1 functionality, which implements the -// core compiler functionality along with a number of additional tools for -// demonstration and testing purposes. -// -//===----------------------------------------------------------------------===// - -#include "llvm/Option/Arg.h" -#include "clang/CodeGen/ObjectFilePCHContainerOperations.h" -#include "clang/Config/config.h" -#include "clang/Basic/Stack.h" -#include "clang/Driver/DriverDiagnostic.h" -#include "clang/Driver/Options.h" -#include "clang/Frontend/CompilerInstance.h" -#include "clang/Frontend/CompilerInvocation.h" -#include "clang/Frontend/FrontendDiagnostic.h" -#include "clang/Frontend/TextDiagnosticBuffer.h" -#include "clang/Frontend/TextDiagnosticPrinter.h" -#include "clang/Frontend/Utils.h" -#include "clang/FrontendTool/Utils.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/Config/llvm-config.h" -#include "llvm/LinkAllPasses.h" -#include "llvm/Option/ArgList.h" -#include "llvm/Option/OptTable.h" -#include "llvm/Support/Compiler.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/ManagedStatic.h" -#include "llvm/Support/Signals.h" -#include "llvm/Support/TargetSelect.h" -#include "llvm/Support/Timer.h" -#include "llvm/Support/raw_ostream.h" -#include - -#ifdef CLANG_HAVE_RLIMITS -#include -#endif - -// have no .a version in packages -#undef LINK_POLLY_INTO_TOOLS - -using namespace clang; -using namespace llvm::opt; - -//===----------------------------------------------------------------------===// -// Main driver -//===----------------------------------------------------------------------===// - -static void LLVMErrorHandler(void *UserData, const std::string &Message, - bool GenCrashDiag) { - DiagnosticsEngine &Diags = *static_cast(UserData); - - Diags.Report(diag::err_fe_error_backend) << Message; - - // Run the interrupt handlers to make sure any special cleanups get done, in - // particular that we remove files registered with RemoveFileOnSignal. - llvm::sys::RunInterruptHandlers(); - - // We cannot recover from llvm errors. When reporting a fatal error, exit - // with status 70 to generate crash diagnostics. For BSD systems this is - // defined as an internal software error. Otherwise, exit with status 1. - exit(GenCrashDiag ? 70 : 1); -} - -#ifdef LINK_POLLY_INTO_TOOLS -namespace polly { -void initializePollyPasses(llvm::PassRegistry &Registry); -} -#endif - -#ifdef CLANG_HAVE_RLIMITS -#if defined(__linux__) && defined(__PIE__) -static size_t getCurrentStackAllocation() { - // If we can't compute the current stack usage, allow for 512K of command - // line arguments and environment. - size_t Usage = 512 * 1024; - if (FILE *StatFile = fopen("/proc/self/stat", "r")) { - // We assume that the stack extends from its current address to the end of - // the environment space. In reality, there is another string literal (the - // program name) after the environment, but this is close enough (we only - // need to be within 100K or so). - unsigned long StackPtr, EnvEnd; - // Disable silly GCC -Wformat warning that complains about length - // modifiers on ignored format specifiers. We want to retain these - // for documentation purposes even though they have no effect. -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wformat" -#endif - if (fscanf(StatFile, - "%*d %*s %*c %*d %*d %*d %*d %*d %*u %*lu %*lu %*lu %*lu %*lu " - "%*lu %*ld %*ld %*ld %*ld %*ld %*ld %*llu %*lu %*ld %*lu %*lu " - "%*lu %*lu %lu %*lu %*lu %*lu %*lu %*lu %*llu %*lu %*lu %*d %*d " - "%*u %*u %*llu %*lu %*ld %*lu %*lu %*lu %*lu %*lu %*lu %lu %*d", - &StackPtr, &EnvEnd) == 2) { -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif - Usage = StackPtr < EnvEnd ? EnvEnd - StackPtr : StackPtr - EnvEnd; - } - fclose(StatFile); - } - return Usage; -} - -#include - -LLVM_ATTRIBUTE_NOINLINE -static void ensureStackAddressSpace() { - // Linux kernels prior to 4.1 will sometimes locate the heap of a PIE binary - // relatively close to the stack (they are only guaranteed to be 128MiB - // apart). This results in crashes if we happen to heap-allocate more than - // 128MiB before we reach our stack high-water mark. - // - // To avoid these crashes, ensure that we have sufficient virtual memory - // pages allocated before we start running. - size_t Curr = getCurrentStackAllocation(); - const int kTargetStack = DesiredStackSize - 256 * 1024; - if (Curr < kTargetStack) { - volatile char *volatile Alloc = - static_cast(alloca(kTargetStack - Curr)); - Alloc[0] = 0; - Alloc[kTargetStack - Curr - 1] = 0; - } -} -#else -static void ensureStackAddressSpace() {} -#endif - -/// Attempt to ensure that we have at least 8MiB of usable stack space. -static void ensureSufficientStack() { - struct rlimit rlim; - if (getrlimit(RLIMIT_STACK, &rlim) != 0) - return; - - // Increase the soft stack limit to our desired level, if necessary and - // possible. - if (rlim.rlim_cur != RLIM_INFINITY && - rlim.rlim_cur < rlim_t(DesiredStackSize)) { - // Try to allocate sufficient stack. - if (rlim.rlim_max == RLIM_INFINITY || - rlim.rlim_max >= rlim_t(DesiredStackSize)) - rlim.rlim_cur = DesiredStackSize; - else if (rlim.rlim_cur == rlim.rlim_max) - return; - else - rlim.rlim_cur = rlim.rlim_max; - - if (setrlimit(RLIMIT_STACK, &rlim) != 0 || - rlim.rlim_cur != DesiredStackSize) - return; - } - - // We should now have a stack of size at least DesiredStackSize. Ensure - // that we can actually use that much, if necessary. - ensureStackAddressSpace(); -} -#else -static void ensureSufficientStack() {} -#endif - -int cc1_main(ArrayRef Argv, const char *Argv0, void *MainAddr) { - ensureSufficientStack(); - - std::unique_ptr Clang(new CompilerInstance()); - IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); - - // Register the support for object-file-wrapped Clang modules. - auto PCHOps = Clang->getPCHContainerOperations(); - PCHOps->registerWriter(llvm::make_unique()); - PCHOps->registerReader(llvm::make_unique()); - - // Initialize targets first, so that --version shows registered targets. - llvm::InitializeAllTargets(); - llvm::InitializeAllTargetMCs(); - llvm::InitializeAllAsmPrinters(); - llvm::InitializeAllAsmParsers(); - -#ifdef LINK_POLLY_INTO_TOOLS - llvm::PassRegistry &Registry = *llvm::PassRegistry::getPassRegistry(); - polly::initializePollyPasses(Registry); -#endif - - // Buffer diagnostics from argument parsing so that we can output them using a - // well formed diagnostic object. - IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); - TextDiagnosticBuffer *DiagsBuffer = new TextDiagnosticBuffer; - DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagsBuffer); - bool Success = CompilerInvocation::CreateFromArgs( - Clang->getInvocation(), Argv.begin(), Argv.end(), Diags); - - // Infer the builtin include path if unspecified. - if (Clang->getHeaderSearchOpts().UseBuiltinIncludes && - Clang->getHeaderSearchOpts().ResourceDir.empty()) - Clang->getHeaderSearchOpts().ResourceDir = - CompilerInvocation::GetResourcesPath(Argv0, MainAddr); - - // Create the actual diagnostics engine. - Clang->createDiagnostics(); - if (!Clang->hasDiagnostics()) - return 1; - - // Set an error handler, so that any LLVM backend diagnostics go through our - // error handler. - llvm::install_fatal_error_handler(LLVMErrorHandler, - static_cast(&Clang->getDiagnostics())); - - DiagsBuffer->FlushDiagnostics(Clang->getDiagnostics()); - if (!Success) - return 1; - - // Execute the frontend actions. - Success = ExecuteCompilerInvocation(Clang.get()); - - // If any timers were active but haven't been destroyed yet, print their - // results now. This happens in -disable-free mode. - llvm::TimerGroup::printAll(llvm::errs()); - - // Our error handler depends on the Diagnostics object, which we're - // potentially about to delete. Uninstall the handler now so that any - // later errors use the default handling behavior instead. - llvm::remove_fatal_error_handler(); - - // When running with -disable-free, don't do any destruction or shutdown. - if (Clang->getFrontendOpts().DisableFree) { - BuryPointer(std::move(Clang)); - return !Success; - } - - return !Success; -} diff --git a/dbms/programs/clang/Compiler-7.0.0/cc1as_main.cpp b/dbms/programs/clang/Compiler-7.0.0/cc1as_main.cpp deleted file mode 100644 index d93b1f5cb1d..00000000000 --- a/dbms/programs/clang/Compiler-7.0.0/cc1as_main.cpp +++ /dev/null @@ -1,572 +0,0 @@ -//===-- cc1as_main.cpp - Clang Assembler ---------------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This is the entry point to the clang -cc1as functionality, which implements -// the direct interface to the LLVM MC based assembler. -// -//===----------------------------------------------------------------------===// - -#include "clang/Basic/Diagnostic.h" -#include "clang/Basic/DiagnosticOptions.h" -#include "clang/Driver/DriverDiagnostic.h" -#include "clang/Driver/Options.h" -#include "clang/Frontend/FrontendDiagnostic.h" -#include "clang/Frontend/TextDiagnosticPrinter.h" -#include "clang/Frontend/Utils.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/StringSwitch.h" -#include "llvm/ADT/Triple.h" -#include "llvm/IR/DataLayout.h" -#include "llvm/MC/MCAsmBackend.h" -#include "llvm/MC/MCAsmInfo.h" -#include "llvm/MC/MCCodeEmitter.h" -#include "llvm/MC/MCContext.h" -#include "llvm/MC/MCInstrInfo.h" -#include "llvm/MC/MCObjectFileInfo.h" -#include "llvm/MC/MCObjectWriter.h" -#include "llvm/MC/MCParser/MCAsmParser.h" -#include "llvm/MC/MCParser/MCTargetAsmParser.h" -#include "llvm/MC/MCRegisterInfo.h" -#include "llvm/MC/MCStreamer.h" -#include "llvm/MC/MCSubtargetInfo.h" -#include "llvm/MC/MCTargetOptions.h" -#include "llvm/Option/Arg.h" -#include "llvm/Option/ArgList.h" -#include "llvm/Option/OptTable.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/FileSystem.h" -#include "llvm/Support/FormattedStream.h" -#include "llvm/Support/Host.h" -#include "llvm/Support/MemoryBuffer.h" -#include "llvm/Support/Path.h" -#include "llvm/Support/Signals.h" -#include "llvm/Support/SourceMgr.h" -#include "llvm/Support/TargetRegistry.h" -#include "llvm/Support/TargetSelect.h" -#include "llvm/Support/Timer.h" -#include "llvm/Support/raw_ostream.h" -#include -#include -using namespace clang; -using namespace clang::driver; -using namespace clang::driver::options; -using namespace llvm; -using namespace llvm::opt; - -namespace { - -/// Helper class for representing a single invocation of the assembler. -struct AssemblerInvocation { - /// @name Target Options - /// @{ - - /// The name of the target triple to assemble for. - std::string Triple; - - /// If given, the name of the target CPU to determine which instructions - /// are legal. - std::string CPU; - - /// The list of target specific features to enable or disable -- this should - /// be a list of strings starting with '+' or '-'. - std::vector Features; - - /// The list of symbol definitions. - std::vector SymbolDefs; - - /// @} - /// @name Language Options - /// @{ - - std::vector IncludePaths; - unsigned NoInitialTextSection : 1; - unsigned SaveTemporaryLabels : 1; - unsigned GenDwarfForAssembly : 1; - unsigned RelaxELFRelocations : 1; - unsigned DwarfVersion; - std::string DwarfDebugFlags; - std::string DwarfDebugProducer; - std::string DebugCompilationDir; - std::map DebugPrefixMap; - llvm::DebugCompressionType CompressDebugSections = - llvm::DebugCompressionType::None; - std::string MainFileName; - std::string SplitDwarfFile; - - /// @} - /// @name Frontend Options - /// @{ - - std::string InputFile; - std::vector LLVMArgs; - std::string OutputPath; - enum FileType { - FT_Asm, ///< Assembly (.s) output, transliterate mode. - FT_Null, ///< No output, for timing purposes. - FT_Obj ///< Object file output. - }; - FileType OutputType; - unsigned ShowHelp : 1; - unsigned ShowVersion : 1; - - /// @} - /// @name Transliterate Options - /// @{ - - unsigned OutputAsmVariant; - unsigned ShowEncoding : 1; - unsigned ShowInst : 1; - - /// @} - /// @name Assembler Options - /// @{ - - unsigned RelaxAll : 1; - unsigned NoExecStack : 1; - unsigned FatalWarnings : 1; - unsigned IncrementalLinkerCompatible : 1; - - /// The name of the relocation model to use. - std::string RelocationModel; - - /// @} - -public: - AssemblerInvocation() { - Triple = ""; - NoInitialTextSection = 0; - InputFile = "-"; - OutputPath = "-"; - OutputType = FT_Asm; - OutputAsmVariant = 0; - ShowInst = 0; - ShowEncoding = 0; - RelaxAll = 0; - NoExecStack = 0; - FatalWarnings = 0; - IncrementalLinkerCompatible = 0; - DwarfVersion = 0; - } - - static bool CreateFromArgs(AssemblerInvocation &Res, - ArrayRef Argv, - DiagnosticsEngine &Diags); -}; - -} - -bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts, - ArrayRef Argv, - DiagnosticsEngine &Diags) { - bool Success = true; - - // Parse the arguments. - std::unique_ptr OptTbl(createDriverOptTable()); - - const unsigned IncludedFlagsBitmask = options::CC1AsOption; - unsigned MissingArgIndex, MissingArgCount; - InputArgList Args = OptTbl->ParseArgs(Argv, MissingArgIndex, MissingArgCount, - IncludedFlagsBitmask); - - // Check for missing argument error. - if (MissingArgCount) { - Diags.Report(diag::err_drv_missing_argument) - << Args.getArgString(MissingArgIndex) << MissingArgCount; - Success = false; - } - - // Issue errors on unknown arguments. - for (const Arg *A : Args.filtered(OPT_UNKNOWN)) { - auto ArgString = A->getAsString(Args); - std::string Nearest; - if (OptTbl->findNearest(ArgString, Nearest, IncludedFlagsBitmask) > 1) - Diags.Report(diag::err_drv_unknown_argument) << ArgString; - else - Diags.Report(diag::err_drv_unknown_argument_with_suggestion) - << ArgString << Nearest; - Success = false; - } - - // Construct the invocation. - - // Target Options - Opts.Triple = llvm::Triple::normalize(Args.getLastArgValue(OPT_triple)); - Opts.CPU = Args.getLastArgValue(OPT_target_cpu); - Opts.Features = Args.getAllArgValues(OPT_target_feature); - - // Use the default target triple if unspecified. - if (Opts.Triple.empty()) - Opts.Triple = llvm::sys::getDefaultTargetTriple(); - - // Language Options - Opts.IncludePaths = Args.getAllArgValues(OPT_I); - Opts.NoInitialTextSection = Args.hasArg(OPT_n); - Opts.SaveTemporaryLabels = Args.hasArg(OPT_msave_temp_labels); - // Any DebugInfoKind implies GenDwarfForAssembly. - Opts.GenDwarfForAssembly = Args.hasArg(OPT_debug_info_kind_EQ); - - if (const Arg *A = Args.getLastArg(OPT_compress_debug_sections, - OPT_compress_debug_sections_EQ)) { - if (A->getOption().getID() == OPT_compress_debug_sections) { - // TODO: be more clever about the compression type auto-detection - Opts.CompressDebugSections = llvm::DebugCompressionType::GNU; - } else { - Opts.CompressDebugSections = - llvm::StringSwitch(A->getValue()) - .Case("none", llvm::DebugCompressionType::None) - .Case("zlib", llvm::DebugCompressionType::Z) - .Case("zlib-gnu", llvm::DebugCompressionType::GNU) - .Default(llvm::DebugCompressionType::None); - } - } - - Opts.RelaxELFRelocations = Args.hasArg(OPT_mrelax_relocations); - Opts.DwarfVersion = getLastArgIntValue(Args, OPT_dwarf_version_EQ, 2, Diags); - Opts.DwarfDebugFlags = Args.getLastArgValue(OPT_dwarf_debug_flags); - Opts.DwarfDebugProducer = Args.getLastArgValue(OPT_dwarf_debug_producer); - Opts.DebugCompilationDir = Args.getLastArgValue(OPT_fdebug_compilation_dir); - Opts.MainFileName = Args.getLastArgValue(OPT_main_file_name); - - for (const auto &Arg : Args.getAllArgValues(OPT_fdebug_prefix_map_EQ)) - Opts.DebugPrefixMap.insert(StringRef(Arg).split('=')); - - // Frontend Options - if (Args.hasArg(OPT_INPUT)) { - bool First = true; - for (const Arg *A : Args.filtered(OPT_INPUT)) { - if (First) { - Opts.InputFile = A->getValue(); - First = false; - } else { - Diags.Report(diag::err_drv_unknown_argument) << A->getAsString(Args); - Success = false; - } - } - } - Opts.LLVMArgs = Args.getAllArgValues(OPT_mllvm); - Opts.OutputPath = Args.getLastArgValue(OPT_o); - Opts.SplitDwarfFile = Args.getLastArgValue(OPT_split_dwarf_file); - if (Arg *A = Args.getLastArg(OPT_filetype)) { - StringRef Name = A->getValue(); - unsigned OutputType = StringSwitch(Name) - .Case("asm", FT_Asm) - .Case("null", FT_Null) - .Case("obj", FT_Obj) - .Default(~0U); - if (OutputType == ~0U) { - Diags.Report(diag::err_drv_invalid_value) << A->getAsString(Args) << Name; - Success = false; - } else - Opts.OutputType = FileType(OutputType); - } - Opts.ShowHelp = Args.hasArg(OPT_help); - Opts.ShowVersion = Args.hasArg(OPT_version); - - // Transliterate Options - Opts.OutputAsmVariant = - getLastArgIntValue(Args, OPT_output_asm_variant, 0, Diags); - Opts.ShowEncoding = Args.hasArg(OPT_show_encoding); - Opts.ShowInst = Args.hasArg(OPT_show_inst); - - // Assemble Options - Opts.RelaxAll = Args.hasArg(OPT_mrelax_all); - Opts.NoExecStack = Args.hasArg(OPT_mno_exec_stack); - Opts.FatalWarnings = Args.hasArg(OPT_massembler_fatal_warnings); - Opts.RelocationModel = Args.getLastArgValue(OPT_mrelocation_model, "pic"); - Opts.IncrementalLinkerCompatible = - Args.hasArg(OPT_mincremental_linker_compatible); - Opts.SymbolDefs = Args.getAllArgValues(OPT_defsym); - - return Success; -} - -static std::unique_ptr -getOutputStream(StringRef Path, DiagnosticsEngine &Diags, bool Binary) { - // Make sure that the Out file gets unlinked from the disk if we get a - // SIGINT. - if (Path != "-") - sys::RemoveFileOnSignal(Path); - - std::error_code EC; - auto Out = llvm::make_unique( - Path, EC, (Binary ? sys::fs::F_None : sys::fs::F_Text)); - if (EC) { - Diags.Report(diag::err_fe_unable_to_open_output) << Path << EC.message(); - return nullptr; - } - - return Out; -} - -static bool ExecuteAssembler(AssemblerInvocation &Opts, - DiagnosticsEngine &Diags) { - // Get the target specific parser. - std::string Error; - const Target *TheTarget = TargetRegistry::lookupTarget(Opts.Triple, Error); - if (!TheTarget) - return Diags.Report(diag::err_target_unknown_triple) << Opts.Triple; - - ErrorOr> Buffer = - MemoryBuffer::getFileOrSTDIN(Opts.InputFile); - - if (std::error_code EC = Buffer.getError()) { - Error = EC.message(); - return Diags.Report(diag::err_fe_error_reading) << Opts.InputFile; - } - - SourceMgr SrcMgr; - - // Tell SrcMgr about this buffer, which is what the parser will pick up. - SrcMgr.AddNewSourceBuffer(std::move(*Buffer), SMLoc()); - - // Record the location of the include directories so that the lexer can find - // it later. - SrcMgr.setIncludeDirs(Opts.IncludePaths); - - std::unique_ptr MRI(TheTarget->createMCRegInfo(Opts.Triple)); - assert(MRI && "Unable to create target register info!"); - - std::unique_ptr MAI(TheTarget->createMCAsmInfo(*MRI, Opts.Triple)); - assert(MAI && "Unable to create target asm info!"); - - // Ensure MCAsmInfo initialization occurs before any use, otherwise sections - // may be created with a combination of default and explicit settings. - MAI->setCompressDebugSections(Opts.CompressDebugSections); - - MAI->setRelaxELFRelocations(Opts.RelaxELFRelocations); - - bool IsBinary = Opts.OutputType == AssemblerInvocation::FT_Obj; - if (Opts.OutputPath.empty()) - Opts.OutputPath = "-"; - std::unique_ptr FDOS = - getOutputStream(Opts.OutputPath, Diags, IsBinary); - if (!FDOS) - return true; - std::unique_ptr DwoOS; - if (!Opts.SplitDwarfFile.empty()) - DwoOS = getOutputStream(Opts.SplitDwarfFile, Diags, IsBinary); - - // FIXME: This is not pretty. MCContext has a ptr to MCObjectFileInfo and - // MCObjectFileInfo needs a MCContext reference in order to initialize itself. - std::unique_ptr MOFI(new MCObjectFileInfo()); - - MCContext Ctx(MAI.get(), MRI.get(), MOFI.get(), &SrcMgr); - - bool PIC = false; - if (Opts.RelocationModel == "static") { - PIC = false; - } else if (Opts.RelocationModel == "pic") { - PIC = true; - } else { - assert(Opts.RelocationModel == "dynamic-no-pic" && - "Invalid PIC model!"); - PIC = false; - } - - MOFI->InitMCObjectFileInfo(Triple(Opts.Triple), PIC, Ctx); - if (Opts.SaveTemporaryLabels) - Ctx.setAllowTemporaryLabels(false); - if (Opts.GenDwarfForAssembly) - Ctx.setGenDwarfForAssembly(true); - if (!Opts.DwarfDebugFlags.empty()) - Ctx.setDwarfDebugFlags(StringRef(Opts.DwarfDebugFlags)); - if (!Opts.DwarfDebugProducer.empty()) - Ctx.setDwarfDebugProducer(StringRef(Opts.DwarfDebugProducer)); - if (!Opts.DebugCompilationDir.empty()) - Ctx.setCompilationDir(Opts.DebugCompilationDir); - if (!Opts.DebugPrefixMap.empty()) - for (const auto &KV : Opts.DebugPrefixMap) - Ctx.addDebugPrefixMapEntry(KV.first, KV.second); - if (!Opts.MainFileName.empty()) - Ctx.setMainFileName(StringRef(Opts.MainFileName)); - Ctx.setDwarfVersion(Opts.DwarfVersion); - - // Build up the feature string from the target feature list. - std::string FS; - if (!Opts.Features.empty()) { - FS = Opts.Features[0]; - for (unsigned i = 1, e = Opts.Features.size(); i != e; ++i) - FS += "," + Opts.Features[i]; - } - - std::unique_ptr Str; - - std::unique_ptr MCII(TheTarget->createMCInstrInfo()); - std::unique_ptr STI( - TheTarget->createMCSubtargetInfo(Opts.Triple, Opts.CPU, FS)); - - raw_pwrite_stream *Out = FDOS.get(); - std::unique_ptr BOS; - - // FIXME: There is a bit of code duplication with addPassesToEmitFile. - if (Opts.OutputType == AssemblerInvocation::FT_Asm) { - MCInstPrinter *IP = TheTarget->createMCInstPrinter( - llvm::Triple(Opts.Triple), Opts.OutputAsmVariant, *MAI, *MCII, *MRI); - - std::unique_ptr CE; - if (Opts.ShowEncoding) - CE.reset(TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx)); - MCTargetOptions MCOptions; - std::unique_ptr MAB( - TheTarget->createMCAsmBackend(*STI, *MRI, MCOptions)); - - auto FOut = llvm::make_unique(*Out); - Str.reset(TheTarget->createAsmStreamer( - Ctx, std::move(FOut), /*asmverbose*/ true, - /*useDwarfDirectory*/ true, IP, std::move(CE), std::move(MAB), - Opts.ShowInst)); - } else if (Opts.OutputType == AssemblerInvocation::FT_Null) { - Str.reset(createNullStreamer(Ctx)); - } else { - assert(Opts.OutputType == AssemblerInvocation::FT_Obj && - "Invalid file type!"); - if (!FDOS->supportsSeeking()) { - BOS = make_unique(*FDOS); - Out = BOS.get(); - } - - std::unique_ptr CE( - TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx)); - MCTargetOptions MCOptions; - std::unique_ptr MAB( - TheTarget->createMCAsmBackend(*STI, *MRI, MCOptions)); - std::unique_ptr OW = - DwoOS ? MAB->createDwoObjectWriter(*Out, *DwoOS) - : MAB->createObjectWriter(*Out); - - Triple T(Opts.Triple); - Str.reset(TheTarget->createMCObjectStreamer( - T, Ctx, std::move(MAB), std::move(OW), std::move(CE), *STI, - Opts.RelaxAll, Opts.IncrementalLinkerCompatible, - /*DWARFMustBeAtTheEnd*/ true)); - Str.get()->InitSections(Opts.NoExecStack); - } - - // Assembly to object compilation should leverage assembly info. - Str->setUseAssemblerInfoForParsing(true); - - bool Failed = false; - - std::unique_ptr Parser( - createMCAsmParser(SrcMgr, Ctx, *Str.get(), *MAI)); - - // FIXME: init MCTargetOptions from sanitizer flags here. - MCTargetOptions Options; - std::unique_ptr TAP( - TheTarget->createMCAsmParser(*STI, *Parser, *MCII, Options)); - if (!TAP) - Failed = Diags.Report(diag::err_target_unknown_triple) << Opts.Triple; - - // Set values for symbols, if any. - for (auto &S : Opts.SymbolDefs) { - auto Pair = StringRef(S).split('='); - auto Sym = Pair.first; - auto Val = Pair.second; - int64_t Value = 1; - // We have already error checked this in the driver. - Val.getAsInteger(0, Value); - Ctx.setSymbolValue(Parser->getStreamer(), Sym, Value); - } - - if (!Failed) { - Parser->setTargetParser(*TAP.get()); - Failed = Parser->Run(Opts.NoInitialTextSection); - } - - // Close Streamer first. - // It might have a reference to the output stream. - Str.reset(); - // Close the output stream early. - BOS.reset(); - FDOS.reset(); - - // Delete output file if there were errors. - if (Failed) { - if (Opts.OutputPath != "-") - sys::fs::remove(Opts.OutputPath); - if (!Opts.SplitDwarfFile.empty() && Opts.SplitDwarfFile != "-") - sys::fs::remove(Opts.SplitDwarfFile); - } - - return Failed; -} - -static void LLVMErrorHandler(void *UserData, const std::string &Message, - bool GenCrashDiag) { - DiagnosticsEngine &Diags = *static_cast(UserData); - - Diags.Report(diag::err_fe_error_backend) << Message; - - // We cannot recover from llvm errors. - exit(1); -} - -int cc1as_main(ArrayRef Argv, const char *Argv0, void *MainAddr) { - // Initialize targets and assembly printers/parsers. - InitializeAllTargetInfos(); - InitializeAllTargetMCs(); - InitializeAllAsmParsers(); - - // Construct our diagnostic client. - IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); - TextDiagnosticPrinter *DiagClient - = new TextDiagnosticPrinter(errs(), &*DiagOpts); - DiagClient->setPrefix("clang -cc1as"); - IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); - DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient); - - // Set an error handler, so that any LLVM backend diagnostics go through our - // error handler. - ScopedFatalErrorHandler FatalErrorHandler - (LLVMErrorHandler, static_cast(&Diags)); - - // Parse the arguments. - AssemblerInvocation Asm; - if (!AssemblerInvocation::CreateFromArgs(Asm, Argv, Diags)) - return 1; - - if (Asm.ShowHelp) { - std::unique_ptr Opts(driver::createDriverOptTable()); - Opts->PrintHelp(llvm::outs(), "clang -cc1as", "Clang Integrated Assembler", - /*Include=*/driver::options::CC1AsOption, /*Exclude=*/0, - /*ShowAllAliases=*/false); - return 0; - } - - // Honor -version. - // - // FIXME: Use a better -version message? - if (Asm.ShowVersion) { - llvm::cl::PrintVersionMessage(); - return 0; - } - - // Honor -mllvm. - // - // FIXME: Remove this, one day. - if (!Asm.LLVMArgs.empty()) { - unsigned NumArgs = Asm.LLVMArgs.size(); - auto Args = llvm::make_unique(NumArgs + 2); - Args[0] = "clang (LLVM option parsing)"; - for (unsigned i = 0; i != NumArgs; ++i) - Args[i + 1] = Asm.LLVMArgs[i].c_str(); - Args[NumArgs + 1] = nullptr; - llvm::cl::ParseCommandLineOptions(NumArgs + 1, Args.get()); - } - - // Execute the invocation, unless there were parsing errors. - bool Failed = Diags.hasErrorOccurred() || ExecuteAssembler(Asm, Diags); - - // If any timers were active but haven't been destroyed yet, print their - // results now. - TimerGroup::printAll(errs()); - - return !!Failed; -} diff --git a/dbms/programs/clang/Compiler-7.0.0/cc1gen_reproducer_main.cpp b/dbms/programs/clang/Compiler-7.0.0/cc1gen_reproducer_main.cpp deleted file mode 100644 index a4c034d8d35..00000000000 --- a/dbms/programs/clang/Compiler-7.0.0/cc1gen_reproducer_main.cpp +++ /dev/null @@ -1,196 +0,0 @@ -//===-- cc1gen_reproducer_main.cpp - Clang reproducer generator ----------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This is the entry point to the clang -cc1gen-reproducer functionality, which -// generates reproducers for invocations for clang-based tools. -// -//===----------------------------------------------------------------------===// - -#include "clang/Basic/Diagnostic.h" -#include "clang/Basic/LLVM.h" -#include "clang/Basic/VirtualFileSystem.h" -#include "clang/Driver/Compilation.h" -#include "clang/Driver/Driver.h" -#include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/Support/FileSystem.h" -#include "llvm/Support/TargetSelect.h" -#include "llvm/Support/YAMLTraits.h" -#include "llvm/Support/raw_ostream.h" - -using namespace clang; - -namespace { - -struct UnsavedFileHash { - std::string Name; - std::string MD5; -}; - -struct ClangInvocationInfo { - std::string Toolchain; - std::string LibclangOperation; - std::string LibclangOptions; - std::vector Arguments; - std::vector InvocationArguments; - std::vector UnsavedFileHashes; - bool Dump = false; -}; - -} // end anonymous namespace - -LLVM_YAML_IS_SEQUENCE_VECTOR(UnsavedFileHash) - -namespace llvm { -namespace yaml { - -template <> struct MappingTraits { - static void mapping(IO &IO, UnsavedFileHash &Info) { - IO.mapRequired("name", Info.Name); - IO.mapRequired("md5", Info.MD5); - } -}; - -template <> struct MappingTraits { - static void mapping(IO &IO, ClangInvocationInfo &Info) { - IO.mapRequired("toolchain", Info.Toolchain); - IO.mapOptional("libclang.operation", Info.LibclangOperation); - IO.mapOptional("libclang.opts", Info.LibclangOptions); - IO.mapRequired("args", Info.Arguments); - IO.mapOptional("invocation-args", Info.InvocationArguments); - IO.mapOptional("unsaved_file_hashes", Info.UnsavedFileHashes); - } -}; - -} // end namespace yaml -} // end namespace llvm - -static std::string generateReproducerMetaInfo(const ClangInvocationInfo &Info) { - std::string Result; - llvm::raw_string_ostream OS(Result); - OS << '{'; - bool NeedComma = false; - auto EmitKey = [&](StringRef Key) { - if (NeedComma) - OS << ", "; - NeedComma = true; - OS << '"' << Key << "\": "; - }; - auto EmitStringKey = [&](StringRef Key, StringRef Value) { - if (Value.empty()) - return; - EmitKey(Key); - OS << '"' << Value << '"'; - }; - EmitStringKey("libclang.operation", Info.LibclangOperation); - EmitStringKey("libclang.opts", Info.LibclangOptions); - if (!Info.InvocationArguments.empty()) { - EmitKey("invocation-args"); - OS << '['; - for (const auto &Arg : llvm::enumerate(Info.InvocationArguments)) { - if (Arg.index()) - OS << ','; - OS << '"' << Arg.value() << '"'; - } - OS << ']'; - } - OS << '}'; - // FIXME: Compare unsaved file hashes and report mismatch in the reproducer. - if (Info.Dump) - llvm::outs() << "REPRODUCER METAINFO: " << OS.str() << "\n"; - return std::move(OS.str()); -} - -/// Generates a reproducer for a set of arguments from a specific invocation. -static llvm::Optional -generateReproducerForInvocationArguments(ArrayRef Argv, - const ClangInvocationInfo &Info) { - using namespace driver; - auto TargetAndMode = ToolChain::getTargetAndModeFromProgramName(Argv[0]); - - IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions; - - IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); - DiagnosticsEngine Diags(DiagID, &*DiagOpts, new IgnoringDiagConsumer()); - ProcessWarningOptions(Diags, *DiagOpts, /*ReportDiags=*/false); - Driver TheDriver(Argv[0], llvm::sys::getDefaultTargetTriple(), Diags); - TheDriver.setTargetAndMode(TargetAndMode); - - std::unique_ptr C(TheDriver.BuildCompilation(Argv)); - if (C && !C->containsError()) { - for (const auto &J : C->getJobs()) { - if (const Command *Cmd = dyn_cast(&J)) { - Driver::CompilationDiagnosticReport Report; - TheDriver.generateCompilationDiagnostics( - *C, *Cmd, generateReproducerMetaInfo(Info), &Report); - return Report; - } - } - } - - return None; -} - -std::string GetExecutablePath(const char *Argv0, bool CanonicalPrefixes); - -static void printReproducerInformation( - llvm::raw_ostream &OS, const ClangInvocationInfo &Info, - const driver::Driver::CompilationDiagnosticReport &Report) { - OS << "REPRODUCER:\n"; - OS << "{\n"; - OS << R"("files":[)"; - for (const auto &File : llvm::enumerate(Report.TemporaryFiles)) { - if (File.index()) - OS << ','; - OS << '"' << File.value() << '"'; - } - OS << "]\n}\n"; -} - -int cc1gen_reproducer_main(ArrayRef Argv, const char *Argv0, - void *MainAddr) { - if (Argv.size() < 1) { - llvm::errs() << "error: missing invocation file\n"; - return 1; - } - // Parse the invocation descriptor. - StringRef Input = Argv[0]; - llvm::ErrorOr> Buffer = - llvm::MemoryBuffer::getFile(Input); - if (!Buffer) { - llvm::errs() << "error: failed to read " << Input << ": " - << Buffer.getError().message() << "\n"; - return 1; - } - llvm::yaml::Input YAML(Buffer.get()->getBuffer()); - ClangInvocationInfo InvocationInfo; - YAML >> InvocationInfo; - if (Argv.size() > 1 && Argv[1] == StringRef("-v")) - InvocationInfo.Dump = true; - - // Create an invocation that will produce the reproducer. - std::vector DriverArgs; - for (const auto &Arg : InvocationInfo.Arguments) - DriverArgs.push_back(Arg.c_str()); - std::string Path = GetExecutablePath(Argv0, /*CanonicalPrefixes=*/true); - DriverArgs[0] = Path.c_str(); - llvm::Optional Report = - generateReproducerForInvocationArguments(DriverArgs, InvocationInfo); - - // Emit the information about the reproduce files to stdout. - int Result = 1; - if (Report) { - printReproducerInformation(llvm::outs(), InvocationInfo, *Report); - Result = 0; - } - - // Remove the input file. - llvm::sys::fs::remove(Input); - return Result; -} diff --git a/dbms/programs/clang/Compiler-7.0.0/driver.cpp b/dbms/programs/clang/Compiler-7.0.0/driver.cpp deleted file mode 100644 index 79d71b08ba7..00000000000 --- a/dbms/programs/clang/Compiler-7.0.0/driver.cpp +++ /dev/null @@ -1,514 +0,0 @@ -//===-- driver.cpp - Clang GCC-Compatible Driver --------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This is the entry point to the clang driver; it is a thin wrapper -// for functionality in the Driver clang library. -// -//===----------------------------------------------------------------------===// - -#include "clang/Driver/Driver.h" -#include "clang/Basic/DiagnosticOptions.h" -#include "clang/Driver/Compilation.h" -#include "clang/Driver/DriverDiagnostic.h" -#include "clang/Driver/Options.h" -#include "clang/Driver/ToolChain.h" -#include "clang/Frontend/ChainedDiagnosticConsumer.h" -#include "clang/Frontend/CompilerInvocation.h" -#include "clang/Frontend/SerializedDiagnosticPrinter.h" -#include "clang/Frontend/TextDiagnosticPrinter.h" -#include "clang/Frontend/Utils.h" -#include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/SmallString.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/Option/ArgList.h" -#include "llvm/Option/OptTable.h" -#include "llvm/Option/Option.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/FileSystem.h" -#include "llvm/Support/Host.h" -#include "llvm/Support/InitLLVM.h" -#include "llvm/Support/Path.h" -#include "llvm/Support/Process.h" -#include "llvm/Support/Program.h" -#include "llvm/Support/Regex.h" -#include "llvm/Support/Signals.h" -#include "llvm/Support/StringSaver.h" -#include "llvm/Support/TargetSelect.h" -#include "llvm/Support/Timer.h" -#include "llvm/Support/raw_ostream.h" -#include -#include -#include -using namespace clang; -using namespace clang::driver; -using namespace llvm::opt; - -std::string GetExecutablePath(const char *Argv0, bool CanonicalPrefixes) { - if (!CanonicalPrefixes) { - SmallString<128> ExecutablePath(Argv0); - // Do a PATH lookup if Argv0 isn't a valid path. - if (!llvm::sys::fs::exists(ExecutablePath)) - if (llvm::ErrorOr P = - llvm::sys::findProgramByName(ExecutablePath)) - ExecutablePath = *P; - return ExecutablePath.str(); - } - - // This just needs to be some symbol in the binary; C++ doesn't - // allow taking the address of ::main however. - void *P = (void*) (intptr_t) GetExecutablePath; - return llvm::sys::fs::getMainExecutable(Argv0, P); -} - -static const char *GetStableCStr(std::set &SavedStrings, - StringRef S) { - return SavedStrings.insert(S).first->c_str(); -} - -/// ApplyQAOverride - Apply a list of edits to the input argument lists. -/// -/// The input string is a space separate list of edits to perform, -/// they are applied in order to the input argument lists. Edits -/// should be one of the following forms: -/// -/// '#': Silence information about the changes to the command line arguments. -/// -/// '^': Add FOO as a new argument at the beginning of the command line. -/// -/// '+': Add FOO as a new argument at the end of the command line. -/// -/// 's/XXX/YYY/': Substitute the regular expression XXX with YYY in the command -/// line. -/// -/// 'xOPTION': Removes all instances of the literal argument OPTION. -/// -/// 'XOPTION': Removes all instances of the literal argument OPTION, -/// and the following argument. -/// -/// 'Ox': Removes all flags matching 'O' or 'O[sz0-9]' and adds 'Ox' -/// at the end of the command line. -/// -/// \param OS - The stream to write edit information to. -/// \param Args - The vector of command line arguments. -/// \param Edit - The override command to perform. -/// \param SavedStrings - Set to use for storing string representations. -static void ApplyOneQAOverride(raw_ostream &OS, - SmallVectorImpl &Args, - StringRef Edit, - std::set &SavedStrings) { - // This does not need to be efficient. - - if (Edit[0] == '^') { - const char *Str = - GetStableCStr(SavedStrings, Edit.substr(1)); - OS << "### Adding argument " << Str << " at beginning\n"; - Args.insert(Args.begin() + 1, Str); - } else if (Edit[0] == '+') { - const char *Str = - GetStableCStr(SavedStrings, Edit.substr(1)); - OS << "### Adding argument " << Str << " at end\n"; - Args.push_back(Str); - } else if (Edit[0] == 's' && Edit[1] == '/' && Edit.endswith("/") && - Edit.slice(2, Edit.size()-1).find('/') != StringRef::npos) { - StringRef MatchPattern = Edit.substr(2).split('/').first; - StringRef ReplPattern = Edit.substr(2).split('/').second; - ReplPattern = ReplPattern.slice(0, ReplPattern.size()-1); - - for (unsigned i = 1, e = Args.size(); i != e; ++i) { - // Ignore end-of-line response file markers - if (Args[i] == nullptr) - continue; - std::string Repl = llvm::Regex(MatchPattern).sub(ReplPattern, Args[i]); - - if (Repl != Args[i]) { - OS << "### Replacing '" << Args[i] << "' with '" << Repl << "'\n"; - Args[i] = GetStableCStr(SavedStrings, Repl); - } - } - } else if (Edit[0] == 'x' || Edit[0] == 'X') { - auto Option = Edit.substr(1); - for (unsigned i = 1; i < Args.size();) { - if (Option == Args[i]) { - OS << "### Deleting argument " << Args[i] << '\n'; - Args.erase(Args.begin() + i); - if (Edit[0] == 'X') { - if (i < Args.size()) { - OS << "### Deleting argument " << Args[i] << '\n'; - Args.erase(Args.begin() + i); - } else - OS << "### Invalid X edit, end of command line!\n"; - } - } else - ++i; - } - } else if (Edit[0] == 'O') { - for (unsigned i = 1; i < Args.size();) { - const char *A = Args[i]; - // Ignore end-of-line response file markers - if (A == nullptr) - continue; - if (A[0] == '-' && A[1] == 'O' && - (A[2] == '\0' || - (A[3] == '\0' && (A[2] == 's' || A[2] == 'z' || - ('0' <= A[2] && A[2] <= '9'))))) { - OS << "### Deleting argument " << Args[i] << '\n'; - Args.erase(Args.begin() + i); - } else - ++i; - } - OS << "### Adding argument " << Edit << " at end\n"; - Args.push_back(GetStableCStr(SavedStrings, '-' + Edit.str())); - } else { - OS << "### Unrecognized edit: " << Edit << "\n"; - } -} - -/// ApplyQAOverride - Apply a comma separate list of edits to the -/// input argument lists. See ApplyOneQAOverride. -static void ApplyQAOverride(SmallVectorImpl &Args, - const char *OverrideStr, - std::set &SavedStrings) { - raw_ostream *OS = &llvm::errs(); - - if (OverrideStr[0] == '#') { - ++OverrideStr; - OS = &llvm::nulls(); - } - - *OS << "### CCC_OVERRIDE_OPTIONS: " << OverrideStr << "\n"; - - // This does not need to be efficient. - - const char *S = OverrideStr; - while (*S) { - const char *End = ::strchr(S, ' '); - if (!End) - End = S + strlen(S); - if (End != S) - ApplyOneQAOverride(*OS, Args, std::string(S, End), SavedStrings); - S = End; - if (*S != '\0') - ++S; - } -} - -extern int cc1_main(ArrayRef Argv, const char *Argv0, - void *MainAddr); -extern int cc1as_main(ArrayRef Argv, const char *Argv0, - void *MainAddr); -extern int cc1gen_reproducer_main(ArrayRef Argv, - const char *Argv0, void *MainAddr); - -static void insertTargetAndModeArgs(const ParsedClangName &NameParts, - SmallVectorImpl &ArgVector, - std::set &SavedStrings) { - // Put target and mode arguments at the start of argument list so that - // arguments specified in command line could override them. Avoid putting - // them at index 0, as an option like '-cc1' must remain the first. - int InsertionPoint = 0; - if (ArgVector.size() > 0) - ++InsertionPoint; - - if (NameParts.DriverMode) { - // Add the mode flag to the arguments. - ArgVector.insert(ArgVector.begin() + InsertionPoint, - GetStableCStr(SavedStrings, NameParts.DriverMode)); - } - - if (NameParts.TargetIsValid) { - const char *arr[] = {"-target", GetStableCStr(SavedStrings, - NameParts.TargetPrefix)}; - ArgVector.insert(ArgVector.begin() + InsertionPoint, - std::begin(arr), std::end(arr)); - } -} - -static void getCLEnvVarOptions(std::string &EnvValue, llvm::StringSaver &Saver, - SmallVectorImpl &Opts) { - llvm::cl::TokenizeWindowsCommandLine(EnvValue, Saver, Opts); - // The first instance of '#' should be replaced with '=' in each option. - for (const char *Opt : Opts) - if (char *NumberSignPtr = const_cast(::strchr(Opt, '#'))) - *NumberSignPtr = '='; -} - -static void SetBackdoorDriverOutputsFromEnvVars(Driver &TheDriver) { - // Handle CC_PRINT_OPTIONS and CC_PRINT_OPTIONS_FILE. - TheDriver.CCPrintOptions = !!::getenv("CC_PRINT_OPTIONS"); - if (TheDriver.CCPrintOptions) - TheDriver.CCPrintOptionsFilename = ::getenv("CC_PRINT_OPTIONS_FILE"); - - // Handle CC_PRINT_HEADERS and CC_PRINT_HEADERS_FILE. - TheDriver.CCPrintHeaders = !!::getenv("CC_PRINT_HEADERS"); - if (TheDriver.CCPrintHeaders) - TheDriver.CCPrintHeadersFilename = ::getenv("CC_PRINT_HEADERS_FILE"); - - // Handle CC_LOG_DIAGNOSTICS and CC_LOG_DIAGNOSTICS_FILE. - TheDriver.CCLogDiagnostics = !!::getenv("CC_LOG_DIAGNOSTICS"); - if (TheDriver.CCLogDiagnostics) - TheDriver.CCLogDiagnosticsFilename = ::getenv("CC_LOG_DIAGNOSTICS_FILE"); -} - -static void FixupDiagPrefixExeName(TextDiagnosticPrinter *DiagClient, - const std::string &Path) { - // If the clang binary happens to be named cl.exe for compatibility reasons, - // use clang-cl.exe as the prefix to avoid confusion between clang and MSVC. - StringRef ExeBasename(llvm::sys::path::filename(Path)); - if (ExeBasename.equals_lower("cl.exe")) - ExeBasename = "clang-cl.exe"; - DiagClient->setPrefix(ExeBasename); -} - -// This lets us create the DiagnosticsEngine with a properly-filled-out -// DiagnosticOptions instance. -static DiagnosticOptions * -CreateAndPopulateDiagOpts(ArrayRef argv) { - auto *DiagOpts = new DiagnosticOptions; - std::unique_ptr Opts(createDriverOptTable()); - unsigned MissingArgIndex, MissingArgCount; - InputArgList Args = - Opts->ParseArgs(argv.slice(1), MissingArgIndex, MissingArgCount); - // We ignore MissingArgCount and the return value of ParseDiagnosticArgs. - // Any errors that would be diagnosed here will also be diagnosed later, - // when the DiagnosticsEngine actually exists. - (void)ParseDiagnosticArgs(*DiagOpts, Args); - return DiagOpts; -} - -static void SetInstallDir(SmallVectorImpl &argv, - Driver &TheDriver, bool CanonicalPrefixes) { - // Attempt to find the original path used to invoke the driver, to determine - // the installed path. We do this manually, because we want to support that - // path being a symlink. - SmallString<128> InstalledPath(argv[0]); - - // Do a PATH lookup, if there are no directory components. - if (llvm::sys::path::filename(InstalledPath) == InstalledPath) - if (llvm::ErrorOr Tmp = llvm::sys::findProgramByName( - llvm::sys::path::filename(InstalledPath.str()))) - InstalledPath = *Tmp; - - // FIXME: We don't actually canonicalize this, we just make it absolute. - if (CanonicalPrefixes) - llvm::sys::fs::make_absolute(InstalledPath); - - StringRef InstalledPathParent(llvm::sys::path::parent_path(InstalledPath)); - if (llvm::sys::fs::exists(InstalledPathParent)) - TheDriver.setInstalledDir(InstalledPathParent); -} - -static int ExecuteCC1Tool(ArrayRef argv, StringRef Tool) { - void *GetExecutablePathVP = (void *)(intptr_t) GetExecutablePath; - if (Tool == "") - return cc1_main(argv.slice(2), argv[0], GetExecutablePathVP); - if (Tool == "as") - return cc1as_main(argv.slice(2), argv[0], GetExecutablePathVP); - if (Tool == "gen-reproducer") - return cc1gen_reproducer_main(argv.slice(2), argv[0], GetExecutablePathVP); - - // Reject unknown tools. - llvm::errs() << "error: unknown integrated tool '" << Tool << "'. " - << "Valid tools include '-cc1' and '-cc1as'.\n"; - return 1; -} - -int mainEntryClickHouseClang(int argc_, /* const */ char **argv_) { - llvm::InitLLVM X(argc_, argv_); - SmallVector argv(argv_, argv_ + argc_); - - if (llvm::sys::Process::FixupStandardFileDescriptors()) - return 1; - - llvm::InitializeAllTargets(); - auto TargetAndMode = ToolChain::getTargetAndModeFromProgramName(argv[0]); - - llvm::BumpPtrAllocator A; - llvm::StringSaver Saver(A); - - // Parse response files using the GNU syntax, unless we're in CL mode. There - // are two ways to put clang in CL compatibility mode: argv[0] is either - // clang-cl or cl, or --driver-mode=cl is on the command line. The normal - // command line parsing can't happen until after response file parsing, so we - // have to manually search for a --driver-mode=cl argument the hard way. - // Finally, our -cc1 tools don't care which tokenization mode we use because - // response files written by clang will tokenize the same way in either mode. - bool ClangCLMode = false; - if (StringRef(TargetAndMode.DriverMode).equals("--driver-mode=cl") || - std::find_if(argv.begin(), argv.end(), [](const char *F) { - return F && strcmp(F, "--driver-mode=cl") == 0; - }) != argv.end()) { - ClangCLMode = true; - } - enum { Default, POSIX, Windows } RSPQuoting = Default; - for (const char *F : argv) { - if (strcmp(F, "--rsp-quoting=posix") == 0) - RSPQuoting = POSIX; - else if (strcmp(F, "--rsp-quoting=windows") == 0) - RSPQuoting = Windows; - } - - // Determines whether we want nullptr markers in argv to indicate response - // files end-of-lines. We only use this for the /LINK driver argument with - // clang-cl.exe on Windows. - bool MarkEOLs = ClangCLMode; - - llvm::cl::TokenizerCallback Tokenizer; - if (RSPQuoting == Windows || (RSPQuoting == Default && ClangCLMode)) - Tokenizer = &llvm::cl::TokenizeWindowsCommandLine; - else - Tokenizer = &llvm::cl::TokenizeGNUCommandLine; - - if (MarkEOLs && argv.size() > 1 && StringRef(argv[1]).startswith("-cc1")) - MarkEOLs = false; - llvm::cl::ExpandResponseFiles(Saver, Tokenizer, argv, MarkEOLs); - - // Handle -cc1 integrated tools, even if -cc1 was expanded from a response - // file. - auto FirstArg = std::find_if(argv.begin() + 1, argv.end(), - [](const char *A) { return A != nullptr; }); - if (FirstArg != argv.end() && StringRef(*FirstArg).startswith("-cc1")) { - // If -cc1 came from a response file, remove the EOL sentinels. - if (MarkEOLs) { - auto newEnd = std::remove(argv.begin(), argv.end(), nullptr); - argv.resize(newEnd - argv.begin()); - } - return ExecuteCC1Tool(argv, argv[1] + 4); - } - - bool CanonicalPrefixes = true; - for (int i = 1, size = argv.size(); i < size; ++i) { - // Skip end-of-line response file markers - if (argv[i] == nullptr) - continue; - if (StringRef(argv[i]) == "-no-canonical-prefixes") { - CanonicalPrefixes = false; - break; - } - } - - // Handle CL and _CL_ which permits additional command line options to be - // prepended or appended. - if (ClangCLMode) { - // Arguments in "CL" are prepended. - llvm::Optional OptCL = llvm::sys::Process::GetEnv("CL"); - if (OptCL.hasValue()) { - SmallVector PrependedOpts; - getCLEnvVarOptions(OptCL.getValue(), Saver, PrependedOpts); - - // Insert right after the program name to prepend to the argument list. - argv.insert(argv.begin() + 1, PrependedOpts.begin(), PrependedOpts.end()); - } - // Arguments in "_CL_" are appended. - llvm::Optional Opt_CL_ = llvm::sys::Process::GetEnv("_CL_"); - if (Opt_CL_.hasValue()) { - SmallVector AppendedOpts; - getCLEnvVarOptions(Opt_CL_.getValue(), Saver, AppendedOpts); - - // Insert at the end of the argument list to append. - argv.append(AppendedOpts.begin(), AppendedOpts.end()); - } - } - - std::set SavedStrings; - // Handle CCC_OVERRIDE_OPTIONS, used for editing a command line behind the - // scenes. - if (const char *OverrideStr = ::getenv("CCC_OVERRIDE_OPTIONS")) { - // FIXME: Driver shouldn't take extra initial argument. - ApplyQAOverride(argv, OverrideStr, SavedStrings); - } - - std::string Path = GetExecutablePath(argv[0], CanonicalPrefixes); - - IntrusiveRefCntPtr DiagOpts = - CreateAndPopulateDiagOpts(argv); - - TextDiagnosticPrinter *DiagClient - = new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts); - FixupDiagPrefixExeName(DiagClient, Path); - - IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); - - DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient); - - if (!DiagOpts->DiagnosticSerializationFile.empty()) { - auto SerializedConsumer = - clang::serialized_diags::create(DiagOpts->DiagnosticSerializationFile, - &*DiagOpts, /*MergeChildRecords=*/true); - Diags.setClient(new ChainedDiagnosticConsumer( - Diags.takeClient(), std::move(SerializedConsumer))); - } - - ProcessWarningOptions(Diags, *DiagOpts, /*ReportDiags=*/false); - - Driver TheDriver(Path, llvm::sys::getDefaultTargetTriple(), Diags); - SetInstallDir(argv, TheDriver, CanonicalPrefixes); - TheDriver.setTargetAndMode(TargetAndMode); - - insertTargetAndModeArgs(TargetAndMode, argv, SavedStrings); - - SetBackdoorDriverOutputsFromEnvVars(TheDriver); - - std::unique_ptr C(TheDriver.BuildCompilation(argv)); - int Res = 1; - if (C && !C->containsError()) { - SmallVector, 4> FailingCommands; - Res = TheDriver.ExecuteCompilation(*C, FailingCommands); - - // Force a crash to test the diagnostics. - if (TheDriver.GenReproducer) { - Diags.Report(diag::err_drv_force_crash) - << !::getenv("FORCE_CLANG_DIAGNOSTICS_CRASH"); - - // Pretend that every command failed. - FailingCommands.clear(); - for (const auto &J : C->getJobs()) - if (const Command *C = dyn_cast(&J)) - FailingCommands.push_back(std::make_pair(-1, C)); - } - - for (const auto &P : FailingCommands) { - int CommandRes = P.first; - const Command *FailingCommand = P.second; - if (!Res) - Res = CommandRes; - - // If result status is < 0, then the driver command signalled an error. - // If result status is 70, then the driver command reported a fatal error. - // On Windows, abort will return an exit code of 3. In these cases, - // generate additional diagnostic information if possible. - bool DiagnoseCrash = CommandRes < 0 || CommandRes == 70; -#ifdef _WIN32 - DiagnoseCrash |= CommandRes == 3; -#endif - if (DiagnoseCrash) { - TheDriver.generateCompilationDiagnostics(*C, *FailingCommand); - break; - } - } - } - - Diags.getClient()->finish(); - - // If any timers were active but haven't been destroyed yet, print their - // results now. This happens in -disable-free mode. - llvm::TimerGroup::printAll(llvm::errs()); - -#ifdef _WIN32 - // Exit status should not be negative on Win32, unless abnormal termination. - // Once abnormal termiation was caught, negative status should not be - // propagated. - if (Res < 0) - Res = 1; -#endif - - // If we have multiple failing commands, we return the result of the first - // failing command. - return Res; -} diff --git a/dbms/programs/clang/Compiler-7.0.0/lld.cpp b/dbms/programs/clang/Compiler-7.0.0/lld.cpp deleted file mode 100644 index 8e118b6e24b..00000000000 --- a/dbms/programs/clang/Compiler-7.0.0/lld.cpp +++ /dev/null @@ -1,150 +0,0 @@ -//===- tools/lld/lld.cpp - Linker Driver Dispatcher -----------------------===// -// -// The LLVM Linker -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file contains the main function of the lld executable. The main -// function is a thin wrapper which dispatches to the platform specific -// driver. -// -// lld is a single executable that contains four different linkers for ELF, -// COFF, WebAssembly and Mach-O. The main function dispatches according to -// argv[0] (i.e. command name). The most common name for each target is shown -// below: -// -// - ld.lld: ELF (Unix) -// - ld64: Mach-O (macOS) -// - lld-link: COFF (Windows) -// - ld-wasm: WebAssembly -// -// lld can be invoked as "lld" along with "-flavor" option. This is for -// backward compatibility and not recommended. -// -//===----------------------------------------------------------------------===// - -#include "lld/Common/Driver.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/StringSwitch.h" -#include "llvm/ADT/Twine.h" -#include "llvm/Support/InitLLVM.h" -#include "llvm/Support/Path.h" -#include -using namespace lld; -using namespace llvm; -using namespace llvm::sys; - -/* - -enum Flavor { - Invalid, - Gnu, // -flavor gnu - WinLink, // -flavor link - Darwin, // -flavor darwin - Wasm, // -flavor wasm -}; - -LLVM_ATTRIBUTE_NORETURN static void die(const Twine &S) { - errs() << S << "\n"; - exit(1); -} - -static Flavor getFlavor(StringRef S) { - return StringSwitch(S) - .CasesLower("ld", "ld.lld", "gnu", Gnu) - .CasesLower("wasm", "ld-wasm", Wasm) - .CaseLower("link", WinLink) - .CasesLower("ld64", "ld64.lld", "darwin", Darwin) - .Default(Invalid); -} - -static bool isPETarget(const std::vector &V) { - for (auto It = V.begin(); It + 1 != V.end(); ++It) { - if (StringRef(*It) != "-m") - continue; - StringRef S = *(It + 1); - return S == "i386pe" || S == "i386pep" || S == "thumb2pe" || S == "arm64pe"; - } - return false; -} - -static Flavor parseProgname(StringRef Progname) { -#if __APPLE__ - // Use Darwin driver for "ld" on Darwin. - if (Progname == "ld") - return Darwin; -#endif - -#if LLVM_ON_UNIX - // Use GNU driver for "ld" on other Unix-like system. - if (Progname == "ld") - return Gnu; -#endif - - // Progname may be something like "lld-gnu". Parse it. - SmallVector V; - Progname.split(V, "-"); - for (StringRef S : V) - if (Flavor F = getFlavor(S)) - return F; - return Invalid; -} - -static Flavor parseFlavor(std::vector &V) { - // Parse -flavor option. - if (V.size() > 1 && V[1] == StringRef("-flavor")) { - if (V.size() <= 2) - die("missing arg value for '-flavor'"); - Flavor F = getFlavor(V[2]); - if (F == Invalid) - die("Unknown flavor: " + StringRef(V[2])); - V.erase(V.begin() + 1, V.begin() + 3); - return F; - } - - // Deduct the flavor from argv[0]. - StringRef Arg0 = path::filename(V[0]); - if (Arg0.endswith_lower(".exe")) - Arg0 = Arg0.drop_back(4); - return parseProgname(Arg0); -} -*/ - -// If this function returns true, lld calls _exit() so that it quickly -// exits without invoking destructors of globally allocated objects. -// -// We don't want to do that if we are running tests though, because -// doing that breaks leak sanitizer. So, lit sets this environment variable, -// and we use it to detect whether we are running tests or not. -static bool canExitEarly() { return StringRef(getenv("LLD_IN_TEST")) != "1"; } - -/// Universal linker main(). This linker emulates the gnu, darwin, or -/// windows linker based on the argv[0] or -flavor option. -int mainEntryClickHouseLLD(int Argc, /* const */ char **Argv) { - InitLLVM X(Argc, Argv); - - std::vector Args(Argv, Argv + Argc); -/* - switch (parseFlavor(Args)) { - case Gnu: - if (isPETarget(Args)) - return !mingw::link(Args); -*/ - return !elf::link(Args, canExitEarly()); -/* - case WinLink: - return !coff::link(Args, canExitEarly()); - case Darwin: - return !mach_o::link(Args, canExitEarly()); - case Wasm: - return !wasm::link(Args, canExitEarly()); - default: - die("lld is a generic driver.\n" - "Invoke ld.lld (Unix), ld64.lld (macOS), lld-link (Windows), wasm-lld" - " (WebAssembly) instead"); - } -*/ -} diff --git a/dbms/programs/clang/Compiler-7.0.0bundled/CMakeLists.txt b/dbms/programs/clang/Compiler-7.0.0bundled/CMakeLists.txt deleted file mode 100644 index a5f8314b862..00000000000 --- a/dbms/programs/clang/Compiler-7.0.0bundled/CMakeLists.txt +++ /dev/null @@ -1,49 +0,0 @@ -add_definitions(-Wno-error -Wno-unused-parameter -Wno-non-virtual-dtor -U_LIBCPP_DEBUG) - -link_directories(${LLVM_LIBRARY_DIRS}) - -add_library(clickhouse-compiler-lib - driver.cpp - cc1_main.cpp - cc1as_main.cpp - lld.cpp) - -target_compile_options(clickhouse-compiler-lib PRIVATE -fno-rtti -fno-exceptions -g0) - -string(REPLACE "${INCLUDE_DEBUG_HELPERS}" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) # cant compile with -fno-rtti - -llvm_libs_all(REQUIRED_LLVM_LIBRARIES) - -message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}") - -target_include_directories(clickhouse-compiler-lib SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS}) - -# This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory. - -target_link_libraries(clickhouse-compiler-lib PRIVATE - -clangBasic clangCodeGen clangDriver -clangFrontend -clangFrontendTool -clangRewriteFrontend clangARCMigrate clangStaticAnalyzerFrontend -clangParse clangSerialization clangSema clangEdit clangStaticAnalyzerCheckers -clangASTMatchers clangStaticAnalyzerCore clangAnalysis clangAST clangRewrite clangLex clangBasic -clangCrossTU clangIndex - -lldCOFF -lldDriver -lldELF -lldMinGW -lldMachO -lldReaderWriter -lldYAML -lldCommon -lldCore - -${REQUIRED_LLVM_LIBRARIES} - -PUBLIC ${ZLIB_LIBRARIES} ${EXECINFO_LIBRARIES} Threads::Threads -${MALLOC_LIBRARIES} -${GLIBC_COMPATIBILITY_LIBRARIES} -${MEMCPY_LIBRARIES} -) diff --git a/dbms/programs/clang/Compiler-7.0.0bundled/cc1_main.cpp b/dbms/programs/clang/Compiler-7.0.0bundled/cc1_main.cpp deleted file mode 100644 index 3686475dd42..00000000000 --- a/dbms/programs/clang/Compiler-7.0.0bundled/cc1_main.cpp +++ /dev/null @@ -1,243 +0,0 @@ -//===-- cc1_main.cpp - Clang CC1 Compiler Frontend ------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This is the entry point to the clang -cc1 functionality, which implements the -// core compiler functionality along with a number of additional tools for -// demonstration and testing purposes. -// -//===----------------------------------------------------------------------===// - -#include "llvm/Option/Arg.h" -#include "clang/CodeGen/ObjectFilePCHContainerOperations.h" -#include "clang/Config/config.h" -#include "clang/Driver/DriverDiagnostic.h" -#include "clang/Driver/Options.h" -#include "clang/Frontend/CompilerInstance.h" -#include "clang/Frontend/CompilerInvocation.h" -#include "clang/Frontend/FrontendDiagnostic.h" -#include "clang/Frontend/TextDiagnosticBuffer.h" -#include "clang/Frontend/TextDiagnosticPrinter.h" -#include "clang/Frontend/Utils.h" -#include "clang/FrontendTool/Utils.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/Config/llvm-config.h" -#include "llvm/LinkAllPasses.h" -#include "llvm/Option/ArgList.h" -#include "llvm/Option/OptTable.h" -#include "llvm/Support/Compiler.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/ManagedStatic.h" -#include "llvm/Support/Signals.h" -#include "llvm/Support/TargetSelect.h" -#include "llvm/Support/Timer.h" -#include "llvm/Support/raw_ostream.h" -#include - -#ifdef CLANG_HAVE_RLIMITS -#include -#endif - -// have no .a version in packages -#undef LINK_POLLY_INTO_TOOLS - -using namespace clang; -using namespace llvm::opt; - -//===----------------------------------------------------------------------===// -// Main driver -//===----------------------------------------------------------------------===// - -static void LLVMErrorHandler(void *UserData, const std::string &Message, - bool GenCrashDiag) { - DiagnosticsEngine &Diags = *static_cast(UserData); - - Diags.Report(diag::err_fe_error_backend) << Message; - - // Run the interrupt handlers to make sure any special cleanups get done, in - // particular that we remove files registered with RemoveFileOnSignal. - llvm::sys::RunInterruptHandlers(); - - // We cannot recover from llvm errors. When reporting a fatal error, exit - // with status 70 to generate crash diagnostics. For BSD systems this is - // defined as an internal software error. Otherwise, exit with status 1. - exit(GenCrashDiag ? 70 : 1); -} - -#ifdef LINK_POLLY_INTO_TOOLS -namespace polly { -void initializePollyPasses(llvm::PassRegistry &Registry); -} -#endif - -#ifdef CLANG_HAVE_RLIMITS -// The amount of stack we think is "sufficient". If less than this much is -// available, we may be unable to reach our template instantiation depth -// limit and other similar limits. -// FIXME: Unify this with the stack we request when spawning a thread to build -// a module. -static const int kSufficientStack = 8 << 20; - -#if defined(__linux__) && defined(__PIE__) -static size_t getCurrentStackAllocation() { - // If we can't compute the current stack usage, allow for 512K of command - // line arguments and environment. - size_t Usage = 512 * 1024; - if (FILE *StatFile = fopen("/proc/self/stat", "r")) { - // We assume that the stack extends from its current address to the end of - // the environment space. In reality, there is another string literal (the - // program name) after the environment, but this is close enough (we only - // need to be within 100K or so). - unsigned long StackPtr, EnvEnd; - // Disable silly GCC -Wformat warning that complains about length - // modifiers on ignored format specifiers. We want to retain these - // for documentation purposes even though they have no effect. -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wformat" -#endif - if (fscanf(StatFile, - "%*d %*s %*c %*d %*d %*d %*d %*d %*u %*lu %*lu %*lu %*lu %*lu " - "%*lu %*ld %*ld %*ld %*ld %*ld %*ld %*llu %*lu %*ld %*lu %*lu " - "%*lu %*lu %lu %*lu %*lu %*lu %*lu %*lu %*llu %*lu %*lu %*d %*d " - "%*u %*u %*llu %*lu %*ld %*lu %*lu %*lu %*lu %*lu %*lu %lu %*d", - &StackPtr, &EnvEnd) == 2) { -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif - Usage = StackPtr < EnvEnd ? EnvEnd - StackPtr : StackPtr - EnvEnd; - } - fclose(StatFile); - } - return Usage; -} - -#include - -LLVM_ATTRIBUTE_NOINLINE -static void ensureStackAddressSpace(int ExtraChunks = 0) { - // Linux kernels prior to 4.1 will sometimes locate the heap of a PIE binary - // relatively close to the stack (they are only guaranteed to be 128MiB - // apart). This results in crashes if we happen to heap-allocate more than - // 128MiB before we reach our stack high-water mark. - // - // To avoid these crashes, ensure that we have sufficient virtual memory - // pages allocated before we start running. - size_t Curr = getCurrentStackAllocation(); - const int kTargetStack = kSufficientStack - 256 * 1024; - if (Curr < kTargetStack) { - volatile char *volatile Alloc = - static_cast(alloca(kTargetStack - Curr)); - Alloc[0] = 0; - Alloc[kTargetStack - Curr - 1] = 0; - } -} -#else -static void ensureStackAddressSpace() {} -#endif - -/// Attempt to ensure that we have at least 8MiB of usable stack space. -static void ensureSufficientStack() { - struct rlimit rlim; - if (getrlimit(RLIMIT_STACK, &rlim) != 0) - return; - - // Increase the soft stack limit to our desired level, if necessary and - // possible. - if (rlim.rlim_cur != RLIM_INFINITY && rlim.rlim_cur < kSufficientStack) { - // Try to allocate sufficient stack. - if (rlim.rlim_max == RLIM_INFINITY || rlim.rlim_max >= kSufficientStack) - rlim.rlim_cur = kSufficientStack; - else if (rlim.rlim_cur == rlim.rlim_max) - return; - else - rlim.rlim_cur = rlim.rlim_max; - - if (setrlimit(RLIMIT_STACK, &rlim) != 0 || - rlim.rlim_cur != kSufficientStack) - return; - } - - // We should now have a stack of size at least kSufficientStack. Ensure - // that we can actually use that much, if necessary. - ensureStackAddressSpace(); -} -#else -static void ensureSufficientStack() {} -#endif - -int cc1_main(ArrayRef Argv, const char *Argv0, void *MainAddr) { - ensureSufficientStack(); - - std::unique_ptr Clang(new CompilerInstance()); - IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); - - // Register the support for object-file-wrapped Clang modules. - auto PCHOps = Clang->getPCHContainerOperations(); - PCHOps->registerWriter(llvm::make_unique()); - PCHOps->registerReader(llvm::make_unique()); - - // Initialize targets first, so that --version shows registered targets. - llvm::InitializeAllTargets(); - llvm::InitializeAllTargetMCs(); - llvm::InitializeAllAsmPrinters(); - llvm::InitializeAllAsmParsers(); - -#ifdef LINK_POLLY_INTO_TOOLS - llvm::PassRegistry &Registry = *llvm::PassRegistry::getPassRegistry(); - polly::initializePollyPasses(Registry); -#endif - - // Buffer diagnostics from argument parsing so that we can output them using a - // well formed diagnostic object. - IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); - TextDiagnosticBuffer *DiagsBuffer = new TextDiagnosticBuffer; - DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagsBuffer); - bool Success = CompilerInvocation::CreateFromArgs( - Clang->getInvocation(), Argv.begin(), Argv.end(), Diags); - - // Infer the builtin include path if unspecified. - if (Clang->getHeaderSearchOpts().UseBuiltinIncludes && - Clang->getHeaderSearchOpts().ResourceDir.empty()) - Clang->getHeaderSearchOpts().ResourceDir = - CompilerInvocation::GetResourcesPath(Argv0, MainAddr); - - // Create the actual diagnostics engine. - Clang->createDiagnostics(); - if (!Clang->hasDiagnostics()) - return 1; - - // Set an error handler, so that any LLVM backend diagnostics go through our - // error handler. - llvm::install_fatal_error_handler(LLVMErrorHandler, - static_cast(&Clang->getDiagnostics())); - - DiagsBuffer->FlushDiagnostics(Clang->getDiagnostics()); - if (!Success) - return 1; - - // Execute the frontend actions. - Success = ExecuteCompilerInvocation(Clang.get()); - - // If any timers were active but haven't been destroyed yet, print their - // results now. This happens in -disable-free mode. - llvm::TimerGroup::printAll(llvm::errs()); - - // Our error handler depends on the Diagnostics object, which we're - // potentially about to delete. Uninstall the handler now so that any - // later errors use the default handling behavior instead. - llvm::remove_fatal_error_handler(); - - // When running with -disable-free, don't do any destruction or shutdown. - if (Clang->getFrontendOpts().DisableFree) { - BuryPointer(std::move(Clang)); - return !Success; - } - - return !Success; -} diff --git a/dbms/programs/clang/Compiler-7.0.0bundled/cc1as_main.cpp b/dbms/programs/clang/Compiler-7.0.0bundled/cc1as_main.cpp deleted file mode 100644 index ce23422077f..00000000000 --- a/dbms/programs/clang/Compiler-7.0.0bundled/cc1as_main.cpp +++ /dev/null @@ -1,555 +0,0 @@ -//===-- cc1as_main.cpp - Clang Assembler ---------------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This is the entry point to the clang -cc1as functionality, which implements -// the direct interface to the LLVM MC based assembler. -// -//===----------------------------------------------------------------------===// - -#include "clang/Basic/Diagnostic.h" -#include "clang/Basic/DiagnosticOptions.h" -#include "clang/Driver/DriverDiagnostic.h" -#include "clang/Driver/Options.h" -#include "clang/Frontend/FrontendDiagnostic.h" -#include "clang/Frontend/TextDiagnosticPrinter.h" -#include "clang/Frontend/Utils.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/StringSwitch.h" -#include "llvm/ADT/Triple.h" -#include "llvm/IR/DataLayout.h" -#include "llvm/MC/MCAsmBackend.h" -#include "llvm/MC/MCAsmInfo.h" -#include "llvm/MC/MCCodeEmitter.h" -#include "llvm/MC/MCContext.h" -#include "llvm/MC/MCInstrInfo.h" -#include "llvm/MC/MCObjectFileInfo.h" -#include "llvm/MC/MCParser/MCAsmParser.h" -#include "llvm/MC/MCParser/MCTargetAsmParser.h" -#include "llvm/MC/MCRegisterInfo.h" -#include "llvm/MC/MCStreamer.h" -#include "llvm/MC/MCSubtargetInfo.h" -#include "llvm/MC/MCTargetOptions.h" -#include "llvm/Option/Arg.h" -#include "llvm/Option/ArgList.h" -#include "llvm/Option/OptTable.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/FileSystem.h" -#include "llvm/Support/FormattedStream.h" -#include "llvm/Support/Host.h" -#include "llvm/Support/MemoryBuffer.h" -#include "llvm/Support/Path.h" -#include "llvm/Support/Signals.h" -#include "llvm/Support/SourceMgr.h" -#include "llvm/Support/TargetRegistry.h" -#include "llvm/Support/TargetSelect.h" -#include "llvm/Support/Timer.h" -#include "llvm/Support/raw_ostream.h" -#include -#include -using namespace clang; -using namespace clang::driver; -using namespace clang::driver::options; -using namespace llvm; -using namespace llvm::opt; - - -namespace { - -/// \brief Helper class for representing a single invocation of the assembler. -struct AssemblerInvocation { - /// @name Target Options - /// @{ - - /// The name of the target triple to assemble for. - std::string Triple; - - /// If given, the name of the target CPU to determine which instructions - /// are legal. - std::string CPU; - - /// The list of target specific features to enable or disable -- this should - /// be a list of strings starting with '+' or '-'. - std::vector Features; - - /// The list of symbol definitions. - std::vector SymbolDefs; - - /// @} - /// @name Language Options - /// @{ - - std::vector IncludePaths; - unsigned NoInitialTextSection : 1; - unsigned SaveTemporaryLabels : 1; - unsigned GenDwarfForAssembly : 1; - unsigned RelaxELFRelocations : 1; - unsigned DwarfVersion; - std::string DwarfDebugFlags; - std::string DwarfDebugProducer; - std::string DebugCompilationDir; - llvm::DebugCompressionType CompressDebugSections = - llvm::DebugCompressionType::None; - std::string MainFileName; - - /// @} - /// @name Frontend Options - /// @{ - - std::string InputFile; - std::vector LLVMArgs; - std::string OutputPath; - enum FileType { - FT_Asm, ///< Assembly (.s) output, transliterate mode. - FT_Null, ///< No output, for timing purposes. - FT_Obj ///< Object file output. - }; - FileType OutputType; - unsigned ShowHelp : 1; - unsigned ShowVersion : 1; - - /// @} - /// @name Transliterate Options - /// @{ - - unsigned OutputAsmVariant; - unsigned ShowEncoding : 1; - unsigned ShowInst : 1; - - /// @} - /// @name Assembler Options - /// @{ - - unsigned RelaxAll : 1; - unsigned NoExecStack : 1; - unsigned FatalWarnings : 1; - unsigned IncrementalLinkerCompatible : 1; - - /// The name of the relocation model to use. - std::string RelocationModel; - - /// @} - -public: - AssemblerInvocation() { - Triple = ""; - NoInitialTextSection = 0; - InputFile = "-"; - OutputPath = "-"; - OutputType = FT_Asm; - OutputAsmVariant = 0; - ShowInst = 0; - ShowEncoding = 0; - RelaxAll = 0; - NoExecStack = 0; - FatalWarnings = 0; - IncrementalLinkerCompatible = 0; - DwarfVersion = 0; - } - - static bool CreateFromArgs(AssemblerInvocation &Res, - ArrayRef Argv, - DiagnosticsEngine &Diags); -}; - -} - -bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts, - ArrayRef Argv, - DiagnosticsEngine &Diags) { - bool Success = true; - - // Parse the arguments. - std::unique_ptr OptTbl(createDriverOptTable()); - - const unsigned IncludedFlagsBitmask = options::CC1AsOption; - unsigned MissingArgIndex, MissingArgCount; - InputArgList Args = OptTbl->ParseArgs(Argv, MissingArgIndex, MissingArgCount, - IncludedFlagsBitmask); - - // Check for missing argument error. - if (MissingArgCount) { - Diags.Report(diag::err_drv_missing_argument) - << Args.getArgString(MissingArgIndex) << MissingArgCount; - Success = false; - } - - // Issue errors on unknown arguments. - for (const Arg *A : Args.filtered(OPT_UNKNOWN)) { - auto ArgString = A->getAsString(Args); - std::string Nearest; - if (OptTbl->findNearest(ArgString, Nearest, IncludedFlagsBitmask) > 1) - Diags.Report(diag::err_drv_unknown_argument) << ArgString; - else - Diags.Report(diag::err_drv_unknown_argument_with_suggestion) - << ArgString << Nearest; - Success = false; - } - - // Construct the invocation. - - // Target Options - Opts.Triple = llvm::Triple::normalize(Args.getLastArgValue(OPT_triple)); - Opts.CPU = Args.getLastArgValue(OPT_target_cpu); - Opts.Features = Args.getAllArgValues(OPT_target_feature); - - // Use the default target triple if unspecified. - if (Opts.Triple.empty()) - Opts.Triple = llvm::sys::getDefaultTargetTriple(); - - // Language Options - Opts.IncludePaths = Args.getAllArgValues(OPT_I); - Opts.NoInitialTextSection = Args.hasArg(OPT_n); - Opts.SaveTemporaryLabels = Args.hasArg(OPT_msave_temp_labels); - // Any DebugInfoKind implies GenDwarfForAssembly. - Opts.GenDwarfForAssembly = Args.hasArg(OPT_debug_info_kind_EQ); - - if (const Arg *A = Args.getLastArg(OPT_compress_debug_sections, - OPT_compress_debug_sections_EQ)) { - if (A->getOption().getID() == OPT_compress_debug_sections) { - // TODO: be more clever about the compression type auto-detection - Opts.CompressDebugSections = llvm::DebugCompressionType::GNU; - } else { - Opts.CompressDebugSections = - llvm::StringSwitch(A->getValue()) - .Case("none", llvm::DebugCompressionType::None) - .Case("zlib", llvm::DebugCompressionType::Z) - .Case("zlib-gnu", llvm::DebugCompressionType::GNU) - .Default(llvm::DebugCompressionType::None); - } - } - - Opts.RelaxELFRelocations = Args.hasArg(OPT_mrelax_relocations); - Opts.DwarfVersion = getLastArgIntValue(Args, OPT_dwarf_version_EQ, 2, Diags); - Opts.DwarfDebugFlags = Args.getLastArgValue(OPT_dwarf_debug_flags); - Opts.DwarfDebugProducer = Args.getLastArgValue(OPT_dwarf_debug_producer); - Opts.DebugCompilationDir = Args.getLastArgValue(OPT_fdebug_compilation_dir); - Opts.MainFileName = Args.getLastArgValue(OPT_main_file_name); - - // Frontend Options - if (Args.hasArg(OPT_INPUT)) { - bool First = true; - for (const Arg *A : Args.filtered(OPT_INPUT)) { - if (First) { - Opts.InputFile = A->getValue(); - First = false; - } else { - Diags.Report(diag::err_drv_unknown_argument) << A->getAsString(Args); - Success = false; - } - } - } - Opts.LLVMArgs = Args.getAllArgValues(OPT_mllvm); - Opts.OutputPath = Args.getLastArgValue(OPT_o); - if (Arg *A = Args.getLastArg(OPT_filetype)) { - StringRef Name = A->getValue(); - unsigned OutputType = StringSwitch(Name) - .Case("asm", FT_Asm) - .Case("null", FT_Null) - .Case("obj", FT_Obj) - .Default(~0U); - if (OutputType == ~0U) { - Diags.Report(diag::err_drv_invalid_value) << A->getAsString(Args) << Name; - Success = false; - } else - Opts.OutputType = FileType(OutputType); - } - Opts.ShowHelp = Args.hasArg(OPT_help); - Opts.ShowVersion = Args.hasArg(OPT_version); - - // Transliterate Options - Opts.OutputAsmVariant = - getLastArgIntValue(Args, OPT_output_asm_variant, 0, Diags); - Opts.ShowEncoding = Args.hasArg(OPT_show_encoding); - Opts.ShowInst = Args.hasArg(OPT_show_inst); - - // Assemble Options - Opts.RelaxAll = Args.hasArg(OPT_mrelax_all); - Opts.NoExecStack = Args.hasArg(OPT_mno_exec_stack); - Opts.FatalWarnings = Args.hasArg(OPT_massembler_fatal_warnings); - Opts.RelocationModel = Args.getLastArgValue(OPT_mrelocation_model, "pic"); - Opts.IncrementalLinkerCompatible = - Args.hasArg(OPT_mincremental_linker_compatible); - Opts.SymbolDefs = Args.getAllArgValues(OPT_defsym); - - return Success; -} - -static std::unique_ptr -getOutputStream(AssemblerInvocation &Opts, DiagnosticsEngine &Diags, - bool Binary) { - if (Opts.OutputPath.empty()) - Opts.OutputPath = "-"; - - // Make sure that the Out file gets unlinked from the disk if we get a - // SIGINT. - if (Opts.OutputPath != "-") - sys::RemoveFileOnSignal(Opts.OutputPath); - - std::error_code EC; - auto Out = llvm::make_unique( - Opts.OutputPath, EC, (Binary ? sys::fs::F_None : sys::fs::F_Text)); - if (EC) { - Diags.Report(diag::err_fe_unable_to_open_output) << Opts.OutputPath - << EC.message(); - return nullptr; - } - - return Out; -} - -static bool ExecuteAssembler(AssemblerInvocation &Opts, - DiagnosticsEngine &Diags) { - // Get the target specific parser. - std::string Error; - const Target *TheTarget = TargetRegistry::lookupTarget(Opts.Triple, Error); - if (!TheTarget) - return Diags.Report(diag::err_target_unknown_triple) << Opts.Triple; - - ErrorOr> Buffer = - MemoryBuffer::getFileOrSTDIN(Opts.InputFile); - - if (std::error_code EC = Buffer.getError()) { - Error = EC.message(); - return Diags.Report(diag::err_fe_error_reading) << Opts.InputFile; - } - - SourceMgr SrcMgr; - - // Tell SrcMgr about this buffer, which is what the parser will pick up. - SrcMgr.AddNewSourceBuffer(std::move(*Buffer), SMLoc()); - - // Record the location of the include directories so that the lexer can find - // it later. - SrcMgr.setIncludeDirs(Opts.IncludePaths); - - std::unique_ptr MRI(TheTarget->createMCRegInfo(Opts.Triple)); - assert(MRI && "Unable to create target register info!"); - - std::unique_ptr MAI(TheTarget->createMCAsmInfo(*MRI, Opts.Triple)); - assert(MAI && "Unable to create target asm info!"); - - // Ensure MCAsmInfo initialization occurs before any use, otherwise sections - // may be created with a combination of default and explicit settings. - MAI->setCompressDebugSections(Opts.CompressDebugSections); - - MAI->setRelaxELFRelocations(Opts.RelaxELFRelocations); - - bool IsBinary = Opts.OutputType == AssemblerInvocation::FT_Obj; - std::unique_ptr FDOS = getOutputStream(Opts, Diags, IsBinary); - if (!FDOS) - return true; - - // FIXME: This is not pretty. MCContext has a ptr to MCObjectFileInfo and - // MCObjectFileInfo needs a MCContext reference in order to initialize itself. - std::unique_ptr MOFI(new MCObjectFileInfo()); - - MCContext Ctx(MAI.get(), MRI.get(), MOFI.get(), &SrcMgr); - - bool PIC = false; - if (Opts.RelocationModel == "static") { - PIC = false; - } else if (Opts.RelocationModel == "pic") { - PIC = true; - } else { - assert(Opts.RelocationModel == "dynamic-no-pic" && - "Invalid PIC model!"); - PIC = false; - } - - MOFI->InitMCObjectFileInfo(Triple(Opts.Triple), PIC, Ctx); - if (Opts.SaveTemporaryLabels) - Ctx.setAllowTemporaryLabels(false); - if (Opts.GenDwarfForAssembly) - Ctx.setGenDwarfForAssembly(true); - if (!Opts.DwarfDebugFlags.empty()) - Ctx.setDwarfDebugFlags(StringRef(Opts.DwarfDebugFlags)); - if (!Opts.DwarfDebugProducer.empty()) - Ctx.setDwarfDebugProducer(StringRef(Opts.DwarfDebugProducer)); - if (!Opts.DebugCompilationDir.empty()) - Ctx.setCompilationDir(Opts.DebugCompilationDir); - if (!Opts.MainFileName.empty()) - Ctx.setMainFileName(StringRef(Opts.MainFileName)); - Ctx.setDwarfVersion(Opts.DwarfVersion); - - // Build up the feature string from the target feature list. - std::string FS; - if (!Opts.Features.empty()) { - FS = Opts.Features[0]; - for (unsigned i = 1, e = Opts.Features.size(); i != e; ++i) - FS += "," + Opts.Features[i]; - } - - std::unique_ptr Str; - - std::unique_ptr MCII(TheTarget->createMCInstrInfo()); - std::unique_ptr STI( - TheTarget->createMCSubtargetInfo(Opts.Triple, Opts.CPU, FS)); - - raw_pwrite_stream *Out = FDOS.get(); - std::unique_ptr BOS; - - // FIXME: There is a bit of code duplication with addPassesToEmitFile. - if (Opts.OutputType == AssemblerInvocation::FT_Asm) { - MCInstPrinter *IP = TheTarget->createMCInstPrinter( - llvm::Triple(Opts.Triple), Opts.OutputAsmVariant, *MAI, *MCII, *MRI); - - std::unique_ptr CE; - if (Opts.ShowEncoding) - CE.reset(TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx)); - MCTargetOptions MCOptions; - std::unique_ptr MAB( - TheTarget->createMCAsmBackend(*STI, *MRI, MCOptions)); - - auto FOut = llvm::make_unique(*Out); - Str.reset(TheTarget->createAsmStreamer( - Ctx, std::move(FOut), /*asmverbose*/ true, - /*useDwarfDirectory*/ true, IP, std::move(CE), std::move(MAB), - Opts.ShowInst)); - } else if (Opts.OutputType == AssemblerInvocation::FT_Null) { - Str.reset(createNullStreamer(Ctx)); - } else { - assert(Opts.OutputType == AssemblerInvocation::FT_Obj && - "Invalid file type!"); - if (!FDOS->supportsSeeking()) { - BOS = make_unique(*FDOS); - Out = BOS.get(); - } - - std::unique_ptr CE( - TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx)); - MCTargetOptions MCOptions; - std::unique_ptr MAB( - TheTarget->createMCAsmBackend(*STI, *MRI, MCOptions)); - - Triple T(Opts.Triple); - Str.reset(TheTarget->createMCObjectStreamer( - T, Ctx, std::move(MAB), *Out, std::move(CE), *STI, Opts.RelaxAll, - Opts.IncrementalLinkerCompatible, - /*DWARFMustBeAtTheEnd*/ true)); - Str.get()->InitSections(Opts.NoExecStack); - } - - // Assembly to object compilation should leverage assembly info. - Str->setUseAssemblerInfoForParsing(true); - - bool Failed = false; - - std::unique_ptr Parser( - createMCAsmParser(SrcMgr, Ctx, *Str.get(), *MAI)); - - // FIXME: init MCTargetOptions from sanitizer flags here. - MCTargetOptions Options; - std::unique_ptr TAP( - TheTarget->createMCAsmParser(*STI, *Parser, *MCII, Options)); - if (!TAP) - Failed = Diags.Report(diag::err_target_unknown_triple) << Opts.Triple; - - // Set values for symbols, if any. - for (auto &S : Opts.SymbolDefs) { - auto Pair = StringRef(S).split('='); - auto Sym = Pair.first; - auto Val = Pair.second; - int64_t Value; - // We have already error checked this in the driver. - Val.getAsInteger(0, Value); - Ctx.setSymbolValue(Parser->getStreamer(), Sym, Value); - } - - if (!Failed) { - Parser->setTargetParser(*TAP.get()); - Failed = Parser->Run(Opts.NoInitialTextSection); - } - - // Close Streamer first. - // It might have a reference to the output stream. - Str.reset(); - // Close the output stream early. - BOS.reset(); - FDOS.reset(); - - // Delete output file if there were errors. - if (Failed && Opts.OutputPath != "-") - sys::fs::remove(Opts.OutputPath); - - return Failed; -} - -static void LLVMErrorHandler(void *UserData, const std::string &Message, - bool /*GenCrashDiag*/) { - DiagnosticsEngine &Diags = *static_cast(UserData); - - Diags.Report(diag::err_fe_error_backend) << Message; - - // We cannot recover from llvm errors. - exit(1); -} - -int cc1as_main(ArrayRef Argv, const char */*Argv0*/, void */*MainAddr*/) { - // Initialize targets and assembly printers/parsers. - InitializeAllTargetInfos(); - InitializeAllTargetMCs(); - InitializeAllAsmParsers(); - - // Construct our diagnostic client. - IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); - TextDiagnosticPrinter *DiagClient - = new TextDiagnosticPrinter(errs(), &*DiagOpts); - DiagClient->setPrefix("clang -cc1as"); - IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); - DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient); - - // Set an error handler, so that any LLVM backend diagnostics go through our - // error handler. - ScopedFatalErrorHandler FatalErrorHandler - (LLVMErrorHandler, static_cast(&Diags)); - - // Parse the arguments. - AssemblerInvocation Asm; - if (!AssemblerInvocation::CreateFromArgs(Asm, Argv, Diags)) - return 1; - - if (Asm.ShowHelp) { - std::unique_ptr Opts(driver::createDriverOptTable()); - Opts->PrintHelp(llvm::outs(), "clang -cc1as", "Clang Integrated Assembler", - /*Include=*/driver::options::CC1AsOption, /*Exclude=*/0, - /*ShowAllAliases=*/false); - return 0; - } - - // Honor -version. - // - // FIXME: Use a better -version message? - if (Asm.ShowVersion) { - llvm::cl::PrintVersionMessage(); - return 0; - } - - // Honor -mllvm. - // - // FIXME: Remove this, one day. - if (!Asm.LLVMArgs.empty()) { - unsigned NumArgs = Asm.LLVMArgs.size(); - auto Args = llvm::make_unique(NumArgs + 2); - Args[0] = "clang (LLVM option parsing)"; - for (unsigned i = 0; i != NumArgs; ++i) - Args[i + 1] = Asm.LLVMArgs[i].c_str(); - Args[NumArgs + 1] = nullptr; - llvm::cl::ParseCommandLineOptions(NumArgs + 1, Args.get()); - } - - // Execute the invocation, unless there were parsing errors. - bool Failed = Diags.hasErrorOccurred() || ExecuteAssembler(Asm, Diags); - - // If any timers were active but haven't been destroyed yet, print their - // results now. - TimerGroup::printAll(errs()); - - return !!Failed; -} diff --git a/dbms/programs/clang/Compiler-7.0.0bundled/driver.cpp b/dbms/programs/clang/Compiler-7.0.0bundled/driver.cpp deleted file mode 100644 index 9a061b9d137..00000000000 --- a/dbms/programs/clang/Compiler-7.0.0bundled/driver.cpp +++ /dev/null @@ -1,512 +0,0 @@ -//===-- driver.cpp - Clang GCC-Compatible Driver --------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This is the entry point to the clang driver; it is a thin wrapper -// for functionality in the Driver clang library. -// -//===----------------------------------------------------------------------===// - -#include "clang/Driver/Driver.h" -#include "clang/Basic/DiagnosticOptions.h" -#include "clang/Driver/Compilation.h" -#include "clang/Driver/DriverDiagnostic.h" -#include "clang/Driver/Options.h" -#include "clang/Driver/ToolChain.h" -#include "clang/Frontend/ChainedDiagnosticConsumer.h" -#include "clang/Frontend/CompilerInvocation.h" -#include "clang/Frontend/SerializedDiagnosticPrinter.h" -#include "clang/Frontend/TextDiagnosticPrinter.h" -#include "clang/Frontend/Utils.h" -#include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/SmallString.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/Option/ArgList.h" -#include "llvm/Option/OptTable.h" -#include "llvm/Option/Option.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/FileSystem.h" -#include "llvm/Support/Host.h" -#include "llvm/Support/InitLLVM.h" -#include "llvm/Support/Path.h" -#include "llvm/Support/Process.h" -#include "llvm/Support/Program.h" -#include "llvm/Support/Regex.h" -#include "llvm/Support/Signals.h" -#include "llvm/Support/StringSaver.h" -#include "llvm/Support/TargetSelect.h" -#include "llvm/Support/Timer.h" -#include "llvm/Support/raw_ostream.h" -#include -#include -#include -using namespace clang; -using namespace clang::driver; -using namespace llvm::opt; - -std::string GetExecutablePath(const char *Argv0, bool CanonicalPrefixes) { - if (!CanonicalPrefixes) { - SmallString<128> ExecutablePath(Argv0); - // Do a PATH lookup if Argv0 isn't a valid path. - if (!llvm::sys::fs::exists(ExecutablePath)) - if (llvm::ErrorOr P = - llvm::sys::findProgramByName(ExecutablePath)) - ExecutablePath = *P; - return ExecutablePath.str(); - } - - // This just needs to be some symbol in the binary; C++ doesn't - // allow taking the address of ::main however. - void *P = (void*) (intptr_t) GetExecutablePath; - return llvm::sys::fs::getMainExecutable(Argv0, P); -} - -static const char *GetStableCStr(std::set &SavedStrings, - StringRef S) { - return SavedStrings.insert(S).first->c_str(); -} - -/// ApplyQAOverride - Apply a list of edits to the input argument lists. -/// -/// The input string is a space separate list of edits to perform, -/// they are applied in order to the input argument lists. Edits -/// should be one of the following forms: -/// -/// '#': Silence information about the changes to the command line arguments. -/// -/// '^': Add FOO as a new argument at the beginning of the command line. -/// -/// '+': Add FOO as a new argument at the end of the command line. -/// -/// 's/XXX/YYY/': Substitute the regular expression XXX with YYY in the command -/// line. -/// -/// 'xOPTION': Removes all instances of the literal argument OPTION. -/// -/// 'XOPTION': Removes all instances of the literal argument OPTION, -/// and the following argument. -/// -/// 'Ox': Removes all flags matching 'O' or 'O[sz0-9]' and adds 'Ox' -/// at the end of the command line. -/// -/// \param OS - The stream to write edit information to. -/// \param Args - The vector of command line arguments. -/// \param Edit - The override command to perform. -/// \param SavedStrings - Set to use for storing string representations. -static void ApplyOneQAOverride(raw_ostream &OS, - SmallVectorImpl &Args, - StringRef Edit, - std::set &SavedStrings) { - // This does not need to be efficient. - - if (Edit[0] == '^') { - const char *Str = - GetStableCStr(SavedStrings, Edit.substr(1)); - OS << "### Adding argument " << Str << " at beginning\n"; - Args.insert(Args.begin() + 1, Str); - } else if (Edit[0] == '+') { - const char *Str = - GetStableCStr(SavedStrings, Edit.substr(1)); - OS << "### Adding argument " << Str << " at end\n"; - Args.push_back(Str); - } else if (Edit[0] == 's' && Edit[1] == '/' && Edit.endswith("/") && - Edit.slice(2, Edit.size()-1).find('/') != StringRef::npos) { - StringRef MatchPattern = Edit.substr(2).split('/').first; - StringRef ReplPattern = Edit.substr(2).split('/').second; - ReplPattern = ReplPattern.slice(0, ReplPattern.size()-1); - - for (unsigned i = 1, e = Args.size(); i != e; ++i) { - // Ignore end-of-line response file markers - if (Args[i] == nullptr) - continue; - std::string Repl = llvm::Regex(MatchPattern).sub(ReplPattern, Args[i]); - - if (Repl != Args[i]) { - OS << "### Replacing '" << Args[i] << "' with '" << Repl << "'\n"; - Args[i] = GetStableCStr(SavedStrings, Repl); - } - } - } else if (Edit[0] == 'x' || Edit[0] == 'X') { - auto Option = Edit.substr(1); - for (unsigned i = 1; i < Args.size();) { - if (Option == Args[i]) { - OS << "### Deleting argument " << Args[i] << '\n'; - Args.erase(Args.begin() + i); - if (Edit[0] == 'X') { - if (i < Args.size()) { - OS << "### Deleting argument " << Args[i] << '\n'; - Args.erase(Args.begin() + i); - } else - OS << "### Invalid X edit, end of command line!\n"; - } - } else - ++i; - } - } else if (Edit[0] == 'O') { - for (unsigned i = 1; i < Args.size();) { - const char *A = Args[i]; - // Ignore end-of-line response file markers - if (A == nullptr) - continue; - if (A[0] == '-' && A[1] == 'O' && - (A[2] == '\0' || - (A[3] == '\0' && (A[2] == 's' || A[2] == 'z' || - ('0' <= A[2] && A[2] <= '9'))))) { - OS << "### Deleting argument " << Args[i] << '\n'; - Args.erase(Args.begin() + i); - } else - ++i; - } - OS << "### Adding argument " << Edit << " at end\n"; - Args.push_back(GetStableCStr(SavedStrings, '-' + Edit.str())); - } else { - OS << "### Unrecognized edit: " << Edit << "\n"; - } -} - -/// ApplyQAOverride - Apply a comma separate list of edits to the -/// input argument lists. See ApplyOneQAOverride. -static void ApplyQAOverride(SmallVectorImpl &Args, - const char *OverrideStr, - std::set &SavedStrings) { - raw_ostream *OS = &llvm::errs(); - - if (OverrideStr[0] == '#') { - ++OverrideStr; - OS = &llvm::nulls(); - } - - *OS << "### CCC_OVERRIDE_OPTIONS: " << OverrideStr << "\n"; - - // This does not need to be efficient. - - const char *S = OverrideStr; - while (*S) { - const char *End = ::strchr(S, ' '); - if (!End) - End = S + strlen(S); - if (End != S) - ApplyOneQAOverride(*OS, Args, std::string(S, End), SavedStrings); - S = End; - if (*S != '\0') - ++S; - } -} - -extern int cc1_main(ArrayRef Argv, const char *Argv0, - void *MainAddr); -extern int cc1as_main(ArrayRef Argv, const char *Argv0, - void *MainAddr); -extern int cc1gen_reproducer_main(ArrayRef Argv, - const char *Argv0, void *MainAddr); - -static void insertTargetAndModeArgs(const ParsedClangName &NameParts, - SmallVectorImpl &ArgVector, - std::set &SavedStrings) { - // Put target and mode arguments at the start of argument list so that - // arguments specified in command line could override them. Avoid putting - // them at index 0, as an option like '-cc1' must remain the first. - int InsertionPoint = 0; - if (ArgVector.size() > 0) - ++InsertionPoint; - - if (NameParts.DriverMode) { - // Add the mode flag to the arguments. - ArgVector.insert(ArgVector.begin() + InsertionPoint, - GetStableCStr(SavedStrings, NameParts.DriverMode)); - } - - if (NameParts.TargetIsValid) { - const char *arr[] = {"-target", GetStableCStr(SavedStrings, - NameParts.TargetPrefix)}; - ArgVector.insert(ArgVector.begin() + InsertionPoint, - std::begin(arr), std::end(arr)); - } -} - -static void getCLEnvVarOptions(std::string &EnvValue, llvm::StringSaver &Saver, - SmallVectorImpl &Opts) { - llvm::cl::TokenizeWindowsCommandLine(EnvValue, Saver, Opts); - // The first instance of '#' should be replaced with '=' in each option. - for (const char *Opt : Opts) - if (char *NumberSignPtr = const_cast(::strchr(Opt, '#'))) - *NumberSignPtr = '='; -} - -static void SetBackdoorDriverOutputsFromEnvVars(Driver &TheDriver) { - // Handle CC_PRINT_OPTIONS and CC_PRINT_OPTIONS_FILE. - TheDriver.CCPrintOptions = !!::getenv("CC_PRINT_OPTIONS"); - if (TheDriver.CCPrintOptions) - TheDriver.CCPrintOptionsFilename = ::getenv("CC_PRINT_OPTIONS_FILE"); - - // Handle CC_PRINT_HEADERS and CC_PRINT_HEADERS_FILE. - TheDriver.CCPrintHeaders = !!::getenv("CC_PRINT_HEADERS"); - if (TheDriver.CCPrintHeaders) - TheDriver.CCPrintHeadersFilename = ::getenv("CC_PRINT_HEADERS_FILE"); - - // Handle CC_LOG_DIAGNOSTICS and CC_LOG_DIAGNOSTICS_FILE. - TheDriver.CCLogDiagnostics = !!::getenv("CC_LOG_DIAGNOSTICS"); - if (TheDriver.CCLogDiagnostics) - TheDriver.CCLogDiagnosticsFilename = ::getenv("CC_LOG_DIAGNOSTICS_FILE"); -} - -static void FixupDiagPrefixExeName(TextDiagnosticPrinter *DiagClient, - const std::string &Path) { - // If the clang binary happens to be named cl.exe for compatibility reasons, - // use clang-cl.exe as the prefix to avoid confusion between clang and MSVC. - StringRef ExeBasename(llvm::sys::path::filename(Path)); - if (ExeBasename.equals_lower("cl.exe")) - ExeBasename = "clang-cl.exe"; - DiagClient->setPrefix(ExeBasename); -} - -// This lets us create the DiagnosticsEngine with a properly-filled-out -// DiagnosticOptions instance. -static DiagnosticOptions * -CreateAndPopulateDiagOpts(ArrayRef argv) { - auto *DiagOpts = new DiagnosticOptions; - std::unique_ptr Opts(createDriverOptTable()); - unsigned MissingArgIndex, MissingArgCount; - InputArgList Args = - Opts->ParseArgs(argv.slice(1), MissingArgIndex, MissingArgCount); - // We ignore MissingArgCount and the return value of ParseDiagnosticArgs. - // Any errors that would be diagnosed here will also be diagnosed later, - // when the DiagnosticsEngine actually exists. - (void)ParseDiagnosticArgs(*DiagOpts, Args); - return DiagOpts; -} - -static void SetInstallDir(SmallVectorImpl &argv, - Driver &TheDriver, bool CanonicalPrefixes) { - // Attempt to find the original path used to invoke the driver, to determine - // the installed path. We do this manually, because we want to support that - // path being a symlink. - SmallString<128> InstalledPath(argv[0]); - - // Do a PATH lookup, if there are no directory components. - if (llvm::sys::path::filename(InstalledPath) == InstalledPath) - if (llvm::ErrorOr Tmp = llvm::sys::findProgramByName( - llvm::sys::path::filename(InstalledPath.str()))) - InstalledPath = *Tmp; - - // FIXME: We don't actually canonicalize this, we just make it absolute. - if (CanonicalPrefixes) - llvm::sys::fs::make_absolute(InstalledPath); - - StringRef InstalledPathParent(llvm::sys::path::parent_path(InstalledPath)); - if (llvm::sys::fs::exists(InstalledPathParent)) - TheDriver.setInstalledDir(InstalledPathParent); -} - -static int ExecuteCC1Tool(ArrayRef argv, StringRef Tool) { - void *GetExecutablePathVP = (void *)(intptr_t) GetExecutablePath; - if (Tool == "") - return cc1_main(argv.slice(2), argv[0], GetExecutablePathVP); - if (Tool == "as") - return cc1as_main(argv.slice(2), argv[0], GetExecutablePathVP); - - // Reject unknown tools. - llvm::errs() << "error: unknown integrated tool '" << Tool << "'. " - << "Valid tools include '-cc1' and '-cc1as'.\n"; - return 1; -} - -int mainEntryClickHouseClang(int argc_, char **argv_) { - llvm::InitLLVM X(argc_, argv_); - SmallVector argv(argv_, argv_ + argc_); - - if (llvm::sys::Process::FixupStandardFileDescriptors()) - return 1; - - llvm::InitializeAllTargets(); - auto TargetAndMode = ToolChain::getTargetAndModeFromProgramName(argv[0]); - - llvm::BumpPtrAllocator A; - llvm::StringSaver Saver(A); - - // Parse response files using the GNU syntax, unless we're in CL mode. There - // are two ways to put clang in CL compatibility mode: argv[0] is either - // clang-cl or cl, or --driver-mode=cl is on the command line. The normal - // command line parsing can't happen until after response file parsing, so we - // have to manually search for a --driver-mode=cl argument the hard way. - // Finally, our -cc1 tools don't care which tokenization mode we use because - // response files written by clang will tokenize the same way in either mode. - bool ClangCLMode = false; - if (StringRef(TargetAndMode.DriverMode).equals("--driver-mode=cl") || - std::find_if(argv.begin(), argv.end(), [](const char *F) { - return F && strcmp(F, "--driver-mode=cl") == 0; - }) != argv.end()) { - ClangCLMode = true; - } - enum { Default, POSIX, Windows } RSPQuoting = Default; - for (const char *F : argv) { - if (strcmp(F, "--rsp-quoting=posix") == 0) - RSPQuoting = POSIX; - else if (strcmp(F, "--rsp-quoting=windows") == 0) - RSPQuoting = Windows; - } - - // Determines whether we want nullptr markers in argv to indicate response - // files end-of-lines. We only use this for the /LINK driver argument with - // clang-cl.exe on Windows. - bool MarkEOLs = ClangCLMode; - - llvm::cl::TokenizerCallback Tokenizer; - if (RSPQuoting == Windows || (RSPQuoting == Default && ClangCLMode)) - Tokenizer = &llvm::cl::TokenizeWindowsCommandLine; - else - Tokenizer = &llvm::cl::TokenizeGNUCommandLine; - - if (MarkEOLs && argv.size() > 1 && StringRef(argv[1]).startswith("-cc1")) - MarkEOLs = false; - llvm::cl::ExpandResponseFiles(Saver, Tokenizer, argv, MarkEOLs); - - // Handle -cc1 integrated tools, even if -cc1 was expanded from a response - // file. - auto FirstArg = std::find_if(argv.begin() + 1, argv.end(), - [](const char *A) { return A != nullptr; }); - if (FirstArg != argv.end() && StringRef(*FirstArg).startswith("-cc1")) { - // If -cc1 came from a response file, remove the EOL sentinels. - if (MarkEOLs) { - auto newEnd = std::remove(argv.begin(), argv.end(), nullptr); - argv.resize(newEnd - argv.begin()); - } - return ExecuteCC1Tool(argv, argv[1] + 4); - } - - bool CanonicalPrefixes = true; - for (int i = 1, size = argv.size(); i < size; ++i) { - // Skip end-of-line response file markers - if (argv[i] == nullptr) - continue; - if (StringRef(argv[i]) == "-no-canonical-prefixes") { - CanonicalPrefixes = false; - break; - } - } - - // Handle CL and _CL_ which permits additional command line options to be - // prepended or appended. - if (ClangCLMode) { - // Arguments in "CL" are prepended. - llvm::Optional OptCL = llvm::sys::Process::GetEnv("CL"); - if (OptCL.hasValue()) { - SmallVector PrependedOpts; - getCLEnvVarOptions(OptCL.getValue(), Saver, PrependedOpts); - - // Insert right after the program name to prepend to the argument list. - argv.insert(argv.begin() + 1, PrependedOpts.begin(), PrependedOpts.end()); - } - // Arguments in "_CL_" are appended. - llvm::Optional Opt_CL_ = llvm::sys::Process::GetEnv("_CL_"); - if (Opt_CL_.hasValue()) { - SmallVector AppendedOpts; - getCLEnvVarOptions(Opt_CL_.getValue(), Saver, AppendedOpts); - - // Insert at the end of the argument list to append. - argv.append(AppendedOpts.begin(), AppendedOpts.end()); - } - } - - std::set SavedStrings; - // Handle CCC_OVERRIDE_OPTIONS, used for editing a command line behind the - // scenes. - if (const char *OverrideStr = ::getenv("CCC_OVERRIDE_OPTIONS")) { - // FIXME: Driver shouldn't take extra initial argument. - ApplyQAOverride(argv, OverrideStr, SavedStrings); - } - - std::string Path = GetExecutablePath(argv[0], CanonicalPrefixes); - - IntrusiveRefCntPtr DiagOpts = - CreateAndPopulateDiagOpts(argv); - - TextDiagnosticPrinter *DiagClient - = new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts); - FixupDiagPrefixExeName(DiagClient, Path); - - IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); - - DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient); - - if (!DiagOpts->DiagnosticSerializationFile.empty()) { - auto SerializedConsumer = - clang::serialized_diags::create(DiagOpts->DiagnosticSerializationFile, - &*DiagOpts, /*MergeChildRecords=*/true); - Diags.setClient(new ChainedDiagnosticConsumer( - Diags.takeClient(), std::move(SerializedConsumer))); - } - - ProcessWarningOptions(Diags, *DiagOpts, /*ReportDiags=*/false); - - Driver TheDriver(Path, llvm::sys::getDefaultTargetTriple(), Diags); - SetInstallDir(argv, TheDriver, CanonicalPrefixes); - TheDriver.setTargetAndMode(TargetAndMode); - - insertTargetAndModeArgs(TargetAndMode, argv, SavedStrings); - - SetBackdoorDriverOutputsFromEnvVars(TheDriver); - - std::unique_ptr C(TheDriver.BuildCompilation(argv)); - int Res = 1; - if (C && !C->containsError()) { - SmallVector, 4> FailingCommands; - Res = TheDriver.ExecuteCompilation(*C, FailingCommands); - - // Force a crash to test the diagnostics. - if (TheDriver.GenReproducer) { - Diags.Report(diag::err_drv_force_crash) - << !::getenv("FORCE_CLANG_DIAGNOSTICS_CRASH"); - - // Pretend that every command failed. - FailingCommands.clear(); - for (const auto &J : C->getJobs()) - if (const Command *C = dyn_cast(&J)) - FailingCommands.push_back(std::make_pair(-1, C)); - } - - for (const auto &P : FailingCommands) { - int CommandRes = P.first; - const Command *FailingCommand = P.second; - if (!Res) - Res = CommandRes; - - // If result status is < 0, then the driver command signalled an error. - // If result status is 70, then the driver command reported a fatal error. - // On Windows, abort will return an exit code of 3. In these cases, - // generate additional diagnostic information if possible. - bool DiagnoseCrash = CommandRes < 0 || CommandRes == 70; -#ifdef _WIN32 - DiagnoseCrash |= CommandRes == 3; -#endif - if (DiagnoseCrash) { - TheDriver.generateCompilationDiagnostics(*C, *FailingCommand); - break; - } - } - } - - Diags.getClient()->finish(); - - // If any timers were active but haven't been destroyed yet, print their - // results now. This happens in -disable-free mode. - llvm::TimerGroup::printAll(llvm::errs()); - -#ifdef _WIN32 - // Exit status should not be negative on Win32, unless abnormal termination. - // Once abnormal termiation was caught, negative status should not be - // propagated. - if (Res < 0) - Res = 1; -#endif - - // If we have multiple failing commands, we return the result of the first - // failing command. - return Res; -} diff --git a/dbms/programs/clang/Compiler-7.0.0bundled/lld.cpp b/dbms/programs/clang/Compiler-7.0.0bundled/lld.cpp deleted file mode 100644 index 203e50d42a9..00000000000 --- a/dbms/programs/clang/Compiler-7.0.0bundled/lld.cpp +++ /dev/null @@ -1,10 +0,0 @@ -#include "lld/Common/Driver.h" -#include "llvm/Support/InitLLVM.h" -#include - -int mainEntryClickHouseLLD(int argc, char ** argv) -{ - llvm::InitLLVM X(argc, argv); - std::vector args(argv, argv + argc); - return !lld::elf::link(args, false); -} diff --git a/dbms/programs/clang/Compiler-7.0.0svn b/dbms/programs/clang/Compiler-7.0.0svn deleted file mode 120000 index eeeb5bbc2c0..00000000000 --- a/dbms/programs/clang/Compiler-7.0.0svn +++ /dev/null @@ -1 +0,0 @@ -Compiler-7.0.0 \ No newline at end of file diff --git a/dbms/programs/clang/Compiler-7.0.1 b/dbms/programs/clang/Compiler-7.0.1 deleted file mode 120000 index eeeb5bbc2c0..00000000000 --- a/dbms/programs/clang/Compiler-7.0.1 +++ /dev/null @@ -1 +0,0 @@ -Compiler-7.0.0 \ No newline at end of file diff --git a/dbms/programs/clang/clickhouse-clang.cpp b/dbms/programs/clang/clickhouse-clang.cpp deleted file mode 100644 index 261ae18b6d3..00000000000 --- a/dbms/programs/clang/clickhouse-clang.cpp +++ /dev/null @@ -1,2 +0,0 @@ -int mainEntryClickHouseClang(int argc, char ** argv); -int main(int argc_, char ** argv_) { return mainEntryClickHouseClang(argc_, argv_); } diff --git a/dbms/programs/clang/clickhouse-lld.cpp b/dbms/programs/clang/clickhouse-lld.cpp deleted file mode 100644 index baa6182d66d..00000000000 --- a/dbms/programs/clang/clickhouse-lld.cpp +++ /dev/null @@ -1,2 +0,0 @@ -int mainEntryClickHouseLLD(int argc, char ** argv); -int main(int argc_, char ** argv_) { return mainEntryClickHouseLLD(argc_, argv_); } diff --git a/dbms/programs/clang/copy_headers.sh b/dbms/programs/clang/copy_headers.sh deleted file mode 100755 index 45a58855c91..00000000000 --- a/dbms/programs/clang/copy_headers.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env bash - -set -e -#set -x -#echo "Args: $*"; env | sort - -# Этот скрипт собирает все заголовочные файлы, нужные для компиляции некоторого translation unit-а -# и копирует их с сохранением путей в директорию DST. -# Это затем может быть использовано, чтобы скомпилировать translation unit на другом сервере, -# используя ровно такой же набор заголовочных файлов. -# -# Требуется clang, желательно наиболее свежий (trunk). -# -# Используется при сборке пакетов. -# Заголовочные файлы записываются в пакет clickhouse-common, в директорию /usr/share/clickhouse/headers. -# -# Если вы хотите установить их самостоятельно, без сборки пакета, -# чтобы clickhouse-server видел их там, где ожидается, выполните: -# -# sudo ./copy_headers.sh . /usr/share/clickhouse/headers/ - -SOURCE_PATH=${1:-../../..} -DST=${2:-$SOURCE_PATH/../headers} -BUILD_PATH=${BUILD_PATH=${3:-$SOURCE_PATH/build}} - -PATH="/usr/local/bin:/usr/local/sbin:/usr/bin:$PATH" - -if [[ -z $CLANG ]]; then - CLANG="clang" -fi - -START_HEADERS=$(echo \ - $BUILD_PATH/dbms/src/Common/config_version.h \ - $SOURCE_PATH/dbms/src/Interpreters/SpecializedAggregator.h \ - $SOURCE_PATH/dbms/src/AggregateFunctions/AggregateFunction*.h) - -for header in $START_HEADERS; do - START_HEADERS_INCLUDE+="-include $header " -done - - -GCC_ROOT=`$CLANG -v 2>&1 | grep "Selected GCC installation"| sed -n -e 's/^.*: //p'` - -# TODO: Does not work on macos? -GCC_ROOT=${GCC_ROOT:=/usr/lib/clang/${CMAKE_CXX_COMPILER_VERSION}} - -# Опция -mcx16 для того, чтобы выбиралось больше заголовочных файлов (с запасом). -# The latter options are the same that are added while building packages. -for src_file in $(echo | $CLANG -M -xc++ -std=c++1z -Wall -Werror -msse2 -msse4 -mcx16 -mpopcnt -O3 -g -fPIC -fstack-protector -D_FORTIFY_SOURCE=2 \ - -I $GCC_ROOT/include \ - -I $GCC_ROOT/include-fixed \ - $(cat "$BUILD_PATH/include_directories.txt") \ - $START_HEADERS_INCLUDE \ - - | - tr -d '\\' | - sed -E -e 's/^-\.o://'); -do - dst_file=$src_file; - [ -n $BUILD_PATH ] && dst_file=$(echo $dst_file | sed -E -e "s!^$BUILD_PATH!!") - [ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed -E -e "s!^$DESTDIR!!") - dst_file=$(echo $dst_file | sed -E -e 's/build\///') # for simplicity reasons, will put generated headers near the rest. - mkdir -p "$DST/$(echo $dst_file | sed -E -e 's/\/[^/]*$/\//')"; - cp "$src_file" "$DST/$dst_file"; -done - - -# Копируем больше заголовочных файлов с интринсиками, так как на серверах, куда будут устанавливаться -# заголовочные файлы, будет использоваться опция -march=native. - -for src_file in $(ls -1 $($CLANG -v -xc++ - <<<'' 2>&1 | grep '^ /' | grep 'include' | grep -E '/lib/clang/|/include/clang/')/*.h | grep -vE 'arm|altivec|Intrin'); -do - dst_file=$src_file; - [ -n $BUILD_PATH ] && dst_file=$(echo $dst_file | sed -E -e "s!^$BUILD_PATH!!") - [ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed -E -e "s!^$DESTDIR!!") - mkdir -p "$DST/$(echo $dst_file | sed -E -e 's/\/[^/]*$/\//')"; - cp "$src_file" "$DST/$dst_file"; -done - -if [ -d "$SOURCE_PATH/contrib/boost/libs/smart_ptr/include/boost/smart_ptr/detail" ]; then - # Even more platform-specific headers - for src_file in $(ls -1 $SOURCE_PATH/contrib/boost/libs/smart_ptr/include/boost/smart_ptr/detail/*); - do - dst_file=$src_file; - [ -n $BUILD_PATH ] && dst_file=$(echo $dst_file | sed -E -e "s!^$BUILD_PATH!!") - [ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed -E -e "s!^$DESTDIR!!") - mkdir -p "$DST/$(echo $dst_file | sed -E -e 's/\/[^/]*$/\//')"; - cp "$src_file" "$DST/$dst_file"; - done -fi - -if [ -d "$SOURCE_PATH/contrib/boost/boost/smart_ptr/detail" ]; then - for src_file in $(ls -1 $SOURCE_PATH/contrib/boost/boost/smart_ptr/detail/*); - do - dst_file=$src_file; - [ -n $BUILD_PATH ] && dst_file=$(echo $dst_file | sed -E -e "s!^$BUILD_PATH!!") - [ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed -E -e "s!^$DESTDIR!!") - mkdir -p "$DST/$(echo $dst_file | sed -E -e 's/\/[^/]*$/\//')"; - cp "$src_file" "$DST/$dst_file"; - done -fi diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index cf72d7a87c3..0bb6cf62f90 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -67,6 +67,7 @@ #include #include #include +#include #if USE_READLINE #include "Suggest.h" @@ -130,7 +131,7 @@ private: bool print_time_to_stderr = false; /// Output execution time to stderr in batch mode. bool stdin_is_not_tty = false; /// stdin is not a terminal. - winsize terminal_size {}; /// Terminal size is needed to render progress bar. + uint16_t terminal_width = 0; /// Terminal width is needed to render progress bar. std::unique_ptr connection; /// Connection to DB. String query_id; /// Current query_id. @@ -671,7 +672,7 @@ private: String text; if (config().has("query")) - text = config().getString("query"); + text = config().getRawString("query"); /// Poco configuration should not process substitutions in form of ${...} inside query. else { /// If 'query' parameter is not set, read a query from stdin. @@ -1465,7 +1466,7 @@ private: if (show_progress_bar) { - ssize_t width_of_progress_bar = static_cast(terminal_size.ws_col) - written_progress_chars - strlen(" 99%"); + ssize_t width_of_progress_bar = static_cast(terminal_width) - written_progress_chars - strlen(" 99%"); if (width_of_progress_bar > 0) { std::string bar = UnicodeBar::render(UnicodeBar::getWidth(progress.read_rows, 0, total_rows_corrected, width_of_progress_bar)); @@ -1642,22 +1643,13 @@ public: stdin_is_not_tty = !isatty(STDIN_FILENO); + if (!stdin_is_not_tty) + terminal_width = getTerminalWidth(); + namespace po = boost::program_options; - unsigned line_length = po::options_description::m_default_line_length; - unsigned min_description_length = line_length / 2; - if (!stdin_is_not_tty) - { - if (ioctl(STDIN_FILENO, TIOCGWINSZ, &terminal_size)) - throwFromErrno("Cannot obtain terminal window size (ioctl TIOCGWINSZ)", ErrorCodes::SYSTEM_ERROR); - line_length = std::max( - static_cast(strlen("--http_native_compression_disable_checksumming_on_decompress ")), - static_cast(terminal_size.ws_col)); - min_description_length = std::min(min_description_length, line_length - 2); - } - /// Main commandline options related to client functionality and all parameters from Settings. - po::options_description main_description("Main options", line_length, min_description_length); + po::options_description main_description = createOptionsDescription("Main options", terminal_width); main_description.add_options() ("help", "produce help message") ("config-file,C", po::value(), "config-file path") @@ -1672,7 +1664,7 @@ public: * the "\n" is used to distinguish this case because there is hardly a chance an user would use "\n" * as the password. */ - ("password", po::value()->implicit_value("\n"), "password") + ("password", po::value()->implicit_value("\n", ""), "password") ("ask-password", "ask-password") ("query_id", po::value(), "query_id") ("query,q", po::value(), "query") @@ -1703,7 +1695,7 @@ public: context.getSettingsRef().addProgramOptions(main_description); /// Commandline options related to external tables. - po::options_description external_description("External tables options"); + po::options_description external_description = createOptionsDescription("External tables options", terminal_width); external_description.add_options() ("file", po::value(), "data file or - for stdin") ("name", po::value()->default_value("_data"), "name of the table") diff --git a/dbms/programs/compressor/Compressor.cpp b/dbms/programs/compressor/Compressor.cpp index 427d58cbdc6..a073a79b416 100644 --- a/dbms/programs/compressor/Compressor.cpp +++ b/dbms/programs/compressor/Compressor.cpp @@ -12,8 +12,9 @@ #include #include #include - #include +#include + namespace DB { @@ -59,7 +60,7 @@ void checkAndWriteHeader(DB::ReadBuffer & in, DB::WriteBuffer & out) int mainEntryClickHouseCompressor(int argc, char ** argv) { - boost::program_options::options_description desc("Allowed options"); + boost::program_options::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth()); desc.add_options() ("help,h", "produce help message") ("decompress,d", "decompress") diff --git a/dbms/programs/format/Format.cpp b/dbms/programs/format/Format.cpp index b7e2629df16..ff415d88e1b 100644 --- a/dbms/programs/format/Format.cpp +++ b/dbms/programs/format/Format.cpp @@ -6,13 +6,13 @@ #include #include #include - +#include int mainEntryClickHouseFormat(int argc, char ** argv) { using namespace DB; - boost::program_options::options_description desc("Allowed options"); + boost::program_options::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth()); desc.add_options() ("help,h", "produce help message") ("hilite", "add syntax highlight with ANSI terminal escape sequences") diff --git a/dbms/programs/local/LocalServer.cpp b/dbms/programs/local/LocalServer.cpp index bed55a0fc5f..1844c037784 100644 --- a/dbms/programs/local/LocalServer.cpp +++ b/dbms/programs/local/LocalServer.cpp @@ -35,6 +35,7 @@ #include #include #include +#include namespace DB @@ -267,7 +268,7 @@ void LocalServer::attachSystemTables() void LocalServer::processQueries() { String initial_create_query = getInitialCreateTableQuery(); - String queries_str = initial_create_query + config().getString("query"); + String queries_str = initial_create_query + config().getRawString("query"); std::vector queries; auto parse_res = splitMultipartQuery(queries_str, queries); @@ -409,17 +410,7 @@ void LocalServer::init(int argc, char ** argv) /// Don't parse options with Poco library, we prefer neat boost::program_options stopOptionsProcessing(); - unsigned line_length = po::options_description::m_default_line_length; - unsigned min_description_length = line_length / 2; - if (isatty(STDIN_FILENO)) - { - winsize terminal_size{}; - ioctl(0, TIOCGWINSZ, &terminal_size); - line_length = std::max(3U, static_cast(terminal_size.ws_col)); - min_description_length = std::min(min_description_length, line_length - 2); - } - - po::options_description description("Main options", line_length, min_description_length); + po::options_description description = createOptionsDescription("Main options", getTerminalWidth()); description.add_options() ("help", "produce help message") ("config-file,c", po::value(), "config-file path") diff --git a/dbms/programs/main.cpp b/dbms/programs/main.cpp index 57821d854e9..3fbbcee0f15 100644 --- a/dbms/programs/main.cpp +++ b/dbms/programs/main.cpp @@ -56,11 +56,6 @@ int mainEntryClickHouseObfuscator(int argc, char ** argv); #endif -#if USE_EMBEDDED_COMPILER - int mainEntryClickHouseClang(int argc, char ** argv); - int mainEntryClickHouseLLD(int argc, char ** argv); -#endif - namespace { @@ -100,12 +95,6 @@ std::pair clickhouse_applications[] = #if ENABLE_CLICKHOUSE_OBFUSCATOR || !defined(ENABLE_CLICKHOUSE_OBFUSCATOR) {"obfuscator", mainEntryClickHouseObfuscator}, #endif - -#if USE_EMBEDDED_COMPILER - {"clang", mainEntryClickHouseClang}, - {"clang++", mainEntryClickHouseClang}, - {"lld", mainEntryClickHouseLLD}, -#endif }; @@ -152,11 +141,6 @@ int main(int argc_, char ** argv_) /// will work only after additional call of this function. updatePHDRCache(); -#if USE_EMBEDDED_COMPILER - if (argc_ >= 2 && 0 == strcmp(argv_[1], "-cc1")) - return mainEntryClickHouseClang(argc_, argv_); -#endif - #if USE_TCMALLOC /** Without this option, tcmalloc returns memory to OS too frequently for medium-sized memory allocations * (like IO buffers, column vectors, hash tables, etc.), diff --git a/dbms/programs/obfuscator/Obfuscator.cpp b/dbms/programs/obfuscator/Obfuscator.cpp index a96c10072dc..febe2b28606 100644 --- a/dbms/programs/obfuscator/Obfuscator.cpp +++ b/dbms/programs/obfuscator/Obfuscator.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -36,6 +37,7 @@ #include #include #include +#include static const char * documantation = R"( @@ -263,11 +265,11 @@ public: ColumnPtr generate(const IColumn & column) override { - const auto & src_data = static_cast &>(column).getData(); + const auto & src_data = assert_cast &>(column).getData(); size_t size = src_data.size(); auto res_column = ColumnVector::create(size); - auto & res_data = static_cast &>(*res_column).getData(); + auto & res_data = assert_cast &>(*res_column).getData(); for (size_t i = 0; i < size; ++i) { @@ -355,7 +357,7 @@ public: ColumnPtr generate(const IColumn & column) override { - const ColumnFixedString & column_fixed_string = static_cast(column); + const ColumnFixedString & column_fixed_string = assert_cast(column); const size_t string_size = column_fixed_string.getN(); const auto & src_data = column_fixed_string.getChars(); @@ -392,11 +394,11 @@ public: ColumnPtr generate(const IColumn & column) override { - const auto & src_data = static_cast &>(column).getData(); + const auto & src_data = assert_cast &>(column).getData(); size_t size = src_data.size(); auto res_column = ColumnVector::create(size); - auto & res_data = static_cast &>(*res_column).getData(); + auto & res_data = assert_cast &>(*res_column).getData(); for (size_t i = 0; i < size; ++i) { @@ -749,7 +751,7 @@ public: void train(const IColumn & column) override { - const ColumnString & column_string = static_cast(column); + const ColumnString & column_string = assert_cast(column); size_t size = column_string.size(); for (size_t i = 0; i < size; ++i) @@ -766,7 +768,7 @@ public: ColumnPtr generate(const IColumn & column) override { - const ColumnString & column_string = static_cast(column); + const ColumnString & column_string = assert_cast(column); size_t size = column_string.size(); auto res_column = ColumnString::create(); @@ -801,7 +803,7 @@ public: void train(const IColumn & column) override { - const ColumnArray & column_array = static_cast(column); + const ColumnArray & column_array = assert_cast(column); const IColumn & nested_column = column_array.getData(); nested_model->train(nested_column); @@ -814,7 +816,7 @@ public: ColumnPtr generate(const IColumn & column) override { - const ColumnArray & column_array = static_cast(column); + const ColumnArray & column_array = assert_cast(column); const IColumn & nested_column = column_array.getData(); ColumnPtr new_nested_column = nested_model->generate(nested_column); @@ -834,7 +836,7 @@ public: void train(const IColumn & column) override { - const ColumnNullable & column_nullable = static_cast(column); + const ColumnNullable & column_nullable = assert_cast(column); const IColumn & nested_column = column_nullable.getNestedColumn(); nested_model->train(nested_column); @@ -847,7 +849,7 @@ public: ColumnPtr generate(const IColumn & column) override { - const ColumnNullable & column_nullable = static_cast(column); + const ColumnNullable & column_nullable = assert_cast(column); const IColumn & nested_column = column_nullable.getNestedColumn(); ColumnPtr new_nested_column = nested_model->generate(nested_column); @@ -948,7 +950,7 @@ try using namespace DB; namespace po = boost::program_options; - po::options_description description("Options"); + po::options_description description = createOptionsDescription("Options", getTerminalWidth()); description.add_options() ("help", "produce help message") ("structure,S", po::value(), "structure of the initial table (list of column and type names)") diff --git a/dbms/programs/odbc-bridge/ODBCBlockInputStream.cpp b/dbms/programs/odbc-bridge/ODBCBlockInputStream.cpp index 70aaba3f137..8aa93c43c2b 100644 --- a/dbms/programs/odbc-bridge/ODBCBlockInputStream.cpp +++ b/dbms/programs/odbc-bridge/ODBCBlockInputStream.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -44,46 +45,46 @@ namespace switch (type) { case ValueType::vtUInt8: - static_cast(column).insertValue(value.convert()); + assert_cast(column).insertValue(value.convert()); break; case ValueType::vtUInt16: - static_cast(column).insertValue(value.convert()); + assert_cast(column).insertValue(value.convert()); break; case ValueType::vtUInt32: - static_cast(column).insertValue(value.convert()); + assert_cast(column).insertValue(value.convert()); break; case ValueType::vtUInt64: - static_cast(column).insertValue(value.convert()); + assert_cast(column).insertValue(value.convert()); break; case ValueType::vtInt8: - static_cast(column).insertValue(value.convert()); + assert_cast(column).insertValue(value.convert()); break; case ValueType::vtInt16: - static_cast(column).insertValue(value.convert()); + assert_cast(column).insertValue(value.convert()); break; case ValueType::vtInt32: - static_cast(column).insertValue(value.convert()); + assert_cast(column).insertValue(value.convert()); break; case ValueType::vtInt64: - static_cast(column).insertValue(value.convert()); + assert_cast(column).insertValue(value.convert()); break; case ValueType::vtFloat32: - static_cast(column).insertValue(value.convert()); + assert_cast(column).insertValue(value.convert()); break; case ValueType::vtFloat64: - static_cast(column).insertValue(value.convert()); + assert_cast(column).insertValue(value.convert()); break; case ValueType::vtString: - static_cast(column).insert(value.convert()); + assert_cast(column).insert(value.convert()); break; case ValueType::vtDate: - static_cast(column).insertValue(UInt16{LocalDate{value.convert()}.getDayNum()}); + assert_cast(column).insertValue(UInt16{LocalDate{value.convert()}.getDayNum()}); break; case ValueType::vtDateTime: - static_cast(column).insertValue(time_t{LocalDateTime{value.convert()}}); + assert_cast(column).insertValue(time_t{LocalDateTime{value.convert()}}); break; case ValueType::vtUUID: - static_cast(column).insert(parse(value.convert())); + assert_cast(column).insert(parse(value.convert())); break; } } @@ -114,7 +115,7 @@ Block ODBCBlockInputStream::readImpl() { if (description.types[idx].second) { - ColumnNullable & column_nullable = static_cast(*columns[idx]); + ColumnNullable & column_nullable = assert_cast(*columns[idx]); insertValue(column_nullable.getNestedColumn(), description.types[idx].first, value); column_nullable.getNullMapData().emplace_back(0); } diff --git a/dbms/programs/performance-test/PerformanceTestSuite.cpp b/dbms/programs/performance-test/PerformanceTestSuite.cpp index 14ea8882a6b..eaa4e24cde9 100644 --- a/dbms/programs/performance-test/PerformanceTestSuite.cpp +++ b/dbms/programs/performance-test/PerformanceTestSuite.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include "TestStopConditions.h" #include "TestStats.h" @@ -324,7 +325,7 @@ try using po::value; using Strings = DB::Strings; - po::options_description desc("Allowed options"); + po::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth()); desc.add_options() ("help", "produce help message") ("lite", "use lite version of output") diff --git a/dbms/programs/server/Server.cpp b/dbms/programs/server/Server.cpp index ef61537e38d..f10dc07ab56 100644 --- a/dbms/programs/server/Server.cpp +++ b/dbms/programs/server/Server.cpp @@ -520,7 +520,18 @@ int Server::main(const std::vector & /*args*/) /// Init trace collector only after trace_log system table was created /// Disable it if we collect test coverage information, because it will work extremely slow. -#if USE_INTERNAL_UNWIND_LIBRARY && !WITH_COVERAGE + /// + /// It also cannot work with sanitizers. + /// Sanitizers are using quick "frame walking" stack unwinding (this implies -fno-omit-frame-pointer) + /// And they do unwiding frequently (on every malloc/free, thread/mutex operations, etc). + /// They change %rbp during unwinding and it confuses libunwind if signal comes during sanitizer unwiding + /// and query profiler decide to unwind stack with libunwind at this moment. + /// + /// Symptoms: you'll get silent Segmentation Fault - without sanitizer message and without usual ClickHouse diagnostics. + /// + /// Look at compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h + /// +#if USE_UNWIND && !WITH_COVERAGE && !defined(SANITIZER) /// QueryProfiler cannot work reliably with any other libunwind or without PHDR cache. if (hasPHDRCache()) global_context->initializeTraceCollector(); diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index 5091258acaf..77b359f2763 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -182,11 +182,11 @@ void TCPHandler::runImpl() /// Should we send internal logs to client? const auto client_logs_level = query_context->getSettingsRef().send_logs_level; if (client_revision >= DBMS_MIN_REVISION_WITH_SERVER_LOGS - && client_logs_level.value != LogsLevel::none) + && client_logs_level != LogsLevel::none) { state.logs_queue = std::make_shared(); state.logs_queue->max_priority = Poco::Logger::parseLevel(client_logs_level.toString()); - CurrentThread::attachInternalTextLogsQueue(state.logs_queue, client_logs_level.value); + CurrentThread::attachInternalTextLogsQueue(state.logs_queue, client_logs_level); } query_context->setExternalTablesInitializer([&connection_settings, this] (Context & context) @@ -329,7 +329,7 @@ void TCPHandler::readData(const Settings & connection_settings) const auto receive_timeout = query_context->getSettingsRef().receive_timeout.value; /// Poll interval should not be greater than receive_timeout - const size_t default_poll_interval = connection_settings.poll_interval.value * 1000000; + const size_t default_poll_interval = connection_settings.poll_interval * 1000000; size_t current_poll_interval = static_cast(receive_timeout.totalMicroseconds()); constexpr size_t min_poll_interval = 5000; // 5 ms size_t poll_interval = std::max(min_poll_interval, std::min(default_poll_interval, current_poll_interval)); @@ -865,7 +865,7 @@ bool TCPHandler::receiveData() if (!(storage = query_context->tryGetExternalTable(external_table_name))) { NamesAndTypesList columns = block.getNamesAndTypesList(); - storage = StorageMemory::create("_external", external_table_name, ColumnsDescription{columns}); + storage = StorageMemory::create("_external", external_table_name, ColumnsDescription{columns}, ConstraintsDescription{}); storage->startup(); query_context->addExternalTable(external_table_name, storage); } diff --git a/dbms/programs/server/config.xml b/dbms/programs/server/config.xml index 44b3c25c19d..814b7dded3c 100644 --- a/dbms/programs/server/config.xml +++ b/dbms/programs/server/config.xml @@ -217,6 +217,7 @@ See https://clickhouse.yandex/docs/en/table_engines/replication/ --> + + 3000 + + diff --git a/dbms/tests/integration/test_atomic_drop_table/configs/remote_servers.xml b/dbms/tests/integration/test_atomic_drop_table/configs/remote_servers.xml new file mode 100644 index 00000000000..538aa72d386 --- /dev/null +++ b/dbms/tests/integration/test_atomic_drop_table/configs/remote_servers.xml @@ -0,0 +1,14 @@ + + + + + true + + shard_0 + node1 + 9000 + + + + + diff --git a/dbms/tests/integration/test_atomic_drop_table/test.py b/dbms/tests/integration/test_atomic_drop_table/test.py new file mode 100644 index 00000000000..7d845baeec6 --- /dev/null +++ b/dbms/tests/integration/test_atomic_drop_table/test.py @@ -0,0 +1,37 @@ +import time +import pytest + +from helpers.network import PartitionManager +from helpers.cluster import ClickHouseCluster + + +cluster = ClickHouseCluster(__file__) +node1 = cluster.add_instance('node1', config_dir="configs", with_zookeeper=True) + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + node1.query("CREATE DATABASE zktest;") + node1.query( + ''' + CREATE TABLE zktest.atomic_drop_table (n UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/zktest/tables/atomic_drop_table', 'node1') + PARTITION BY n ORDER BY n + ''' + ) + yield cluster + finally: + cluster.shutdown() + +def test_atomic_delete_with_stopped_zookeeper(start_cluster): + node1.query("insert into zktest.atomic_drop_table values (8192)") + + with PartitionManager() as pm: + pm.drop_instance_zk_connections(node1) + error = node1.query_and_get_error("DROP TABLE zktest.atomic_drop_table") #Table won't drop + assert error != "" + + time.sleep(5) + assert '8192' in node1.query("select * from zktest.atomic_drop_table") diff --git a/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_cache_xypairs.xml b/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_cache_xypairs.xml new file mode 100644 index 00000000000..4142706259a --- /dev/null +++ b/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_cache_xypairs.xml @@ -0,0 +1,31 @@ + + + cache_xypairs + + + localhost + 9000 + default + + test + xypairs
+ + + 1 + + + 5 + + + + + x + + + y + UInt64 + 0 + + + + diff --git a/dbms/tests/integration/test_dictionaries/test.py b/dbms/tests/integration/test_dictionaries/test.py index 251fe7f31ee..95f82f65c0d 100644 --- a/dbms/tests/integration/test_dictionaries/test.py +++ b/dbms/tests/integration/test_dictionaries/test.py @@ -17,6 +17,10 @@ def get_status(dictionary_name): return instance.query("SELECT status FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip("\n") +def get_last_exception(dictionary_name): + return instance.query("SELECT last_exception FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip("\n").replace("\\'", "'") + + def get_loading_start_time(dictionary_name): s = instance.query("SELECT loading_start_time FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip("\n") if s == "0000-00-00 00:00:00": @@ -350,3 +354,58 @@ def test_reload_after_fail_by_timer(started_cluster): time.sleep(6); query("SELECT dictGetInt32('no_file_2', 'a', toUInt64(9))") == "10\n" assert get_status("no_file_2") == "LOADED" + + +def test_reload_after_fail_in_cache_dictionary(started_cluster): + query = instance.query + query_and_get_error = instance.query_and_get_error + + # Can't get a value from the cache dictionary because the source (table `test.xypairs`) doesn't respond. + expected_error = "Table test.xypairs doesn't exist" + assert expected_error in query_and_get_error("SELECT dictGetUInt64('cache_xypairs', 'y', toUInt64(1))") + assert get_status("cache_xypairs") == "LOADED" + assert expected_error in get_last_exception("cache_xypairs") + + # Create table `test.xypairs`. + query(''' + drop table if exists test.xypairs; + create table test.xypairs (x UInt64, y UInt64) engine=Log; + insert into test.xypairs values (1, 56), (3, 78); + ''') + + # Cache dictionary now works. + assert_eq_with_retry(instance, "SELECT dictGet('cache_xypairs', 'y', toUInt64(1))", "56", ignore_error=True) + query("SELECT dictGet('cache_xypairs', 'y', toUInt64(2))") == "0" + assert get_last_exception("cache_xypairs") == "" + + # Drop table `test.xypairs`. + query('drop table if exists test.xypairs') + + # Values are cached so we can get them. + query("SELECT dictGet('cache_xypairs', 'y', toUInt64(1))") == "56" + query("SELECT dictGet('cache_xypairs', 'y', toUInt64(2))") == "0" + assert get_last_exception("cache_xypairs") == "" + + # But we can't get a value from the source table which isn't cached. + assert expected_error in query_and_get_error("SELECT dictGetUInt64('cache_xypairs', 'y', toUInt64(3))") + assert expected_error in get_last_exception("cache_xypairs") + + # Passed time should not spoil the cache. + time.sleep(5); + query("SELECT dictGet('cache_xypairs', 'y', toUInt64(1))") == "56" + query("SELECT dictGet('cache_xypairs', 'y', toUInt64(2))") == "0" + assert expected_error in query_and_get_error("SELECT dictGetUInt64('cache_xypairs', 'y', toUInt64(3))") + assert expected_error in get_last_exception("cache_xypairs") + + # Create table `test.xypairs` again with changed values. + query(''' + drop table if exists test.xypairs; + create table test.xypairs (x UInt64, y UInt64) engine=Log; + insert into test.xypairs values (1, 57), (3, 79); + ''') + + # The cache dictionary returns new values now. + assert_eq_with_retry(instance, "SELECT dictGet('cache_xypairs', 'y', toUInt64(1))", "57") + query("SELECT dictGet('cache_xypairs', 'y', toUInt64(2))") == "0" + query("SELECT dictGet('cache_xypairs', 'y', toUInt64(3))") == "79" + assert get_last_exception("cache_xypairs") == "" diff --git a/dbms/tests/queries/0_stateless/00962_temporary_live_view_watch_live.reference b/dbms/tests/integration/test_partition/__init__.py similarity index 100% rename from dbms/tests/queries/0_stateless/00962_temporary_live_view_watch_live.reference rename to dbms/tests/integration/test_partition/__init__.py diff --git a/dbms/tests/integration/test_partition/test.py b/dbms/tests/integration/test_partition/test.py new file mode 100644 index 00000000000..3365343b6fb --- /dev/null +++ b/dbms/tests/integration/test_partition/test.py @@ -0,0 +1,244 @@ +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV + + +cluster = ClickHouseCluster(__file__) +instance = cluster.add_instance('instance') +q = instance.query +path_to_data = '/var/lib/clickhouse/' + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + q('CREATE DATABASE test') + + yield cluster + + finally: + cluster.shutdown() + + +@pytest.fixture +def partition_table_simple(started_cluster): + q("DROP TABLE IF EXISTS test.partition") + q("CREATE TABLE test.partition (date MATERIALIZED toDate(0), x UInt64, sample_key MATERIALIZED intHash64(x)) " + "ENGINE=MergeTree PARTITION BY date SAMPLE BY sample_key ORDER BY (date,x,sample_key) " + "SETTINGS index_granularity=8192, index_granularity_bytes=0") + q("INSERT INTO test.partition ( x ) VALUES ( now() )") + q("INSERT INTO test.partition ( x ) VALUES ( now()+1 )") + + yield + + q('DROP TABLE test.partition') + + +def test_partition_simple(partition_table_simple): + q("ALTER TABLE test.partition DETACH PARTITION 197001") + q("ALTER TABLE test.partition ATTACH PARTITION 197001") + q("OPTIMIZE TABLE test.partition") + + +def exec_bash(cmd): + cmd = '/bin/bash -c "{}"'.format(cmd.replace('"', '\\"')) + return instance.exec_in_container(cmd) + + +def partition_complex_assert_columns_txt(): + path_to_parts = path_to_data + 'data/test/partition/' + parts = TSV(q("SELECT name FROM system.parts WHERE database='test' AND table='partition'")) + for part_name in parts.lines: + path_to_columns = path_to_parts + part_name + '/columns.txt' + # 2 header lines + 3 columns + assert exec_bash('cat {} | wc -l'.format(path_to_columns)) == u'5\n' + + +def partition_complex_assert_checksums(): + # Do `cd` for consistent output for reference + # Do not check increment.txt - it can be changed by other tests with FREEZE + cmd = 'cd ' + path_to_data + " && find shadow -type f -exec md5sum {} \\;" \ + " | grep partition" \ + " | sed 's!shadow/[0-9]*/data/[a-z0-9_-]*/!shadow/1/data/test/!g'" \ + " | sort" \ + " | uniq" + + checksums = "082814b5aa5109160d5c0c5aff10d4df\tshadow/1/data/test/partition/19700102_2_2_0/k.bin\n" \ + "082814b5aa5109160d5c0c5aff10d4df\tshadow/1/data/test/partition/19700201_1_1_0/v1.bin\n" \ + "13cae8e658e0ca4f75c56b1fc424e150\tshadow/1/data/test/partition/19700102_2_2_0/minmax_p.idx\n" \ + "25daad3d9e60b45043a70c4ab7d3b1c6\tshadow/1/data/test/partition/19700102_2_2_0/partition.dat\n" \ + "3726312af62aec86b64a7708d5751787\tshadow/1/data/test/partition/19700201_1_1_0/partition.dat\n" \ + "37855b06a39b79a67ea4e86e4a3299aa\tshadow/1/data/test/partition/19700102_2_2_0/checksums.txt\n" \ + "38e62ff37e1e5064e9a3f605dfe09d13\tshadow/1/data/test/partition/19700102_2_2_0/v1.bin\n" \ + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition/19700102_2_2_0/k.mrk\n" \ + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition/19700102_2_2_0/p.mrk\n" \ + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition/19700102_2_2_0/v1.mrk\n" \ + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition/19700201_1_1_0/k.mrk\n" \ + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition/19700201_1_1_0/p.mrk\n" \ + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition/19700201_1_1_0/v1.mrk\n" \ + "55a54008ad1ba589aa210d2629c1df41\tshadow/1/data/test/partition/19700201_1_1_0/primary.idx\n" \ + "5f087cb3e7071bf9407e095821e2af8f\tshadow/1/data/test/partition/19700201_1_1_0/checksums.txt\n" \ + "77d5af402ada101574f4da114f242e02\tshadow/1/data/test/partition/19700102_2_2_0/columns.txt\n" \ + "77d5af402ada101574f4da114f242e02\tshadow/1/data/test/partition/19700201_1_1_0/columns.txt\n" \ + "88cdc31ded355e7572d68d8cde525d3a\tshadow/1/data/test/partition/19700201_1_1_0/p.bin\n" \ + "9e688c58a5487b8eaf69c9e1005ad0bf\tshadow/1/data/test/partition/19700102_2_2_0/primary.idx\n" \ + "c4ca4238a0b923820dcc509a6f75849b\tshadow/1/data/test/partition/19700102_2_2_0/count.txt\n" \ + "c4ca4238a0b923820dcc509a6f75849b\tshadow/1/data/test/partition/19700201_1_1_0/count.txt\n" \ + "cfcb770c3ecd0990dcceb1bde129e6c6\tshadow/1/data/test/partition/19700102_2_2_0/p.bin\n" \ + "e2af3bef1fd129aea73a890ede1e7a30\tshadow/1/data/test/partition/19700201_1_1_0/k.bin\n" \ + "f2312862cc01adf34a93151377be2ddf\tshadow/1/data/test/partition/19700201_1_1_0/minmax_p.idx\n" + + assert TSV(exec_bash(cmd).replace(' ', '\t')) == TSV(checksums) + + +@pytest.fixture +def partition_table_complex(started_cluster): + q("DROP TABLE IF EXISTS test.partition") + q("CREATE TABLE test.partition (p Date, k Int8, v1 Int8 MATERIALIZED k + 1) " + "ENGINE = MergeTree PARTITION BY p ORDER BY k SETTINGS index_granularity=1, index_granularity_bytes=0") + q("INSERT INTO test.partition (p, k) VALUES(toDate(31), 1)") + q("INSERT INTO test.partition (p, k) VALUES(toDate(1), 2)") + + yield + + q("DROP TABLE test.partition") + + +def test_partition_complex(partition_table_complex): + + partition_complex_assert_columns_txt() + + q("ALTER TABLE test.partition FREEZE") + + partition_complex_assert_checksums() + + q("ALTER TABLE test.partition DETACH PARTITION 197001") + q("ALTER TABLE test.partition ATTACH PARTITION 197001") + + partition_complex_assert_columns_txt() + + q("ALTER TABLE test.partition MODIFY COLUMN v1 Int8") + + # Check the backup hasn't changed + partition_complex_assert_checksums() + + q("OPTIMIZE TABLE test.partition") + + expected = TSV('31\t1\t2\n' + '1\t2\t3') + res = q("SELECT toUInt16(p), k, v1 FROM test.partition ORDER BY k") + assert(TSV(res) == expected) + + +@pytest.fixture +def cannot_attach_active_part_table(started_cluster): + q("DROP TABLE IF EXISTS test.attach_active") + q("CREATE TABLE test.attach_active (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 4) ORDER BY n") + q("INSERT INTO test.attach_active SELECT number FROM system.numbers LIMIT 16") + + yield + + q("DROP TABLE test.attach_active") + + +def test_cannot_attach_active_part(cannot_attach_active_part_table): + error = instance.client.query_and_get_error("ALTER TABLE test.attach_active ATTACH PART '../1_2_2_0'") + print error + assert 0 <= error.find('Invalid part name') + + res = q("SElECT name FROM system.parts WHERE table='attach_active' AND database='test' ORDER BY name") + assert TSV(res) == TSV('0_1_1_0\n1_2_2_0\n2_3_3_0\n3_4_4_0') + assert TSV(q("SElECT count(), sum(n) FROM test.attach_active")) == TSV('16\t120') + + +@pytest.fixture +def attach_check_all_parts_table(started_cluster): + q("SYSTEM STOP MERGES") + q("DROP TABLE IF EXISTS test.attach_partition") + q("CREATE TABLE test.attach_partition (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 8) ORDER BY n") + q("INSERT INTO test.attach_partition SELECT number FROM system.numbers WHERE number % 2 = 0 LIMIT 8") + q("INSERT INTO test.attach_partition SELECT number FROM system.numbers WHERE number % 2 = 1 LIMIT 8") + + yield + + q("DROP TABLE test.attach_partition") + q("SYSTEM START MERGES") + + +def test_attach_check_all_parts(attach_check_all_parts_table): + q("ALTER TABLE test.attach_partition DETACH PARTITION 0") + + path_to_detached = path_to_data + 'data/test/attach_partition/detached/' + exec_bash('mkdir {}'.format(path_to_detached + '0_5_5_0')) + exec_bash('cp -pr {} {}'.format(path_to_detached + '0_1_1_0', path_to_detached + 'attaching_0_6_6_0')) + exec_bash('cp -pr {} {}'.format(path_to_detached + '0_3_3_0', path_to_detached + 'deleting_0_7_7_0')) + + error = instance.client.query_and_get_error("ALTER TABLE test.attach_partition ATTACH PARTITION 0") + assert 0 <= error.find('No columns in part 0_5_5_0') + + parts = q("SElECT name FROM system.parts WHERE table='attach_partition' AND database='test' ORDER BY name") + assert TSV(parts) == TSV('1_2_2_0\n1_4_4_0') + detached = q("SELECT name FROM system.detached_parts " + "WHERE table='attach_partition' AND database='test' ORDER BY name") + assert TSV(detached) == TSV('0_1_1_0\n0_3_3_0\n0_5_5_0\nattaching_0_6_6_0\ndeleting_0_7_7_0') + + exec_bash('rm -r {}'.format(path_to_detached + '0_5_5_0')) + + q("ALTER TABLE test.attach_partition ATTACH PARTITION 0") + parts = q("SElECT name FROM system.parts WHERE table='attach_partition' AND database='test' ORDER BY name") + expected = '0_5_5_0\n0_6_6_0\n1_2_2_0\n1_4_4_0' + assert TSV(parts) == TSV(expected) + assert TSV(q("SElECT count(), sum(n) FROM test.attach_partition")) == TSV('16\t120') + + detached = q("SELECT name FROM system.detached_parts " + "WHERE table='attach_partition' AND database='test' ORDER BY name") + assert TSV(detached) == TSV('attaching_0_6_6_0\ndeleting_0_7_7_0') + + +@pytest.fixture +def drop_detached_parts_table(started_cluster): + q("SYSTEM STOP MERGES") + q("DROP TABLE IF EXISTS test.drop_detached") + q("CREATE TABLE test.drop_detached (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 8) ORDER BY n") + q("INSERT INTO test.drop_detached SELECT number FROM system.numbers WHERE number % 2 = 0 LIMIT 8") + q("INSERT INTO test.drop_detached SELECT number FROM system.numbers WHERE number % 2 = 1 LIMIT 8") + + yield + + q("DROP TABLE test.drop_detached") + q("SYSTEM START MERGES") + + +def test_drop_detached_parts(drop_detached_parts_table): + s = {"allow_drop_detached": 1} + q("ALTER TABLE test.drop_detached DETACH PARTITION 0") + q("ALTER TABLE test.drop_detached DETACH PARTITION 1") + + path_to_detached = path_to_data + 'data/test/drop_detached/detached/' + exec_bash('mkdir {}'.format(path_to_detached + 'attaching_0_6_6_0')) + exec_bash('mkdir {}'.format(path_to_detached + 'deleting_0_7_7_0')) + exec_bash('mkdir {}'.format(path_to_detached + 'any_other_name')) + exec_bash('mkdir {}'.format(path_to_detached + 'prefix_1_2_2_0_0')) + + error = instance.client.query_and_get_error("ALTER TABLE test.drop_detached DROP DETACHED PART '../1_2_2_0'", settings=s) + assert 0 <= error.find('Invalid part name') + + q("ALTER TABLE test.drop_detached DROP DETACHED PART '0_1_1_0'", settings=s) + + error = instance.client.query_and_get_error("ALTER TABLE test.drop_detached DROP DETACHED PART 'attaching_0_6_6_0'", settings=s) + assert 0 <= error.find('Cannot drop part') + + error = instance.client.query_and_get_error("ALTER TABLE test.drop_detached DROP DETACHED PART 'deleting_0_7_7_0'", settings=s) + assert 0 <= error.find('Cannot drop part') + + q("ALTER TABLE test.drop_detached DROP DETACHED PART 'any_other_name'", settings=s) + + detached = q("SElECT name FROM system.detached_parts WHERE table='drop_detached' AND database='test' ORDER BY name") + assert TSV(detached) == TSV('0_3_3_0\n1_2_2_0\n1_4_4_0\nattaching_0_6_6_0\ndeleting_0_7_7_0\nprefix_1_2_2_0_0') + + q("ALTER TABLE test.drop_detached DROP DETACHED PARTITION 1", settings=s) + detached = q("SElECT name FROM system.detached_parts WHERE table='drop_detached' AND database='test' ORDER BY name") + assert TSV(detached) == TSV('0_3_3_0\nattaching_0_6_6_0\ndeleting_0_7_7_0') + diff --git a/dbms/tests/integration/test_replicated_mutations/configs/merge_tree_max_parts.xml b/dbms/tests/integration/test_replicated_mutations/configs/merge_tree_max_parts.xml new file mode 100644 index 00000000000..60047dcab2c --- /dev/null +++ b/dbms/tests/integration/test_replicated_mutations/configs/merge_tree_max_parts.xml @@ -0,0 +1,6 @@ + + + 50 + 50 + + \ No newline at end of file diff --git a/dbms/tests/integration/test_replicated_mutations/test.py b/dbms/tests/integration/test_replicated_mutations/test.py index 351ceff3608..0347ba4782c 100644 --- a/dbms/tests/integration/test_replicated_mutations/test.py +++ b/dbms/tests/integration/test_replicated_mutations/test.py @@ -10,21 +10,29 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=True) +node1 = cluster.add_instance('node1', macros={'cluster': 'test1'}, with_zookeeper=True) # Check, that limits on max part size for merges doesn`t affect mutations -node2 = cluster.add_instance('node2', main_configs=["configs/merge_tree.xml"], with_zookeeper=True) -nodes = [node1, node2] +node2 = cluster.add_instance('node2', macros={'cluster': 'test1'}, main_configs=["configs/merge_tree.xml"], with_zookeeper=True) + +node3 = cluster.add_instance('node3', macros={'cluster': 'test2'}, main_configs=["configs/merge_tree_max_parts.xml"], with_zookeeper=True) +node4 = cluster.add_instance('node4', macros={'cluster': 'test2'}, main_configs=["configs/merge_tree_max_parts.xml"], with_zookeeper=True) + +node5 = cluster.add_instance('node5', macros={'cluster': 'test3'}, main_configs=["configs/merge_tree_max_parts.xml"]) + +all_nodes = [node1, node2, node3, node4, node5] @pytest.fixture(scope="module") def started_cluster(): try: cluster.start() - for node in nodes: + for node in all_nodes: node.query("DROP TABLE IF EXISTS test_mutations") - for node in nodes: - node.query("CREATE TABLE test_mutations(d Date, x UInt32, i UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/test/test_mutations', '{instance}') ORDER BY x PARTITION BY toYYYYMM(d)") + for node in [node1, node2, node3, node4]: + node.query("CREATE TABLE test_mutations(d Date, x UInt32, i UInt32) ENGINE ReplicatedMergeTree('/clickhouse/{cluster}/tables/test/test_mutations', '{instance}') ORDER BY x PARTITION BY toYYYYMM(d)") + + node5.query("CREATE TABLE test_mutations(d Date, x UInt32, i UInt32) ENGINE MergeTree() ORDER BY x PARTITION BY toYYYYMM(d)") yield cluster @@ -33,7 +41,8 @@ def started_cluster(): class Runner: - def __init__(self): + def __init__(self, nodes): + self.nodes = nodes self.mtx = threading.Lock() self.total_inserted_xs = 0 self.total_inserted_rows = 0 @@ -49,7 +58,9 @@ class Runner: self.stop_ev = threading.Event() - def do_insert(self, thread_num): + self.exceptions = [] + + def do_insert(self, thread_num, partitions_num): self.stop_ev.wait(random.random()) # Each thread inserts a small random number of rows with random year, month 01 and day determined @@ -67,7 +78,7 @@ class Runner: for x in xs: self.currently_inserting_xs[x] += 1 - year = 2000 + random.randint(0, 10) + year = 2000 + random.randint(0, partitions_num) date_str = '{year}-{month}-{day}'.format(year=year, month=month, day=day) payload = '' for x in xs: @@ -76,7 +87,7 @@ class Runner: try: print 'thread {}: insert for {}: {}'.format(thread_num, date_str, ','.join(str(x) for x in xs)) - random.choice(nodes).query("INSERT INTO test_mutations FORMAT TSV", payload) + random.choice(self.nodes).query("INSERT INTO test_mutations FORMAT TSV", payload) with self.mtx: for x in xs: @@ -86,6 +97,7 @@ class Runner: except Exception, e: print 'Exception while inserting,', e + self.exceptions.append(e) finally: with self.mtx: for x in xs: @@ -113,7 +125,7 @@ class Runner: try: print 'thread {}: delete {} * {}'.format(thread_num, to_delete_count, x) - random.choice(nodes).query("ALTER TABLE test_mutations DELETE WHERE x = {}".format(x)) + random.choice(self.nodes).query("ALTER TABLE test_mutations DELETE WHERE x = {}".format(x)) with self.mtx: self.total_mutations += 1 @@ -130,14 +142,27 @@ class Runner: self.stop_ev.wait(1.0 + random.random() * 2) +def wait_for_mutations(nodes, number_of_mutations): + for i in range(100): # wait for replication 80 seconds max + time.sleep(0.8) + + def get_done_mutations(node): + return int(node.query("SELECT sum(is_done) FROM system.mutations WHERE table = 'test_mutations'").rstrip()) + + if all([get_done_mutations(n) == number_of_mutations for n in nodes]): + return True + return False + + def test_mutations(started_cluster): DURATION_SECONDS = 30 + nodes = [node1, node2] - runner = Runner() + runner = Runner(nodes) threads = [] for thread_num in range(5): - threads.append(threading.Thread(target=runner.do_insert, args=(thread_num, ))) + threads.append(threading.Thread(target=runner.do_insert, args=(thread_num, 10))) for thread_num in (11, 12, 13): threads.append(threading.Thread(target=runner.do_delete, args=(thread_num,))) @@ -155,18 +180,11 @@ def test_mutations(started_cluster): assert runner.total_inserted_rows > 0 assert runner.total_mutations > 0 - all_done = False - for i in range(100): # wait for replication 80 seconds max - time.sleep(0.8) + all_done = wait_for_mutations(nodes, runner.total_mutations) - def get_done_mutations(node): - return int(node.query("SELECT sum(is_done) FROM system.mutations WHERE table = 'test_mutations'").rstrip()) - - if all([get_done_mutations(n) == runner.total_mutations for n in nodes]): - all_done = True - break - - print node1.query("SELECT mutation_id, command, parts_to_do, is_done FROM system.mutations WHERE table = 'test_mutations' FORMAT TSVWithNames") + print "Total mutations: ", runner.total_mutations + for node in nodes: + print node.query("SELECT mutation_id, command, parts_to_do, is_done FROM system.mutations WHERE table = 'test_mutations' FORMAT TSVWithNames") assert all_done expected_sum = runner.total_inserted_xs - runner.total_deleted_xs @@ -174,3 +192,44 @@ def test_mutations(started_cluster): for i, node in enumerate(nodes): actual_sums.append(int(node.query("SELECT sum(x) FROM test_mutations").rstrip())) assert actual_sums[i] == expected_sum + + +@pytest.mark.parametrize( + ('nodes', ), + [ + ([node5, ], ), # MergeTree + ([node3, node4], ), # ReplicatedMergeTree + ] +) +def test_mutations_dont_prevent_merges(started_cluster, nodes): + for year in range(2000, 2016): + rows = '' + date_str = '{}-01-{}'.format(year, random.randint(1, 10)) + for i in range(10): + rows += '{} {} {}\n'.format(date_str, random.randint(1, 10), i) + nodes[0].query("INSERT INTO test_mutations FORMAT TSV", rows) + + # will run mutations of 16 parts in parallel, mutations will sleep for about 20 seconds + nodes[0].query("ALTER TABLE test_mutations UPDATE i = sleepEachRow(2) WHERE 1") + + runner = Runner(nodes) + threads = [] + for thread_num in range(2): + threads.append(threading.Thread(target=runner.do_insert, args=(thread_num, 0))) + + # will insert approx 8-10 new parts per 1 second into one partition + for t in threads: + t.start() + + all_done = wait_for_mutations(nodes, 1) + + runner.stop_ev.set() + for t in threads: + t.join() + + for node in nodes: + print node.query("SELECT mutation_id, command, parts_to_do, is_done FROM system.mutations WHERE table = 'test_mutations' FORMAT TSVWithNames") + print node.query("SELECT partition, count(name), sum(active), sum(active*rows) FROM system.parts WHERE table ='test_mutations' GROUP BY partition FORMAT TSVWithNames") + + assert all_done + assert all([str(e).find("Too many parts") < 0 for e in runner.exceptions]) diff --git a/dbms/tests/integration/test_storage_kafka/test.py b/dbms/tests/integration/test_storage_kafka/test.py index 0f0cf6996ef..24c5a3ac3bf 100644 --- a/dbms/tests/integration/test_storage_kafka/test.py +++ b/dbms/tests/integration/test_storage_kafka/test.py @@ -504,6 +504,7 @@ def test_kafka_virtual_columns_with_materialized_view(kafka_cluster): kafka_check_result(result, True, 'test_kafka_virtual2.reference') +@pytest.mark.timeout(60) def test_kafka_insert(kafka_cluster): instance.query(''' CREATE TABLE test.kafka (key UInt64, value UInt64) @@ -540,6 +541,7 @@ def test_kafka_insert(kafka_cluster): kafka_check_result(result, True) +@pytest.mark.timeout(60) def test_kafka_produce_consume(kafka_cluster): instance.query(''' CREATE TABLE test.kafka (key UInt64, value UInt64) @@ -603,6 +605,78 @@ def test_kafka_produce_consume(kafka_cluster): assert int(result) == messages_num * threads_num, 'ClickHouse lost some messages: {}'.format(result) +@pytest.mark.timeout(300) +def test_kafka_commit_on_block_write(kafka_cluster): + instance.query(''' + DROP TABLE IF EXISTS test.view; + DROP TABLE IF EXISTS test.consumer; + CREATE TABLE test.kafka (key UInt64, value UInt64) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'kafka1:19092', + kafka_topic_list = 'block', + kafka_group_name = 'block', + kafka_format = 'JSONEachRow', + kafka_max_block_size = 100, + kafka_row_delimiter = '\\n'; + CREATE TABLE test.view (key UInt64, value UInt64) + ENGINE = MergeTree() + ORDER BY key; + CREATE MATERIALIZED VIEW test.consumer TO test.view AS + SELECT * FROM test.kafka; + ''') + + cancel = threading.Event() + + i = [0] + def produce(): + while not cancel.is_set(): + messages = [] + for _ in range(101): + messages.append(json.dumps({'key': i[0], 'value': i[0]})) + i[0] += 1 + kafka_produce('block', messages) + + kafka_thread = threading.Thread(target=produce) + kafka_thread.start() + + while int(instance.query('SELECT count() FROM test.view')) == 0: + time.sleep(1) + + cancel.set() + + instance.query(''' + DROP TABLE test.kafka; + ''') + + while int(instance.query("SELECT count() FROM system.tables WHERE database='test' AND name='kafka'")) == 1: + time.sleep(1) + + instance.query(''' + CREATE TABLE test.kafka (key UInt64, value UInt64) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'kafka1:19092', + kafka_topic_list = 'block', + kafka_group_name = 'block', + kafka_format = 'JSONEachRow', + kafka_max_block_size = 100, + kafka_row_delimiter = '\\n'; + ''') + + while int(instance.query('SELECT uniqExact(key) FROM test.view')) < i[0]: + time.sleep(1) + + result = int(instance.query('SELECT count() == uniqExact(key) FROM test.view')) + + instance.query(''' + DROP TABLE test.consumer; + DROP TABLE test.view; + ''') + + kafka_thread.join() + + assert result == 1, 'Messages from kafka get duplicated!' + + if __name__ == '__main__': cluster.start() raw_input("Cluster created, press any key to destroy...") diff --git a/dbms/tests/queries/0_stateless/00093_union_race_conditions_4.sh b/dbms/tests/queries/0_stateless/00093_union_race_conditions_4.sh index fcdbe4cbcdd..2d255a0c2f6 100755 --- a/dbms/tests/queries/0_stateless/00093_union_race_conditions_4.sh +++ b/dbms/tests/queries/0_stateless/00093_union_race_conditions_4.sh @@ -6,4 +6,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -o errexit set -o pipefail -for i in {1..10}; do seq 1 10 | sed 's/.*/SELECT 1 % ((number + 500) % 1000) FROM system.numbers_mt LIMIT 1000;/' | $CLICKHOUSE_CLIENT -n --receive_timeout=1 --max_block_size=1 >/dev/null 2>&1 && echo 'Fail!' && break; echo -n '.'; done; echo +for i in {1..10}; do seq 1 10 | sed 's/.*/SELECT 1 % ((number + 500) % 1000) FROM numbers_mt(1000);/' | $CLICKHOUSE_CLIENT -n --receive_timeout=1 --max_block_size=1 >/dev/null 2>&1 && echo 'Fail!' && break; echo -n '.'; done; echo diff --git a/dbms/tests/queries/0_stateless/00281_compile_sizeof_packed.reference b/dbms/tests/queries/0_stateless/00281_compile_sizeof_packed.reference deleted file mode 100644 index 207dc069e43..00000000000 --- a/dbms/tests/queries/0_stateless/00281_compile_sizeof_packed.reference +++ /dev/null @@ -1,2 +0,0 @@ -1 Hello -2 Hello diff --git a/dbms/tests/queries/0_stateless/00281_compile_sizeof_packed.sql b/dbms/tests/queries/0_stateless/00281_compile_sizeof_packed.sql deleted file mode 100644 index 5902b94b753..00000000000 --- a/dbms/tests/queries/0_stateless/00281_compile_sizeof_packed.sql +++ /dev/null @@ -1,2 +0,0 @@ -SET compile = 1, min_count_to_compile = 0, max_threads = 1, send_logs_level = 'none'; -SELECT arrayJoin([1, 2, 1]) AS UserID, argMax('Hello', today()) AS res GROUP BY UserID; diff --git a/dbms/tests/queries/0_stateless/00284_external_aggregation.sql b/dbms/tests/queries/0_stateless/00284_external_aggregation.sql index 75d2c0b9bc2..cd9abec59a8 100644 --- a/dbms/tests/queries/0_stateless/00284_external_aggregation.sql +++ b/dbms/tests/queries/0_stateless/00284_external_aggregation.sql @@ -1,5 +1,5 @@ SET max_bytes_before_external_group_by = 100000000; -SET max_memory_usage = 201000000; +SET max_memory_usage = 351000000; SELECT sum(k), sum(c) FROM (SELECT number AS k, count() AS c FROM (SELECT * FROM system.numbers LIMIT 10000000) GROUP BY k); SELECT sum(k), sum(c), max(u) FROM (SELECT number AS k, count() AS c, uniqArray(range(number % 16)) AS u FROM (SELECT * FROM system.numbers LIMIT 1000000) GROUP BY k); diff --git a/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.reference b/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.reference index e838f583cdf..a2c79e66928 100644 --- a/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.reference +++ b/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.reference @@ -3,17 +3,18 @@ < X-ClickHouse-Progress: {"read_rows":"10","read_bytes":"80","written_rows":"0","written_bytes":"0","total_rows_to_read":"10"} < X-ClickHouse-Progress: {"read_rows":"10","read_bytes":"80","written_rows":"0","written_bytes":"0","total_rows_to_read":"10"} 9 -< X-ClickHouse-Progress: {"read_rows":"1","read_bytes":"8","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} -< X-ClickHouse-Progress: {"read_rows":"2","read_bytes":"16","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} -< X-ClickHouse-Progress: {"read_rows":"3","read_bytes":"24","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} -< X-ClickHouse-Progress: {"read_rows":"4","read_bytes":"32","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} -< X-ClickHouse-Progress: {"read_rows":"5","read_bytes":"40","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} -< X-ClickHouse-Progress: {"read_rows":"6","read_bytes":"48","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} -< X-ClickHouse-Progress: {"read_rows":"7","read_bytes":"56","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} -< X-ClickHouse-Progress: {"read_rows":"8","read_bytes":"64","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} -< X-ClickHouse-Progress: {"read_rows":"9","read_bytes":"72","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} -< X-ClickHouse-Progress: {"read_rows":"10","read_bytes":"80","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} -< X-ClickHouse-Progress: {"read_rows":"10","read_bytes":"80","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} +< X-ClickHouse-Progress: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"10"} +< X-ClickHouse-Progress: {"read_rows":"1","read_bytes":"8","written_rows":"0","written_bytes":"0","total_rows_to_read":"10"} +< X-ClickHouse-Progress: {"read_rows":"2","read_bytes":"16","written_rows":"0","written_bytes":"0","total_rows_to_read":"10"} +< X-ClickHouse-Progress: {"read_rows":"3","read_bytes":"24","written_rows":"0","written_bytes":"0","total_rows_to_read":"10"} +< X-ClickHouse-Progress: {"read_rows":"4","read_bytes":"32","written_rows":"0","written_bytes":"0","total_rows_to_read":"10"} +< X-ClickHouse-Progress: {"read_rows":"5","read_bytes":"40","written_rows":"0","written_bytes":"0","total_rows_to_read":"10"} +< X-ClickHouse-Progress: {"read_rows":"6","read_bytes":"48","written_rows":"0","written_bytes":"0","total_rows_to_read":"10"} +< X-ClickHouse-Progress: {"read_rows":"7","read_bytes":"56","written_rows":"0","written_bytes":"0","total_rows_to_read":"10"} +< X-ClickHouse-Progress: {"read_rows":"8","read_bytes":"64","written_rows":"0","written_bytes":"0","total_rows_to_read":"10"} +< X-ClickHouse-Progress: {"read_rows":"9","read_bytes":"72","written_rows":"0","written_bytes":"0","total_rows_to_read":"10"} +< X-ClickHouse-Progress: {"read_rows":"10","read_bytes":"80","written_rows":"0","written_bytes":"0","total_rows_to_read":"10"} +< X-ClickHouse-Progress: {"read_rows":"10","read_bytes":"80","written_rows":"0","written_bytes":"0","total_rows_to_read":"10"} 0 1 2 diff --git a/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh b/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh index d95798bc95c..c86154a8402 100755 --- a/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh +++ b/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh @@ -6,9 +6,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) ${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}?max_block_size=5&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0" -d 'SELECT max(number) FROM numbers(10)' 2>&1 | grep -E 'Content-Encoding|X-ClickHouse-Progress|^[0-9]' # This test will fail with external poco (progress not supported) -# "grep -v 11" in order to skip extra progress header for 11-th row (for processors pipeline) -${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}?max_block_size=1&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0&experimental_use_processors=0" -d 'SELECT number FROM system.numbers LIMIT 10' 2>&1 | grep -E 'Content-Encoding|X-ClickHouse-Progress|^[0-9]' | grep -v 11 -${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?max_block_size=1&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0&enable_http_compression=1&experimental_use_processors=0" -H 'Accept-Encoding: gzip' -d 'SELECT number FROM system.numbers LIMIT 10' | gzip -d +${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}?max_block_size=1&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0" -d 'SELECT number FROM numbers(10)' 2>&1 | grep -E 'Content-Encoding|X-ClickHouse-Progress|^[0-9]' +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?max_block_size=1&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0&enable_http_compression=1" -H 'Accept-Encoding: gzip' -d 'SELECT number FROM system.numbers LIMIT 10' | gzip -d # 'send_progress_in_http_headers' is false by default ${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}?max_block_size=1&http_headers_progress_interval_ms=0" -d 'SELECT number FROM system.numbers LIMIT 10' 2>&1 | grep -q 'X-ClickHouse-Progress' && echo 'Fail' || true diff --git a/dbms/tests/queries/0_stateless/00428_partition.reference b/dbms/tests/queries/0_stateless/00428_partition.reference deleted file mode 100644 index c777fd7a5c3..00000000000 --- a/dbms/tests/queries/0_stateless/00428_partition.reference +++ /dev/null @@ -1,54 +0,0 @@ -5 -5 -082814b5aa5109160d5c0c5aff10d4df shadow/1/data/test/partition_428/19700102_2_2_0/k.bin -082814b5aa5109160d5c0c5aff10d4df shadow/1/data/test/partition_428/19700201_1_1_0/v1.bin -13cae8e658e0ca4f75c56b1fc424e150 shadow/1/data/test/partition_428/19700102_2_2_0/minmax_p.idx -25daad3d9e60b45043a70c4ab7d3b1c6 shadow/1/data/test/partition_428/19700102_2_2_0/partition.dat -3726312af62aec86b64a7708d5751787 shadow/1/data/test/partition_428/19700201_1_1_0/partition.dat -37855b06a39b79a67ea4e86e4a3299aa shadow/1/data/test/partition_428/19700102_2_2_0/checksums.txt -38e62ff37e1e5064e9a3f605dfe09d13 shadow/1/data/test/partition_428/19700102_2_2_0/v1.bin -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700102_2_2_0/k.mrk -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700102_2_2_0/p.mrk -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700102_2_2_0/v1.mrk -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700201_1_1_0/k.mrk -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700201_1_1_0/p.mrk -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700201_1_1_0/v1.mrk -55a54008ad1ba589aa210d2629c1df41 shadow/1/data/test/partition_428/19700201_1_1_0/primary.idx -5f087cb3e7071bf9407e095821e2af8f shadow/1/data/test/partition_428/19700201_1_1_0/checksums.txt -77d5af402ada101574f4da114f242e02 shadow/1/data/test/partition_428/19700102_2_2_0/columns.txt -77d5af402ada101574f4da114f242e02 shadow/1/data/test/partition_428/19700201_1_1_0/columns.txt -88cdc31ded355e7572d68d8cde525d3a shadow/1/data/test/partition_428/19700201_1_1_0/p.bin -9e688c58a5487b8eaf69c9e1005ad0bf shadow/1/data/test/partition_428/19700102_2_2_0/primary.idx -c4ca4238a0b923820dcc509a6f75849b shadow/1/data/test/partition_428/19700102_2_2_0/count.txt -c4ca4238a0b923820dcc509a6f75849b shadow/1/data/test/partition_428/19700201_1_1_0/count.txt -cfcb770c3ecd0990dcceb1bde129e6c6 shadow/1/data/test/partition_428/19700102_2_2_0/p.bin -e2af3bef1fd129aea73a890ede1e7a30 shadow/1/data/test/partition_428/19700201_1_1_0/k.bin -f2312862cc01adf34a93151377be2ddf shadow/1/data/test/partition_428/19700201_1_1_0/minmax_p.idx -5 -5 -082814b5aa5109160d5c0c5aff10d4df shadow/1/data/test/partition_428/19700102_2_2_0/k.bin -082814b5aa5109160d5c0c5aff10d4df shadow/1/data/test/partition_428/19700201_1_1_0/v1.bin -13cae8e658e0ca4f75c56b1fc424e150 shadow/1/data/test/partition_428/19700102_2_2_0/minmax_p.idx -25daad3d9e60b45043a70c4ab7d3b1c6 shadow/1/data/test/partition_428/19700102_2_2_0/partition.dat -3726312af62aec86b64a7708d5751787 shadow/1/data/test/partition_428/19700201_1_1_0/partition.dat -37855b06a39b79a67ea4e86e4a3299aa shadow/1/data/test/partition_428/19700102_2_2_0/checksums.txt -38e62ff37e1e5064e9a3f605dfe09d13 shadow/1/data/test/partition_428/19700102_2_2_0/v1.bin -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700102_2_2_0/k.mrk -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700102_2_2_0/p.mrk -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700102_2_2_0/v1.mrk -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700201_1_1_0/k.mrk -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700201_1_1_0/p.mrk -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700201_1_1_0/v1.mrk -55a54008ad1ba589aa210d2629c1df41 shadow/1/data/test/partition_428/19700201_1_1_0/primary.idx -5f087cb3e7071bf9407e095821e2af8f shadow/1/data/test/partition_428/19700201_1_1_0/checksums.txt -77d5af402ada101574f4da114f242e02 shadow/1/data/test/partition_428/19700102_2_2_0/columns.txt -77d5af402ada101574f4da114f242e02 shadow/1/data/test/partition_428/19700201_1_1_0/columns.txt -88cdc31ded355e7572d68d8cde525d3a shadow/1/data/test/partition_428/19700201_1_1_0/p.bin -9e688c58a5487b8eaf69c9e1005ad0bf shadow/1/data/test/partition_428/19700102_2_2_0/primary.idx -c4ca4238a0b923820dcc509a6f75849b shadow/1/data/test/partition_428/19700102_2_2_0/count.txt -c4ca4238a0b923820dcc509a6f75849b shadow/1/data/test/partition_428/19700201_1_1_0/count.txt -cfcb770c3ecd0990dcceb1bde129e6c6 shadow/1/data/test/partition_428/19700102_2_2_0/p.bin -e2af3bef1fd129aea73a890ede1e7a30 shadow/1/data/test/partition_428/19700201_1_1_0/k.bin -f2312862cc01adf34a93151377be2ddf shadow/1/data/test/partition_428/19700201_1_1_0/minmax_p.idx -31,1,2 -1,2,3 diff --git a/dbms/tests/queries/0_stateless/00428_partition.sh b/dbms/tests/queries/0_stateless/00428_partition.sh deleted file mode 100755 index 033d5e24c13..00000000000 --- a/dbms/tests/queries/0_stateless/00428_partition.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash - -set -e - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. $CURDIR/../shell_config.sh - -# Not found column date in block. There are only columns: x. - -# Test 1. Complex test checking columns.txt - -chl="$CLICKHOUSE_CLIENT -q" -ch_dir=`${CLICKHOUSE_EXTRACT_CONFIG} -k path` - -$chl "DROP TABLE IF EXISTS test.partition_428" -$chl "CREATE TABLE test.partition_428 (p Date, k Int8, v1 Int8 MATERIALIZED k + 1) ENGINE = MergeTree PARTITION BY p ORDER BY k SETTINGS index_granularity=1, index_granularity_bytes=0" -$chl "INSERT INTO test.partition_428 (p, k) VALUES(toDate(31), 1)" -$chl "INSERT INTO test.partition_428 (p, k) VALUES(toDate(1), 2)" - -for part in `$chl "SELECT name FROM system.parts WHERE database='test' AND table='partition_428'"`; do - # 2 header lines + 3 columns - (sudo -n cat $ch_dir/data/test/partition_428/$part/columns.txt 2>/dev/null || \ - cat $ch_dir/data/test/partition_428/$part/columns.txt) | wc -l -done - -$chl "ALTER TABLE test.partition_428 FREEZE" - -# Do `cd` for consistent output for reference -# Do not check increment.txt - it can be changed by other tests with FREEZE -cd $ch_dir && find shadow -type f -exec md5sum {} \; | grep "partition_428" | sed 's!shadow/[0-9]*/data/[a-z0-9_-]*/!shadow/1/data/test/!g' | sort | uniq - -$chl "ALTER TABLE test.partition_428 DETACH PARTITION 197001" -$chl "ALTER TABLE test.partition_428 ATTACH PARTITION 197001" - -for part in `$chl "SELECT name FROM system.parts WHERE database='test' AND table='partition_428'"`; do - # 2 header lines + 3 columns - (sudo -n cat $ch_dir/data/test/partition_428/$part/columns.txt 2>/dev/null || \ - cat $ch_dir/data/test/partition_428/$part/columns.txt) | wc -l -done - -$chl "ALTER TABLE test.partition_428 MODIFY COLUMN v1 Int8" - -# Check the backup hasn't changed -cd $ch_dir && find shadow -type f -exec md5sum {} \; | grep "partition_428" | sed 's!shadow/[0-9]*/data/[a-z0-9_-]*/!shadow/1/data/test/!g' | sort | uniq - -$chl "OPTIMIZE TABLE test.partition_428" - -$chl "SELECT toUInt16(p), k, v1 FROM test.partition_428 ORDER BY k FORMAT CSV" -$chl "DROP TABLE test.partition_428" - -# Test 2. Simple test - -$chl "drop table if exists test.partition_428" -$chl "create table test.partition_428 (date MATERIALIZED toDate(0), x UInt64, sample_key MATERIALIZED intHash64(x)) ENGINE=MergeTree PARTITION BY date SAMPLE BY sample_key ORDER BY (date,x,sample_key) SETTINGS index_granularity=8192, index_granularity_bytes=0" -$chl "insert into test.partition_428 ( x ) VALUES ( now() )" -$chl "insert into test.partition_428 ( x ) VALUES ( now()+1 )" -$chl "alter table test.partition_428 detach partition 197001" -$chl "alter table test.partition_428 attach partition 197001" -$chl "optimize table test.partition_428" -$chl "drop table test.partition_428" diff --git a/dbms/tests/queries/0_stateless/00568_compile_catch_throw.sh b/dbms/tests/queries/0_stateless/00568_compile_catch_throw.sh deleted file mode 100755 index fbf5efcda2c..00000000000 --- a/dbms/tests/queries/0_stateless/00568_compile_catch_throw.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. $CURDIR/../shell_config.sh - -SETTINGS="--compile=1 --min_count_to_compile=0 --max_threads=1 --max_memory_usage=8000000 --server_logs_file=/dev/null" -output=$($CLICKHOUSE_CLIENT -q "SELECT length(groupArray(number)) FROM (SELECT * FROM system.numbers LIMIT 1000000)" $SETTINGS 2>&1) - -[[ $? -eq 0 ]] && echo "Expected non-zero RC" -if ! echo "$output" | grep -Fc -e 'Memory limit (for query) exceeded' -e 'Cannot compile code' ; then - echo -e 'There is no expected exception "Memory limit (for query) exceeded: would use..." or "Cannot compile code..."' "Whereas got:\n$output" -fi - -$CLICKHOUSE_CLIENT -q "SELECT 1" diff --git a/dbms/tests/queries/0_stateless/00600_replace_running_query.reference b/dbms/tests/queries/0_stateless/00600_replace_running_query.reference index 237dd6b5309..a01672aae85 100644 --- a/dbms/tests/queries/0_stateless/00600_replace_running_query.reference +++ b/dbms/tests/queries/0_stateless/00600_replace_running_query.reference @@ -1,5 +1,6 @@ 0 -1 0 -3 0 -2 0 +1 +1 +1 +1 44 diff --git a/dbms/tests/queries/0_stateless/00600_replace_running_query.sh b/dbms/tests/queries/0_stateless/00600_replace_running_query.sh index 8e4677bb1d5..9fc25291548 100755 --- a/dbms/tests/queries/0_stateless/00600_replace_running_query.sh +++ b/dbms/tests/queries/0_stateless/00600_replace_running_query.sh @@ -1,17 +1,18 @@ #!/usr/bin/env bash +CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=none + CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh -set -e -o pipefail function wait_for_query_to_start() { while [[ $($CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" -d "SELECT count() FROM system.processes WHERE query_id = '$1'") == 0 ]]; do sleep 0.1; done } -$CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL?query_id=hello&replace_running_query=1" -d 'SELECT sum(ignore(*)) FROM (SELECT number % 1000 AS k, groupArray(number) FROM numbers(100000000) GROUP BY k)' 2>&1 > /dev/null & +$CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL?query_id=hello&replace_running_query=1" -d 'SELECT 1, count() FROM system.numbers' 2>&1 > /dev/null & wait_for_query_to_start 'hello' # Replace it @@ -20,15 +21,20 @@ $CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL?query_id=hello&replace_running_query=1" -d # Wait for it to be replaced wait -${CLICKHOUSE_CLIENT} --user=readonly --query_id=42 --query='SELECT 1, sleep(1)' & +${CLICKHOUSE_CLIENT} --user=readonly --query_id=42 --query='SELECT 2, count() FROM system.numbers' 2>&1 | grep -cF 'was cancelled' & wait_for_query_to_start '42' -( ${CLICKHOUSE_CLIENT} --query_id=42 --query='SELECT 43' ||: ) 2>&1 | grep -F 'is already running by user' > /dev/null + +# Trying to run another query with the same query_id +${CLICKHOUSE_CLIENT} --query_id=42 --query='SELECT 43' 2>&1 | grep -cF 'is already running by user' + +# Trying to replace query of a different user +$CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL?query_id=42&replace_running_query=1" -d 'SELECT 1' | grep -cF 'is already running by user' + +$CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" -d "KILL QUERY WHERE query_id = '42' SYNC" > /dev/null wait -${CLICKHOUSE_CLIENT} --query='SELECT 3, sleep(1)' & -sleep 0.1 -${CLICKHOUSE_CLIENT} --query_id=42 --query='SELECT 2, sleep(1)' & +${CLICKHOUSE_CLIENT} --query_id=42 --query='SELECT 3, count() FROM system.numbers' 2>&1 | grep -cF 'was cancelled' & wait_for_query_to_start '42' -( ${CLICKHOUSE_CLIENT} --query_id=42 --replace_running_query=1 --queue_max_wait_ms=500 --query='SELECT 43' ||: ) 2>&1 | grep -F "can't be stopped" > /dev/null +${CLICKHOUSE_CLIENT} --query_id=42 --replace_running_query=1 --replace_running_query_max_wait_ms=500 --query='SELECT 43' 2>&1 | grep -F "can't be stopped" > /dev/null ${CLICKHOUSE_CLIENT} --query_id=42 --replace_running_query=1 --query='SELECT 44' wait diff --git a/dbms/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh b/dbms/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh index 5d1a7338e46..75d6a0dd59b 100755 --- a/dbms/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh +++ b/dbms/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh @@ -1,5 +1,11 @@ #!/usr/bin/env bash +# Because REPLACE PARTITION does not forces immediate removal of replaced data parts from local filesystem +# (it tries to do it as quick as possible, but it still performed in separate thread asynchronously) +# and when we do DETACH TABLE / ATTACH TABLE or SYSTEM RESTART REPLICA, these files may be discovered +# and discarded after restart with Warning/Error messages in log. This is Ok. +CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=none + CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh @@ -101,7 +107,7 @@ $CLICKHOUSE_CLIENT --query="DROP TABLE test.src;" $CLICKHOUSE_CLIENT --query="CREATE TABLE test.src (p UInt64, k String, d UInt64) ENGINE = MergeTree PARTITION BY p ORDER BY k;" $CLICKHOUSE_CLIENT --query="INSERT INTO test.src VALUES (1, '0', 1);" $CLICKHOUSE_CLIENT --query="INSERT INTO test.src VALUES (1, '1', 1);" -$CLICKHOUSE_CLIENT --query="INSERT INTO test.dst_r1 VALUES (1, '1', 2);" -- trash part to be +$CLICKHOUSE_CLIENT --query="INSERT INTO test.dst_r1 VALUES (1, '1', 2); -- trash part to be deleted" # Stop replication at the second replica and remove source table to use fetch instead of copying $CLICKHOUSE_CLIENT --query="SYSTEM STOP REPLICATION QUEUES test.dst_r2;" diff --git a/dbms/tests/queries/0_stateless/00704_drop_truncate_memory_table.sh b/dbms/tests/queries/0_stateless/00704_drop_truncate_memory_table.sh index 170259f0e24..1bbb69f4dd7 100755 --- a/dbms/tests/queries/0_stateless/00704_drop_truncate_memory_table.sh +++ b/dbms/tests/queries/0_stateless/00704_drop_truncate_memory_table.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash set -e +CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=none + CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh diff --git a/dbms/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh b/dbms/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh index 17779b73add..965408065cf 100755 --- a/dbms/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh +++ b/dbms/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh @@ -8,14 +8,58 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) echo "DROP TABLE IF EXISTS concurrent_alter_column" | ${CLICKHOUSE_CLIENT} echo "CREATE TABLE concurrent_alter_column (ts DATETIME) ENGINE = MergeTree PARTITION BY toStartOfDay(ts) ORDER BY tuple()" | ${CLICKHOUSE_CLIENT} -for i in {1..500}; do echo "ALTER TABLE concurrent_alter_column ADD COLUMN c$i DOUBLE;"; done | ${CLICKHOUSE_CLIENT} -n +function thread1() +{ + while true; do + for i in {1..500}; do echo "ALTER TABLE concurrent_alter_column ADD COLUMN c$i DOUBLE;"; done | ${CLICKHOUSE_CLIENT} -n --query_id=alter1 + done +} -for i in {1..100}; do echo "ALTER TABLE concurrent_alter_column ADD COLUMN d DOUBLE" | ${CLICKHOUSE_CLIENT}; sleep `echo 0.0$RANDOM`; echo "ALTER TABLE concurrent_alter_column DROP COLUMN d" | ${CLICKHOUSE_CLIENT} -n; done & -for i in {1..100}; do echo "ALTER TABLE concurrent_alter_column ADD COLUMN e DOUBLE" | ${CLICKHOUSE_CLIENT}; sleep `echo 0.0$RANDOM`; echo "ALTER TABLE concurrent_alter_column DROP COLUMN e" | ${CLICKHOUSE_CLIENT} -n; done & -for i in {1..100}; do echo "ALTER TABLE concurrent_alter_column ADD COLUMN f DOUBLE" | ${CLICKHOUSE_CLIENT}; sleep `echo 0.0$RANDOM`; echo "ALTER TABLE concurrent_alter_column DROP COLUMN f" | ${CLICKHOUSE_CLIENT} -n; done & +function thread2() +{ + while true; do + echo "ALTER TABLE concurrent_alter_column ADD COLUMN d DOUBLE" | ${CLICKHOUSE_CLIENT} --query_id=alter2; + sleep `echo 0.0$RANDOM`; + echo "ALTER TABLE concurrent_alter_column DROP COLUMN d" | ${CLICKHOUSE_CLIENT} --query_id=alter2; + done +} + +function thread3() +{ + while true; do + echo "ALTER TABLE concurrent_alter_column ADD COLUMN e DOUBLE" | ${CLICKHOUSE_CLIENT} --query_id=alter3; + sleep `echo 0.0$RANDOM`; + echo "ALTER TABLE concurrent_alter_column DROP COLUMN e" | ${CLICKHOUSE_CLIENT} --query_id=alter3; + done +} + +function thread4() +{ + while true; do + echo "ALTER TABLE concurrent_alter_column ADD COLUMN f DOUBLE" | ${CLICKHOUSE_CLIENT} --query_id=alter4; + sleep `echo 0.0$RANDOM`; + echo "ALTER TABLE concurrent_alter_column DROP COLUMN f" | ${CLICKHOUSE_CLIENT} --query_id=alter4; + done +} + +# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout +export -f thread1; +export -f thread2; +export -f thread3; +export -f thread4; + +TIMEOUT=30 + +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread4 2> /dev/null & wait echo "DROP TABLE concurrent_alter_column" | ${CLICKHOUSE_CLIENT} +# Check for deadlocks +echo "SELECT * FROM system.processes WHERE query_id LIKE 'alter%'" | ${CLICKHOUSE_CLIENT} + echo 'did not crash' diff --git a/dbms/tests/queries/0_stateless/00972_live_view_select_1.reference b/dbms/tests/queries/0_stateless/00881_unknown_identifier_in_in.reference similarity index 100% rename from dbms/tests/queries/0_stateless/00972_live_view_select_1.reference rename to dbms/tests/queries/0_stateless/00881_unknown_identifier_in_in.reference diff --git a/dbms/tests/queries/0_stateless/00881_unknown_identifier_in_in.sql b/dbms/tests/queries/0_stateless/00881_unknown_identifier_in_in.sql new file mode 100644 index 00000000000..2ce709c45be --- /dev/null +++ b/dbms/tests/queries/0_stateless/00881_unknown_identifier_in_in.sql @@ -0,0 +1,4 @@ +SELECT toUInt64(1) x FROM (select 1) +GROUP BY 1 +HAVING x +IN ( SELECT countIf(y, z == 1) FROM (SELECT 1 y, 1 z) ); diff --git a/dbms/tests/queries/0_stateless/00900_orc_load.reference b/dbms/tests/queries/0_stateless/00900_orc_load.reference new file mode 100644 index 00000000000..fe79e37ee18 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00900_orc_load.reference @@ -0,0 +1,2 @@ +0 0 0 0 0 2019-01-01 test1 +2147483647 -1 9223372036854775806 123.345345 345345.3453451212 2019-01-01 test2 diff --git a/dbms/tests/queries/0_stateless/00900_orc_load.sh b/dbms/tests/queries/0_stateless/00900_orc_load.sh new file mode 100755 index 00000000000..cd553f6d234 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00900_orc_load.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CUR_DIR/../shell_config.sh + +CB_DIR=$(dirname "$CLICKHOUSE_CLIENT_BINARY") +[ "$CB_DIR" == "." ] && ROOT_DIR=$CUR_DIR/../../../.. +[ "$CB_DIR" != "." ] && BUILD_DIR=$CB_DIR/../.. +[ -z "$ROOT_DIR" ] && ROOT_DIR=$CB_DIR/../../.. + +DATA_FILE=$CUR_DIR/data_orc/test.orc + +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS orc_load" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE orc_load (int Int32, smallint Int8, bigint Int64, float Float32, double Float64, date Date, y String) ENGINE = Memory" +cat $DATA_FILE | ${CLICKHOUSE_CLIENT} -q "insert into orc_load format ORC" +${CLICKHOUSE_CLIENT} --query="select * from orc_load" + diff --git a/dbms/tests/queries/0_stateless/00900_parquet_load.reference b/dbms/tests/queries/0_stateless/00900_parquet_load.reference index 83d0e8efde9..4e3977e0e96 100644 --- a/dbms/tests/queries/0_stateless/00900_parquet_load.reference +++ b/dbms/tests/queries/0_stateless/00900_parquet_load.reference @@ -39,7 +39,7 @@ 23.0 24.0 === Try load data from datapage_v2.snappy.parquet -Code: 33. DB::Ex---tion: Error while reading parquet data: IOError: Arrow error: IOError: Corrupt snappy compressed data. +Code: 33. DB::Ex---tion: Error while reading Parquet data: IOError: Arrow error: IOError: Corrupt snappy compressed data. === Try load data from fixed_length_decimal_1.parquet 1.0 @@ -171,19 +171,19 @@ Code: 33. DB::Ex---tion: Error while reading parquet data: IOError: Arrow error: Code: 8. DB::Ex---tion: Column "element" is not presented in input data === Try load data from nested_maps.snappy.parquet -Code: 33. DB::Ex---tion: Error while reading parquet data: NotImplemented: Currently only nesting with Lists is supported. +Code: 33. DB::Ex---tion: Error while reading Parquet data: NotImplemented: Currently only nesting with Lists is supported. === Try load data from nonnullable.impala.parquet -Code: 33. DB::Ex---tion: Error while reading parquet data: NotImplemented: Currently only nesting with Lists is supported. +Code: 33. DB::Ex---tion: Error while reading Parquet data: NotImplemented: Currently only nesting with Lists is supported. === Try load data from nullable.impala.parquet -Code: 33. DB::Ex---tion: Error while reading parquet data: NotImplemented: Currently only nesting with Lists is supported. +Code: 33. DB::Ex---tion: Error while reading Parquet data: NotImplemented: Currently only nesting with Lists is supported. === Try load data from nulls.snappy.parquet Code: 8. DB::Ex---tion: Column "b_c_int" is not presented in input data === Try load data from repeated_no_annotation.parquet -Code: 33. DB::Ex---tion: Error while reading parquet data: NotImplemented: Currently only nesting with Lists is supported. +Code: 33. DB::Ex---tion: Error while reading Parquet data: NotImplemented: Currently only nesting with Lists is supported. === Try load data from userdata1.parquet 1454486129 1 Amanda Jordan ajordan0@com.com Female 1.197.201.2 6759521864920116 Indonesia 3/8/1971 49756.53 Internal Auditor 1E+02 diff --git a/dbms/tests/queries/0_stateless/00918_json_functions.reference b/dbms/tests/queries/0_stateless/00918_json_functions.reference index 44d4dc1c9bf..a23b177d468 100644 --- a/dbms/tests/queries/0_stateless/00918_json_functions.reference +++ b/dbms/tests/queries/0_stateless/00918_json_functions.reference @@ -46,6 +46,13 @@ hello 1 Thursday Friday +(3,5) +(7,3) +(5,0) +(3,5) +(3,0) +(3,5) +(3,0) --JSONExtractKeysAndValues-- [('a','hello')] [('b',[-100,200,300])] @@ -121,6 +128,13 @@ hello 1 Thursday Friday +(3,5) +(7,3) +(5,0) +(3,5) +(3,0) +(3,5) +(3,0) --JSONExtractKeysAndValues-- [('a','hello')] [('b',[-100,200,300])] diff --git a/dbms/tests/queries/0_stateless/00918_json_functions.sql b/dbms/tests/queries/0_stateless/00918_json_functions.sql index 83f6d1578f9..38bf0a7ffec 100644 --- a/dbms/tests/queries/0_stateless/00918_json_functions.sql +++ b/dbms/tests/queries/0_stateless/00918_json_functions.sql @@ -54,6 +54,13 @@ SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 4, 'Nullable( SELECT JSONExtract('{"passed": true}', 'passed', 'UInt8'); SELECT JSONExtract('{"day": "Thursday"}', 'day', 'Enum8(\'Sunday\' = 0, \'Monday\' = 1, \'Tuesday\' = 2, \'Wednesday\' = 3, \'Thursday\' = 4, \'Friday\' = 5, \'Saturday\' = 6)'); SELECT JSONExtract('{"day": 5}', 'day', 'Enum8(\'Sunday\' = 0, \'Monday\' = 1, \'Tuesday\' = 2, \'Wednesday\' = 3, \'Thursday\' = 4, \'Friday\' = 5, \'Saturday\' = 6)'); +SELECT JSONExtract('{"a":3,"b":5,"c":7}', 'Tuple(a Int, b Int)'); +SELECT JSONExtract('{"a":3,"b":5,"c":7}', 'Tuple(c Int, a Int)'); +SELECT JSONExtract('{"a":3,"b":5,"c":7}', 'Tuple(b Int, d Int)'); +SELECT JSONExtract('{"a":3,"b":5,"c":7}', 'Tuple(Int, Int)'); +SELECT JSONExtract('{"a":3}', 'Tuple(Int, Int)'); +SELECT JSONExtract('[3,5,7]', 'Tuple(Int, Int)'); +SELECT JSONExtract('[3]', 'Tuple(Int, Int)'); SELECT '--JSONExtractKeysAndValues--'; SELECT JSONExtractKeysAndValues('{"a": "hello", "b": [-100, 200.0, 300]}', 'String'); @@ -138,6 +145,13 @@ SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 4, 'Nullable( SELECT JSONExtract('{"passed": true}', 'passed', 'UInt8'); SELECT JSONExtract('{"day": "Thursday"}', 'day', 'Enum8(\'Sunday\' = 0, \'Monday\' = 1, \'Tuesday\' = 2, \'Wednesday\' = 3, \'Thursday\' = 4, \'Friday\' = 5, \'Saturday\' = 6)'); SELECT JSONExtract('{"day": 5}', 'day', 'Enum8(\'Sunday\' = 0, \'Monday\' = 1, \'Tuesday\' = 2, \'Wednesday\' = 3, \'Thursday\' = 4, \'Friday\' = 5, \'Saturday\' = 6)'); +SELECT JSONExtract('{"a":3,"b":5,"c":7}', 'Tuple(a Int, b Int)'); +SELECT JSONExtract('{"a":3,"b":5,"c":7}', 'Tuple(c Int, a Int)'); +SELECT JSONExtract('{"a":3,"b":5,"c":7}', 'Tuple(b Int, d Int)'); +SELECT JSONExtract('{"a":3,"b":5,"c":7}', 'Tuple(Int, Int)'); +SELECT JSONExtract('{"a":3}', 'Tuple(Int, Int)'); +SELECT JSONExtract('[3,5,7]', 'Tuple(Int, Int)'); +SELECT JSONExtract('[3]', 'Tuple(Int, Int)'); SELECT '--JSONExtractKeysAndValues--'; SELECT JSONExtractKeysAndValues('{"a": "hello", "b": [-100, 200.0, 300]}', 'String'); diff --git a/dbms/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh b/dbms/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh index b0fd9a70bd4..1f7571a2404 100755 --- a/dbms/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh +++ b/dbms/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh @@ -19,7 +19,7 @@ $CLICKHOUSE_CLIENT --use_uncompressed_cache=1 --query_id="test-query-uncompresse sleep 1 $CLICKHOUSE_CLIENT --query="SYSTEM FLUSH LOGS" -$CLICKHOUSE_CLIENT --query="SELECT ProfileEvents.Values[indexOf(ProfileEvents.Names, 'Seek')], ProfileEvents.Values[indexOf(ProfileEvents.Names, 'ReadCompressedBytes')], ProfileEvents.Values[indexOf(ProfileEvents.Names, 'UncompressedCacheHits')] AS hit FROM system.query_log WHERE (query_id = 'test-query-uncompressed-cache') AND (type = 2) ORDER BY event_time DESC LIMIT 1" +$CLICKHOUSE_CLIENT --query="SELECT ProfileEvents.Values[indexOf(ProfileEvents.Names, 'Seek')], ProfileEvents.Values[indexOf(ProfileEvents.Names, 'ReadCompressedBytes')], ProfileEvents.Values[indexOf(ProfileEvents.Names, 'UncompressedCacheHits')] AS hit FROM system.query_log WHERE (query_id = 'test-query-uncompressed-cache') AND (type = 2) AND event_date >= yesterday() ORDER BY event_time DESC LIMIT 1" $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS small_table" diff --git a/dbms/tests/queries/0_stateless/00942_mutate_index.sh b/dbms/tests/queries/0_stateless/00942_mutate_index.sh index c6dd1dfb836..467eb9ab671 100755 --- a/dbms/tests/queries/0_stateless/00942_mutate_index.sh +++ b/dbms/tests/queries/0_stateless/00942_mutate_index.sh @@ -2,6 +2,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh +. $CURDIR/mergetree_mutations.lib $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.minmax_idx;" @@ -35,8 +36,7 @@ $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 1;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 5;" $CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx UPDATE i64 = 5 WHERE i64 = 1;" - -sleep 0.1 +wait_for_mutation "minmax_idx" "mutation_2.txt" "test" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 1;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 5;" diff --git a/dbms/tests/queries/0_stateless/00943_materialize_index.sh b/dbms/tests/queries/0_stateless/00943_materialize_index.sh index f51b66993aa..bc59b41b005 100755 --- a/dbms/tests/queries/0_stateless/00943_materialize_index.sh +++ b/dbms/tests/queries/0_stateless/00943_materialize_index.sh @@ -2,6 +2,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh +. $CURDIR/mergetree_mutations.lib $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.minmax_idx;" @@ -38,22 +39,26 @@ SET allow_experimental_data_skipping_indices=1; ALTER TABLE test.minmax_idx ADD INDEX idx (i64, u64 * i64) TYPE minmax GRANULARITY 1;" $CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx MATERIALIZE INDEX idx IN PARTITION 1;" +wait_for_mutation "minmax_idx" "mutation_3.txt" "test" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" $CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx MATERIALIZE INDEX idx IN PARTITION 2;" +wait_for_mutation "minmax_idx" "mutation_4.txt" "test" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" $CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx CLEAR INDEX idx IN PARTITION 1;" $CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx CLEAR INDEX idx IN PARTITION 2;" +sleep 0.5 $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" $CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx MATERIALIZE INDEX idx;" +wait_for_mutation "minmax_idx" "mutation_5.txt" "test" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" diff --git a/dbms/tests/queries/0_stateless/00944_clear_index_in_partition.sh b/dbms/tests/queries/0_stateless/00944_clear_index_in_partition.sh index 9047bbb3a72..74f15e63545 100755 --- a/dbms/tests/queries/0_stateless/00944_clear_index_in_partition.sh +++ b/dbms/tests/queries/0_stateless/00944_clear_index_in_partition.sh @@ -2,6 +2,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh +. $CURDIR/mergetree_mutations.lib $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.minmax_idx;" @@ -42,7 +43,7 @@ $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" $CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx MATERIALIZE INDEX idx IN PARTITION 1;" -sleep 0.5 +wait_for_mutation "minmax_idx" "mutation_3.txt" "test" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" diff --git a/dbms/tests/queries/0_stateless/00952_basic_constraints.reference b/dbms/tests/queries/0_stateless/00952_basic_constraints.reference new file mode 100644 index 00000000000..4d98efd8939 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00952_basic_constraints.reference @@ -0,0 +1,7 @@ +1 2 +ok +1 2 +ok +ok +0 11 +7 18 diff --git a/dbms/tests/queries/0_stateless/00952_basic_constraints.sh b/dbms/tests/queries/0_stateless/00952_basic_constraints.sh new file mode 100755 index 00000000000..8ba3b082f12 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00952_basic_constraints.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +EXCEPTION_TEXT=violated +EXCEPTION_SUCCESS_TEXT=ok +$CLICKHOUSE_CLIENT --query="CREATE DATABASE IF NOT EXISTS test;" +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test_constraints;" + +$CLICKHOUSE_CLIENT --query="CREATE TABLE test_constraints +( + a UInt32, + b UInt32, + CONSTRAINT b_constraint CHECK b > 0 +) +ENGINE = MergeTree ORDER BY (a);" + +# This one must succeed +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" +$CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" + +# This one must throw and exception + +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (3, 4), (1, 0), (3, 4), (6, 0);" 2>&1 \ + | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" +$CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" + +$CLICKHOUSE_CLIENT --query="DROP TABLE test_constraints;" + +# Test two constraints on one table +$CLICKHOUSE_CLIENT --query="CREATE TABLE test_constraints +( + a UInt32, + b UInt32, + CONSTRAINT a_constraint CHECK a < 10, + CONSTRAINT b_constraint CHECK b > 10 +) +ENGINE = MergeTree ORDER BY (a);" + +# This one must throw an exception +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" 2>&1 \ + | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" +$CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" + +# This one must throw an exception +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (5, 16), (10, 11), (9, 11), (8, 12);" 2>&1 \ + | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" +$CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" + +# This one must succeed +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (7, 18), (0, 11);" +$CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" + +$CLICKHOUSE_CLIENT --query="DROP TABLE test_constraints;" \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/00953_constraints_operations.reference b/dbms/tests/queries/0_stateless/00953_constraints_operations.reference new file mode 100644 index 00000000000..5713da9fef5 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00953_constraints_operations.reference @@ -0,0 +1,4 @@ +1 2 +ok +1 2 +ok diff --git a/dbms/tests/queries/0_stateless/00953_constraints_operations.sh b/dbms/tests/queries/0_stateless/00953_constraints_operations.sh new file mode 100755 index 00000000000..3b415848501 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00953_constraints_operations.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +EXCEPTION_TEXT=violated +EXCEPTION_SUCCESS_TEXT=ok + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test_constraints;" + +$CLICKHOUSE_CLIENT --query="CREATE TABLE test_constraints +( + a UInt32, + b UInt32, + CONSTRAINT b_constraint CHECK b > 0 +) +ENGINE = MergeTree ORDER BY (a);" + +# This one must succeed +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" +$CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" + +# This one must throw and exception + +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 0);" 2>&1 \ + | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" +$CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" + +$CLICKHOUSE_CLIENT --query="ALTER TABLE test_constraints DROP CONSTRAINT b_constraint;" + +# This one must suceed now +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 0);" + +$CLICKHOUSE_CLIENT --query="ALTER TABLE test_constraints ADD CONSTRAINT b_constraint CHECK b > 10;" + +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 10);" 2>&1 \ + | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" + +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 11);" + +$CLICKHOUSE_CLIENT --query="DROP TABLE test_constraints;" \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/00957_neighbor.reference b/dbms/tests/queries/0_stateless/00957_neighbor.reference new file mode 100644 index 00000000000..d25d727da5d --- /dev/null +++ b/dbms/tests/queries/0_stateless/00957_neighbor.reference @@ -0,0 +1,69 @@ +Zero offset +0 0 +1 1 +2 2 +Nullable values +\N 0 \N +\N 1 2 +2 2 \N +Result with different type +0 1 +1 2 +2 -10 +Offset > block +0 0 +1 0 +2 0 +Abs(Offset) > block +0 0 +1 0 +2 0 +Positive offset +0 1 +1 2 +2 0 +Negative offset +0 1 +1 2 +2 0 +Positive offset with defaults +0 2 +1 3 +2 12 +3 13 +Negative offset with defaults +0 10 +1 11 +2 0 +3 1 +Positive offset with const defaults +0 1 +1 2 +2 1000 +Negative offset with const defaults +0 1000 +1 0 +2 1 +Dynamic column and offset, out of bounds +0 0 0 +1 2 3 +2 4 20 +3 6 30 +Dynamic column and offset, negative +0 0 0 +1 -2 10 +2 -4 20 +3 -6 30 +4 -8 40 +5 -10 50 +Dynamic column and offset, without defaults +0 4 4 +1 2 3 +2 0 2 +3 -2 1 +4 -4 0 +5 -6 0 +Constant column +0 0 +1 0 +2 0 diff --git a/dbms/tests/queries/0_stateless/00957_neighbor.sql b/dbms/tests/queries/0_stateless/00957_neighbor.sql new file mode 100644 index 00000000000..c4b801c80cb --- /dev/null +++ b/dbms/tests/queries/0_stateless/00957_neighbor.sql @@ -0,0 +1,42 @@ +-- no arguments +select neighbor(); -- { serverError 42 } +-- single argument +select neighbor(1); -- { serverError 42 } +-- greater than 3 arguments +select neighbor(1,2,3,4); -- { serverError 42 } +-- bad default value +select neighbor(dummy, 1, 'hello'); -- { serverError 386 } +-- types without common supertype (UInt64 and Int8) +select number, neighbor(number, 1, -10) from numbers(3); -- { serverError 386 } +-- nullable offset is not allowed +select number, if(number > 1, number, null) as offset, neighbor(number, offset) from numbers(3); -- { serverError 43 } +select 'Zero offset'; +select number, neighbor(number, 0) from numbers(3); +select 'Nullable values'; +select if(number > 1, number, null) as value, number as offset, neighbor(value, offset) as neighbor from numbers(3); +select 'Result with different type'; +select toInt32(number) as n, neighbor(n, 1, -10) from numbers(3); +select 'Offset > block'; +select number, neighbor(number, 10) from numbers(3); +select 'Abs(Offset) > block'; +select number, neighbor(number, -10) from numbers(3); +select 'Positive offset'; +select number, neighbor(number, 1) from numbers(3); +select 'Negative offset'; +select number, neighbor(number, 1) from numbers(3); +select 'Positive offset with defaults'; +select number, neighbor(number, 2, number + 10) from numbers(4); +select 'Negative offset with defaults'; +select number, neighbor(number, -2, number + 10) from numbers(4); +select 'Positive offset with const defaults'; +select number, neighbor(number, 1, 1000) from numbers(3); +select 'Negative offset with const defaults'; +select number, neighbor(number, -1, 1000) from numbers(3); +select 'Dynamic column and offset, out of bounds'; +select number, number * 2 as offset, neighbor(number, offset, number * 10) from numbers(4); +select 'Dynamic column and offset, negative'; +select number, -number * 2 as offset, neighbor(number, offset, number * 10) from numbers(6); +select 'Dynamic column and offset, without defaults'; +select number, -(number - 2) * 2 as offset, neighbor(number, offset) from numbers(6); +select 'Constant column'; +select number, neighbor(1000, 10) from numbers(3); \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/00960_live_view_watch_events_live.py b/dbms/tests/queries/0_stateless/00960_live_view_watch_events_live.py deleted file mode 100755 index b7fc3f4e3a6..00000000000 --- a/dbms/tests/queries/0_stateless/00960_live_view_watch_events_live.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python -import os -import sys -import signal - -CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) - -from client import client, prompt, end_of_block - -log = None -# uncomment the line below for debugging -#log=sys.stdout - -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: - client1.expect(prompt) - client2.expect(prompt) - - client1.send('DROP TABLE IF EXISTS test.lv') - client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') - client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') - client1.expect(prompt) - client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') - client1.expect(prompt) - client1.send('WATCH test.lv EVENTS') - client1.expect('1.*' + end_of_block) - client2.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect('2.*' + end_of_block) - client2.send('INSERT INTO test.mt VALUES (4),(5),(6)') - client1.expect('3.*' + end_of_block) - # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) - if match.groups()[1]: - client1.send(client1.command) - client1.expect(prompt) - client1.send('DROP TABLE test.lv') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.reference b/dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.reference deleted file mode 100644 index 6fbbedf1b21..00000000000 --- a/dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.reference +++ /dev/null @@ -1,3 +0,0 @@ -0 1 -6 2 -21 3 diff --git a/dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.sql b/dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.sql deleted file mode 100644 index c3e2ab8d102..00000000000 --- a/dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.sql +++ /dev/null @@ -1,18 +0,0 @@ -DROP TABLE IF EXISTS test.lv; -DROP TABLE IF EXISTS test.mt; - -CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); -CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt; - -WATCH test.lv LIMIT 0; - -INSERT INTO test.mt VALUES (1),(2),(3); - -WATCH test.lv LIMIT 0; - -INSERT INTO test.mt VALUES (4),(5),(6); - -WATCH test.lv LIMIT 0; - -DROP TABLE test.lv; -DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00962_temporary_live_view_watch_live.py b/dbms/tests/queries/0_stateless/00962_temporary_live_view_watch_live.py deleted file mode 100755 index f27b1213c70..00000000000 --- a/dbms/tests/queries/0_stateless/00962_temporary_live_view_watch_live.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python -import os -import sys -import signal - -CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) - -from client import client, prompt, end_of_block - -log = None -# uncomment the line below for debugging -#log=sys.stdout - -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: - client1.expect(prompt) - client2.expect(prompt) - - client1.send('DROP TABLE IF EXISTS test.lv') - client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.mt') - client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') - client1.expect(prompt) - client1.send('CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') - client1.expect(prompt) - client1.send('WATCH test.lv') - client1.expect(r'0.*1' + end_of_block) - client2.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect(r'6.*2' + end_of_block) - client2.send('INSERT INTO test.mt VALUES (4),(5),(6)') - client1.expect(r'21.*3' + end_of_block) - # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) - if match.groups()[1]: - client1.send(client1.command) - client1.expect(prompt) - client1.send('DROP TABLE test.lv') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00963_temporary_live_view_watch_live_timeout.py.disabled b/dbms/tests/queries/0_stateless/00963_temporary_live_view_watch_live_timeout.py.disabled deleted file mode 100755 index df627c84e49..00000000000 --- a/dbms/tests/queries/0_stateless/00963_temporary_live_view_watch_live_timeout.py.disabled +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python -import os -import sys -import signal - -CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) - -from client import client, prompt, end_of_block - -log = None -# uncomment the line below for debugging -#log=sys.stdout - -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: - client1.expect(prompt) - client2.expect(prompt) - - client1.send('DROP TABLE IF EXISTS test.lv') - client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.mt') - client1.expect(prompt) - client1.send('SET temporary_live_view_timeout=1') - client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') - client1.expect(prompt) - client1.send('CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') - client1.expect(prompt) - client1.send('WATCH test.lv') - client1.expect(r'0.*1' + end_of_block) - client2.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client2.expect(prompt) - client1.expect(r'6.*2' + end_of_block) - client2.send('INSERT INTO test.mt VALUES (4),(5),(6)') - client2.expect(prompt) - client1.expect(r'21.*3' + end_of_block) - # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) - if match.groups()[1]: - client1.send(client1.command) - client1.expect(prompt) - client1.send('SELECT sleep(1)') - client1.expect(prompt) - client1.send('DROP TABLE test.lv') - client1.expect('Table test.lv doesn\'t exist') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.py b/dbms/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.py deleted file mode 100755 index 5664c0e6c6d..00000000000 --- a/dbms/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python -import os -import sys -import signal - -CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) - -from client import client, prompt, end_of_block - -log = None -# uncomment the line below for debugging -#log=sys.stdout - -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: - client1.expect(prompt) - client2.expect(prompt) - - client1.send('DROP TABLE IF EXISTS test.lv') - client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') - client1.expect(prompt) - client1.send('SET live_view_heartbeat_interval=1') - client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') - client1.expect(prompt) - client1.send('CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') - client1.expect(prompt) - client1.send('WATCH test.lv EVENTS') - client2.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect('2.*' + end_of_block) - client1.expect('Progress: 2.00 rows.*\)') - # wait for heartbeat - client1.expect('Progress: 2.00 rows.*\)') - # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) - if match.groups()[1]: - client1.send(client1.command) - client1.expect(prompt) - client1.send('DROP TABLE test.lv') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00965_live_view_watch_heartbeat.py b/dbms/tests/queries/0_stateless/00965_live_view_watch_heartbeat.py deleted file mode 100755 index 03e22175dff..00000000000 --- a/dbms/tests/queries/0_stateless/00965_live_view_watch_heartbeat.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python -import os -import sys -import signal - -CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) - -from client import client, prompt, end_of_block - -log = None -# uncomment the line below for debugging -#log=sys.stdout - -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: - client1.expect(prompt) - client2.expect(prompt) - - client1.send('DROP TABLE IF EXISTS test.lv') - client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') - client1.expect(prompt) - client1.send('SET live_view_heartbeat_interval=1') - client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') - client1.expect(prompt) - client1.send('CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') - client1.expect(prompt) - client1.send('WATCH test.lv') - client1.expect(r'0.*1' + end_of_block) - client2.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect(r'6.*2' + end_of_block) - client1.expect('Progress: 2.00 rows.*\)') - # wait for heartbeat - client1.expect('Progress: 2.00 rows.*\)') - # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) - if match.groups()[1]: - client1.send(client1.command) - client1.expect(prompt) - client1.send('DROP TABLE test.lv') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00966_live_view_watch_events_http.py b/dbms/tests/queries/0_stateless/00966_live_view_watch_events_http.py deleted file mode 100755 index bb9d6152200..00000000000 --- a/dbms/tests/queries/0_stateless/00966_live_view_watch_events_http.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python -import os -import sys - -CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) - -from client import client, prompt, end_of_block -from httpclient import client as http_client - -log = None -# uncomment the line below for debugging -#log=sys.stdout - -with client(name='client1>', log=log) as client1: - client1.expect(prompt) - - client1.send('DROP TABLE IF EXISTS test.lv') - client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') - client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') - client1.expect(prompt) - client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') - client1.expect(prompt) - - - with http_client({'method':'GET', 'url': '/?query=WATCH%20test.lv%20EVENTS'}, name='client2>', log=log) as client2: - client2.expect('.*1\n') - client1.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect(prompt) - client2.expect('.*2\n') - - client1.send('DROP TABLE test.lv') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00967_live_view_watch_http.py b/dbms/tests/queries/0_stateless/00967_live_view_watch_http.py deleted file mode 100755 index d3439431eb3..00000000000 --- a/dbms/tests/queries/0_stateless/00967_live_view_watch_http.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python -import os -import sys - -CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) - -from client import client, prompt, end_of_block -from httpclient import client as http_client - -log = None -# uncomment the line below for debugging -#log=sys.stdout - -with client(name='client1>', log=log) as client1: - client1.expect(prompt) - - client1.send('DROP TABLE IF EXISTS test.lv') - client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') - client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') - client1.expect(prompt) - client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') - client1.expect(prompt) - - - with http_client({'method':'GET', 'url':'/?query=WATCH%20test.lv'}, name='client2>', log=log) as client2: - client2.expect('.*0\t1\n') - client1.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect(prompt) - client2.expect('.*6\t2\n') - - client1.send('DROP TABLE test.lv') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.reference b/dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.reference deleted file mode 100644 index 5ae423d90d1..00000000000 --- a/dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.reference +++ /dev/null @@ -1,4 +0,0 @@ -{"row":{"a":1}} -{"row":{"a":2}} -{"row":{"a":3}} -{"progress":{"read_rows":"3","read_bytes":"36","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}} diff --git a/dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.sql b/dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.sql deleted file mode 100644 index 8c6f4197d54..00000000000 --- a/dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.sql +++ /dev/null @@ -1,12 +0,0 @@ -DROP TABLE IF EXISTS test.lv; -DROP TABLE IF EXISTS test.mt; - -CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); -CREATE LIVE VIEW test.lv AS SELECT * FROM test.mt; - -INSERT INTO test.mt VALUES (1),(2),(3); - -SELECT * FROM test.lv FORMAT JSONEachRowWithProgress; - -DROP TABLE test.lv; -DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.reference b/dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.reference deleted file mode 100644 index 287a1ced92d..00000000000 --- a/dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.reference +++ /dev/null @@ -1,6 +0,0 @@ -{"row":{"sum(a)":"0","_version":"1"}} -{"progress":{"read_rows":"1","read_bytes":"16","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}} -{"row":{"sum(a)":"6","_version":"2"}} -{"progress":{"read_rows":"1","read_bytes":"16","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}} -{"row":{"sum(a)":"21","_version":"3"}} -{"progress":{"read_rows":"1","read_bytes":"16","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}} diff --git a/dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.sql b/dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.sql deleted file mode 100644 index 725a4ad00ed..00000000000 --- a/dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.sql +++ /dev/null @@ -1,18 +0,0 @@ -DROP TABLE IF EXISTS test.lv; -DROP TABLE IF EXISTS test.mt; - -CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); -CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt; - -WATCH test.lv LIMIT 0 FORMAT JSONEachRowWithProgress; - -INSERT INTO test.mt VALUES (1),(2),(3); - -WATCH test.lv LIMIT 0 FORMAT JSONEachRowWithProgress; - -INSERT INTO test.mt VALUES (4),(5),(6); - -WATCH test.lv LIMIT 0 FORMAT JSONEachRowWithProgress; - -DROP TABLE test.lv; -DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.py b/dbms/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.py deleted file mode 100755 index 63628c4a76f..00000000000 --- a/dbms/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python -import os -import sys - -CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) - -from client import client, prompt, end_of_block -from httpclient import client as http_client - -log = None -# uncomment the line below for debugging -#log=sys.stdout - -with client(name='client1>', log=log) as client1: - client1.expect(prompt) - - client1.send('DROP TABLE IF EXISTS test.lv') - client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') - client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') - client1.expect(prompt) - client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') - client1.expect(prompt) - - - with http_client({'method':'GET', 'url': '/?live_view_heartbeat_interval=1&query=WATCH%20test.lv%20EVENTS%20FORMAT%20JSONEachRowWithProgress'}, name='client2>', log=log) as client2: - client2.expect('{"progress":{"read_rows":"1","read_bytes":"8","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}}\n', escape=True) - client2.expect('{"row":{"version":"1"}', escape=True) - client2.expect('{"progress":{"read_rows":"1","read_bytes":"8","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}}', escape=True) - # heartbeat is provided by progress message - client2.expect('{"progress":{"read_rows":"1","read_bytes":"8","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}}', escape=True) - - client1.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect(prompt) - - client2.expect('{"row":{"version":"2"}}\n', escape=True) - - client1.send('DROP TABLE test.lv') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.py b/dbms/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.py deleted file mode 100755 index 7bdb47b7caa..00000000000 --- a/dbms/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python -import os -import sys - -CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) - -from client import client, prompt, end_of_block -from httpclient import client as http_client - -log = None -# uncomment the line below for debugging -#log=sys.stdout - -with client(name='client1>', log=log) as client1: - client1.expect(prompt) - - client1.send('DROP TABLE IF EXISTS test.lv') - client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') - client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') - client1.expect(prompt) - client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') - client1.expect(prompt) - - with http_client({'method':'GET', 'url':'/?live_view_heartbeat_interval=1&query=WATCH%20test.lv%20FORMAT%20JSONEachRowWithProgress'}, name='client2>', log=log) as client2: - client2.expect('"progress".*',) - client2.expect('{"row":{"sum(a)":"0","_version":"1"}}\n', escape=True) - client2.expect('"progress".*\n') - # heartbeat is provided by progress message - client2.expect('"progress".*\n') - - client1.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect(prompt) - - client2.expect('"progress".*"read_rows":"2".*\n') - client2.expect('{"row":{"sum(a)":"6","_version":"2"}}\n', escape=True) - - client1.send('DROP TABLE test.lv') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00972_live_view_select_1.sql b/dbms/tests/queries/0_stateless/00972_live_view_select_1.sql deleted file mode 100644 index 661080b577b..00000000000 --- a/dbms/tests/queries/0_stateless/00972_live_view_select_1.sql +++ /dev/null @@ -1,7 +0,0 @@ -DROP TABLE IF EXISTS test.lv; - -CREATE LIVE VIEW test.lv AS SELECT 1; - -SELECT * FROM test.lv; - -DROP TABLE test.lv; diff --git a/dbms/tests/queries/0_stateless/00973_live_view_select.reference b/dbms/tests/queries/0_stateless/00973_live_view_select.reference deleted file mode 100644 index 75236c0daf7..00000000000 --- a/dbms/tests/queries/0_stateless/00973_live_view_select.reference +++ /dev/null @@ -1,4 +0,0 @@ -6 1 -6 1 -12 2 -12 2 diff --git a/dbms/tests/queries/0_stateless/00973_live_view_select.sql b/dbms/tests/queries/0_stateless/00973_live_view_select.sql deleted file mode 100644 index ff4a45ffcc1..00000000000 --- a/dbms/tests/queries/0_stateless/00973_live_view_select.sql +++ /dev/null @@ -1,18 +0,0 @@ -DROP TABLE IF EXISTS test.lv; -DROP TABLE IF EXISTS test.mt; - -CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); -CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt; - -INSERT INTO test.mt VALUES (1),(2),(3); - -SELECT *,_version FROM test.lv; -SELECT *,_version FROM test.lv; - -INSERT INTO test.mt VALUES (1),(2),(3); - -SELECT *,_version FROM test.lv; -SELECT *,_version FROM test.lv; - -DROP TABLE test.lv; -DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.reference b/dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.reference deleted file mode 100644 index 6d50f0e9c3a..00000000000 --- a/dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.reference +++ /dev/null @@ -1,2 +0,0 @@ -6 -21 diff --git a/dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.sql b/dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.sql deleted file mode 100644 index 3c11f855c9d..00000000000 --- a/dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.sql +++ /dev/null @@ -1,16 +0,0 @@ -DROP TABLE IF EXISTS test.lv; -DROP TABLE IF EXISTS test.mt; - -CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); -CREATE LIVE VIEW test.lv AS SELECT * FROM test.mt; - -INSERT INTO test.mt VALUES (1),(2),(3); - -SELECT sum(a) FROM test.lv; - -INSERT INTO test.mt VALUES (4),(5),(6); - -SELECT sum(a) FROM test.lv; - -DROP TABLE test.lv; -DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sh b/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sh index 149f0668bd1..c3cde4c08bb 100755 --- a/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sh +++ b/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sh @@ -10,7 +10,7 @@ do ${CLICKHOUSE_CLIENT} --query="SYSTEM FLUSH LOGS" sleep 0.1; -if [[ $($CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" -d "SELECT count() > 0 FROM system.text_log WHERE position(system.text_log.message, 'SELECT 6103') > 0") == 1 ]]; then echo 1; exit; fi; +if [[ $($CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" -d "SELECT count() > 0 FROM system.text_log WHERE position(system.text_log.message, 'SELECT 6103') > 0 AND event_date >= yesterday()") == 1 ]]; then echo 1; exit; fi; done; diff --git a/dbms/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh b/dbms/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh index 613226a3fb7..765dfb6abe5 100755 --- a/dbms/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh +++ b/dbms/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh @@ -2,6 +2,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh +. $CURDIR/mergetree_mutations.lib $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.indices_mutaions1;" $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.indices_mutaions2;" @@ -44,19 +45,20 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO test.indices_mutaions1 VALUES (9, 1, 2)" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2;" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2 FORMAT JSON" | grep "rows_read" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2 FORMAT JSON;" | grep "rows_read" $CLICKHOUSE_CLIENT --query="ALTER TABLE test.indices_mutaions1 CLEAR INDEX idx IN PARTITION 1;" sleep 1 $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2;" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2 FORMAT JSON" | grep "rows_read" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2 FORMAT JSON;" | grep "rows_read" $CLICKHOUSE_CLIENT --query="ALTER TABLE test.indices_mutaions1 MATERIALIZE INDEX idx IN PARTITION 1;" -sleep 1 +wait_for_mutation "indices_mutaions1" "0000000000" "test" +wait_for_mutation "indices_mutaions2" "0000000000" "test" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2;" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2 FORMAT JSON" | grep "rows_read" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2 FORMAT JSON;" | grep "rows_read" $CLICKHOUSE_CLIENT --query="DROP TABLE test.indices_mutaions1" $CLICKHOUSE_CLIENT --query="DROP TABLE test.indices_mutaions2" diff --git a/dbms/tests/queries/0_stateless/00975_live_view_create.reference b/dbms/tests/queries/0_stateless/00975_live_view_create.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/tests/queries/0_stateless/00975_live_view_create.sql b/dbms/tests/queries/0_stateless/00975_live_view_create.sql deleted file mode 100644 index 1c929b15b00..00000000000 --- a/dbms/tests/queries/0_stateless/00975_live_view_create.sql +++ /dev/null @@ -1,7 +0,0 @@ -DROP TABLE IF EXISTS test.mt; - -CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); -CREATE LIVE VIEW test.lv AS SELECT * FROM test.mt; - -DROP TABLE test.lv; -DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00976_live_view_select_version.reference b/dbms/tests/queries/0_stateless/00976_live_view_select_version.reference deleted file mode 100644 index 453bd800469..00000000000 --- a/dbms/tests/queries/0_stateless/00976_live_view_select_version.reference +++ /dev/null @@ -1,3 +0,0 @@ -1 1 -2 1 -3 1 diff --git a/dbms/tests/queries/0_stateless/00976_live_view_select_version.sql b/dbms/tests/queries/0_stateless/00976_live_view_select_version.sql deleted file mode 100644 index 5f3ab1f7546..00000000000 --- a/dbms/tests/queries/0_stateless/00976_live_view_select_version.sql +++ /dev/null @@ -1,12 +0,0 @@ -DROP TABLE IF EXISTS test.lv; -DROP TABLE IF EXISTS test.mt; - -CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); -CREATE LIVE VIEW test.lv AS SELECT * FROM test.mt; - -INSERT INTO test.mt VALUES (1),(2),(3); - -SELECT *,_version FROM test.lv; - -DROP TABLE test.lv; -DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00977_live_view_watch_events.reference b/dbms/tests/queries/0_stateless/00977_live_view_watch_events.reference deleted file mode 100644 index 01e79c32a8c..00000000000 --- a/dbms/tests/queries/0_stateless/00977_live_view_watch_events.reference +++ /dev/null @@ -1,3 +0,0 @@ -1 -2 -3 diff --git a/dbms/tests/queries/0_stateless/00977_live_view_watch_events.sql b/dbms/tests/queries/0_stateless/00977_live_view_watch_events.sql deleted file mode 100644 index a3b84e8d4c1..00000000000 --- a/dbms/tests/queries/0_stateless/00977_live_view_watch_events.sql +++ /dev/null @@ -1,18 +0,0 @@ -DROP TABLE IF EXISTS test.lv; -DROP TABLE IF EXISTS test.mt; - -CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); -CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt; - -WATCH test.lv EVENTS LIMIT 0; - -INSERT INTO test.mt VALUES (1),(2),(3); - -WATCH test.lv EVENTS LIMIT 0; - -INSERT INTO test.mt VALUES (4),(5),(6); - -WATCH test.lv EVENTS LIMIT 0; - -DROP TABLE test.lv; -DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00978_live_view_watch.reference b/dbms/tests/queries/0_stateless/00978_live_view_watch.reference deleted file mode 100644 index 6fbbedf1b21..00000000000 --- a/dbms/tests/queries/0_stateless/00978_live_view_watch.reference +++ /dev/null @@ -1,3 +0,0 @@ -0 1 -6 2 -21 3 diff --git a/dbms/tests/queries/0_stateless/00978_live_view_watch.sql b/dbms/tests/queries/0_stateless/00978_live_view_watch.sql deleted file mode 100644 index abe4a6c32ae..00000000000 --- a/dbms/tests/queries/0_stateless/00978_live_view_watch.sql +++ /dev/null @@ -1,18 +0,0 @@ -DROP TABLE IF EXISTS test.lv; -DROP TABLE IF EXISTS test.mt; - -CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); -CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt; - -WATCH test.lv LIMIT 0; - -INSERT INTO test.mt VALUES (1),(2),(3); - -WATCH test.lv LIMIT 0; - -INSERT INTO test.mt VALUES (4),(5),(6); - -WATCH test.lv LIMIT 0; - -DROP TABLE test.lv; -DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00979_live_view_watch_live.py b/dbms/tests/queries/0_stateless/00979_live_view_watch_live.py deleted file mode 100755 index 948e4c93662..00000000000 --- a/dbms/tests/queries/0_stateless/00979_live_view_watch_live.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python -import os -import sys -import signal - -CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) - -from client import client, prompt, end_of_block - -log = None -# uncomment the line below for debugging -#log=sys.stdout - -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: - client1.expect(prompt) - client2.expect(prompt) - - client1.send('DROP TABLE IF EXISTS test.lv') - client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') - client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') - client1.expect(prompt) - client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') - client1.expect(prompt) - client1.send('WATCH test.lv') - client1.expect(r'0.*1' + end_of_block) - client2.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect(r'6.*2' + end_of_block) - client2.expect(prompt) - client2.send('INSERT INTO test.mt VALUES (4),(5),(6)') - client1.expect(r'21.*3' + end_of_block) - client2.expect(prompt) - for i in range(1,129): - client2.send('INSERT INTO test.mt VALUES (1)') - client1.expect(r'%d.*%d' % (21+i, 3+i) + end_of_block) - client2.expect(prompt) - # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) - if match.groups()[1]: - client1.send(client1.command) - client1.expect(prompt) - client1.send('DROP TABLE test.lv') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00979_live_view_watch_live.reference b/dbms/tests/queries/0_stateless/00979_live_view_watch_live.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/tests/queries/0_stateless/00980_alter_settings_race.reference b/dbms/tests/queries/0_stateless/00980_alter_settings_race.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00980_alter_settings_race.reference @@ -0,0 +1 @@ +1 diff --git a/dbms/tests/queries/0_stateless/00980_alter_settings_race.sh b/dbms/tests/queries/0_stateless/00980_alter_settings_race.sh new file mode 100755 index 00000000000..3a9e854210d --- /dev/null +++ b/dbms/tests/queries/0_stateless/00980_alter_settings_race.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS table_for_concurrent_alter" + +$CLICKHOUSE_CLIENT --query="CREATE TABLE table_for_concurrent_alter (id UInt64, Data String) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity=4096;"; + +n=0 +while [ "$n" -lt 50 ]; +do + n=$(( n + 1 )) + $CLICKHOUSE_CLIENT --query="INSERT INTO table_for_concurrent_alter VALUES(1, 'Hello')" > /dev/null 2> /dev/null & + $CLICKHOUSE_CLIENT --query="OPTIMIZE TABLE table_for_concurrent_alter FINAL" > /dev/null 2> /dev/null & +done & + + +q=0 +while [ "$q" -lt 50 ]; +do + q=$(( q + 1 )) + counter=$(( 100 + q )) + $CLICKHOUSE_CLIENT --query="ALTER TABLE table_for_concurrent_alter MODIFY SETTING parts_to_throw_insert = $counter, parts_to_delay_insert = $counter, min_merge_bytes_to_use_direct_io = $counter" > /dev/null 2> /dev/null & +done & + +sleep 4 + +# we just test race conditions, not logic +$CLICKHOUSE_CLIENT --query "SELECT 1" + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS table_for_concurrent_alter" diff --git a/dbms/tests/queries/0_stateless/00980_create_temporary_live_view.reference b/dbms/tests/queries/0_stateless/00980_create_temporary_live_view.reference deleted file mode 100644 index 7f9fcbb2e9c..00000000000 --- a/dbms/tests/queries/0_stateless/00980_create_temporary_live_view.reference +++ /dev/null @@ -1,3 +0,0 @@ -temporary_live_view_timeout 5 -live_view_heartbeat_interval 15 -0 diff --git a/dbms/tests/queries/0_stateless/00980_create_temporary_live_view.sql b/dbms/tests/queries/0_stateless/00980_create_temporary_live_view.sql deleted file mode 100644 index 8cd6ee06ace..00000000000 --- a/dbms/tests/queries/0_stateless/00980_create_temporary_live_view.sql +++ /dev/null @@ -1,15 +0,0 @@ -DROP TABLE IF EXISTS test.lv; -DROP TABLE IF EXISTS test.mt; - -SELECT name, value from system.settings WHERE name = 'temporary_live_view_timeout'; -SELECT name, value from system.settings WHERE name = 'live_view_heartbeat_interval'; - -SET temporary_live_view_timeout=1; -CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); -CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt; - -SHOW TABLES LIKE 'lv'; -SELECT sleep(2); -SHOW TABLES LIKE 'lv'; - -DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00980_merge_alter_settings.reference b/dbms/tests/queries/0_stateless/00980_merge_alter_settings.reference new file mode 100644 index 00000000000..ee3818d25dc --- /dev/null +++ b/dbms/tests/queries/0_stateless/00980_merge_alter_settings.reference @@ -0,0 +1,6 @@ +CREATE TABLE default.table_for_alter (`id` UInt64, `Data` String) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity = 4096 +CREATE TABLE default.table_for_alter (`id` UInt64, `Data` String) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity = 4096, parts_to_throw_insert = 1, parts_to_delay_insert = 1 +CREATE TABLE default.table_for_alter (`id` UInt64, `Data` String) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity = 4096, parts_to_throw_insert = 100, parts_to_delay_insert = 100 +2 +CREATE TABLE default.table_for_alter (`id` UInt64, `Data` String) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity = 4096, parts_to_throw_insert = 100, parts_to_delay_insert = 100, check_delay_period = 30 +CREATE TABLE default.table_for_alter (`id` UInt64, `Data` String, `Data2` UInt64) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity = 4096, parts_to_throw_insert = 100, parts_to_delay_insert = 100, check_delay_period = 15 diff --git a/dbms/tests/queries/0_stateless/00980_merge_alter_settings.sql b/dbms/tests/queries/0_stateless/00980_merge_alter_settings.sql new file mode 100644 index 00000000000..ed42a79ebbf --- /dev/null +++ b/dbms/tests/queries/0_stateless/00980_merge_alter_settings.sql @@ -0,0 +1,55 @@ +DROP TABLE IF EXISTS log_for_alter; + +CREATE TABLE log_for_alter ( + id UInt64, + Data String +) ENGINE = Log(); + +ALTER TABLE log_for_alter MODIFY SETTING aaa=123; -- { serverError 471 } + +DROP TABLE IF EXISTS log_for_alter; + +DROP TABLE IF EXISTS table_for_alter; + +CREATE TABLE table_for_alter ( + id UInt64, + Data String +) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity=4096; + +ALTER TABLE table_for_alter MODIFY SETTING index_granularity=555; -- { serverError 472 } + +SHOW CREATE TABLE table_for_alter; + +ALTER TABLE table_for_alter MODIFY SETTING parts_to_throw_insert = 1, parts_to_delay_insert = 1; + +SHOW CREATE TABLE table_for_alter; + +INSERT INTO table_for_alter VALUES (1, '1'); +INSERT INTO table_for_alter VALUES (2, '2'); -- { serverError 252 } + +DETACH TABLE table_for_alter; + +ATTACH TABLE table_for_alter; + +INSERT INTO table_for_alter VALUES (2, '2'); -- { serverError 252 } + +ALTER TABLE table_for_alter MODIFY SETTING xxx_yyy=124; -- { serverError 115 } + +ALTER TABLE table_for_alter MODIFY SETTING parts_to_throw_insert = 100, parts_to_delay_insert = 100; + +INSERT INTO table_for_alter VALUES (2, '2'); + +SHOW CREATE TABLE table_for_alter; + +SELECT COUNT() FROM table_for_alter; + +ALTER TABLE table_for_alter MODIFY SETTING check_delay_period=10, check_delay_period=20, check_delay_period=30; + +SHOW CREATE TABLE table_for_alter; + +ALTER TABLE table_for_alter ADD COLUMN Data2 UInt64, MODIFY SETTING check_delay_period=5, check_delay_period=10, check_delay_period=15; + +SHOW CREATE TABLE table_for_alter; + +DROP TABLE IF EXISTS table_for_alter; + diff --git a/dbms/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.reference b/dbms/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.reference new file mode 100644 index 00000000000..159102e1ca7 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.reference @@ -0,0 +1,12 @@ +CREATE TABLE default.replicated_table_for_alter1 (`id` UInt64, `Data` String) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'1\') ORDER BY id SETTINGS index_granularity = 8192 +CREATE TABLE default.replicated_table_for_alter1 (`id` UInt64, `Data` String) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'1\') ORDER BY id SETTINGS index_granularity = 8192 +4 +4 +4 +4 +6 +6 +CREATE TABLE default.replicated_table_for_alter1 (`id` UInt64, `Data` String) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'1\') ORDER BY id SETTINGS index_granularity = 8192, use_minimalistic_part_header_in_zookeeper = 1 +CREATE TABLE default.replicated_table_for_alter2 (`id` UInt64, `Data` String) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'2\') ORDER BY id SETTINGS index_granularity = 8192, parts_to_throw_insert = 1, parts_to_delay_insert = 1 +CREATE TABLE default.replicated_table_for_alter1 (`id` UInt64, `Data` String, `Data2` UInt64) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'1\') ORDER BY id SETTINGS index_granularity = 8192, use_minimalistic_part_header_in_zookeeper = 1, check_delay_period = 15 +CREATE TABLE default.replicated_table_for_alter2 (`id` UInt64, `Data` String, `Data2` UInt64) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'2\') ORDER BY id SETTINGS index_granularity = 8192, parts_to_throw_insert = 1, parts_to_delay_insert = 1 diff --git a/dbms/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.sql b/dbms/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.sql new file mode 100644 index 00000000000..f2e453c99d2 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.sql @@ -0,0 +1,67 @@ +DROP TABLE IF EXISTS replicated_table_for_alter1; +DROP TABLE IF EXISTS replicated_table_for_alter2; + +CREATE TABLE replicated_table_for_alter1 ( + id UInt64, + Data String +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/replicated_table_for_alter', '1') ORDER BY id; + +CREATE TABLE replicated_table_for_alter2 ( + id UInt64, + Data String +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/replicated_table_for_alter', '2') ORDER BY id; + +SHOW CREATE TABLE replicated_table_for_alter1; + +ALTER TABLE replicated_table_for_alter1 MODIFY SETTING index_granularity = 4096; -- { serverError 472 } + +SHOW CREATE TABLE replicated_table_for_alter1; + +INSERT INTO replicated_table_for_alter2 VALUES (1, '1'), (2, '2'); + +SYSTEM SYNC REPLICA replicated_table_for_alter1; + +ALTER TABLE replicated_table_for_alter1 MODIFY SETTING use_minimalistic_part_header_in_zookeeper = 1; + +INSERT INTO replicated_table_for_alter1 VALUES (3, '3'), (4, '4'); + +SYSTEM SYNC REPLICA replicated_table_for_alter2; + +SELECT COUNT() FROM replicated_table_for_alter1; +SELECT COUNT() FROM replicated_table_for_alter2; + +DETACH TABLE replicated_table_for_alter2; +ATTACH TABLE replicated_table_for_alter2; + +DETACH TABLE replicated_table_for_alter1; +ATTACH TABLE replicated_table_for_alter1; + +SELECT COUNT() FROM replicated_table_for_alter1; +SELECT COUNT() FROM replicated_table_for_alter2; + +ALTER TABLE replicated_table_for_alter2 MODIFY SETTING parts_to_throw_insert = 1, parts_to_delay_insert = 1; +INSERT INTO replicated_table_for_alter2 VALUES (3, '1'), (4, '2'); -- { serverError 252 } + +INSERT INTO replicated_table_for_alter1 VALUES (5, '5'), (6, '6'); + +SYSTEM SYNC REPLICA replicated_table_for_alter2; + +SELECT COUNT() FROM replicated_table_for_alter1; +SELECT COUNT() FROM replicated_table_for_alter2; + +DETACH TABLE replicated_table_for_alter2; +ATTACH TABLE replicated_table_for_alter2; + +DETACH TABLE replicated_table_for_alter1; +ATTACH TABLE replicated_table_for_alter1; + +SHOW CREATE TABLE replicated_table_for_alter1; +SHOW CREATE TABLE replicated_table_for_alter2; + +ALTER TABLE replicated_table_for_alter1 ADD COLUMN Data2 UInt64, MODIFY SETTING check_delay_period=5, check_delay_period=10, check_delay_period=15; + +SHOW CREATE TABLE replicated_table_for_alter1; +SHOW CREATE TABLE replicated_table_for_alter2; + +DROP TABLE IF EXISTS replicated_table_for_alter2; +DROP TABLE IF EXISTS replicated_table_for_alter1; diff --git a/dbms/tests/queries/0_stateless/00984_parser_stack_overflow.sh b/dbms/tests/queries/0_stateless/00984_parser_stack_overflow.sh index 64fae3fb0f9..14f2a8e31fb 100755 --- a/dbms/tests/queries/0_stateless/00984_parser_stack_overflow.sh +++ b/dbms/tests/queries/0_stateless/00984_parser_stack_overflow.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +CLICKHOUSE_CURL_TIMEOUT=30 + CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh diff --git a/dbms/tests/queries/0_stateless/00963_temporary_live_view_watch_live_timeout.reference b/dbms/tests/queries/0_stateless/00988_constraints_replication_zookeeper.reference similarity index 100% rename from dbms/tests/queries/0_stateless/00963_temporary_live_view_watch_live_timeout.reference rename to dbms/tests/queries/0_stateless/00988_constraints_replication_zookeeper.reference diff --git a/dbms/tests/queries/0_stateless/00988_constraints_replication_zookeeper.sql b/dbms/tests/queries/0_stateless/00988_constraints_replication_zookeeper.sql new file mode 100644 index 00000000000..bc0e8a30670 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00988_constraints_replication_zookeeper.sql @@ -0,0 +1,43 @@ +DROP TABLE IF EXISTS replicated_constraints1; +DROP TABLE IF EXISTS replicated_constraints2; + +CREATE TABLE replicated_constraints1 +( + a UInt32, + b UInt32, + CONSTRAINT a_constraint CHECK a < 10 +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/alter_constraints', 'r1') ORDER BY (a); + +CREATE TABLE replicated_constraints2 +( + a UInt32, + b UInt32, + CONSTRAINT a_constraint CHECK a < 10 +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/alter_constraints', 'r2') ORDER BY (a); + +INSERT INTO replicated_constraints1 VALUES (1, 2); +INSERT INTO replicated_constraints2 VALUES (3, 4); + +SYSTEM SYNC REPLICA replicated_constraints1; +SYSTEM SYNC REPLICA replicated_constraints2; + +INSERT INTO replicated_constraints1 VALUES (10, 10); -- { serverError 469 } + +ALTER TABLE replicated_constraints1 DROP CONSTRAINT a_constraint; + +SYSTEM SYNC REPLICA replicated_constraints2; + +INSERT INTO replicated_constraints1 VALUES (10, 10); +INSERT INTO replicated_constraints2 VALUES (10, 10); + +ALTER TABLE replicated_constraints1 ADD CONSTRAINT b_constraint CHECK b > 10; +ALTER TABLE replicated_constraints2 ADD CONSTRAINT a_constraint CHECK a < 10; + +SYSTEM SYNC REPLICA replicated_constraints1; +SYSTEM SYNC REPLICA replicated_constraints2; + +INSERT INTO replicated_constraints1 VALUES (10, 11); -- { serverError 469 } +INSERT INTO replicated_constraints2 VALUES (9, 10); -- { serverError 469 } + +DROP TABLE replicated_constraints1; +DROP TABLE replicated_constraints2; \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/00990_hasToken.python b/dbms/tests/queries/0_stateless/00990_hasToken.python new file mode 100755 index 00000000000..cd2a284655f --- /dev/null +++ b/dbms/tests/queries/0_stateless/00990_hasToken.python @@ -0,0 +1,160 @@ +#!/usr/bin/env python +# encoding: utf-8 + +import re +from string import Template + +HAYSTACKS = [ + "hay hay hay hay hay hay hay hay hay needle hay hay hay hay hay hay hay hay hay", + "hay hay hay hay hay hay hay hay hay hay hay hay hay hay hay hay hay hay needle", + "needle hay hay hay hay hay hay hay hay hay hay hay hay hay hay hay hay hay hay", +] + +NEEDLE = "needle" + +HAY_RE = re.compile(r'\bhay\b', re.IGNORECASE) +NEEDLE_RE = re.compile(r'\bneedle\b', re.IGNORECASE) + +def replace_follow_case(replacement): + def func(match): + g = match.group() + if g.islower(): return replacement.lower() + if g.istitle(): return replacement.title() + if g.isupper(): return replacement.upper() + return replacement + return func + +def replace_separators(query, new_sep): + SEP_RE = re.compile('\\s+') + result = SEP_RE.sub(new_sep, query) + return result + +def enlarge_haystack(query, times, separator=''): + return HAY_RE.sub(replace_follow_case(('hay' + separator) * times), query) + +def small_needle(query): + return NEEDLE_RE.sub(replace_follow_case('n'), query) + +def remove_needle(query): + return NEEDLE_RE.sub('', query) + +def replace_needle(query, new_needle): + return NEEDLE_RE.sub(new_needle, query) + +# with str.lower, str.uppert, str.title and such +def transform_needle(query, string_transformation_func): + def replace_with_transformation(match): + g = match.group() + return string_transformation_func(g) + + return NEEDLE_RE.sub(replace_with_transformation, query) + +def create_cases(case_sensitive_func, case_insensitive_func, table_row_template, table_query_template, const_query_template): + const_queries = [] + table_rows = [] + table_queries = set() + + def add_case(func, haystack, needle, match): + match = int(match) + args = dict( + func = func, + haystack = haystack, + needle = needle, + match = match + ) + const_queries.append(const_query_template.substitute(args)) + table_queries.add(table_query_template.substitute(args)) + table_rows.append(table_row_template.substitute(args)) + + def add_case_sensitive(haystack, needle, match): + add_case(case_sensitive_func, haystack, needle, match) + if match: + add_case(case_sensitive_func, transform_needle(haystack, str.swapcase), transform_needle(needle, str.swapcase), match) + + def add_case_insensitive(haystack, needle, match): + add_case(case_insensitive_func, haystack, needle, match) + if match: + add_case(case_insensitive_func, transform_needle(haystack, str.swapcase), needle, match) + add_case(case_insensitive_func, haystack, transform_needle(needle, str.swapcase), match) + + + # Negative cases + add_case_sensitive(remove_needle(HAYSTACKS[0]), NEEDLE, False) + add_case_insensitive(remove_needle(HAYSTACKS[0]), NEEDLE, False) + + for haystack in HAYSTACKS: + add_case_sensitive(transform_needle(haystack, str.swapcase), NEEDLE, False) + + sep = '' + h = replace_separators(haystack, sep) + + add_case_sensitive(h, NEEDLE, False) + add_case_insensitive(h, NEEDLE, False) + + add_case_sensitive(small_needle(h), small_needle(NEEDLE), False) + add_case_insensitive(small_needle(h), small_needle(NEEDLE), False) + + add_case_sensitive(enlarge_haystack(h, 10, sep), NEEDLE, False) + add_case_insensitive(enlarge_haystack(h, 10, sep), NEEDLE, False) + + # positive cases + for haystack in HAYSTACKS: + add_case_sensitive(haystack, NEEDLE, True) + add_case_insensitive(haystack, NEEDLE, True) + + + for sep in list(''' ,'''): + h = replace_separators(haystack, sep) + add_case_sensitive(h, NEEDLE, True) + add_case_sensitive(small_needle(h), small_needle(NEEDLE), True) + add_case_sensitive(enlarge_haystack(h, 200, sep), NEEDLE, True) + + add_case_insensitive(h, NEEDLE, True) + add_case_insensitive(small_needle(h), small_needle(NEEDLE), True) + add_case_insensitive(enlarge_haystack(h, 200, sep), NEEDLE, True) + + # case insesitivity works only on ASCII strings + add_case_sensitive(replace_needle(h, 'иголка'), replace_needle(NEEDLE, 'иголка'), True) + add_case_sensitive(replace_needle(h, '指针'), replace_needle(NEEDLE, '指针'), True) + + for sep in list('''~!@$%^&*()-=+|]}[{";:/?.><\t''') + [r'\\\\']: + h = replace_separators(HAYSTACKS[0], sep) + add_case(case_sensitive_func, h, NEEDLE, True) + + return table_rows, table_queries, const_queries + +def main(): + + def query(x): + print x + + CONST_QUERY = Template("""SELECT ${func}('${haystack}', '${needle}'), ' expecting ', ${match};""") + TABLE_QUERY = Template("""WITH '${needle}' as n + SELECT haystack, needle, ${func}(haystack, n) as result + FROM ht + WHERE func = '${func}' AND needle = n AND result != match;""") + TABLE_ROW = Template("""('${haystack}', '${needle}', ${match}, '${func}')""") + + rows, table_queries, const_queries = create_cases('hasToken', 'hasTokenCaseInsensitive', TABLE_ROW, TABLE_QUERY, CONST_QUERY) + for q in const_queries: + query(q) + + query("""DROP TABLE IF EXISTS ht; + CREATE TABLE IF NOT EXISTS + ht +( + haystack String, + needle String, + match UInt8, + func String +) +ENGINE MergeTree() +ORDER BY haystack; +INSERT INTO ht VALUES {values};""".format(values=", ".join(rows))) + for q in sorted(table_queries): + query(q) + + query("""DROP TABLE ht""") + +if __name__ == '__main__': + main() diff --git a/dbms/tests/queries/0_stateless/00990_hasToken.reference b/dbms/tests/queries/0_stateless/00990_hasToken.reference new file mode 100644 index 00000000000..1e8c067362c --- /dev/null +++ b/dbms/tests/queries/0_stateless/00990_hasToken.reference @@ -0,0 +1,180 @@ +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 diff --git a/dbms/tests/queries/0_stateless/00990_hasToken.sh b/dbms/tests/queries/0_stateless/00990_hasToken.sh new file mode 100755 index 00000000000..4ccb77b8ecc --- /dev/null +++ b/dbms/tests/queries/0_stateless/00990_hasToken.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +# We should have correct env vars from shell_config.sh to run this test + +python $CURDIR/00990_hasToken.python | ${CLICKHOUSE_CLIENT} -nm diff --git a/dbms/tests/queries/0_stateless/00990_hasToken_and_tokenbf.reference b/dbms/tests/queries/0_stateless/00990_hasToken_and_tokenbf.reference new file mode 100644 index 00000000000..10e8f0d2c59 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00990_hasToken_and_tokenbf.reference @@ -0,0 +1,3 @@ +2007 +2007 +2007 diff --git a/dbms/tests/queries/0_stateless/00990_hasToken_and_tokenbf.sql b/dbms/tests/queries/0_stateless/00990_hasToken_and_tokenbf.sql new file mode 100644 index 00000000000..60e4d959417 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00990_hasToken_and_tokenbf.sql @@ -0,0 +1,33 @@ +SET allow_experimental_data_skipping_indices = 1; + +DROP TABLE IF EXISTS bloom_filter; + +CREATE TABLE bloom_filter +( + id UInt64, + s String, + INDEX tok_bf (s, lower(s)) TYPE tokenbf_v1(512, 3, 0) GRANULARITY 1 +) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity = 8; + +insert into bloom_filter select number, 'yyy,uuu' from numbers(1024); +insert into bloom_filter select number+2000, 'abc,def,zzz' from numbers(8); +insert into bloom_filter select number+3000, 'yyy,uuu' from numbers(1024); +insert into bloom_filter select number+3000, 'abcdefzzz' from numbers(1024); + +set max_rows_to_read = 16; + +SELECT max(id) FROM bloom_filter WHERE hasToken(s, 'abc'); +SELECT max(id) FROM bloom_filter WHERE hasToken(s, 'def'); +SELECT max(id) FROM bloom_filter WHERE hasToken(s, 'zzz'); + +-- invert result +-- this does not work as expected, reading more rows that it should +-- SELECT max(id) FROM bloom_filter WHERE NOT hasToken(s, 'yyy'); + +-- accessing to many rows +SELECT max(id) FROM bloom_filter WHERE hasToken(s, 'yyy'); -- { serverError 158 } + +-- this syntax is not supported by tokenbf +SELECT max(id) FROM bloom_filter WHERE hasToken(s, 'zzz') == 1; -- { serverError 158 } + +DROP TABLE bloom_filter; \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.python b/dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.python deleted file mode 100644 index 782671cdfaf..00000000000 --- a/dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.python +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python - -import subprocess -import threading -import Queue as queue -import os -import sys -import signal - - -CLICKHOUSE_CLIENT = os.environ.get('CLICKHOUSE_CLIENT') -CLICKHOUSE_CURL = os.environ.get('CLICKHOUSE_CURL') -CLICKHOUSE_URL = os.environ.get('CLICKHOUSE_URL') - - -def send_query(query): - cmd = list(CLICKHOUSE_CLIENT.split()) - cmd += ['--query', query] - # print(cmd) - return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout - - -def send_query_in_process_group(query): - cmd = list(CLICKHOUSE_CLIENT.split()) - cmd += ['--query', query] - # print(cmd) - return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, preexec_fn=os.setsid) - - -def read_lines_and_push_to_queue(pipe, queue): - try: - for line in iter(pipe.readline, ''): - line = line.strip() - print(line) - sys.stdout.flush() - queue.put(line) - except KeyboardInterrupt: - pass - - queue.put(None) - - -def test(): - send_query('DROP TABLE IF EXISTS test.lv').read() - send_query('DROP TABLE IF EXISTS test.mt').read() - send_query('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()').read() - send_query('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt').read() - - q = queue.Queue() - p = send_query_in_process_group('WATCH test.lv') - thread = threading.Thread(target=read_lines_and_push_to_queue, args=(p.stdout, q)) - thread.start() - - line = q.get() - print(line) - assert (line == '0\t1') - - send_query('INSERT INTO test.mt VALUES (1),(2),(3)').read() - line = q.get() - print(line) - assert (line == '6\t2') - - send_query('INSERT INTO test.mt VALUES (4),(5),(6)').read() - line = q.get() - print(line) - assert (line == '21\t3') - - # Send Ctrl+C to client. - os.killpg(os.getpgid(p.pid), signal.SIGINT) - # This insert shouldn't affect lv. - send_query('INSERT INTO test.mt VALUES (7),(8),(9)').read() - line = q.get() - print(line) - assert (line is None) - - send_query('DROP TABLE if exists test.lv').read() - send_query('DROP TABLE if exists test.lv').read() - - thread.join() - -test() diff --git a/dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.reference b/dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.reference deleted file mode 100644 index 1e94cdade41..00000000000 --- a/dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.reference +++ /dev/null @@ -1,7 +0,0 @@ -0 1 -0 1 -6 2 -6 2 -21 3 -21 3 -None diff --git a/dbms/tests/queries/0_stateless/00991_live_view_watch_http.python b/dbms/tests/queries/0_stateless/00991_live_view_watch_http.python deleted file mode 100755 index 938547ca0cb..00000000000 --- a/dbms/tests/queries/0_stateless/00991_live_view_watch_http.python +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python - -import subprocess -import threading -import Queue as queue -import os -import sys - - -CLICKHOUSE_CLIENT = os.environ.get('CLICKHOUSE_CLIENT') -CLICKHOUSE_CURL = os.environ.get('CLICKHOUSE_CURL') -CLICKHOUSE_URL = os.environ.get('CLICKHOUSE_URL') - - -def send_query(query): - cmd = list(CLICKHOUSE_CLIENT.split()) - cmd += ['--query', query] - # print(cmd) - return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout - - -def send_http_query(query): - cmd = list(CLICKHOUSE_CURL.split()) # list(['curl', '-sSN', '--max-time', '10']) - cmd += ['-sSN', CLICKHOUSE_URL, '-d', query] - return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout - - -def read_lines_and_push_to_queue(pipe, queue): - for line in iter(pipe.readline, ''): - line = line.strip() - print(line) - sys.stdout.flush() - queue.put(line) - - queue.put(None) - - -def test(): - send_query('DROP TABLE IF EXISTS test.lv').read() - send_query('DROP TABLE IF EXISTS test.mt').read() - send_query('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()').read() - send_query('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt').read() - - q = queue.Queue() - pipe = send_http_query('WATCH test.lv') - thread = threading.Thread(target=read_lines_and_push_to_queue, args=(pipe, q)) - thread.start() - - line = q.get() - print(line) - assert (line == '0\t1') - - send_query('INSERT INTO test.mt VALUES (1),(2),(3)').read() - line = q.get() - print(line) - assert (line == '6\t2') - - send_query('DROP TABLE if exists test.lv').read() - send_query('DROP TABLE if exists test.lv').read() - - thread.join() - -test() diff --git a/dbms/tests/queries/0_stateless/00991_live_view_watch_http.reference b/dbms/tests/queries/0_stateless/00991_live_view_watch_http.reference deleted file mode 100644 index 489457d751b..00000000000 --- a/dbms/tests/queries/0_stateless/00991_live_view_watch_http.reference +++ /dev/null @@ -1,4 +0,0 @@ -0 1 -0 1 -6 2 -6 2 diff --git a/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.python b/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.python deleted file mode 100644 index 70063adc6e3..00000000000 --- a/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.python +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python - -import subprocess -import threading -import Queue as queue -import os -import sys -import signal - - -CLICKHOUSE_CLIENT = os.environ.get('CLICKHOUSE_CLIENT') -CLICKHOUSE_CURL = os.environ.get('CLICKHOUSE_CURL') -CLICKHOUSE_URL = os.environ.get('CLICKHOUSE_URL') - - -def send_query(query): - cmd = list(CLICKHOUSE_CLIENT.split()) - cmd += ['--query', query] - # print(cmd) - return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout - - -def send_query_in_process_group(query): - cmd = list(CLICKHOUSE_CLIENT.split()) - cmd += ['--query', query, '--live_view_heartbeat_interval=1', '--progress'] - # print(cmd) - return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, preexec_fn=os.setsid) - - -def read_lines_and_push_to_queue(pipe, queue): - try: - for line in iter(pipe.readline, ''): - line = line.strip() - # print(line) - sys.stdout.flush() - queue.put(line) - except KeyboardInterrupt: - pass - - queue.put(None) - - -def test(): - send_query('DROP TABLE IF EXISTS test.lv').read() - send_query('DROP TABLE IF EXISTS test.mt').read() - send_query('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()').read() - send_query('CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt').read() - - q = queue.Queue() - p = send_query_in_process_group('WATCH test.lv') - thread = threading.Thread(target=read_lines_and_push_to_queue, args=(p.stdout, q)) - thread.start() - - line = q.get() - # print(line) - assert (line.endswith('0\t1')) - assert ('Progress: 0.00 rows' in line) - - send_query('INSERT INTO test.mt VALUES (1),(2),(3)').read() - line = q.get() - assert (line.endswith('6\t2')) - assert ('Progress: 1.00 rows' in line) - - # send_query('INSERT INTO test.mt VALUES (4),(5),(6)').read() - # line = q.get() - # print(line) - # assert (line.endswith('6\t2')) - # assert ('Progress: 1.00 rows' in line) - - # Send Ctrl+C to client. - os.killpg(os.getpgid(p.pid), signal.SIGINT) - # This insert shouldn't affect lv. - send_query('INSERT INTO test.mt VALUES (7),(8),(9)').read() - line = q.get() - # print(line) - # assert (line is None) - - send_query('DROP TABLE if exists test.lv').read() - send_query('DROP TABLE if exists test.lv').read() - - thread.join() - -test() diff --git a/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.reference b/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.python b/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.python deleted file mode 100644 index d290018a02c..00000000000 --- a/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.python +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python - -import subprocess -import threading -import Queue as queue -import os -import sys -import signal - - -CLICKHOUSE_CLIENT = os.environ.get('CLICKHOUSE_CLIENT') -CLICKHOUSE_CURL = os.environ.get('CLICKHOUSE_CURL') -CLICKHOUSE_URL = os.environ.get('CLICKHOUSE_URL') - - -def send_query(query): - cmd = list(CLICKHOUSE_CLIENT.split()) - cmd += ['--query', query] - # print(cmd) - return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout - - -def send_query_in_process_group(query): - cmd = list(CLICKHOUSE_CLIENT.split()) - cmd += ['--query', query] - # print(cmd) - return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, preexec_fn=os.setsid) - - -def read_lines_and_push_to_queue(pipe, queue): - try: - for line in iter(pipe.readline, ''): - line = line.strip() - print(line) - sys.stdout.flush() - queue.put(line) - except KeyboardInterrupt: - pass - - queue.put(None) - - -def test(): - send_query('DROP TABLE IF EXISTS test.lv').read() - send_query('DROP TABLE IF EXISTS test.mt').read() - send_query('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()').read() - send_query('CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt').read() - - q = queue.Queue() - p = send_query_in_process_group('WATCH test.lv') - thread = threading.Thread(target=read_lines_and_push_to_queue, args=(p.stdout, q)) - thread.start() - - line = q.get() - print(line) - assert (line == '0\t1') - - send_query('INSERT INTO test.mt VALUES (1),(2),(3)').read() - line = q.get() - print(line) - assert (line == '6\t2') - - send_query('INSERT INTO test.mt VALUES (4),(5),(6)').read() - line = q.get() - print(line) - assert (line == '21\t3') - - # Send Ctrl+C to client. - os.killpg(os.getpgid(p.pid), signal.SIGINT) - # This insert shouldn't affect lv. - send_query('INSERT INTO test.mt VALUES (7),(8),(9)').read() - line = q.get() - print(line) - assert (line is None) - - send_query('DROP TABLE if exists test.lv').read() - send_query('DROP TABLE if exists test.lv').read() - - thread.join() - -test() diff --git a/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.reference b/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.reference deleted file mode 100644 index 1e94cdade41..00000000000 --- a/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.reference +++ /dev/null @@ -1,7 +0,0 @@ -0 1 -0 1 -6 2 -6 2 -21 3 -21 3 -None diff --git a/dbms/tests/queries/0_stateless/00995_optimize_read_in_order_with_aggregation.reference b/dbms/tests/queries/0_stateless/00995_optimize_read_in_order_with_aggregation.reference new file mode 100644 index 00000000000..d567f8a0b01 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00995_optimize_read_in_order_with_aggregation.reference @@ -0,0 +1 @@ +4950 diff --git a/dbms/tests/queries/0_stateless/00995_optimize_read_in_order_with_aggregation.sql b/dbms/tests/queries/0_stateless/00995_optimize_read_in_order_with_aggregation.sql new file mode 100644 index 00000000000..93c907811a5 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00995_optimize_read_in_order_with_aggregation.sql @@ -0,0 +1,6 @@ +SET optimize_read_in_order = 1; +DROP TABLE IF EXISTS order_with_aggr; +CREATE TABLE order_with_aggr(a Int) ENGINE = MergeTree ORDER BY a; + +INSERT INTO order_with_aggr SELECT * FROM numbers(100); +SELECT sum(a) as s FROM order_with_aggr ORDER BY s; diff --git a/dbms/tests/queries/0_stateless/00996_neighbor.reference b/dbms/tests/queries/0_stateless/00996_neighbor.reference new file mode 100644 index 00000000000..ebdb8c9e684 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00996_neighbor.reference @@ -0,0 +1,270 @@ +0 0 +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +0 5 +1 6 +2 7 +3 8 +4 9 +5 +6 +7 +8 +9 +0 +1 +2 +3 +4 +5 0 +6 1 +7 2 +8 3 +9 4 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 5 +1 6 +2 7 +3 8 +4 9 +5 Hello +6 Hello +7 Hello +8 Hello +9 Hello +0 World +1 World +2 World +3 World +4 World +5 0 +6 1 +7 2 +8 3 +9 4 +0 5 +1 6 +2 7 +3 8 +4 9 +5 Hello 5 +6 Hello 6 +7 Hello 7 +8 Hello 8 +9 Hello 9 +0 World 0 +1 World 1 +2 World 2 +3 World 3 +4 World 4 +5 0 +6 1 +7 2 +8 3 +9 4 +0 ClickHouse +1 ClickHouse +2 ClickHouse +3 ClickHouse +4 ClickHouse +5 ClickHouse +6 ClickHouse +7 ClickHouse +8 ClickHouse +9 ClickHouse +0 ClickHouse +1 ClickHouse +2 ClickHouse +3 ClickHouse +4 ClickHouse +5 +6 +7 +8 +9 +0 +1 +2 +3 +4 +5 ClickHouse +6 ClickHouse +7 ClickHouse +8 ClickHouse +9 ClickHouse +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 ClickHouse +1 ClickHouse +2 ClickHouse +3 ClickHouse +4 ClickHouse +5 Hello +6 Hello +7 Hello +8 Hello +9 Hello +0 World +1 World +2 World +3 World +4 World +5 ClickHouse +6 ClickHouse +7 ClickHouse +8 ClickHouse +9 ClickHouse +0 ClickHouse +1 ClickHouse +2 ClickHouse +3 ClickHouse +4 ClickHouse +5 Hello 5 +6 Hello 6 +7 Hello 7 +8 Hello 8 +9 Hello 9 +0 World 0 +1 World 1 +2 World 2 +3 World 3 +4 World 4 +5 ClickHouse +6 ClickHouse +7 ClickHouse +8 ClickHouse +9 ClickHouse +0 0 +1 2 +2 4 +3 6 +4 8 +5 +6 +7 +8 +9 +0 0 +1 1 +2 3 +3 4 +4 6 +5 7 +6 9 +7 +8 +9 +0 Hello +1 Hello +2 Hello +3 Hello +4 Hello +5 +6 +7 +8 +9 +0 +1 +2 +3 Hello +4 Hello +5 Hello +6 Hello +7 Hello +8 Hello +9 Hello +0 World +1 World +2 World +3 Hello +4 Hello +5 Hello +6 Hello +7 Hello +8 Hello +9 Hello diff --git a/dbms/tests/queries/0_stateless/00996_neighbor.sql b/dbms/tests/queries/0_stateless/00996_neighbor.sql new file mode 100644 index 00000000000..25c20b1b896 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00996_neighbor.sql @@ -0,0 +1,42 @@ +SELECT number, neighbor(toString(number), 0) FROM numbers(10); + +SELECT number, neighbor(toString(number), 5) FROM numbers(10); +SELECT number, neighbor(toString(number), -5) FROM numbers(10); + +SELECT number, neighbor(toString(number), 10) FROM numbers(10); +SELECT number, neighbor(toString(number), -10) FROM numbers(10); + +SELECT number, neighbor(toString(number), 15) FROM numbers(10); +SELECT number, neighbor(toString(number), -15) FROM numbers(10); + +SELECT number, neighbor(toString(number), 5, 'Hello') FROM numbers(10); +SELECT number, neighbor(toString(number), -5, 'World') FROM numbers(10); + +SELECT number, neighbor(toString(number), 5, concat('Hello ', toString(number))) FROM numbers(10); +SELECT number, neighbor(toString(number), -5, concat('World ', toString(number))) FROM numbers(10); + + +SELECT number, neighbor('ClickHouse', 0) FROM numbers(10); + +SELECT number, neighbor('ClickHouse', 5) FROM numbers(10); +SELECT number, neighbor('ClickHouse', -5) FROM numbers(10); + +SELECT number, neighbor('ClickHouse', 10) FROM numbers(10); +SELECT number, neighbor('ClickHouse', -10) FROM numbers(10); + +SELECT number, neighbor('ClickHouse', 15) FROM numbers(10); +SELECT number, neighbor('ClickHouse', -15) FROM numbers(10); + +SELECT number, neighbor('ClickHouse', 5, 'Hello') FROM numbers(10); +SELECT number, neighbor('ClickHouse', -5, 'World') FROM numbers(10); + +SELECT number, neighbor('ClickHouse', 5, concat('Hello ', toString(number))) FROM numbers(10); +SELECT number, neighbor('ClickHouse', -5, concat('World ', toString(number))) FROM numbers(10); + + +SELECT number, neighbor(toString(number), number) FROM numbers(10); +SELECT number, neighbor(toString(number), intDiv(number, 2)) FROM numbers(10); + +SELECT number, neighbor('Hello', number) FROM numbers(10); +SELECT number, neighbor('Hello', -3) FROM numbers(10); +SELECT number, neighbor('Hello', -3, 'World') FROM numbers(10); diff --git a/dbms/tests/queries/0_stateless/00997_extract_all_crash_6627.reference b/dbms/tests/queries/0_stateless/00997_extract_all_crash_6627.reference new file mode 100644 index 00000000000..acb53e80e6d --- /dev/null +++ b/dbms/tests/queries/0_stateless/00997_extract_all_crash_6627.reference @@ -0,0 +1 @@ +['9'] diff --git a/dbms/tests/queries/0_stateless/00997_extract_all_crash_6627.sql b/dbms/tests/queries/0_stateless/00997_extract_all_crash_6627.sql new file mode 100644 index 00000000000..06de4ec8afb --- /dev/null +++ b/dbms/tests/queries/0_stateless/00997_extract_all_crash_6627.sql @@ -0,0 +1 @@ +SELECT extractAll('Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.143 YaBrowser/19.7.2.455 Yowser/2.5 Safari/537.36', '[Y][a-zA-Z]{8}/[1-9]([1-9]+)?(((.?)([0-9]+)?){0,4})?'); diff --git a/dbms/tests/queries/0_stateless/00997_set_index_array.reference b/dbms/tests/queries/0_stateless/00997_set_index_array.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00997_set_index_array.reference @@ -0,0 +1 @@ +1 diff --git a/dbms/tests/queries/0_stateless/00997_set_index_array.sql b/dbms/tests/queries/0_stateless/00997_set_index_array.sql new file mode 100644 index 00000000000..c57507ce22d --- /dev/null +++ b/dbms/tests/queries/0_stateless/00997_set_index_array.sql @@ -0,0 +1,24 @@ +SET allow_experimental_data_skipping_indices = 1; + +DROP TABLE IF EXISTS test.set_array; + +CREATE TABLE test.set_array +( + primary_key String, + index_array Array(UInt64), + INDEX additional_index_array (index_array) TYPE set(10000) GRANULARITY 1 +) ENGINE = MergeTree() +ORDER BY (primary_key); + +INSERT INTO test.set_array +select + toString(intDiv(number, 1000000)) as primary_key, + array(number) as index_array +from system.numbers +limit 10000000; + +SET max_rows_to_read = 8192; + +select count() from test.set_array where has(index_array, 333); + +DROP TABLE test.set_array; \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.reference b/dbms/tests/queries/0_stateless/00997_trim.reference similarity index 100% rename from dbms/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.reference rename to dbms/tests/queries/0_stateless/00997_trim.reference diff --git a/dbms/tests/queries/0_stateless/00997_trim.sql b/dbms/tests/queries/0_stateless/00997_trim.sql new file mode 100644 index 00000000000..7519877ec5e --- /dev/null +++ b/dbms/tests/queries/0_stateless/00997_trim.sql @@ -0,0 +1,20 @@ +WITH + '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' AS x, + replaceRegexpAll(x, '.', ' ') AS spaces, + concat(substring(spaces, 1, rand(1) % 62), substring(x, 1, rand(2) % 62), substring(spaces, 1, rand(3) % 62)) AS s, + trimLeft(s) AS sl, + trimRight(s) AS sr, + trimBoth(s) AS t, + replaceRegexpOne(s, '^ +', '') AS slr, + replaceRegexpOne(s, ' +$', '') AS srr, + replaceRegexpOne(s, '^ *(.*?) *$', '\\1') AS tr +SELECT + replaceAll(s, ' ', '_'), + replaceAll(sl, ' ', '_'), + replaceAll(slr, ' ', '_'), + replaceAll(sr, ' ', '_'), + replaceAll(srr, ' ', '_'), + replaceAll(t, ' ', '_'), + replaceAll(tr, ' ', '_') +FROM numbers(100000) +WHERE NOT ((sl = slr) AND (sr = srr) AND (t = tr)) diff --git a/dbms/tests/queries/0_stateless/00998_constraints_all_tables.reference b/dbms/tests/queries/0_stateless/00998_constraints_all_tables.reference new file mode 100644 index 00000000000..730df555af3 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00998_constraints_all_tables.reference @@ -0,0 +1,14 @@ +0 +0 +3 +0 +0 +3 +0 +0 +3 +0 +0 +3 +CREATE TABLE default.constrained (`URL` String, CONSTRAINT is_yandex CHECK domainWithoutWWW(URL) = 'yandex.ru', CONSTRAINT is_utf8 CHECK isValidUTF8(URL)) ENGINE = Log +CREATE TABLE default.constrained2 (`URL` String, CONSTRAINT is_yandex CHECK domainWithoutWWW(URL) = 'yandex.ru', CONSTRAINT is_utf8 CHECK isValidUTF8(URL)) ENGINE = Log diff --git a/dbms/tests/queries/0_stateless/00998_constraints_all_tables.sql b/dbms/tests/queries/0_stateless/00998_constraints_all_tables.sql new file mode 100644 index 00000000000..66b93fca97b --- /dev/null +++ b/dbms/tests/queries/0_stateless/00998_constraints_all_tables.sql @@ -0,0 +1,53 @@ +DROP TABLE IF EXISTS constrained; +CREATE TABLE constrained (URL String, CONSTRAINT is_yandex CHECK domainWithoutWWW(URL) = 'yandex.ru', CONSTRAINT is_utf8 CHECK isValidUTF8(URL)) ENGINE = Null; +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('Hello'), ('test'); -- { serverError 469 } +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('ftp://yandex.ru/Hello'), ('https://yandex.ru/te\xFFst'); -- { serverError 469 } +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('ftp://yandex.ru/Hello'), (toValidUTF8('https://yandex.ru/te\xFFst')); +DROP TABLE constrained; + +CREATE TABLE constrained (URL String, CONSTRAINT is_yandex CHECK domainWithoutWWW(URL) = 'yandex.ru', CONSTRAINT is_utf8 CHECK isValidUTF8(URL)) ENGINE = Memory; +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('Hello'), ('test'); -- { serverError 469 } +SELECT count() FROM constrained; +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('ftp://yandex.ru/Hello'), ('https://yandex.ru/te\xFFst'); -- { serverError 469 } +SELECT count() FROM constrained; +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('ftp://yandex.ru/Hello'), (toValidUTF8('https://yandex.ru/te\xFFst')); +SELECT count() FROM constrained; +DROP TABLE constrained; + +CREATE TABLE constrained (URL String, CONSTRAINT is_yandex CHECK domainWithoutWWW(URL) = 'yandex.ru', CONSTRAINT is_utf8 CHECK isValidUTF8(URL)) ENGINE = StripeLog; +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('Hello'), ('test'); -- { serverError 469 } +SELECT count() FROM constrained; +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('ftp://yandex.ru/Hello'), ('https://yandex.ru/te\xFFst'); -- { serverError 469 } +SELECT count() FROM constrained; +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('ftp://yandex.ru/Hello'), (toValidUTF8('https://yandex.ru/te\xFFst')); +SELECT count() FROM constrained; +DROP TABLE constrained; + +CREATE TABLE constrained (URL String, CONSTRAINT is_yandex CHECK domainWithoutWWW(URL) = 'yandex.ru', CONSTRAINT is_utf8 CHECK isValidUTF8(URL)) ENGINE = TinyLog; +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('Hello'), ('test'); -- { serverError 469 } +SELECT count() FROM constrained; +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('ftp://yandex.ru/Hello'), ('https://yandex.ru/te\xFFst'); -- { serverError 469 } +SELECT count() FROM constrained; +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('ftp://yandex.ru/Hello'), (toValidUTF8('https://yandex.ru/te\xFFst')); +SELECT count() FROM constrained; +DROP TABLE constrained; + +CREATE TABLE constrained (URL String, CONSTRAINT is_yandex CHECK domainWithoutWWW(URL) = 'yandex.ru', CONSTRAINT is_utf8 CHECK isValidUTF8(URL)) ENGINE = Log; +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('Hello'), ('test'); -- { serverError 469 } +SELECT count() FROM constrained; +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('ftp://yandex.ru/Hello'), ('https://yandex.ru/te\xFFst'); -- { serverError 469 } +SELECT count() FROM constrained; +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('ftp://yandex.ru/Hello'), (toValidUTF8('https://yandex.ru/te\xFFst')); +SELECT count() FROM constrained; +DROP TABLE constrained; + + +DROP TABLE IF EXISTS constrained2; +CREATE TABLE constrained (URL String, CONSTRAINT is_yandex CHECK domainWithoutWWW(URL) = 'yandex.ru', CONSTRAINT is_utf8 CHECK isValidUTF8(URL)) ENGINE = Log; +CREATE TABLE constrained2 AS constrained; +SHOW CREATE TABLE constrained FORMAT TSVRaw; +SHOW CREATE TABLE constrained2 FORMAT TSVRaw; +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('Hello'), ('test'); -- { serverError 469 } +INSERT INTO constrained2 VALUES ('https://www.yandex.ru/?q=upyachka'), ('Hello'), ('test'); -- { serverError 469 } +DROP TABLE constrained; +DROP TABLE constrained2; diff --git a/dbms/tests/queries/0_stateless/00999_join_not_nullable_types.reference b/dbms/tests/queries/0_stateless/00999_join_not_nullable_types.reference new file mode 100644 index 00000000000..7b6947fa9a2 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00999_join_not_nullable_types.reference @@ -0,0 +1,8 @@ +0 ['left'] 0 ['left'] \N +1 ['left'] 1 ['left'] 1 +2 [] \N [] 2 +['left'] 0 ['left'] \N +['left'] 1 ['left'] 1 +[] \N [] 2 +['left'] 42 \N +['right'] \N 42 diff --git a/dbms/tests/queries/0_stateless/00999_join_not_nullable_types.sql b/dbms/tests/queries/0_stateless/00999_join_not_nullable_types.sql new file mode 100644 index 00000000000..2a24c6dd296 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00999_join_not_nullable_types.sql @@ -0,0 +1,34 @@ +SET join_use_nulls = 1; + +SELECT * FROM +( + SELECT number, ['left'] as ar, number AS left_number FROM system.numbers LIMIT 2 +) +FULL JOIN +( + SELECT number, ['right'] as ar, number AS right_number FROM system.numbers LIMIT 1, 2 +) +USING (number) +ORDER BY number; + +SELECT * FROM +( + SELECT ['left'] as ar, number AS left_number FROM system.numbers LIMIT 2 +) +FULL JOIN +( + SELECT ['right'] as ar, number AS right_number FROM system.numbers LIMIT 1, 2 +) +ON left_number = right_number +ORDER BY left_number; + +SELECT * FROM +( + SELECT ['left'] as ar, 42 AS left_number +) +FULL JOIN +( + SELECT ['right'] as ar, 42 AS right_number +) +USING(ar) +ORDER BY left_number; diff --git a/dbms/tests/queries/0_stateless/00999_nullable_nested_types_4877.reference b/dbms/tests/queries/0_stateless/00999_nullable_nested_types_4877.reference new file mode 100644 index 00000000000..1e2036c94c7 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00999_nullable_nested_types_4877.reference @@ -0,0 +1,26 @@ +a ('b','c') ('b','c') +d ('e','f') ('','') +a +x +a ('b','c') ('b','c') +x ('','') ('y','z') +a +d +a +x +a ('b','c') ('b','c') +d ('e','f') ('','') +a ('b','c') ('b','c') +x ('','') ('y','z') +a b ['b','c'] +d e [] +a b ['b','c'] +x ['y','z'] +a +d +a +x +a b ['b','c'] +d e [] +a b ['b','c'] +x \N ['y','z'] diff --git a/dbms/tests/queries/0_stateless/00999_nullable_nested_types_4877.sql b/dbms/tests/queries/0_stateless/00999_nullable_nested_types_4877.sql new file mode 100644 index 00000000000..ca523d77235 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00999_nullable_nested_types_4877.sql @@ -0,0 +1,41 @@ +DROP TABLE IF EXISTS l; +DROP TABLE IF EXISTS r; + +CREATE TABLE l (a String, b Tuple(String, String)) ENGINE = Memory(); +CREATE TABLE r (a String, c Tuple(String, String)) ENGINE = Memory(); + +INSERT INTO l (a, b) VALUES ('a', ('b', 'c')), ('d', ('e', 'f')); +INSERT INTO r (a, c) VALUES ('a', ('b', 'c')), ('x', ('y', 'z')); + +SET join_use_nulls = 0; +SELECT * from l LEFT JOIN r USING a ORDER BY a; +SELECT a from l RIGHT JOIN r USING a ORDER BY a; +SELECT * from l RIGHT JOIN r USING a ORDER BY a; + +SET join_use_nulls = 1; +SELECT a from l LEFT JOIN r USING a ORDER BY a; +SELECT a from l RIGHT JOIN r USING a ORDER BY a; +SELECT * from l LEFT JOIN r USING a ORDER BY a; +SELECT * from l RIGHT JOIN r USING a ORDER BY a; + +DROP TABLE l; +DROP TABLE r; + +CREATE TABLE l (a String, b String) ENGINE = Memory(); +CREATE TABLE r (a String, c Array(String)) ENGINE = Memory(); + +INSERT INTO l (a, b) VALUES ('a', 'b'), ('d', 'e'); +INSERT INTO r (a, c) VALUES ('a', ['b', 'c']), ('x', ['y', 'z']); + +SET join_use_nulls = 0; +SELECT * from l LEFT JOIN r USING a ORDER BY a; +SELECT * from l RIGHT JOIN r USING a ORDER BY a; + +SET join_use_nulls = 1; +SELECT a from l LEFT JOIN r USING a ORDER BY a; +SELECT a from l RIGHT JOIN r USING a ORDER BY a; +SELECT * from l LEFT JOIN r USING a ORDER BY a; +SELECT * from l RIGHT JOIN r USING a ORDER BY a; + +DROP TABLE l; +DROP TABLE r; diff --git a/dbms/tests/queries/0_stateless/00999_settings_no_extra_quotes.reference b/dbms/tests/queries/0_stateless/00999_settings_no_extra_quotes.reference new file mode 100644 index 00000000000..573541ac970 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00999_settings_no_extra_quotes.reference @@ -0,0 +1 @@ +0 diff --git a/dbms/tests/queries/0_stateless/00999_settings_no_extra_quotes.sql b/dbms/tests/queries/0_stateless/00999_settings_no_extra_quotes.sql new file mode 100644 index 00000000000..55d9ff2780d --- /dev/null +++ b/dbms/tests/queries/0_stateless/00999_settings_no_extra_quotes.sql @@ -0,0 +1 @@ +SELECT DISTINCT description LIKE '"%"' FROM system.settings; diff --git a/dbms/tests/queries/0_stateless/00999_test_skip_indices_with_alter_and_merge.reference b/dbms/tests/queries/0_stateless/00999_test_skip_indices_with_alter_and_merge.reference new file mode 100644 index 00000000000..5aad65487ae --- /dev/null +++ b/dbms/tests/queries/0_stateless/00999_test_skip_indices_with_alter_and_merge.reference @@ -0,0 +1,2 @@ +201 +201 diff --git a/dbms/tests/queries/0_stateless/00999_test_skip_indices_with_alter_and_merge.sql b/dbms/tests/queries/0_stateless/00999_test_skip_indices_with_alter_and_merge.sql new file mode 100644 index 00000000000..55b2f21dc32 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00999_test_skip_indices_with_alter_and_merge.sql @@ -0,0 +1,22 @@ +SET allow_experimental_data_skipping_indices=1; +DROP TABLE IF EXISTS test_vertical_merge; + +CREATE TABLE test_vertical_merge ( + k UInt64, + val1 UInt64, + val2 UInt64, + INDEX idx1 val1 * val2 TYPE minmax GRANULARITY 1, + INDEX idx2 val1 * k TYPE minmax GRANULARITY 1 +) ENGINE MergeTree() +ORDER BY k +SETTINGS vertical_merge_algorithm_min_rows_to_activate = 1, vertical_merge_algorithm_min_columns_to_activate = 1; + +INSERT INTO test_vertical_merge SELECT number, number + 5, number * 12 from numbers(1000); + +SELECT COUNT() from test_vertical_merge WHERE val2 <= 2400; + +OPTIMIZE TABLE test_vertical_merge FINAL; + +SELECT COUNT() from test_vertical_merge WHERE val2 <= 2400; + +DROP TABLE IF EXISTS test_vertical_merge; diff --git a/dbms/tests/queries/0_stateless/00568_compile_catch_throw.reference b/dbms/tests/queries/0_stateless/01000_bad_size_of_marks_skip_idx.reference similarity index 100% rename from dbms/tests/queries/0_stateless/00568_compile_catch_throw.reference rename to dbms/tests/queries/0_stateless/01000_bad_size_of_marks_skip_idx.reference diff --git a/dbms/tests/queries/0_stateless/01000_bad_size_of_marks_skip_idx.sql b/dbms/tests/queries/0_stateless/01000_bad_size_of_marks_skip_idx.sql new file mode 100644 index 00000000000..7af19fec695 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01000_bad_size_of_marks_skip_idx.sql @@ -0,0 +1,28 @@ +SET allow_experimental_data_skipping_indices=1; + +DROP TABLE IF EXISTS bad_skip_idx; + +CREATE TABLE bad_skip_idx +( + id UInt64, + value String +) ENGINE MergeTree() +ORDER BY id SETTINGS index_granularity_bytes = 64, vertical_merge_algorithm_min_rows_to_activate = 0, vertical_merge_algorithm_min_columns_to_activate = 0; -- actually vertical merge is not required condition for this bug, but it's more easy to reproduce (becuse we don't recalc granularities) + +-- 7 rows per granule +INSERT INTO bad_skip_idx SELECT number, concat('x', toString(number)) FROM numbers(1000); + +-- 3 rows per granule +INSERT INTO bad_skip_idx SELECT number, concat('xxxxxxxxxx', toString(number)) FROM numbers(1000,1000); + +SELECT COUNT(*) from bad_skip_idx WHERE value = 'xxxxxxxxxx1015'; -- check no exception + +INSERT INTO bad_skip_idx SELECT number, concat('x', toString(number)) FROM numbers(1000); + +ALTER TABLE bad_skip_idx ADD INDEX idx value TYPE bloom_filter(0.01) GRANULARITY 4; + +OPTIMIZE TABLE bad_skip_idx FINAL; + +SELECT COUNT(*) from bad_skip_idx WHERE value = 'xxxxxxxxxx1015'; -- check no exception + +DROP TABLE IF EXISTS bad_skip_idx; diff --git a/dbms/tests/queries/0_stateless/01000_unneeded_substitutions_client.reference b/dbms/tests/queries/0_stateless/01000_unneeded_substitutions_client.reference new file mode 100644 index 00000000000..13a393df666 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01000_unneeded_substitutions_client.reference @@ -0,0 +1 @@ +${} diff --git a/dbms/tests/queries/0_stateless/01000_unneeded_substitutions_client.sh b/dbms/tests/queries/0_stateless/01000_unneeded_substitutions_client.sh new file mode 100755 index 00000000000..f6517fc2a42 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01000_unneeded_substitutions_client.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "SELECT '\${}'" diff --git a/dbms/tests/queries/0_stateless/00965_live_view_watch_heartbeat.reference b/dbms/tests/queries/0_stateless/01001_rename_merge_race_condition.reference similarity index 100% rename from dbms/tests/queries/0_stateless/00965_live_view_watch_heartbeat.reference rename to dbms/tests/queries/0_stateless/01001_rename_merge_race_condition.reference diff --git a/dbms/tests/queries/0_stateless/01001_rename_merge_race_condition.sh b/dbms/tests/queries/0_stateless/01001_rename_merge_race_condition.sh new file mode 100755 index 00000000000..b0f1dda7c45 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01001_rename_merge_race_condition.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +set -e + +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test1"; +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test2"; +$CLICKHOUSE_CLIENT --query "CREATE TABLE test1 (x UInt64) ENGINE = Memory"; + + +function thread1() +{ + while true; do + seq 1 1000 | sed -r -e 's/.+/RENAME TABLE test1 TO test2; RENAME TABLE test2 TO test1;/' | $CLICKHOUSE_CLIENT -n + done +} + +function thread2() +{ + while true; do + $CLICKHOUSE_CLIENT --query "SELECT * FROM merge(currentDatabase(), '^test[12]$')" + done +} + +# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout +export -f thread1; +export -f thread2; + +TIMEOUT=10 + +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & + +wait + +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test1"; +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test2"; diff --git a/dbms/tests/queries/0_stateless/00966_live_view_watch_events_http.reference b/dbms/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity.reference similarity index 100% rename from dbms/tests/queries/0_stateless/00966_live_view_watch_events_http.reference rename to dbms/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity.reference diff --git a/dbms/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity.sh b/dbms/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity.sh new file mode 100755 index 00000000000..85fc847f3f3 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +set -e + +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test"; +$CLICKHOUSE_CLIENT --query "CREATE TABLE test (x UInt8, s String MATERIALIZED toString(rand64())) ENGINE = MergeTree ORDER BY s"; + +function thread1() +{ + while true; do + $CLICKHOUSE_CLIENT --query "INSERT INTO test SELECT rand() FROM numbers(1000)"; + done +} + +function thread2() +{ + while true; do + $CLICKHOUSE_CLIENT -n --query "ALTER TABLE test MODIFY COLUMN x Nullable(UInt8);"; + sleep 0.0$RANDOM + $CLICKHOUSE_CLIENT -n --query "ALTER TABLE test MODIFY COLUMN x UInt8;"; + sleep 0.0$RANDOM + done +} + +function thread3() +{ + while true; do + $CLICKHOUSE_CLIENT -n --query "SELECT count() FROM test FORMAT Null"; + done +} + +function thread4() +{ + while true; do + $CLICKHOUSE_CLIENT -n --query "OPTIMIZE TABLE test FINAL"; + sleep 0.1$RANDOM + done +} + +# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout +export -f thread1; +export -f thread2; +export -f thread3; +export -f thread4; + +TIMEOUT=10 + +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread4 2> /dev/null & + +wait + +$CLICKHOUSE_CLIENT -q "DROP TABLE test" diff --git a/dbms/tests/queries/0_stateless/00967_live_view_watch_http.reference b/dbms/tests/queries/0_stateless/01003_kill_query_race_condition.reference similarity index 100% rename from dbms/tests/queries/0_stateless/00967_live_view_watch_http.reference rename to dbms/tests/queries/0_stateless/01003_kill_query_race_condition.reference diff --git a/dbms/tests/queries/0_stateless/01003_kill_query_race_condition.sh b/dbms/tests/queries/0_stateless/01003_kill_query_race_condition.sh new file mode 100755 index 00000000000..d8a73ac24a4 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01003_kill_query_race_condition.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +set -e + +function thread1() +{ + while true; do + $CLICKHOUSE_CLIENT --query_id=hello --query "SELECT count() FROM numbers(1000000000)" --format Null; + done +} + +function thread2() +{ + while true; do + $CLICKHOUSE_CLIENT --query "KILL QUERY WHERE query_id = 'hello'" --format Null; + sleep 0.$RANDOM + done +} + +function thread3() +{ + while true; do + $CLICKHOUSE_CLIENT --query "SHOW PROCESSLIST" --format Null; + $CLICKHOUSE_CLIENT --query "SELECT * FROM system.processes" --format Null; + done +} + +# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout +export -f thread1; +export -f thread2; +export -f thread3; + +TIMEOUT=10 + +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & + +timeout $TIMEOUT bash -c thread2 2> /dev/null & + +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & + +wait diff --git a/dbms/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.reference b/dbms/tests/queries/0_stateless/01004_rename_deadlock.reference similarity index 100% rename from dbms/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.reference rename to dbms/tests/queries/0_stateless/01004_rename_deadlock.reference diff --git a/dbms/tests/queries/0_stateless/01004_rename_deadlock.sh b/dbms/tests/queries/0_stateless/01004_rename_deadlock.sh new file mode 100755 index 00000000000..5d5726bb001 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01004_rename_deadlock.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +set -e + +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test1"; +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test2"; +$CLICKHOUSE_CLIENT --query "CREATE TABLE test1 (x UInt8) ENGINE = MergeTree ORDER BY x"; +$CLICKHOUSE_CLIENT --query "CREATE TABLE test2 (x UInt8) ENGINE = MergeTree ORDER BY x"; + +function thread1() +{ + while true; do + $CLICKHOUSE_CLIENT --query "RENAME TABLE test1 TO test_tmp, test2 TO test1, test_tmp TO test2" + done +} + +function thread2() +{ + while true; do + $CLICKHOUSE_CLIENT --query "SELECT * FROM test1 UNION ALL SELECT * FROM test2" --format Null + done +} + +function thread3() +{ + while true; do + $CLICKHOUSE_CLIENT --query "SELECT * FROM system.tables" --format Null + done +} + +# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout +export -f thread1; +export -f thread2; +export -f thread3; + +TIMEOUT=10 + +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & + +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & + +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & + +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & + +wait + +$CLICKHOUSE_CLIENT -q "DROP TABLE test1" +$CLICKHOUSE_CLIENT -q "DROP TABLE test2" diff --git a/dbms/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.reference b/dbms/tests/queries/0_stateless/01005_rwr_shard_deadlock.reference similarity index 100% rename from dbms/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.reference rename to dbms/tests/queries/0_stateless/01005_rwr_shard_deadlock.reference diff --git a/dbms/tests/queries/0_stateless/01005_rwr_shard_deadlock.sh b/dbms/tests/queries/0_stateless/01005_rwr_shard_deadlock.sh new file mode 100755 index 00000000000..1afd3acd324 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01005_rwr_shard_deadlock.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +set -e + +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test1"; +$CLICKHOUSE_CLIENT --query "CREATE TABLE test1 (x UInt8) ENGINE = MergeTree ORDER BY tuple()"; + +function thread1() +{ + while true; do + $CLICKHOUSE_CLIENT --query "ALTER TABLE test1 MODIFY COLUMN x Nullable(UInt8)" + $CLICKHOUSE_CLIENT --query "ALTER TABLE test1 MODIFY COLUMN x UInt8" + done +} + +function thread2() +{ + while true; do + $CLICKHOUSE_CLIENT --query "SELECT x FROM test1 WHERE x IN (SELECT x FROM remote('127.0.0.2', currentDatabase(), test1))" --format Null + done +} + +# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout +export -f thread1; +export -f thread2; + +TIMEOUT=10 + +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & + +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & + +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & + +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & + +wait + +$CLICKHOUSE_CLIENT -q "DROP TABLE test1" diff --git a/dbms/tests/queries/0_stateless/data_orc/test.orc b/dbms/tests/queries/0_stateless/data_orc/test.orc new file mode 100644 index 00000000000..1b2c9aa4922 Binary files /dev/null and b/dbms/tests/queries/0_stateless/data_orc/test.orc differ diff --git a/dbms/tests/queries/shell_config.sh b/dbms/tests/queries/shell_config.sh index d4ab11be927..b3058a6cdbe 100644 --- a/dbms/tests/queries/shell_config.sh +++ b/dbms/tests/queries/shell_config.sh @@ -46,7 +46,8 @@ export CLICKHOUSE_PORT_INTERSERVER=${CLICKHOUSE_PORT_INTERSERVER:="9009"} export CLICKHOUSE_URL_INTERSERVER=${CLICKHOUSE_URL_INTERSERVER:="${CLICKHOUSE_PORT_HTTP_PROTO}://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_INTERSERVER}/"} export CLICKHOUSE_CURL_COMMAND=${CLICKHOUSE_CURL_COMMAND:="curl"} -export CLICKHOUSE_CURL=${CLICKHOUSE_CURL:="${CLICKHOUSE_CURL_COMMAND} --max-time 10"} +export CLICKHOUSE_CURL_TIMEOUT=${CLICKHOUSE_CURL_TIMEOUT:="10"} +export CLICKHOUSE_CURL=${CLICKHOUSE_CURL:="${CLICKHOUSE_CURL_COMMAND} --max-time ${CLICKHOUSE_CURL_TIMEOUT}"} export CLICKHOUSE_TMP=${CLICKHOUSE_TMP:="."} mkdir -p ${CLICKHOUSE_TMP} diff --git a/dbms/tests/stress b/dbms/tests/stress new file mode 100755 index 00000000000..4a6ad411298 --- /dev/null +++ b/dbms/tests/stress @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# https://stackoverflow.com/questions/360201/how-do-i-kill-background-processes-jobs-when-my-shell-script-exits +trap 'kill -9 $(jobs -p)' EXIT + +function thread() +{ + while true; do + ./clickhouse-test --order random 2>&1 | awk '/^\w+:/ { printf("\033[0;%s%sm \033[0m", ('$1' % 2 ? "4" : "10"), (int('$1' / 2) % 8)) }' + done +} + +# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout +export -f thread; + +NUM_THREADS=${1:-"16"} +TIMEOUT=${2:-"300"} + +for i in $(seq 1 $NUM_THREADS); do + timeout $TIMEOUT bash -c "thread $i" 2> /dev/null & +done + +wait diff --git a/dbms/tests/tsan_suppressions.txt b/dbms/tests/tsan_suppressions.txt index 476e135de14..3dc306ee133 100644 --- a/dbms/tests/tsan_suppressions.txt +++ b/dbms/tests/tsan_suppressions.txt @@ -1,2 +1,5 @@ # libc++ race:locale + +# Too many mutexes: https://github.com/google/sanitizers/issues/950 +deadlock:DB::MergeTreeReadPool::fillPerPartInfo diff --git a/debian/clickhouse-client.install b/debian/clickhouse-client.install index 7331f3eb5ed..5e730db669f 100644 --- a/debian/clickhouse-client.install +++ b/debian/clickhouse-client.install @@ -3,5 +3,6 @@ usr/bin/clickhouse-local usr/bin/clickhouse-compressor usr/bin/clickhouse-benchmark usr/bin/clickhouse-format +usr/bin/clickhouse-obfuscator etc/clickhouse-client/config.xml usr/bin/clickhouse-extract-from-config diff --git a/debian/clickhouse-common-static.install b/debian/clickhouse-common-static.install index 6666b090272..81b1dc4eb1b 100644 --- a/debian/clickhouse-common-static.install +++ b/debian/clickhouse-common-static.install @@ -1,4 +1,3 @@ usr/bin/clickhouse usr/bin/clickhouse-odbc-bridge etc/security/limits.d/clickhouse.conf -usr/share/clickhouse/* diff --git a/debian/clickhouse-server.docs b/debian/clickhouse-server.docs index 95969d08c43..e12d6533be2 100644 --- a/debian/clickhouse-server.docs +++ b/debian/clickhouse-server.docs @@ -1,3 +1,4 @@ LICENSE AUTHORS README.md +CHANGELOG.md diff --git a/debian/clickhouse-server.install b/debian/clickhouse-server.install index f69969a6084..b1475fdf162 100644 --- a/debian/clickhouse-server.install +++ b/debian/clickhouse-server.install @@ -1,6 +1,4 @@ usr/bin/clickhouse-server -usr/bin/clickhouse-clang -usr/bin/clickhouse-lld usr/bin/clickhouse-copier usr/bin/clickhouse-report etc/clickhouse-server/config.xml diff --git a/debian/clickhouse-server.postinst b/debian/clickhouse-server.postinst index 944b1e394f1..c47a8ef4be2 100644 --- a/debian/clickhouse-server.postinst +++ b/debian/clickhouse-server.postinst @@ -20,7 +20,7 @@ OS=${OS=`lsb_release -is 2>/dev/null ||:`} [ -f /usr/share/debconf/confmodule ] && . /usr/share/debconf/confmodule [ -f /etc/default/clickhouse ] && . /etc/default/clickhouse -if [ "$OS" = "rhel" ] || [ "$OS" = "centos" ] || [ "$OS" = "fedora" ] || [ "$OS" = "CentOS" ] || [ "$OS" = "Fedora" ]; then +if [ "$OS" = "rhel" ] || [ "$OS" = "centos" ] || [ "$OS" = "fedora" ] || [ "$OS" = "CentOS" ] || [ "$OS" = "Fedora" ] || [ "$OS" = "ol" ]; then is_rh=1 fi diff --git a/debian/rules b/debian/rules index a49ffc3f66e..c21f0999bbc 100755 --- a/debian/rules +++ b/debian/rules @@ -32,11 +32,6 @@ endif CMAKE_FLAGS += -DENABLE_UTILS=0 -#DEB_CLANG ?= $(shell which clang-6.0 || which clang-5.0 || which clang-4.0 || which clang || which clang-3.9 || which clang-3.8) - -#DEB_CC ?= gcc-7 -#DEB_CXX ?= g++-7 - ifdef DEB_CXX DEB_BUILD_GNU_TYPE := $(shell dpkg-architecture -qDEB_BUILD_GNU_TYPE) DEB_HOST_GNU_TYPE := $(shell dpkg-architecture -qDEB_HOST_GNU_TYPE) @@ -88,12 +83,8 @@ override_dh_auto_configure: override_dh_auto_build: # Fix for ninja. Do not add -O. $(MAKE) $(THREADS_COUNT) -C $(BUILDDIR) $(MAKE_TARGET) -# #cd $(BUILDDIR) && cmake --build . -- -j$(THREADS_COUNT) # cmake return true on error override_dh_auto_test: -# #TODO, use ENABLE_TESTS=1 -# #./debian/tests_wrapper.sh -# cd $(BUILDDIR) && ctest $(THREADS_COUNT) -V -R GLIBC_required_version cd $(BUILDDIR) && ctest $(THREADS_COUNT) -V -E with_server override_dh_clean: @@ -117,11 +108,6 @@ override_dh_install: mkdir -p $(DESTDIR)/etc/systemd/system/ cp debian/clickhouse-server.service $(DESTDIR)/etc/systemd/system/ - # fake metrika files when private dir is empty - mkdir -p $(DESTDIR)/etc/clickhouse-server/metrika - touch $(DESTDIR)/etc/clickhouse-server/metrika/config.xml - touch $(DESTDIR)/etc/clickhouse-server/metrika/users.xml - dh_install --list-missing --sourcedir=$(DESTDIR) override_dh_auto_install: @@ -130,7 +116,5 @@ override_dh_auto_install: override_dh_shlibdeps: true # We depend only on libc and dh_shlibdeps gives us wrong (too strict) dependency. -#TODO: faster packing of non-release builds: ifdef RELEASE_COMPATIBLE override_dh_builddeb: dh_builddeb -- -Z gzip # Older systems don't have "xz", so use "gzip" instead. -#TODO: endif diff --git a/docker/builder/Dockerfile b/docker/builder/Dockerfile index 41a558f9eb8..5978dcd08d0 100644 --- a/docker/builder/Dockerfile +++ b/docker/builder/Dockerfile @@ -28,6 +28,10 @@ RUN apt-get update -y \ tzdata \ gperf +RUN apt install -y wget +RUN printf "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main\ndeb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main" >> /etc/apt/sources.list \ + && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt-get install -y clang-8 lldb-8 lld-8 + COPY build.sh / CMD ["/bin/bash", "/build.sh"] diff --git a/docker/builder/Makefile b/docker/builder/Makefile index 779e944b723..a9a7cddf3f2 100644 --- a/docker/builder/Makefile +++ b/docker/builder/Makefile @@ -1,6 +1,6 @@ build: image mkdir -p $(HOME)/.ccache - docker run --network=host --rm --workdir /server --volume $(realpath ../..):/server --mount=type=bind,source=$(HOME)/.ccache,destination=/ccache -e CCACHE_DIR=/ccache -it yandex/clickhouse-builder + docker run --network=host --rm --workdir /server --volume $(realpath ../..):/server --cap-add=SYS_PTRACE --mount=type=bind,source=$(HOME)/.ccache,destination=/ccache -e CCACHE_DIR=/ccache -it yandex/clickhouse-builder pull: docker pull yandex/clickhouse-builder diff --git a/docker/builder/build.sh b/docker/builder/build.sh index 57999a4b483..8af31cb3d43 100755 --- a/docker/builder/build.sh +++ b/docker/builder/build.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash -#ccache -s +#ccache -s # uncomment to display CCache statistics mkdir -p /server/build_docker cd /server/build_docker -cmake -G Ninja /server -DENABLE_TESTS=1 -DCMAKE_C_COMPILER=`which gcc-9` -DCMAKE_CXX_COMPILER=`which g++-9` +cmake -G Ninja /server -DCMAKE_C_COMPILER=`which gcc-8` -DCMAKE_CXX_COMPILER=`which g++-8` # Set the number of build jobs to the half of number of virtual CPU cores (rounded up). # By default, ninja use all virtual CPU cores, that leads to very high memory consumption without much improvement in build time. diff --git a/docker/packager/deb/Dockerfile b/docker/packager/deb/Dockerfile index f08c2dc3eab..7410920a462 100644 --- a/docker/packager/deb/Dockerfile +++ b/docker/packager/deb/Dockerfile @@ -71,7 +71,9 @@ RUN apt-get --allow-unauthenticated update -y \ odbcinst \ tzdata \ gperf \ - alien + alien \ + libcapnp-dev + COPY build.sh / CMD ["/bin/bash", "/build.sh"] diff --git a/docker/packager/packager b/docker/packager/packager index 0e8bf6ea98d..d0d46fd5f51 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -146,7 +146,7 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, cache, di cmake_flags.append('-DUNBUNDLED=1 -DENABLE_MYSQL=0 -DENABLE_POCO_ODBC=0 -DENABLE_ODBC=0') if split_binary: - cmake_flags.append('-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1 -DGLIBC_COMPATIBILITY=ON') + cmake_flags.append('-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1') if with_coverage: cmake_flags.append('-DWITH_COVERAGE=1') diff --git a/docs/en/data_types/enum.md b/docs/en/data_types/enum.md index 247ec070190..692cbad39b1 100644 --- a/docs/en/data_types/enum.md +++ b/docs/en/data_types/enum.md @@ -1,13 +1,15 @@ # Enum -Enumerated type storing pairs of the `'string' = integer` format. +Enumerated type consisting of named values. + +Named values must be delcared as `'string' = integer` pairs. ClickHouse stores only numbers, but supports operations with the values through their names. ClickHouse supports: -- 8-bit `Enum`. It can contain up to 256 values with enumeration of `[-128, 127]`. -- 16-bit `Enum`. It can contain up to 65536 values with enumeration of `[-32768, 32767]`. +- 8-bit `Enum`. It can contain up to 256 values enumerated in the `[-128, 127]` range. +- 16-bit `Enum`. It can contain up to 65536 values enumerated in the `[-32768, 32767]` range. -ClickHouse automatically chooses a type for `Enum` at data insertion. Also, you can use `Enum8` or `Enum16` types to be sure in size of storage. +ClickHouse automatically chooses the type of `Enum` when data is inserted. You can also use `Enum8` or `Enum16` types to be sure in the size of storage. ## Usage examples @@ -21,7 +23,7 @@ CREATE TABLE t_enum ENGINE = TinyLog ``` -This column `x` can only store the values that are listed in the type definition: `'hello'` or `'world'`. If you try to save any other value, ClickHouse will generate an exception. ClickHouse automatically chooses the 8-bit size for enumeration of this `Enum`. +Column `x` can only store values that are listed in the type definition: `'hello'` or `'world'`. If you try to save any other value, ClickHouse will raise an exception. 8-bit size for this `Enum` is chosen automatically. ```sql :) INSERT INTO t_enum VALUES ('hello'), ('world'), ('hello') @@ -94,7 +96,7 @@ ENGINE = TinyLog it can store not only `'hello'` and `'world'`, but `NULL`, as well. ``` -INSERT INTO t_enum_null Values('hello'),('world'),(NULL) +INSERT INTO t_enum_nullable Values('hello'),('world'),(NULL) ``` In RAM, an `Enum` column is stored in the same way as `Int8` or `Int16` of the corresponding numerical values. diff --git a/docs/en/interfaces/cli.md b/docs/en/interfaces/cli.md index 9f9448f27c8..b6e59c4aa50 100644 --- a/docs/en/interfaces/cli.md +++ b/docs/en/interfaces/cli.md @@ -67,22 +67,22 @@ The command-line client allows passing external data (external temporary tables) ### Queries with Parameters {#cli-queries-with-parameters} -You can create a query with parameters, and pass values for these parameters with the parameters of the client app. For example: +You can create a query with parameters and pass values to them from client application. This allows to avoid formatting query with specific dynamic values on client side. For example: ```bash clickhouse-client --param_parName="[1, 2]" -q "SELECT * FROM table WHERE a = {parName:Array(UInt16)}" ``` -#### Syntax of a Query {#cli-queries-with-parameters-syntax} +#### Query Syntax {#cli-queries-with-parameters-syntax} -Format a query by the standard method. Values that you want to put into the query from the app parameters place in braces and format as follows: +Format a query as usual, then place the values that you want to pass from the app parameters to the query in braces in the following format: ``` {:} ``` -- `name` — Identifier of a placeholder that should be used in app parameter as `--param_name = value`. -- `data type` — A data type of app parameter value. For example, data structure like `(integer, ('string', integer))` can have a data type `Tuple(UInt8, Tuple(String, UInt8))` (also you can use another [integer](../data_types/int_uint.md) types). +- `name` — Placeholder identifier. In the console client it should be used in app parameters as `--param_ = value`. +- `data type` — [Data type](../data_types/index.md) of the app parameter value. For example, a data structure like `(integer, ('string', integer))` can have the `Tuple(UInt8, Tuple(String, UInt8))` data type (you can also use another [integer](../data_types/int_uint.md) types). #### Example @@ -118,6 +118,8 @@ You can pass parameters to `clickhouse-client` (all parameters have a default va - `--stacktrace` – If specified, also print the stack trace if an exception occurs. - `--config-file` – The name of the configuration file. - `--secure` – If specified, will connect to server over secure connection. +- `--param_` — Value for a [query with parameters](#cli-queries-with-parameters). + ### Configuration Files diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index c5dbbce674d..2f409706c61 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -146,7 +146,7 @@ SELECT * FROM t_null FORMAT TSKV x=1 y=\N ``` -When there is a large number of small columns, this format is ineffective, and there is generally no reason to use it. It is used in some departments of Yandex. +When there is a large number of small columns, this format is ineffective, and there is generally no reason to use it. Nevertheless, it is no worse than JSONEachRow in terms of efficiency. Both data output and parsing are supported in this format. For parsing, any order is supported for the values of different columns. It is acceptable for some values to be omitted – they are treated as equal to their default values. In this case, zeros and blank rows are used as default values. Complex values that could be specified in the table are not supported as defaults. diff --git a/docs/en/interfaces/http.md b/docs/en/interfaces/http.md index ef32b101b71..80cf72ec0e2 100644 --- a/docs/en/interfaces/http.md +++ b/docs/en/interfaces/http.md @@ -246,7 +246,7 @@ Use buffering to avoid situations where a query processing error occurred after ### Queries with Parameters {#cli-queries-with-parameters} -You can create a query with parameters, and pass values for these parameters with the parameters of the HTTP request. For more information, see [CLI Formatted Queries](cli.md#cli-queries-with-parameters). +You can create a query with parameters and pass values for them from the corresponding HTTP request parameters. For more information, see [Queries with Parameters for CLI](cli.md#cli-queries-with-parameters). ### Example diff --git a/docs/en/operations/settings/query_complexity.md b/docs/en/operations/settings/query_complexity.md index 77699c868b4..c00f2132ebd 100644 --- a/docs/en/operations/settings/query_complexity.md +++ b/docs/en/operations/settings/query_complexity.md @@ -79,7 +79,7 @@ Enables or disables execution of `GROUP BY` clauses in external memory. See [GRO Possible values: -- Maximum volume or RAM (in bytes) that can be used by the single [GROUP BY](../../query_language/select.md#select-group-by-clause) operation. +- Maximum volume of RAM (in bytes) that can be used by the single [GROUP BY](../../query_language/select.md#select-group-by-clause) operation. - 0 — `GROUP BY` in external memory disabled. Default value: 0. diff --git a/docs/en/operations/system_tables.md b/docs/en/operations/system_tables.md index 9f246c20725..0f0917a876c 100644 --- a/docs/en/operations/system_tables.md +++ b/docs/en/operations/system_tables.md @@ -87,10 +87,9 @@ This table contains a single String column called 'name' – the name of a datab Each database that the server knows about has a corresponding entry in the table. This system table is used for implementing the `SHOW DATABASES` query. -## system.detached_parts - -Contains information about detached parts of [MergeTree](table_engines/mergetree.md) tables. The `reason` column specifies why the part was detached. For user-detached parts, the reason is empty. Such parts can be attached with [ALTER TABLE ATTACH PARTITION|PART](../query_language/query_language/alter/#alter_attach-partition) command. For the description of other columns, see [system.parts](#system_tables-parts). +## system.detached_parts {#system_tables-detached_parts} +Contains information about detached parts of [MergeTree](table_engines/mergetree.md) tables. The `reason` column specifies why the part was detached. For user-detached parts, the reason is empty. Such parts can be attached with [ALTER TABLE ATTACH PARTITION|PART](../query_language/query_language/alter/#alter_attach-partition) command. For the description of other columns, see [system.parts](#system_tables-parts). If part name is invalid, values of some columns may be `NULL`. Such parts can be deleted with [ALTER TABLE DROP DETACHED PART](../query_language/query_language/alter/#alter_drop-detached). ## system.dictionaries diff --git a/docs/en/operations/table_engines/index.md b/docs/en/operations/table_engines/index.md index 41680a5b3af..ce8cf5b4678 100644 --- a/docs/en/operations/table_engines/index.md +++ b/docs/en/operations/table_engines/index.md @@ -2,7 +2,7 @@ The table engine (type of table) determines: -- How and where data is stored, where to write it to, and from where to read it. +- How and where data is stored, where to write it to, and where to read it from. - Which queries are supported, and how. - Concurrent data access. - Use of indexes, if present. @@ -11,13 +11,13 @@ The table engine (type of table) determines: ## Engine Families -### *MergeTree +### MergeTree -The most universal and functional table engines for high-load tasks. The common property of these engines is quick data insertion with subsequent data processing in the background. The `*MergeTree` engines support data replication (with [Replicated*](replication.md) versions of engines), partitioning and other features not supported in other engines. +The most universal and functional table engines for high-load tasks. The property shared by these engines is quick data insertion with subsequent background data processing. `MergeTree` family engines support data replication (with [Replicated*](replication.md) versions of engines), partitioning, and other features not supported in other engines. -Engines of the family: +Engines in the family: -- [MergTree](mergetree.md) +- [MergeTree](mergetree.md) - [ReplacingMergeTree](replacingmergetree.md) - [SummingMergeTree](summingmergetree.md) - [AggregatingMergeTree](aggregatingmergetree.md) @@ -25,11 +25,11 @@ Engines of the family: - [VersionedCollapsingMergeTree](versionedcollapsingmergetree.md) - [GraphiteMergeTree](graphitemergetree.md) -### *Log +### Log -Lightweight [engines](log_family.md) with minimum functionality. They are the most effective in scenarios when you need to quickly write many small tables (up to about 1 million rows) and read them later as a whole. +Lightweight [engines](log_family.md) with minimum functionality. They're the most effective when you need to quickly write many small tables (up to approximately 1 million rows) and read them later as a whole. -Engines of the family: +Engines in the family: - [TinyLog](tinylog.md) - [StripeLog](stripelog.md) @@ -39,7 +39,7 @@ Engines of the family: Engines for communicating with other data storage and processing systems. -Engines of the family: +Engines in the family: - [Kafka](kafka.md) - [MySQL](mysql.md) @@ -48,9 +48,7 @@ Engines of the family: ### Special engines -Engines solving special tasks. - -Engines of the family: +Engines in the family: - [Distributed](distributed.md) - [MaterializedView](materializedview.md) @@ -67,12 +65,12 @@ Engines of the family: ## Virtual columns {#table_engines-virtual_columns} -Virtual column is an integral attribute of a table engine that is defined in the source code of the engine. +Virtual column is an integral table engine attribute that is defined in the engine source code. -You should not specify virtual columns in the `CREATE TABLE` query, and you cannot see them in the results of `SHOW CREATE TABLE` and `DESCRIBE TABLE` queries. Also, virtual columns are read-only, so you can't insert data into virtual columns. +You shouldn't specify virtual columns in the `CREATE TABLE` query and you can't see them in `SHOW CREATE TABLE` and `DESCRIBE TABLE` query results. Virtual columns are also read-only, so you can't insert data into virtual columns. -To select data from a virtual column, you must specify its name in the `SELECT` query. The `SELECT *` doesn't return values from virtual columns. +To select data from a virtual column, you must specify its name in the `SELECT` query. `SELECT *` doesn't return values from virtual columns. -If you create a table with a column that has the same name as one of the table virtual columns, the virtual column becomes inaccessible. Doing so is not recommended. To help avoiding conflicts virtual column names are usually prefixed with an underscore. +If you create a table with a column that has the same name as one of the table virtual columns, the virtual column becomes inaccessible. We don't recommend doing this. To help avoid conflicts, virtual column names are usually prefixed with an underscore. [Original article](https://clickhouse.yandex/docs/en/operations/table_engines/) diff --git a/docs/en/query_language/agg_functions/parametric_functions.md b/docs/en/query_language/agg_functions/parametric_functions.md index 84898a61133..3ea26de5937 100644 --- a/docs/en/query_language/agg_functions/parametric_functions.md +++ b/docs/en/query_language/agg_functions/parametric_functions.md @@ -45,7 +45,7 @@ FROM ( └─────────────────────────────────────────────────────────────────────────┘ ``` -You can visualize a histogram with the [bar](../other_functions.md#function-bar) function, for example: +You can visualize a histogram with the [bar](../functions/other_functions.md#function-bar) function, for example: ```sql WITH histogram(5)(rand() % 100) AS hist diff --git a/docs/en/query_language/alter.md b/docs/en/query_language/alter.md index 102c65528e2..84ff4f390a8 100644 --- a/docs/en/query_language/alter.md +++ b/docs/en/query_language/alter.md @@ -29,7 +29,7 @@ These actions are described in detail below. ADD COLUMN [IF NOT EXISTS] name [type] [default_expr] [AFTER name_after] ``` -Adds a new column to the table with the specified `name`, `type`, and `default_expr` (see the section [Default expressions](create.md#create-default-values)). +Adds a new column to the table with the specified `name`, `type`, and `default_expr` (see the section [Default expressions](create.md#create-default-values)). If the `IF NOT EXISTS` clause is included, the query won't return an error if the column already exists. If you specify `AFTER name_after` (the name of another column), the column is added after the specified one in the list of table columns. Otherwise, the column is added to the end of the table. Note that there is no way to add a column to the beginning of a table. For a chain of actions, `name_after` can be the name of a column that is added in one of the previous actions. @@ -66,7 +66,7 @@ CLEAR COLUMN [IF EXISTS] name IN PARTITION partition_name ``` Resets all data in a column for a specified partition. Read more about setting the partition name in the section [How to specify the partition expression](#alter-how-to-specify-part-expr). - + If the `IF EXISTS` clause is specified, the query won't return an error if the column doesn't exist. Example: @@ -85,7 +85,7 @@ Adds a comment to the column. If the `IF EXISTS` clause is specified, the query Each column can have one comment. If a comment already exists for the column, a new comment overwrites the previous comment. -Comments are stored in the `comment_expression` column returned by the [DESCRIBE TABLE](misc.md#misc-describe-table) query. +Comments are stored in the `comment_expression` column returned by the [DESCRIBE TABLE](misc.md#misc-describe-table) query. Example: @@ -129,7 +129,7 @@ The `ALTER` query lets you create and delete separate elements (columns) in nest There is no support for deleting columns in the primary key or the sampling key (columns that are used in the `ENGINE` expression). Changing the type for columns that are included in the primary key is only possible if this change does not cause the data to be modified (for example, you are allowed to add values to an Enum or to change a type from `DateTime` to `UInt32`). -If the `ALTER` query is not sufficient to make the table changes you need, you can create a new table, copy the data to it using the [INSERT SELECT](insert_into.md#insert_query_insert-select) query, then switch the tables using the [RENAME](misc.md#misc_operations-rename) query and delete the old table. You can use the [clickhouse-copier](../operations/utils/clickhouse-copier.md) as an alternative to the `INSERT SELECT` query. +If the `ALTER` query is not sufficient to make the table changes you need, you can create a new table, copy the data to it using the [INSERT SELECT](insert_into.md#insert_query_insert-select) query, then switch the tables using the [RENAME](misc.md#misc_operations-rename) query and delete the old table. You can use the [clickhouse-copier](../operations/utils/clickhouse-copier.md) as an alternative to the `INSERT SELECT` query. The `ALTER` query blocks all reads and writes for the table. In other words, if a long `SELECT` is running at the time of the `ALTER` query, the `ALTER` query will wait for it to complete. At the same time, all new queries to the same table will wait while this `ALTER` is running. @@ -166,6 +166,22 @@ are available: These commands are lightweight in a sense that they only change metadata or remove files. Also, they are replicated (syncing indices metadata through ZooKeeper). +### Manipulations with constraints + +See more on [constraints](create.md#constraints) + +Constraints could be added or deleted using following syntax: +``` +ALTER TABLE [db].name ADD CONSTRAINT constraint_name CHECK expression; +ALTER TABLE [db].name DROP CONSTRAINT constraint_name; +``` + +Queries will add or remove metadata about constraints from table so they are processed immediately. + +Constraint check *will not be executed* on existing data if it was added. + +All changes on replicated tables are broadcasting to ZooKeeper so will be applied on other replicas. + ### Manipulations With Partitions and Parts {#alter_manipulations-with-partitions} The following operations with [partitions](../operations/table_engines/custom_partitioning_key.md) are available: @@ -211,6 +227,16 @@ Read about setting the partition expression in a section [How to specify the par The query is replicated – it deletes data on all replicas. +#### DROP DETACHED PARTITION|PART {#alter_drop-detached} + +```sql +ALTER TABLE table_name DROP DETACHED PARTITION|PART partition_expr +``` + +Removes the specified part or all parts of the specified partition from `detached`. +Read more about setting the partition expression in a section [How to specify the partition expression](#alter-how-to-specify-part-expr). + + #### ATTACH PARTITION|PART {#alter_attach-partition} ``` sql @@ -241,7 +267,7 @@ This query copies the data partition from the `table1` to `table2`. Note that da For the query to run successfully, the following conditions must be met: - Both tables must have the same structure. -- Both tables must have the same partition key. +- Both tables must have the same partition key. #### CLEAR COLUMN IN PARTITION {#alter_clear-column-partition} @@ -263,13 +289,13 @@ ALTER TABLE visits CLEAR COLUMN hour in PARTITION 201902 ALTER TABLE table_name FREEZE [PARTITION partition_expr] ``` -This query creates a local backup of a specified partition. If the `PARTITION` clause is omitted, the query creates the backup of all partitions at once. +This query creates a local backup of a specified partition. If the `PARTITION` clause is omitted, the query creates the backup of all partitions at once. Note that for old-styled tables you can specify the prefix of the partition name (for example, '2019') - then the query creates the backup for all the corresponding partitions. Read about setting the partition expression in a section [How to specify the partition expression](#alter-how-to-specify-part-expr). !!! note The entire backup process is performed without stopping the server. - + At the time of execution, for a data snapshot, the query creates hardlinks to a table data. Hardlinks are placed in the directory `/var/lib/clickhouse/shadow/N/...`, where: - `/var/lib/clickhouse/` is the working ClickHouse directory specified in the config. @@ -322,7 +348,7 @@ ALTER TABLE users ATTACH PARTITION 201902; ``` Note that: -- The `ALTER ... FETCH PARTITION` query isn't replicated. It places the partition to the `detached` directory only on the local server. +- The `ALTER ... FETCH PARTITION` query isn't replicated. It places the partition to the `detached` directory only on the local server. - The `ALTER TABLE ... ATTACH` query is replicated. It adds the data to all replicas. The data is added to one of the replicas from the `detached` directory, and to the others - from neighboring replicas. Before downloading, the system checks if the partition exists and the table structure matches. The most appropriate replica is selected automatically from the healthy replicas. @@ -336,7 +362,7 @@ You can specify the partition expression in `ALTER ... PARTITION` queries in dif - As a value from the `partition` column of the `system.parts` table. For example, `ALTER TABLE visits DETACH PARTITION 201901`. - As the expression from the table column. Constants and constant expressions are supported. For example, `ALTER TABLE visits DETACH PARTITION toYYYYMM(toDate('2019-01-25'))`. - Using the partition ID. Partition ID is a string identifier of the partition (human-readable, if possible) that is used as the names of partitions in the file system and in ZooKeeper. The partition ID must be specified in the `PARTITION ID` clause, in a single quotes. For example, `ALTER TABLE visits DETACH PARTITION ID '201901'`. -- In the [ALTER ATTACH PART](#alter_attach-partition) query, to specify the name of a part, use a value from the `name` column of the `system.parts` table. For example, `ALTER TABLE visits ATTACH PART 201901_1_1_0`. +- In the [ALTER ATTACH PART](#alter_attach-partition) and [DROP DETACHED PART](#alter_drop-detached) query, to specify the name of a part, use string literal with a value from the `name` column of the [system.detached_parts](../operations/system_tables.md#system_tables-detached_parts) table. For example, `ALTER TABLE visits ATTACH PART '201901_1_1_0'`. Usage of quotes when specifying the partition depends on the type of partition expression. For example, for the `String` type, you have to specify its name in quotes (`'`). For the `Date` and `Int*` types no quotes are needed. diff --git a/docs/en/query_language/create.md b/docs/en/query_language/create.md index e40a452cfe0..8060dfbe09d 100644 --- a/docs/en/query_language/create.md +++ b/docs/en/query_language/create.md @@ -103,6 +103,26 @@ If you add a new column to a table but later change its default expression, the It is not possible to set default values for elements in nested data structures. +### Constraints {#constraints} + +WARNING: This feature is experimental. Correct work is not guaranteed on non-MergeTree family engines. + +Along with columns descriptions constraints could be defined: + +``sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [compression_codec] [TTL expr1], + ... + CONSTRAINT constraint_name_1 CHECK boolean_expr_1, + ... +) ENGINE = engine +``` + +`boolean_expr_1` could by any boolean expression. If constraints are defined for the table, each of them will be checked for every row in `INSERT` query. If any constraint is not satisfied — server will raise an exception with constraint name and checking expression. + +Adding large amount of constraints can negatively affect performance of big `INSERT` queries. + ### TTL expression Defines storage time for values. Can be specified only for MergeTree-family tables. For the detailed description, see [TTL for columns and tables](../operations/table_engines/mergetree.md#table_engine-mergetree-ttl). @@ -144,19 +164,30 @@ These codecs are designed to make compression more effective using specifities o Specialized codecs: -- `Delta(delta_bytes)` — Compression approach, when raw values are replaced with the difference of two neighbor values. Up to `delta_bytes` are used for storing delta value, so `delta_bytes` is a maximum size of raw values. Possible `delta_bytes` values: 1, 2, 4, 8. Default value for `delta_bytes` is `sizeof(type)`, if it is equals to 1, 2, 4, 8. Otherwise it equals 1. -- `DoubleDelta` — Compresses values down to 1 bit (in the best case), using deltas calculation. Best compression rates are achieved on monotonic sequences with constant stride, for example, time series data. Can be used with any fixed-width type. Implements the algorithm used in Gorilla TSDB, extending it to support 64 bit types. Uses 1 extra bit for 32 byte deltas: 5 bit prefix instead of 4 bit prefix. For additional information, see the "Compressing time stamps" section of the [Gorilla: A Fast, Scalable, In-Memory Time Series Database](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf) document. -- `Gorilla` — Compresses values down to 1 bit (in the best case). The codec is efficient when storing series of floating point values that change slowly, because the best compression rate is achieved when neighboring values are binary equal. Implements the algorithm used in Gorilla TSDB, extending it to support 64 bit types. For additional information, see the "Compressing values" section of the [Gorilla: A Fast, Scalable, In-Memory Time Series Database](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf) document. +- `Delta(delta_bytes)` — Compression approach in which raw values are replaced by the difference of two neighboring values, except for the first value that stays unchanged. Up to `delta_bytes` are used for storing delta values, so `delta_bytes` is the maximum size of raw values. Possible `delta_bytes` values: 1, 2, 4, 8. The default value for `delta_bytes` is `sizeof(type)` if equal to 1, 2, 4, or 8. In all other cases, it's 1. +- `DoubleDelta` — Calculates delta of deltas and writes it in compact binary form. Optimal compression rates are achieved for monotonic sequences with a constant stride, such as time series data. Can be used with any fixed-width type. Implements the algorithm used in Gorilla TSDB, extending it to support 64-bit types. Uses 1 extra bit for 32-byte deltas: 5-bit prefixes instead of 4-bit prefixes. For additional information, see Compressing Time Stamps in [Gorilla: A Fast, Scalable, In-Memory Time Series Database](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf). +- `Gorilla` — Calculates XOR between current and previous value and writes it in compact binary form. Efficient when storing a series of floating point values that change slowly, because the best compression rate is achieved when neighboring values are binary equal. Implements the algorithm used in Gorilla TSDB, extending it to support 64-bit types. For additional information, see Compressing Values in [Gorilla: A Fast, Scalable, In-Memory Time Series Database](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf). - `T64` — Compression approach that crops unused high bits of values in integer data types (including `Enum`, `Date` and `DateTime`). At each step of its algorithm, codec takes a block of 64 values, puts them into 64x64 bit matrix, transposes it, crops the unused bits of values and returns the rest as a sequence. Unused bits are the bits, that don't differ between maximum and minimum values in the whole data part for which the compression is used. +`DoubleDelta` and `Gorilla` codecs are used in Gorilla TSDB as the components of its compressing algorithm. Gorilla approach is effective in scenarios when there is a sequence of slowly changing values with their timestamps. Timestamps are effectively compressed by the `DoubleDelta` codec, and values are effectively compressed by the `Gorilla` codec. For example, to get an effectively stored table, you can create it in the following configuration: + +```sql +CREATE TABLE codec_example +( + timestamp DateTime CODEC(DoubleDelta), + slow_values Float32 CODEC(Gorilla) +) +ENGINE = MergeTree() +``` + ### Common purpose codecs {#create-query-common-purpose-codecs} Codecs: - `NONE` — No compression. - `LZ4` — Lossless [data compression algorithm](https://github.com/lz4/lz4) used by default. Applies LZ4 fast compression. -- `LZ4HC[(level)]` — LZ4 CH (high compression) algorithm with configurable level. Default level: 9. If you set `level <= 0`, the default level is applied. Possible levels: [1, 12]. Recommended levels are in range: [4, 9]. -- `ZSTD[(level)]` — [ZSTD compression algorithm](https://en.wikipedia.org/wiki/Zstandard) with configurable `level`. Possible levels: [1, 22]. Default level: 1. +- `LZ4HC[(level)]` — LZ4 HC (high compression) algorithm with configurable level. Default level: 9. Setting `level <= 0` applies the default level. Possible levels: [1, 12]. Recommended level range: [4, 9]. +- `ZSTD[(level)]` — [ZSTD compression algorithm](https://en.wikipedia.org/wiki/Zstandard) with configurable `level`. Possible levels: [1, 22]. Default value: 1. High compression levels useful for asymmetric scenarios, like compress once, decompress a lot of times. Greater levels stands for better compression and higher CPU usage. diff --git a/docs/en/query_language/dicts/external_dicts_dict_layout.md b/docs/en/query_language/dicts/external_dicts_dict_layout.md index 03279688d6c..c3096544d25 100644 --- a/docs/en/query_language/dicts/external_dicts_dict_layout.md +++ b/docs/en/query_language/dicts/external_dicts_dict_layout.md @@ -95,7 +95,7 @@ Configuration example: The dictionary is stored in memory in the form of a hash table with an ordered array of ranges and their corresponding values. -This storage method works the same way as hashed and allows using date/time ranges in addition to the key, if they appear in the dictionary. +This storage method works the same way as hashed and allows using date/time (arbitrary numeric type) ranges in addition to the key. Example: The table contains discounts for each advertiser in the format: @@ -111,7 +111,7 @@ Example: The table contains discounts for each advertiser in the format: +---------------+---------------------+-------------------+--------+ ``` -To use a sample for date ranges, define the `range_min` and `range_max` elements in the [structure](external_dicts_dict_structure.md). +To use a sample for date ranges, define the `range_min` and `range_max` elements in the [structure](external_dicts_dict_structure.md). These elements must contain elements `name` and` type` (if `type` is not specified, the default type will be used - Date). `type` can be any numeric type (Date / DateTime / UInt64 / Int32 / others). Example: @@ -122,14 +122,16 @@ Example: first + Date last + Date ... ``` -To work with these dictionaries, you need to pass an additional date argument to the `dictGetT` function: +To work with these dictionaries, you need to pass an additional argument to the `dictGetT` function, for which a range is selected: ``` dictGetT('dict_name', 'attr_name', id, date) @@ -160,10 +162,12 @@ Configuration example: Abcdef - StartDate + StartTimeStamp + UInt64 - EndDate + EndTimeStamp + UInt64 XXXType diff --git a/docs/en/query_language/functions/other_functions.md b/docs/en/query_language/functions/other_functions.md index 598ac0d193a..349397059af 100644 --- a/docs/en/query_language/functions/other_functions.md +++ b/docs/en/query_language/functions/other_functions.md @@ -314,6 +314,49 @@ Returns the ordinal number of the row in the data block. Different data blocks a Returns the ordinal number of the row in the data block. This function only considers the affected data blocks. +## neighbor(column, offset\[, default_value\]) + +Returns value for `column`, in `offset` distance from current row. +This function is a partial implementation of [window functions](https://en.wikipedia.org/wiki/SQL_window_function) LEAD() and LAG(). + +The result of the function depends on the affected data blocks and the order of data in the block. +If you make a subquery with ORDER BY and call the function from outside the subquery, you can get the expected result. + +If `offset` value is outside block bounds, a default value for `column` returned. If `default_value` is given, then it will be used. +This function can be used to compute year-over-year metric value: + +``` sql +WITH toDate('2018-01-01') AS start_date +SELECT + toStartOfMonth(start_date + (number * 32)) AS month, + toInt32(month) % 100 AS money, + neighbor(money, -12) AS prev_year, + round(prev_year / money, 2) AS year_over_year +FROM numbers(16) +``` + +``` +┌──────month─┬─money─┬─prev_year─┬─year_over_year─┐ +│ 2018-01-01 │ 32 │ 0 │ 0 │ +│ 2018-02-01 │ 63 │ 0 │ 0 │ +│ 2018-03-01 │ 91 │ 0 │ 0 │ +│ 2018-04-01 │ 22 │ 0 │ 0 │ +│ 2018-05-01 │ 52 │ 0 │ 0 │ +│ 2018-06-01 │ 83 │ 0 │ 0 │ +│ 2018-07-01 │ 13 │ 0 │ 0 │ +│ 2018-08-01 │ 44 │ 0 │ 0 │ +│ 2018-09-01 │ 75 │ 0 │ 0 │ +│ 2018-10-01 │ 5 │ 0 │ 0 │ +│ 2018-11-01 │ 36 │ 0 │ 0 │ +│ 2018-12-01 │ 66 │ 0 │ 0 │ +│ 2019-01-01 │ 97 │ 32 │ 0.33 │ +│ 2019-02-01 │ 28 │ 63 │ 2.25 │ +│ 2019-03-01 │ 56 │ 91 │ 1.62 │ +│ 2019-04-01 │ 87 │ 22 │ 0.25 │ +└────────────┴───────┴───────────┴────────────────┘ +``` + + ## runningDifference(x) {#other_functions-runningdifference} Calculates the difference between successive row values ​​in the data block. diff --git a/docs/en/query_language/functions/url_functions.md b/docs/en/query_language/functions/url_functions.md index a21c2cfa0bf..f4ff63ce021 100644 --- a/docs/en/query_language/functions/url_functions.md +++ b/docs/en/query_language/functions/url_functions.md @@ -2,17 +2,19 @@ All these functions don't follow the RFC. They are maximally simplified for improved performance. -## Functions that extract part of a URL +## Functions that Extract Parts of a URL -If there isn't anything similar in a URL, an empty string is returned. +If the relevant part isn't present in a URL, an empty string is returned. ### protocol -Returns the protocol. Examples: http, ftp, mailto, magnet... +Extracts the protocol from a URL. + +Examples of typical returned values: http, https, ftp, mailto, tel, magnet... ### domain -Extracts the host part from URL. +Extracts the hostname from a URL. ``` domain(url) @@ -23,7 +25,7 @@ domain(url) - `url` — URL. Type: [String](../../data_types/string.md). -URL can be specified with or without scheme. Examples: +The URL can be specified with or without a scheme. Examples: ``` svn+ssh://some.svn-hosting.com:80/repo/trunk @@ -31,10 +33,18 @@ some.svn-hosting.com:80/repo/trunk https://yandex.com/time/ ``` +For these examples, the `domain` function returns the following results: + +``` +some.svn-hosting.com +some.svn-hosting.com +yandex.com +``` + **Returned values** -- Host name. If ClickHouse can parse input string as URL. -- Empty string. If ClickHouse cannot parse input string as URL. +- Host name. If ClickHouse can parse the input string as a URL. +- Empty string. If ClickHouse can't parse the input string as a URL. Type: `String`. @@ -55,7 +65,7 @@ Returns the domain and removes no more than one 'www.' from the beginning of it, ### topLevelDomain -Extracts the the top-level domain from URL. +Extracts the the top-level domain from a URL. ``` topLevelDomain(url) @@ -65,7 +75,7 @@ topLevelDomain(url) - `url` — URL. Type: [String](../../data_types/string.md). -URL can be specified with or without scheme. Examples: +The URL can be specified with or without a scheme. Examples: ``` svn+ssh://some.svn-hosting.com:80/repo/trunk @@ -75,8 +85,8 @@ https://yandex.com/time/ **Returned values** -- Domain name. If ClickHouse can parse input string as URL. -- Empty string. If ClickHouse cannot parse input string as URL. +- Domain name. If ClickHouse can parse the input string as a URL. +- Empty string. If ClickHouse cannot parse the input string as a URL. Type: `String`. diff --git a/docs/en/query_language/insert_into.md b/docs/en/query_language/insert_into.md index 914c3b2917f..c0cb9f8c3b1 100644 --- a/docs/en/query_language/insert_into.md +++ b/docs/en/query_language/insert_into.md @@ -40,6 +40,9 @@ INSERT INTO t FORMAT TabSeparated You can insert data separately from the query by using the command-line client or the HTTP interface. For more information, see the section "[Interfaces](../interfaces/index.md#interfaces)". +### Constraints + +If table has [constraints](create.md#constraints), their expressions will be checked for each row of inserted data. If any of those constraints is not satisfied — server will raise an exception containing constraint name and expression, the query will be stopped. ### Inserting The Results of `SELECT` {#insert_query_insert-select} diff --git a/docs/en/query_language/misc.md b/docs/en/query_language/misc.md index 514f5d9f823..08e8f819b8c 100644 --- a/docs/en/query_language/misc.md +++ b/docs/en/query_language/misc.md @@ -201,7 +201,7 @@ All tables are renamed under global locking. Renaming tables is a light operatio SET param = value ``` -Assigns `value` to the `param` configurations settings for the current session. You cannot change [server settings](../operations/server_settings/index.md) this way. +Assigns `value` to the `param` [setting](../operations/settings/index.md) for the current session. You cannot change [server settings](../operations/server_settings/index.md) this way. You can also set all the values from the specified settings profile in a single query. diff --git a/docs/ru/data_types/enum.md b/docs/ru/data_types/enum.md index 7ed0150e65f..8d32cce1648 100644 --- a/docs/ru/data_types/enum.md +++ b/docs/ru/data_types/enum.md @@ -1,27 +1,32 @@ +# Enum -# Enum8, Enum16 +Перечисляемый тип данных, содержащий именованные значения. -Включает в себя типы `Enum8` и `Enum16`. `Enum` сохраняет конечный набор пар `'строка' = целое число`. Все операции с данными типа `Enum` ClickHouse выполняет как с числами, однако пользователь при этом работает со строковыми константами. Это более эффективно с точки зрения производительности, чем работа с типом данных `String`. +Именованные значения задаются парами `'string' = integer`. ClickHouse хранить только числа, но допускает операции над ними с помощью заданных имён. -- `Enum8` описывается парами `'String' = Int8`. -- `Enum16` описывается парами `'String' = Int16`. +ClickHouse поддерживает: -## Примеры применения +- 8-битный `Enum`. Может содержать до 256 значений, пронумерованных в диапазоне `[-128, 127]`. +- 16-битный `Enum`. Может содержать до 65536 значений, пронумерованных в диапазоне `[-32768, 32767]`. + +ClickHouse автоматически выбирает размерность `Enum` при вставке данных. Чтобы точно понимать размер хранимых данных можно использовать типы `Enum8` или `Enum16`. + +## Примеры использования Создадим таблицу со столбцом типа `Enum8('hello' = 1, 'world' = 2)`. -``` +```sql CREATE TABLE t_enum ( - x Enum8('hello' = 1, 'world' = 2) + x Enum('hello' = 1, 'world' = 2) ) ENGINE = TinyLog ``` -В столбец `x` можно сохранять только значения, перечисленные при определении типа, т.е. `'hello'` или `'world'`. Если попытаться сохранить другое значение, ClickHouse сгенерирует исключение. +В столбец `x` можно сохранять только значения, перечисленные при определении типа, т.е. `'hello'` или `'world'`. Если вы попытаетесь сохранить любое другое значение, ClickHouse сгенерирует исключение. ClickHouse автоматически выберет размерность 8-bit для этого `Enum`. -``` -:) INSERT INTO t_enum Values('hello'),('world'),('hello') +```sql +:) INSERT INTO t_enum VALUES ('hello'), ('world'), ('hello') INSERT INTO t_enum VALUES @@ -35,12 +40,12 @@ INSERT INTO t_enum VALUES Exception on client: -Code: 49. DB::Exception: Unknown element 'a' for type Enum8('hello' = 1, 'world' = 2) +Code: 49. DB::Exception: Unknown element 'a' for type Enum('hello' = 1, 'world' = 2) ``` При запросе данных из таблицы ClickHouse выдаст строковые значения из `Enum`. -``` +```sql SELECT * FROM t_enum ┌─x─────┐ @@ -49,10 +54,11 @@ SELECT * FROM t_enum │ hello │ └───────┘ ``` -Если необходимо увидеть цифровые эквиваленты строкам, то необходимо привести тип. -``` -SELECT CAST(x, 'Int8') FROM t_enum +Если необходимо увидеть цифровые эквиваленты строкам, то необходимо привести тип `Enum` к целочисленному. + +```sql +SELECT CAST(x AS Int8) FROM t_enum ┌─CAST(x, 'Int8')─┐ │ 1 │ @@ -61,14 +67,14 @@ SELECT CAST(x, 'Int8') FROM t_enum └─────────────────┘ ``` -Чтобы создать значение типа Enum в запросе, также необходима функция `CAST`. +Чтобы создать значение типа `Enum` в запросе, также необходимо использовать функцию `CAST`. -``` -SELECT toTypeName(CAST('a', 'Enum8(\'a\' = 1, \'b\' = 2)')) +```sql +SELECT toTypeName(CAST('a', 'Enum(\'a\' = 1, \'b\' = 2)')) -┌─toTypeName(CAST('a', 'Enum8(\'a\' = 1, \'b\' = 2)'))─┐ -│ Enum8('a' = 1, 'b' = 2) │ -└──────────────────────────────────────────────────────┘ +┌─toTypeName(CAST('a', 'Enum(\'a\' = 1, \'b\' = 2)'))─┐ +│ Enum8('a' = 1, 'b' = 2) │ +└─────────────────────────────────────────────────────┘ ``` ## Общие правила и особенности использования @@ -90,7 +96,7 @@ ENGINE = TinyLog , то в ней можно будет хранить не только `'hello'` и `'world'`, но и `NULL`. ``` -INSERT INTO t_enum_null Values('hello'),('world'),(NULL) +INSERT INTO t_enum_nullable Values('hello'),('world'),(NULL) ``` В оперативке столбец типа `Enum` представлен так же, как `Int8` или `Int16` соответствующими числовыми значениями. diff --git a/docs/ru/interfaces/cli.md b/docs/ru/interfaces/cli.md index 0f3ce84345e..59980109240 100644 --- a/docs/ru/interfaces/cli.md +++ b/docs/ru/interfaces/cli.md @@ -54,8 +54,7 @@ cat file.csv | clickhouse-client --database=test --query="INSERT INTO test FORMA По умолчанию, в качестве формата, используется формат PrettyCompact (красивые таблички). Вы можете изменить формат с помощью секции FORMAT запроса, или с помощью указания `\G` на конце запроса, с помощью аргумента командной строки `--format` или `--vertical`, или с помощью конфигурационного файла клиента. -Чтобы выйти из клиента, нажмите Ctrl+D (или Ctrl+C), или наберите вместо запроса одно из: -"exit", "quit", "logout", "учше", "йгше", "дщпщге", "exit;", "quit;", "logout;", "учшеж", "йгшеж", "дщпщгеж", "q", "й", "q", "Q", ":q", "й", "Й", "Жй" +Чтобы выйти из клиента, нажмите Ctrl+D (или Ctrl+C), или наберите вместо запроса одно из: "exit", "quit", "logout", "учше", "йгше", "дщпщге", "exit;", "quit;", "logout;", "учшеж", "йгшеж", "дщпщгеж", "q", "й", "q", "Q", ":q", "й", "Й", "Жй" При выполнении запроса, клиент показывает: @@ -68,38 +67,64 @@ cat file.csv | clickhouse-client --database=test --query="INSERT INTO test FORMA Клиент командной строки позволяет передать внешние данные (внешние временные таблицы) для использования запроса. Подробнее смотрите раздел "Внешние данные для обработки запроса" +### Запросы с параметрами {#cli-queries-with-parameters} + +Вы можете создать запрос с параметрами и передавать в них значения из приложения. Это позволяет избежать форматирования запросов на стороне клиента, если известно, какие из параметров запроса динамически меняются. Например: + +```bash +clickhouse-client --param_parName="[1, 2]" -q "SELECT * FROM table WHERE a = {parName:Array(UInt16)}" +``` + +#### Cинтаксис запроса {#cli-queries-with-parameters-syntax} + +Отформатируйте запрос обычным способом. Представьте значения, которые вы хотите передать из параметров приложения в запрос в следующем формате: + +``` +{:} +``` + +- `name` — идентификатор подстановки. В консольном клиенте его следует использовать как часть имени параметра `--param_ = value`. +- `data type` — [тип данных](../data_types/index.md) значения. Например, структура данных `(integer, ('string', integer))` может иметь тип данных `Tuple(UInt8, Tuple(String, UInt8))` ([целочисленный](../data_types/int_uint.md) тип может быть и другим). + +#### Пример + +```bash +clickhouse-client --param_tuple_in_tuple="(10, ('dt', 10))" -q "SELECT * FROM table WHERE val = {tuple_in_tuple:Tuple(UInt8, Tuple(String, UInt8))}" +``` + ## Конфигурирование {#interfaces_cli_configuration} В `clickhouse-client` можно передавать различные параметры (все параметры имеют значения по умолчанию) с помощью: - - Командной строки. - - Параметры командной строки переопределяют значения по умолчанию и параметры конфигурационных файлов. - - - Конфигурационных файлов. - +- Командной строки. + + Параметры командной строки переопределяют значения по умолчанию и параметры конфигурационных файлов. + +- Конфигурационных файлов. + Параметры в конфигурационных файлах переопределяют значения по умолчанию. ### Параметры командной строки -- `--host, -h` - имя сервера, по умолчанию - localhost. Вы можете использовать как имя, так и IPv4 или IPv6 адрес. -- `--port` - порт, к которому соединяться, по умолчанию - 9000. Замечу, что для HTTP и родного интерфейса используются разные порты. -- `--user, -u` - имя пользователя, по умолчанию - default. -- `--password` - пароль, по умолчанию - пустая строка. -- `--query, -q` - запрос для выполнения, при использовании в неинтерактивном режиме. -- `--database, -d` - выбрать текущую БД, по умолчанию - текущая БД из настроек сервера (по умолчанию - БД default). -- `--multiline, -m` - если указано - разрешить многострочные запросы, не отправлять запрос по нажатию Enter. -- `--multiquery, -n` - если указано - разрешить выполнять несколько запросов, разделённых точкой с запятой. Работает только в неинтерактивном режиме. -- `--format, -f` - использовать указанный формат по умолчанию для вывода результата. -- `--vertical, -E` - если указано, использовать формат Vertical по умолчанию для вывода результата. То же самое, что --format=Vertical. В этом формате каждое значение выводится на отдельной строке, что удобно для отображения широких таблиц. -- `--time, -t` - если указано, в неинтерактивном режиме вывести время выполнения запроса в stderr. -- `--stacktrace` - если указано, в случае исключения, выводить также его стек трейс. -- `--config-file` - имя конфигурационного файла. -- `--secure` - если указано, будет использован безопасный канал. +- `--host, -h` — имя сервера, по умолчанию — localhost. Вы можете использовать как имя, так и IPv4 или IPv6 адрес. +- `--port` — порт, к которому соединяться, по умолчанию — 9000. Замечу, что для HTTP и родного интерфейса используются разные порты. +- `--user, -u` — имя пользователя, по умолчанию — default. +- `--password` — пароль, по умолчанию — пустая строка. +- `--query, -q` — запрос для выполнения, при использовании в неинтерактивном режиме. +- `--database, -d` — выбрать текущую БД, по умолчанию — текущая БД из настроек сервера (по умолчанию — БД default). +- `--multiline, -m` — если указано — разрешить многострочные запросы, не отправлять запрос по нажатию Enter. +- `--multiquery, -n` — если указано — разрешить выполнять несколько запросов, разделённых точкой с запятой. Работает только в неинтерактивном режиме. +- `--format, -f` — использовать указанный формат по умолчанию для вывода результата. +- `--vertical, -E` — если указано, использовать формат Vertical по умолчанию для вывода результата. То же самое, что --format=Vertical. В этом формате каждое значение выводится на отдельной строке, что удобно для отображения широких таблиц. +- `--time, -t` — если указано, в неинтерактивном режиме вывести время выполнения запроса в stderr. +- `--stacktrace` — если указано, в случае исключения, выводить также его стек трейс. +- `--config-file` — имя конфигурационного файла. +- `--secure` — если указано, будет использован безопасный канал. +- `--param_` — значение параметра для [запроса с параметрами](#cli-queries-with-parameters). ### Конфигурационные файлы -`clickhouse-client` использует первый существующий файл из: +`clickhouse—client` использует первый существующий файл из: - Определенного параметром `--config-file`. - `./clickhouse-client.xml` diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index 20c919665d5..130a32d63fa 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -145,7 +145,7 @@ SELECT * FROM t_null FORMAT TSKV x=1 y=\N ``` -При большом количестве маленьких столбцов, этот формат существенно неэффективен, и обычно нет причин его использовать. Он реализован, так как используется в некоторых отделах Яндекса. +При большом количестве маленьких столбцов, этот формат существенно неэффективен, и обычно нет причин его использовать. Впрочем, он не хуже формата JSONEachRow по производительности. Поддерживается как вывод, так и парсинг данных в этом формате. При парсинге, поддерживается расположение значений разных столбцов в произвольном порядке. Допустимо отсутствие некоторых значений - тогда они воспринимаются как равные значениям по умолчанию. В этом случае в качестве значений по умолчанию используются нули и пустые строки. Сложные значения, которые могут быть заданы в таблице не поддерживаются как значения по умолчанию. diff --git a/docs/ru/interfaces/http.md b/docs/ru/interfaces/http.md index 20067942dd8..77eb984e8f4 100644 --- a/docs/ru/interfaces/http.md +++ b/docs/ru/interfaces/http.md @@ -245,5 +245,15 @@ curl -sS 'http://localhost:8123/?max_result_bytes=4000000&buffer_size=3000000&wa Буферизация позволяет избежать ситуации когда код ответа и HTTP-заголовки были отправлены клиенту, после чего возникла ошибка выполнения запроса. В такой ситуации сообщение об ошибке записывается в конце тела ответа, и на стороне клиента ошибка может быть обнаружена только на этапе парсинга. +### Запросы с параметрами {#cli-queries-with-parameters} + +Можно создать запрос с параметрами и передать для них значения из соответствующих параметров HTTP-запроса. Дополнительную информацию смотрите в [Запросы с параметрами для консольного клиента](cli.md#cli-queries-with-parameters). + +### Пример + +```bash +curl -sS "
?param_id=2¶m_phrase=test" -d "SELECT * FROM table WHERE int_column = {id:UInt8} and string_column = {phrase:String}" +``` + [Оригинальная статья](https://clickhouse.yandex/docs/ru/interfaces/http_interface/) diff --git a/docs/ru/operations/system_tables.md b/docs/ru/operations/system_tables.md index 707f74c0b22..bcf61a7f0da 100644 --- a/docs/ru/operations/system_tables.md +++ b/docs/ru/operations/system_tables.md @@ -87,6 +87,12 @@ user String — имя пользователя, которого использ Для каждой базы данных, о которой знает сервер, будет присутствовать соответствующая запись в таблице. Эта системная таблица используется для реализации запроса `SHOW DATABASES`. +## system.detached_parts {#system_tables-detached_parts} + +Содержит информацию об отсоединённых кусках таблиц семейства [MergeTree](table_engines/mergetree.md). Столбец `reason` содержит причину, по которой кусок был отсоединён. Для кусов, отсоединённых пользователем, `reason` содержит пустую строку. +Такие куски могут быть присоединены с помощью [ALTER TABLE ATTACH PARTITION|PART](../query_language/query_language/alter/#alter_attach-partition). Остальные столбцы описаны в [system.parts](#system_tables-parts). +Если имя куска некорректно, значения некоторых столбцов могут быть `NULL`. Такие куски могут быть удалены с помощью [ALTER TABLE DROP DETACHED PART](../query_language/query_language/alter/#alter_drop-detached). + ## system.dictionaries Содержит информацию о внешних словарях. diff --git a/docs/ru/operations/table_engines/collapsingmergetree.md b/docs/ru/operations/table_engines/collapsingmergetree.md index f09d84cf8e8..59fdf66324f 100644 --- a/docs/ru/operations/table_engines/collapsingmergetree.md +++ b/docs/ru/operations/table_engines/collapsingmergetree.md @@ -2,7 +2,7 @@ Движок наследует функциональность от [MergeTree](mergetree.md) и добавляет в алгоритм слияния кусков данных логику сворачивания (удаления) строк. -`CollapsingMergeTree` асинхронно удаляет (сворачивает) пары строк, если все поля в строке эквивалентны, за исключением специального поля `Sign`, которое может принимать значения `1` и `-1`. Строки без пары сохраняются. Подробнее смотрите раздел [Сворачивание (удаление) строк](#table_engine-collapsingmergetree-collapsing). +`CollapsingMergeTree` асинхронно удаляет (сворачивает) пары строк, если все поля в ключе сортировки (`ORDER BY`) эквивалентны, за исключением специального поля `Sign`, которое может принимать значения `1` и `-1`. Строки без пары сохраняются. Подробнее смотрите в разделе [Сворачивание (удаление) строк](#table_engine-collapsingmergetree-collapsing). Движок может значительно уменьшить объем хранения и, как следствие, повысить эффективность запросов `SELECT`. @@ -30,7 +30,8 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] Тип данных столбца — `Int8`. **Секции запроса** -При создании таблицы `CollapsingMergeTree` используются те же [секции](mergetree.md#table_engine-mergetree-creating-a-table) запроса, что и при создании таблицы `MergeTree`. + +При создании таблицы с движком `CollapsingMergeTree` используются те же [секции запроса](mergetree.md#table_engine-mergetree-creating-a-table) что и при создании таблицы с движком `MergeTree`.
Устаревший способ создания таблицы @@ -79,7 +80,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] └─────────────────────┴───────────┴──────────┴──────┘ ``` -Первая строка отменяет предыдущее состояние объекта (пользователя). Она должен повторять все поля отменённого состояния за исключением `Sign`. +Первая строка отменяет предыдущее состояние объекта (пользователя). Она должен повторять все поля из ключа сортировки для отменённого состояния за исключением `Sign`. Вторая строка содержит текущее состояние. @@ -94,34 +95,34 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] можно удалить, сворачивая (удаляя) устаревшее состояние объекта. `CollapsingMergeTree` выполняет это при слиянии кусков данных. -Зачем нужны 2 строки для каждого изменения, читайте в параграфе [Алгоритм](#table_engine-collapsingmergetree-collapsing-algorithm). +Зачем нужны две строки для каждого изменения описано в разделе [Алгоритм](#table_engine-collapsingmergetree-collapsing-algorithm). **Особенности подхода** -1. Программа, которая записывает данные, должна помнить состояние объекта, чтобы иметь возможность отменить его. Строка отмены состояния должна быть копией предыдущей строки состояния с противоположным значением `Sign`. Это увеличивает начальный размер хранилища, но позволяет быстро записывать данные. +1. Программа, которая записывает данные, должна помнить состояние объекта, чтобы иметь возможность отменить его. Строка отмены состояния должна содержать копию полей сортировочного ключа предыдущей строки состояния с противоположным значением `Sign`. Это увеличивает начальный размер хранилища, но позволяет быстро записывать данные. 2. Длинные растущие массивы в Столбцах снижают эффективность работы движка за счёт нагрузки на запись. Чем проще данные, тем выше эффективность. 3. Результаты запроса `SELECT` сильно зависят от согласованности истории изменений объекта. Будьте точны при подготовке данных для вставки. Можно получить непредсказуемые результаты для несогласованных данных, например отрицательные значения для неотрицательных метрик, таких как глубина сеанса. ### Алгоритм {#table_engine-collapsingmergetree-collapsing-algorithm} -Когда ClickHouse объединяет куски данных, каждая группа последовательных строк с одним и тем же первичным ключом уменьшается до не более чем двух строк, одна из которых имеет `Sign = 1` (строка состояния), а другая строка с `Sign = -1` (строка отмены состояния). Другими словами, записи сворачиваются. +Во время объединения кусков данных, каждая группа последовательных строк с одинаковым сортировочным ключом (`ORDER BY`) уменьшается до не более чем двух строк, одна из которых имеет `Sign = 1` (строка состояния), а другая строка с `Sign = -1` (строка отмены состояния). Другими словами, записи сворачиваются. Для каждого результирующего куска данных ClickHouse сохраняет: 1. Первую строку отмены состояния и последнюю строку состояния, если количество строк обоих видов совпадает. - 1. Последнюю строку состояния, если строк состояния на одну больше, чем строк отмены состояния. + 2. Последнюю строку состояния, если строк состояния на одну больше, чем строк отмены состояния. - 1. Первую строку отмены состояния, если их на одну больше, чем строк состояния. + 3. Первую строку отмены состояния, если их на одну больше, чем строк состояния. - 1. Ни в одну из строк во всех остальных случаях. + 4. Ни в одну из строк во всех остальных случаях. Слияние продолжается, но ClickHouse рассматривает эту ситуацию как логическую ошибку и записывает её в журнал сервера. Эта ошибка может возникать, если одни и те же данные вставлялись несколько раз. Как видно, от сворачивания не должны меняться результаты расчётов статистик. Изменения постепенно сворачиваются так, что остаются лишь последнее состояние почти каждого объекта. -Столбец `Sign` необходим, поскольку алгоритм слияния не гарантирует, что все строки с одинаковым первичным ключом будут находиться в одном результирующем куске данных и даже на одном физическом сервере. ClickHouse выполняет запросы `SELECT` несколькими потоками, и он не может предсказать порядок строк в результате. Если необходимо получить полностью свёрнутые данные из таблицы `CollapsingMergeTree`, то необходимо агрегирование. +Столбец `Sign` необходим, поскольку алгоритм слияния не гарантирует, что все строки с одинаковым ключом сортировки будут находиться в одном результирующем куске данных и даже на одном физическом сервере. ClickHouse выполняет запросы `SELECT` несколькими потоками, и он не может предсказать порядок строк в результате. Если необходимо получить полностью свёрнутые данные из таблицы `CollapsingMergeTree`, то необходимо агрегирование. Для завершения свертывания добавьте в запрос секцию`GROUP BY` и агрегатные функции, которые учитывают знак. Например, для расчета количества используйте `sum(Sign)` вместо`count()`. Чтобы вычислить сумму чего-либо, используйте `sum(Sign * x)` вместо`sum(х)`, и так далее, а также добавьте `HAVING sum(Sign) > 0` . @@ -131,7 +132,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ## Пример использования -Example data: +Исходные данные: ``` ┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ @@ -220,3 +221,4 @@ SELECT * FROM UAct FINAL Такой способ выбора данных очень неэффективен. Не используйте его для больших таблиц. [Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/collapsingmergetree/) + diff --git a/docs/ru/operations/table_engines/index.md b/docs/ru/operations/table_engines/index.md index cf38d90b55f..ce414a3c0d1 100644 --- a/docs/ru/operations/table_engines/index.md +++ b/docs/ru/operations/table_engines/index.md @@ -9,8 +9,69 @@ - Возможно ли многопоточное выполнение запроса. - Параметры репликации данных. -При чтении, движок обязан лишь выдать запрошенные столбцы, но в некоторых случаях движок может частично обрабатывать данные при ответе на запрос. +## Семейства движков -Для большинства серьёзных задач, следует использовать движки семейства `MergeTree`. +### MergeTree + +Наиболее универсальные и функциональные движки таблиц для задач с высокой загрузкой. Общим свойством этих движков является быстрая вставка данных с последующей фоновой обработкой данных. Движки `*MergeTree` поддерживают репликацию данных (в [Replicated*](replication.md) версиях движков), партиционирование, и другие возможности не поддержанные для других движков. + +Движки семейства: + +- [MergeTree](mergetree.md) +- [ReplacingMergeTree](replacingmergetree.md) +- [SummingMergeTree](summingmergetree.md) +- [AggregatingMergeTree](aggregatingmergetree.md) +- [CollapsingMergeTree](collapsingmergetree.md) +- [VersionedCollapsingMergeTree](versionedcollapsingmergetree.md) +- [GraphiteMergeTree](graphitemergetree.md) + +### Log + +Простые [движки](log_family.md) с минимальной функциональностью. Они наиболее эффективны, когда вам нужно быстро записать много небольших таблиц (до примерно 1 миллиона строк) и прочитать их позже целиком. + +Движки семейства: + +- [TinyLog](tinylog.md) +- [StripeLog](stripelog.md) +- [Log](log.md) + +### Движки для интергации + +Движки для связи с другими системами хранения и обработки данных. + +Движки семейства: + +- [Kafka](kafka.md) +- [MySQL](mysql.md) +- [ODBC](odbc.md) +- [JDBC](jdbc.md) + +### Специальные движки + +Движки семейства: + +- [Distributed](distributed.md) +- [MaterializedView](materializedview.md) +- [Dictionary](dictionary.md) +- [Merge](merge.md) +- [File](file.md) +- [Null](null.md) +- [Set](set.md) +- [Join](join.md) +- [URL](url.md) +- [View](view.md) +- [Memory](memory.md) +- [Buffer](buffer.md) + +## Виртуальные столбцы {#table_engines-virtual_columns} + +Виртуальный столбец — это неотъемлемый атрибут движка таблиц, определенный в исходном коде движка. + +Виртуальные столбцы не надо указывать в запросе `CREATE TABLE` и их не отображаются в результатах запросов `SHOW CREATE TABLE` и `DESCRIBE TABLE`. Также виртуальные столбцы доступны только для чтения, поэтому вы не можете вставлять в них данные. + +Чтобы получить данные из виртуального столбца, необходимо указать его название в запросе `SELECT`. `SELECT *` не отображает данные из виртуальных столбцов. + +При создании таблицы со столбцом, имя которого совпадает с именем одного из виртуальных столбцов таблицы, виртуальный столбец становится недоступным. Не делайте так. Чтобы помочь избежать конфликтов, имена виртуальных столбцов обычно предваряются подчеркиванием. [Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/) + diff --git a/docs/ru/operations/table_engines/kafka.md b/docs/ru/operations/table_engines/kafka.md index 086d4fb4f08..f2318d824e2 100644 --- a/docs/ru/operations/table_engines/kafka.md +++ b/docs/ru/operations/table_engines/kafka.md @@ -152,4 +152,17 @@ Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format В документе [librdkafka configuration reference](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md) можно увидеть список возможных опций конфигурации. Используйте подчеркивание (`_`) вместо точки в конфигурации ClickHouse. Например, `check.crcs=true` будет соответствовать `true`. +## Виртуальные столбцы + +- `_topic` — топик Kafka. +- `_key` — ключ сообщения. +- `_offset` — оффсет сообщения. +- `_timestamp` — временная метка сообщения. +- `_partition` — секция топика Kafka. + +**Смотрите также** + +- [Виртуальные столбцы](index.md#table_engines-virtual_columns) + [Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/kafka/) + diff --git a/docs/ru/operations/table_engines/merge.md b/docs/ru/operations/table_engines/merge.md index 28065e5d120..b87bfdafd75 100644 --- a/docs/ru/operations/table_engines/merge.md +++ b/docs/ru/operations/table_engines/merge.md @@ -52,18 +52,12 @@ FROM WatchLog ## Виртуальные столбцы -Виртуальные столбцы — столбцы, предоставляемые движком таблиц независимо от определения таблицы. То есть, такие столбцы не указываются в `CREATE TABLE`, но доступны для `SELECT`. +- `_table` — содержит имя таблицы, из которой данные были прочитаны. Тип — [String](../../data_types/string.md). -Виртуальные столбцы отличаются от обычных следующими особенностями: + В секции `WHERE/PREWHERE` можно установить константное условие на столбец `_table` (например, `WHERE _table='xyz'`). В этом случае операции чтения выполняются только для тех таблиц, для которых выполняется условие на значение `_table`, таким образом, столбец `_table` работает как индекс. -- они не указываются в определении таблицы; -- в них нельзя вставить данные при `INSERT`; -- при `INSERT` без указания списка столбцов виртуальные столбцы не учитываются; -- они не выбираются при использовании звёздочки (`SELECT *`); -- виртуальные столбцы не показываются в запросах `SHOW CREATE TABLE` и `DESC TABLE`; +**Смотрите также** -Таблица типа `Merge` содержит виртуальный столбец `_table` типа `String`. (Если в таблице уже есть столбец `_table`, то виртуальный столбец называется `_table1`; если уже есть `_table1`, то `_table2` и т. п.) Он содержит имя таблицы, из которой были прочитаны данные. - -Если секция `WHERE/PREWHERE` содержит (в качестве одного из элементов конъюнкции или в качестве всего выражения) условия на столбец `_table`, не зависящие от других столбцов таблицы, то эти условия используются как индекс: условия выполняются над множеством имён таблиц, из которых нужно читать данные, и чтение будет производиться только из тех таблиц, для которых условия сработали. +- [Виртуальные столбцы](index.md#table_engines-virtual_columns) [Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/merge/) diff --git a/docs/ru/query_language/alter.md b/docs/ru/query_language/alter.md index a172134fd16..c078d35f805 100644 --- a/docs/ru/query_language/alter.md +++ b/docs/ru/query_language/alter.md @@ -29,7 +29,7 @@ ALTER TABLE [db].name [ON CLUSTER cluster] ADD|DROP|CLEAR|COMMENT|MODIFY COLUMN ADD COLUMN [IF NOT EXISTS] name [type] [default_expr] [AFTER name_after] ``` -Добавляет в таблицу новый столбец с именем `name`, типом `type` и выражением для умолчания `default_expr` (смотрите раздел [Значения по умолчанию](create.md#create-default-values)). +Добавляет в таблицу новый столбец с именем `name`, типом `type` и выражением для умолчания `default_expr` (смотрите раздел [Значения по умолчанию](create.md#create-default-values)). Если указано `IF NOT EXISTS`, запрос не будет возвращать ошибку, если столбец уже существует. Если указано `AFTER name_after` (имя другого столбца), то столбец добавляется (в список столбцов таблицы) после указанного. Иначе, столбец добавляется в конец таблицы. Обратите внимание, ClickHouse не позволяет добавлять столбцы в начало таблицы. Для цепочки действий, `name_after` может быть именем столбца, который добавляется в одном из предыдущих действий. @@ -84,7 +84,7 @@ COMMENT COLUMN [IF EXISTS] name 'Text comment' Каждый столбец может содержать только один комментарий. При выполнении запроса существующий комментарий заменяется на новый. -Посмотреть комментарии можно в столбце `comment_expression` из запроса [DESCRIBE TABLE](misc.md#misc-describe-table). +Посмотреть комментарии можно в столбце `comment_expression` из запроса [DESCRIBE TABLE](misc.md#misc-describe-table). Пример: @@ -165,6 +165,22 @@ ALTER TABLE [db].name DROP INDEX name Запрос на изменение индексов реплицируется, сохраняя новые метаданные в ZooKeeper и применяя изменения на всех репликах. +### Манипуляции с ограничениями (constraints) + +Про ограничения подробнее написано [тут](create.md#constraints). + +Добавить или удалить ограничение можно с помощью запросов +``` +ALTER TABLE [db].name ADD CONSTRAINT constraint_name CHECK expression; +ALTER TABLE [db].name DROP CONSTRAINT constraint_name; +``` + +Запросы выполняют добавление или удаление метаданных об ограничениях таблицы `[db].name`, поэтому выполняются мнгновенно. + +Если ограничение появилось для непустой таблицы, то *проверка ограничения для имеющихся данных не производится*. + +Запрос на изменение ограничений для Replicated таблиц реплицируется, сохраняя новые метаданные в ZooKeeper и применяя изменения на всех репликах. + ### Манипуляции с партициями и кусками {#alter_manipulations-with-partitions} Для работы с [партициями](../operations/table_engines/custom_partitioning_key.md) доступны следующие операции: @@ -210,6 +226,15 @@ ALTER TABLE table_name DROP PARTITION partition_expr Запрос реплицируется — данные будут удалены на всех репликах. +#### DROP DETACHED PARTITION|PART {#alter_drop-detached} + +```sql +ALTER TABLE table_name DROP DETACHED PARTITION|PART partition_expr +``` + +Удаляет из `detached` кусок или все куски, принадлежащие партиции. +Подробнее о том, как корректно задать имя партиции, см. в разделе [Как задавать имя партиции в запросах ALTER](#alter-how-to-specify-part-expr). + #### ATTACH PARTITION|PART {#alter_attach-partition} ```sql @@ -235,7 +260,7 @@ ALTER TABLE visits ATTACH PART 201901_2_2_0; ALTER TABLE table2 REPLACE PARTITION partition_expr FROM table1 ``` -Копирует партицию из таблицы `table1` в таблицу `table2`. Данные из `table1` не удаляются. +Копирует партицию из таблицы `table1` в таблицу `table2`. Данные из `table1` не удаляются. Следует иметь в виду: @@ -272,19 +297,19 @@ ALTER TABLE table_name CLEAR INDEX index_name IN PARTITION partition_expr ALTER TABLE table_name FREEZE [PARTITION partition_expr] ``` -Создаёт резервную копию для заданной партиции. Если выражение `PARTITION` опущено, резервные копии будут созданы для всех партиций. +Создаёт резервную копию для заданной партиции. Если выражение `PARTITION` опущено, резервные копии будут созданы для всех партиций. !!! note - Создание резервной копии не требует остановки сервера. - + Создание резервной копии не требует остановки сервера. + Для таблиц старого стиля имя партиций можно задавать в виде префикса (например, '2019'). В этом случае резервные копии будут созданы для всех соответствующих партиций. Подробнее о том, как корректно задать имя партиции, см. в разделе [Как задавать имя партиции в запросах ALTER](#alter-how-to-specify-part-expr). - + Запрос делает следующее — для текущего состояния таблицы он формирует жесткие ссылки на данные в этой таблице. Ссылки размещаются в директории `/var/lib/clickhouse/shadow/N/...`, где: - `/var/lib/clickhouse/` — рабочая директория ClickHouse, заданная в конфигурационном файле; - `N` — инкрементальный номер резервной копии. -Структура директорий внутри резервной копии такая же, как внутри `/var/lib/clickhouse/`. Запрос выполнит 'chmod' для всех файлов, запрещая запись в них. +Структура директорий внутри резервной копии такая же, как внутри `/var/lib/clickhouse/`. Запрос выполнит 'chmod' для всех файлов, запрещая запись в них. Обратите внимание, запрос `ALTER TABLE t FREEZE PARTITION` не реплицируется. Он создает резервную копию только на локальном сервере. После создания резервной копии данные из `/var/lib/clickhouse/shadow/` можно скопировать на удалённый сервер, а локальную копию удалить. @@ -332,12 +357,12 @@ ALTER TABLE users ATTACH PARTITION 201902; #### Как задавать имя партиции в запросах ALTER {#alter-how-to-specify-part-expr} -Чтобы задать нужную партицию в запросах `ALTER ... PARTITION`, можно использовать: +Чтобы задать нужную партицию в запросах `ALTER ... PARTITION`, можно использовать: - Имя партиции. Посмотреть имя партиции можно в столбце `partition` системной таблицы [system.parts](../operations/system_tables.md#system_tables-parts). Например, `ALTER TABLE visits DETACH PARTITION 201901`. - Произвольное выражение из столбцов исходной таблицы. Также поддерживаются константы и константные выражения. Например, `ALTER TABLE visits DETACH PARTITION toYYYYMM(toDate('2019-01-25'))`. - Строковый идентификатор партиции. Идентификатор партиции используется для именования кусков партиции на файловой системе и в ZooKeeper. В запросах `ALTER` идентификатор партиции нужно указывать в секции `PARTITION ID`, в одинарных кавычках. Например, `ALTER TABLE visits DETACH PARTITION ID '201901'`. -- Для запросов [ATTACH PART](#alter_attach-partition): чтобы задать имя куска партиции, используйте значение из столбца `name` системной таблицы `system.parts`. Например, `ALTER TABLE visits ATTACH PART 201901_1_1_0`. +- Для запросов [ATTACH PART](#alter_attach-partition) и [DROP DETACHED PART](#alter_drop-detached): чтобы задать имя куска партиции, используйте строковой литерал со значением из столбца `name` системной таблицы [system.detached_parts](../operations/system_tables.md#system_tables-detached_parts). Например, `ALTER TABLE visits ATTACH PART '201901_1_1_0'`. Использование кавычек в имени партиций зависит от типа данных столбца, по которому задано партиционирование. Например, для столбца с типом `String` имя партиции необходимо указывать в кавычках (одинарных). Для типов `Date` и `Int*` кавычки указывать не нужно. diff --git a/docs/ru/query_language/create.md b/docs/ru/query_language/create.md index ebbef603390..d32afb7b9d9 100644 --- a/docs/ru/query_language/create.md +++ b/docs/ru/query_language/create.md @@ -103,28 +103,52 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name ENGINE = engine AS SELECT ... Отсутствует возможность задать значения по умолчанию для элементов вложенных структур данных. +### Ограничения (constraints) {#constraints} + +WARNING: Находится в экспериментальном режиме, поддержано в MergeTree (работоспособность на других типах движков таблиц не гарантируется). + +Наряду с объявлением столбцов можно объявить ограчения на значения в столбцах таблицы: + +```sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [compression_codec] [TTL expr1], + ... + CONSTRAINT constraint_name_1 CHECK boolean_expr_1, + ... +) ENGINE = engine +``` + +`boolean_expr_1` может быть любым булевым выражением, состоящим из операторов сравнения или функций. При наличии одного или нескольких ограничений в момент вставки данных выражения ограничений будут проверяться на истинность для каждой вставляемой строки данных. В случае, если в теле INSERT запроса придут некорректные данные — клиент получит исключение с описанием нарушенного ограничения. + +Добавление большого числа ограничений может негативно повлиять на производительность `INSERT` запросов. + ### Выражение для TTL Определяет время хранения значений. Может быть указано только для таблиц семейства MergeTree. Подробнее смотрите в [TTL для столбцов и таблиц](../operations/table_engines/mergetree.md#table_engine-mergetree-ttl). -## Форматы сжатия для колонок +## Кодеки сжатия столбцов -Помимо сжатия для колонок по умолчанию, определяемого в [настройках сервера](../operations/server_settings/settings.md#compression), -существует возможность указать формат сжатия индивидуально для каждой колонки. +Помимо сжатия данных по умолчанию, определяемого [конфигурационными параметрами сервера](../operations/server_settings/settings.md#compression), можно задать сжатие для каждого отдельного столбца. -Поддерживаемые форматы: +Поддерживаемые алгоритмы сжатия: -- `NONE` - сжатие отсутствует -- `LZ4` -- `LZ4HC(level)` - алгоритм сжатия LZ4\_HC с указанным уровнем компрессии `level`. -Возможный диапазон значений `level`: \[3, 12\]. Значение по умолчанию: 9. Чем выше уровень, тем лучше сжатие, но тратится больше времени. Рекомендованный диапазон \[4, 9\]. -- `ZSTD(level)` - алгоритм сжатия ZSTD с указанным уровнем компрессии `level`. Возможный диапазон значений `level`: \[1, 22\]. Значение по умолчанию: 1. -Чем выше уровень, тем лучше сжатие, но тратится больше времени. -- `Delta(delta_bytes)` - способ сжатия, при котором вместо числовых значений поля сохраняется разность между двумя соседними значениями. Значение `delta_bytes` - число байт для хранения дельты. -Возможные значения: 1, 2, 4, 8. Значение по умолчанию: если `sizeof(type)` равен 1, 2, 4, 8 - `sizeof(type)`, иначе - 1. +- `NONE` — без сжатия. +- `LZ4` — [алгоритм сжатия данных](https://github.com/lz4/lz4) без потерь, используемый по умолчанию. Применяет быстрое сжатие LZ4. +- `LZ4HC[(level)]` — алгоритм сильного сжатия LZ4 HC с настраиваемым уровнем. Уровень по умолчанию — 9. Настройка `level <= 0` устанавливает уровень по умолчанию. Возможные уровни: [1, 12]. Рекомендуемый диапазон уровней: [4, 9]. +- `ZSTD[(level)]` — [Алгоритм сжатия ZSTD](https://en.wikipedia.org/wiki/Zstandard) с настаиваемым уровнем `level`. Возможные уровни: [1, 22]. Значение по умолчанию — 1. +- `Delta(delta_bytes)` — способ сжатия, при котором исходные значения заменяются разностью двух соседних значений. Для хранение разностей используется до `delta_bytes` байтов, т.е. `delta_bytes` — это максимальный размер исходных значений. Возможные значения `delta_bytes` — 1, 2, 4, 8. Значение `delta_bytes` по умолчанию равно `sizeof(type)`, если вычисленный размер равен 1, 2, 4 или 8. Во всех остальных случаях — 1. +- `DoubleDelta` — Сжимает значения вплоть до размера в 1 бит благодаря сохранению разностей. Оптимальные уровни сжатия достигаются для монотонных последовательностей с постоянным шагом, например, временные ряды. Может использоваться с любым типом данных фиксированного размера. Реализует алгоритм, используемый в Gorilla TSDB, расширяя его для поддержки 64-битных типов. Использует 1 дополнительный бит для 32-байтовых значений: 5-битные префиксы вместо 4-битных префиксов. Подробнее смотрите в разделе "Compressing Time Stamps" в [Gorilla: A Fast, Scalable, In-Memory Time Series Database](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf). +- `Gorilla` — Сжимает значения вплоть до размера в 1 bit. Эффективен при хранении рядов медленно изменяющихся чисел с плавающей запятой, потому, что лучшее сжатие достигается, когда соседние значения бинарно равны. Реализует алгоритм, используемый в Gorilla TSDB, расширяя его для поддержки 64-битных типов. Подробнее смотрите в разделе "Compressing Values" в [Gorilla: A Fast, Scalable, In-Memory Time Series Database](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf). + +Высокие уровни сжатия полезны для асимметричных сценариев, например, для таких, в которых требуется однократное сжатие и многократная распаковка. Более высокие уровни обеспечивают лучшее сжатие, но более высокое потребление вычислительных ресурсов. + +!!! warning "Предупреждение" + Базу данных ClickHouse не получится распаковать с помощью внешних утилит типа `lz4`. Используйте специальную программу [clickhouse-compressor](https://github.com/yandex/ClickHouse/tree/master/dbms/programs/compressor). Пример использования: -``` + +```sql CREATE TABLE codec_example ( dt Date CODEC(ZSTD), /* используется уровень сжатия по умолчанию */ @@ -137,9 +161,10 @@ PARTITION BY tuple() ORDER BY dt ``` -Кодеки могут комбинироваться между собой. Если для колонки указана своя последовательность кодеков, то общий табличный кодек не применяется (должен быть указан в последовательности принудительно, если нужен). В примере ниже - оптимизация для хранения timeseries метрик. +Кодеки можно комбинировать. Если для колонки указана своя последовательность кодеков, то общий табличный кодек не применяется (должен быть указан в последовательности принудительно, если нужен). В примере ниже - оптимизация для хранения timeseries метрик. Как правило, значения одной и той же метрики `path` не сильно различаются между собой, и выгоднее использовать дельта-компрессию вместо записи всего числа: -``` + +```sql CREATE TABLE timeseries_example ( dt Date, diff --git a/docs/ru/query_language/dicts/external_dicts_dict_layout.md b/docs/ru/query_language/dicts/external_dicts_dict_layout.md index aafcf531860..826d9b78ae9 100644 --- a/docs/ru/query_language/dicts/external_dicts_dict_layout.md +++ b/docs/ru/query_language/dicts/external_dicts_dict_layout.md @@ -95,7 +95,7 @@ Словарь хранится в оперативной памяти в виде хэш-таблицы с упорядоченным массивом диапазонов и соответствующих им значений. -Этот способ размещения работает также как и hashed и позволяет дополнительно к ключу использовать дипазоны по дате/времени, если они указаны в словаре. +Этот способ размещения работает также как и hashed и позволяет дополнительно к ключу использовать дипазоны по дате/времени (произвольному числовому типу). Пример: таблица содержит скидки для каждого рекламодателя в виде: @@ -111,7 +111,7 @@ +---------------+---------------------+-------------------+--------+ ``` -Чтобы использовать выборку по диапазонам дат, необходимо в [structure](external_dicts_dict_structure.md) определить элементы `range_min`, `range_max`. +Чтобы использовать выборку по диапазонам дат, необходимо в [structure](external_dicts_dict_structure.md) определить элементы `range_min`, `range_max`. В этих элементах должны присутствовать элементы `name` и `type` (если `type` не указан, будет использован тип по умолчанию -- Date). `type` может быть любым численным типом (Date/DateTime/UInt64/Int32/др.). Пример: @@ -122,14 +122,16 @@ first + Date last + Date ... ``` -Для работы с такими словарями в функцию `dictGetT` необходимо передавать дополнительный аргумент - дату: : +Для работы с такими словарями в функцию `dictGetT` необходимо передавать дополнительный аргумент, для которого подбирается диапазон: dictGetT('dict_name', 'attr_name', id, date) @@ -158,10 +160,12 @@ Abcdef - StartDate + StartTimeStamp + UInt64 - EndDate + EndTimeStamp + UInt64 XXXType diff --git a/docs/ru/query_language/functions/other_functions.md b/docs/ru/query_language/functions/other_functions.md index 00242eeb8c7..fdc46a0d4ee 100644 --- a/docs/ru/query_language/functions/other_functions.md +++ b/docs/ru/query_language/functions/other_functions.md @@ -291,6 +291,48 @@ SELECT ## rowNumberInAllBlocks() Возвращает порядковый номер строки в блоке данных. Функция учитывает только задействованные блоки данных. +## neighbor(column, offset\[, default_value\]) + +Функция позволяет получить доступ к значению в колонке `column`, находящемуся на смещении `offset` относительно текущей строки. +Является частичной реализацией [оконных функций](https://en.wikipedia.org/wiki/SQL_window_function) LEAD() и LAG(). + +Результат функции зависит от затронутых блоков данных и порядка данных в блоке. +Если сделать подзапрос с ORDER BY и вызывать функцию извне подзапроса, можно будет получить ожидаемый результат. + +Если значение `offset` выходит за пределы блока данных, то берётся значение по-умолчанию для колонки `column`. Если передан параметр `default_value`, то значение берётся из него. +Например, эта функция может использоваться чтобы оценить year-over-year значение показателя: + +``` sql +WITH toDate('2018-01-01') AS start_date +SELECT + toStartOfMonth(start_date + (number * 32)) AS month, + toInt32(month) % 100 AS money, + neighbor(money, -12) AS prev_year, + round(prev_year / money, 2) AS year_over_year +FROM numbers(16) +``` + +``` +┌──────month─┬─money─┬─prev_year─┬─year_over_year─┐ +│ 2018-01-01 │ 32 │ 0 │ 0 │ +│ 2018-02-01 │ 63 │ 0 │ 0 │ +│ 2018-03-01 │ 91 │ 0 │ 0 │ +│ 2018-04-01 │ 22 │ 0 │ 0 │ +│ 2018-05-01 │ 52 │ 0 │ 0 │ +│ 2018-06-01 │ 83 │ 0 │ 0 │ +│ 2018-07-01 │ 13 │ 0 │ 0 │ +│ 2018-08-01 │ 44 │ 0 │ 0 │ +│ 2018-09-01 │ 75 │ 0 │ 0 │ +│ 2018-10-01 │ 5 │ 0 │ 0 │ +│ 2018-11-01 │ 36 │ 0 │ 0 │ +│ 2018-12-01 │ 66 │ 0 │ 0 │ +│ 2019-01-01 │ 97 │ 32 │ 0.33 │ +│ 2019-02-01 │ 28 │ 63 │ 2.25 │ +│ 2019-03-01 │ 56 │ 91 │ 1.62 │ +│ 2019-04-01 │ 87 │ 22 │ 0.25 │ +└────────────┴───────┴───────────┴────────────────┘ +``` + ## runningDifference(x) Считает разницу между последовательными значениями строк в блоке данных. Возвращает 0 для первой строки и разницу с предыдущей строкой для каждой последующей строки. diff --git a/docs/ru/query_language/functions/url_functions.md b/docs/ru/query_language/functions/url_functions.md index 1897d1b28a3..7002273d5cb 100644 --- a/docs/ru/query_language/functions/url_functions.md +++ b/docs/ru/query_language/functions/url_functions.md @@ -10,13 +10,94 @@ Возвращает протокол. Примеры: http, ftp, mailto, magnet... ### domain -Возвращает домен. Отсекает схему размером не более 16 байт. + +Извлекает имя хоста из URL. + +``` +domain(url) +``` + +**Параметры** + +- `url` — URL. Тип — [String](../../data_types/string.md). + +URL может быть указан со схемой или без неё. Примеры: + +``` +svn+ssh://some.svn-hosting.com:80/repo/trunk +some.svn-hosting.com:80/repo/trunk +https://yandex.com/time/ +``` + +Для указанных примеров функция `domain` возвращает следующие результаты: + +``` +some.svn-hosting.com +some.svn-hosting.com +yandex.com +``` + +**Возвращаемые значения** + +- Имя хоста. Если ClickHouse может распарсить входную строку как URL. +- Пустая строка. Если ClickHouse не может распарсить входную строку как URL. + +Тип — `String`. + +**Пример** + +```sql +SELECT domain('svn+ssh://some.svn-hosting.com:80/repo/trunk') +``` + +```text +┌─domain('svn+ssh://some.svn-hosting.com:80/repo/trunk')─┐ +│ some.svn-hosting.com │ +└────────────────────────────────────────────────────────┘ +``` ### domainWithoutWWW -Возвращает домен, удалив не более одного 'www.' с начала, если есть. + +Возвращает домен, удалив префикс 'www.', если он присутствовал. ### topLevelDomain -Возвращает домен верхнего уровня. Пример: .ru. + +Извлекает домен верхнего уровня из URL. + +``` +topLevelDomain(url) +``` + +**Параметры** + +- `url` — URL. Тип — [String](../../data_types/string.md). + +URL может быть указан со схемой или без неё. Примеры: + +``` +svn+ssh://some.svn-hosting.com:80/repo/trunk +some.svn-hosting.com:80/repo/trunk +https://yandex.com/time/ +``` + +**Возвращаемые значения** + +- Имя домена. Если ClickHouse может распарсить входную строку как URL. +- Пустая строка. Если ClickHouse не может распарсить входную строку как URL. + +Тип — `String`. + +**Пример** + +```sql +SELECT topLevelDomain('svn+ssh://www.some.svn-hosting.com:80/repo/trunk') +``` + +```text +┌─topLevelDomain('svn+ssh://www.some.svn-hosting.com:80/repo/trunk')─┐ +│ com │ +└────────────────────────────────────────────────────────────────────┘ +``` ### firstSignificantSubdomain Возвращает "первый существенный поддомен". Это понятие является нестандартным и специфично для Яндекс.Метрики. Первый существенный поддомен - это домен второго уровня, если он не равен одному из com, net, org, co, или домен третьего уровня, иначе. Например, firstSignificantSubdomain('') = 'yandex', firstSignificantSubdomain('') = 'yandex'. Список "несущественных" доменов второго уровня и другие детали реализации могут изменяться в будущем. diff --git a/docs/ru/query_language/insert_into.md b/docs/ru/query_language/insert_into.md index 356b720e157..454339ebcdb 100644 --- a/docs/ru/query_language/insert_into.md +++ b/docs/ru/query_language/insert_into.md @@ -40,6 +40,10 @@ INSERT INTO t FORMAT TabSeparated С помощью консольного клиента или HTTP интерфейса можно вставлять данные отдельно от запроса. Как это сделать, читайте в разделе "[Интерфейсы](../interfaces/index.md#interfaces)". +### Ограничения (constraints) + +Если в таблице объявлены [ограничения](create.md#constraints), то их выполнимость будет проверена для каждой вставляемой строки. Если для хотя бы одной строки ограничения не будут выполнены, запрос будет остановлен. + ### Вставка результатов `SELECT` {#insert_query_insert-select} ``` sql diff --git a/docs/ru/query_language/misc.md b/docs/ru/query_language/misc.md index 93f548bf73c..ab19e559649 100644 --- a/docs/ru/query_language/misc.md +++ b/docs/ru/query_language/misc.md @@ -193,18 +193,21 @@ RENAME TABLE [db11.]name11 TO [db12.]name12, [db21.]name21 TO [db22.]name22, ... Все таблицы переименовываются под глобальной блокировкой. Переименовывание таблицы является лёгкой операцией. Если вы указали после TO другую базу данных, то таблица будет перенесена в эту базу данных. При этом, директории с базами данных должны быть расположены в одной файловой системе (иначе возвращается ошибка). -## SET +## SET {#query-set} ```sql SET param = value ``` -Позволяет установить настройку `param` в значение `value`. Также можно одним запросом установить все настройки из заданного профиля настроек. Для этого укажите 'profile' в качестве имени настройки. Подробнее смотрите в разделе "Настройки". -Настройка устанавливается на сессию, или на сервер (глобально), если указано `GLOBAL`. -При установке глобальных настроек, эти настройки не применяются к уже запущенной сессии, включая текущую сессию. Она будет использована только для новых сессий. +Устанавливает значение `value` для [настройки](../operations/settings/index.md) `param` в текущей сессии. [Конфигурационные параметры сервера](../operations/server_settings/index.md) нельзя изменить подобным образом. -При перезапуске сервера теряются настройки, установленные с помощью `SET`. -Установить настройки, которые переживут перезапуск сервера, можно только с помощью конфигурационного файла сервера. +Можно одним запросом установить все настройки из заданного профиля настроек. + +```sql +SET profile = 'profile-name-from-the-settings-file' +``` + +Подробности смотрите в разделе [Настройки](../operations/settings/settings.md). ## SHOW CREATE TABLE diff --git a/docs/zh/data_types/enum.md b/docs/zh/data_types/enum.md index ca8488b4345..41133b56d45 100644 --- a/docs/zh/data_types/enum.md +++ b/docs/zh/data_types/enum.md @@ -91,7 +91,7 @@ ENGINE = TinyLog 不仅可以存储 `'hello'` 和 `'world'` ，还可以存储 `NULL`。 ``` -INSERT INTO t_enum_null Values('hello'),('world'),(NULL) +INSERT INTO t_enum_nullable Values('hello'),('world'),(NULL) ``` 在内存中，`Enum` 列的存储方式与相应数值的 `Int8` 或 `Int16` 相同。 diff --git a/docs/zh/database_engines/index.md b/docs/zh/database_engines/index.md deleted file mode 120000 index bbdb762a4ad..00000000000 --- a/docs/zh/database_engines/index.md +++ /dev/null @@ -1 +0,0 @@ -../../en/database_engines/index.md \ No newline at end of file diff --git a/docs/zh/database_engines/index.md b/docs/zh/database_engines/index.md new file mode 100644 index 00000000000..f8ae05e2520 --- /dev/null +++ b/docs/zh/database_engines/index.md @@ -0,0 +1,11 @@ +# 数据库引擎 + +您使用的所有表都是由数据库引擎所提供的 + +默认情况下，ClickHouse使用自己的数据库引擎，该引擎提供可配置的[表引擎](../operations/table_engines/index.md)和[所有支持的SQL语法](../query_language/syntax.md). + +除此之外，您还可以选择使用以下的数据库引擎： + +- [MySQL](mysql.md) + +[来源文章](https://clickhouse.yandex/docs/en/database_engines/) diff --git a/docs/zh/database_engines/mysql.md b/docs/zh/database_engines/mysql.md deleted file mode 120000 index 51ac4126e2d..00000000000 --- a/docs/zh/database_engines/mysql.md +++ /dev/null @@ -1 +0,0 @@ -../../en/database_engines/mysql.md \ No newline at end of file diff --git a/docs/zh/database_engines/mysql.md b/docs/zh/database_engines/mysql.md new file mode 100644 index 00000000000..38dfcb5ef64 --- /dev/null +++ b/docs/zh/database_engines/mysql.md @@ -0,0 +1,124 @@ +# MySQL + +MySQL引擎用于将远程的MySQL服务器中的表映射到ClickHouse中，并允许您对表进行`INSERT`和`SELECT`查询，以方便您在ClickHouse与MySQL之间进行数据交换。 + +`MySQL`数据库引擎会将对其的查询转换为MySQL语法并发送到MySQL服务器中，因此您可以执行诸如`SHOW TABLES`或`SHOW CREATE TABLE`之类的操作。 + +但您无法对其执行以下操作： + +- `ATTACH`/`DETACH` +- `DROP` +- `RENAME` +- `CREATE TABLE` +- `ALTER` + + +## CREATE DATABASE + +``` sql +CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster] +ENGINE = MySQL('host:port', 'database', 'user', 'password') +``` + +**MySQL数据库引擎参数** + +- `host:port` — 链接的MySQL地址。 +- `database` — 链接的MySQL数据库。 +- `user` — 链接的MySQL用户。 +- `password` — 链接的MySQL用户密码。 + + +## 支持的类型对应 + +MySQL | ClickHouse +------|------------ +UNSIGNED TINYINT | [UInt8](../data_types/int_uint.md) +TINYINT | [Int8](../data_types/int_uint.md) +UNSIGNED SMALLINT | [UInt16](../data_types/int_uint.md) +SMALLINT | [Int16](../data_types/int_uint.md) +UNSIGNED INT, UNSIGNED MEDIUMINT | [UInt32](../data_types/int_uint.md) +INT, MEDIUMINT | [Int32](../data_types/int_uint.md) +UNSIGNED BIGINT | [UInt64](../data_types/int_uint.md) +BIGINT | [Int64](../data_types/int_uint.md) +FLOAT | [Float32](../data_types/float.md) +DOUBLE | [Float64](../data_types/float.md) +DATE | [Date](../data_types/date.md) +DATETIME, TIMESTAMP | [DateTime](../data_types/datetime.md) +BINARY | [FixedString](../data_types/fixedstring.md) + +其他的MySQL数据类型将全部都转换为[String](../data_types/string.md)。 + +同时以上的所有类型都支持[Nullable](../data_types/nullable.md)。 + + +## 使用示例 + +在MySQL中创建表: + +``` +mysql> USE test; +Database changed + +mysql> CREATE TABLE `mysql_table` ( + -> `int_id` INT NOT NULL AUTO_INCREMENT, + -> `float` FLOAT NOT NULL, + -> PRIMARY KEY (`int_id`)); +Query OK, 0 rows affected (0,09 sec) + +mysql> insert into mysql_table (`int_id`, `float`) VALUES (1,2); +Query OK, 1 row affected (0,00 sec) + +mysql> select * from mysql_table; ++--------+-------+ +| int_id | value | ++--------+-------+ +| 1 | 2 | ++--------+-------+ +1 row in set (0,00 sec) +``` + +在ClickHouse中创建MySQL类型的数据库，同时与MySQL服务器交换数据： + +```sql +CREATE DATABASE mysql_db ENGINE = MySQL('localhost:3306', 'test', 'my_user', 'user_password') +``` +```sql +SHOW DATABASES +``` +```text +┌─name─────┐ +│ default │ +│ mysql_db │ +│ system │ +└──────────┘ +``` +```sql +SHOW TABLES FROM mysql_db +``` +```text +┌─name─────────┐ +│ mysql_table │ +└──────────────┘ +``` +```sql +SELECT * FROM mysql_db.mysql_table +``` +```text +┌─int_id─┬─value─┐ +│ 1 │ 2 │ +└────────┴───────┘ +``` +```sql +INSERT INTO mysql_db.mysql_table VALUES (3,4) +``` +```sql +SELECT * FROM mysql_db.mysql_table +``` +```text +┌─int_id─┬─value─┐ +│ 1 │ 2 │ +│ 3 │ 4 │ +└────────┴───────┘ +``` + +[来源文章](https://clickhouse.yandex/docs/en/database_engines/mysql/) diff --git a/docs/zh/query_language/create.md b/docs/zh/query_language/create.md index 1b1abef47db..d3a6c2e841b 100644 --- a/docs/zh/query_language/create.md +++ b/docs/zh/query_language/create.md @@ -80,7 +80,95 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name ENGINE = engine AS SELECT ... 不能够为nested类型的列设置默认值。 -### 临时表 +### Constraints {#constraints} + +WARNING: This feature is experimental. Correct work is not guaranteed on non-MergeTree family engines. + +Along with columns descriptions constraints could be defined: + +``sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [compression_codec] [TTL expr1], + ... + CONSTRAINT constraint_name_1 CHECK boolean_expr_1, + ... +) ENGINE = engine +``` + +`boolean_expr_1` could by any boolean expression. If constraints are defined for the table, each of them will be checked for every row in `INSERT` query. If any constraint is not satisfied — server will raise an exception with constraint name and checking expression. + +Adding large amount of constraints can negatively affect performance of big `INSERT` queries. + +### TTL expression + +Defines storage time for values. Can be specified only for MergeTree-family tables. For the detailed description, see [TTL for columns and tables](../operations/table_engines/mergetree.md#table_engine-mergetree-ttl). + +## Column Compression Codecs + +By default, ClickHouse applies to columns the compression method, defined in [server settings](../operations/server_settings/settings.md#compression). Also, you can define compression method for each individual column in the `CREATE TABLE` query. + +``` +CREATE TABLE codec_example +( + dt Date CODEC(ZSTD), + ts DateTime CODEC(LZ4HC), + float_value Float32 CODEC(NONE), + double_value Float64 CODEC(LZ4HC(9)) + value Float32 CODEC(Delta, ZSTD) +) +ENGINE = +... +``` + +If a codec is specified, the default codec doesn't apply. Codecs can be combined in a pipeline, for example, `CODEC(Delta, ZSTD)`. To select the best codecs combination for you project, pass benchmarks, similar to described in the Altinity [New Encodings to Improve ClickHouse Efficiency](https://www.altinity.com/blog/2019/7/new-encodings-to-improve-clickhouse) article. + +!!!warning + You cannot decompress ClickHouse database files with external utilities, for example, `lz4`. Use the special utility, [clickhouse-compressor](https://github.com/yandex/ClickHouse/tree/master/dbms/programs/compressor). + +Compression is supported for the table engines: + +- [*MergeTree](../operations/table_engines/mergetree.md) family +- [*Log](../operations/table_engines/log_family.md) family +- [Set](../operations/table_engines/set.md) +- [Join](../operations/table_engines/join.md) + +ClickHouse supports common purpose codecs and specialized codecs. + +### Specialized codecs {#create-query-specialized-codecs} + +These codecs are designed to make compression more effective using specifities of the data. Some of this codecs don't compress data by itself, but they prepare data to be compressed better by common purpose codecs. + +Specialized codecs: + +- `Delta(delta_bytes)` — Compression approach in which raw values are replaced by the difference of two neighboring values, except for the first value that stays unchanged. Up to `delta_bytes` are used for storing delta values, so `delta_bytes` is the maximum size of raw values. Possible `delta_bytes` values: 1, 2, 4, 8. The default value for `delta_bytes` is `sizeof(type)` if equal to 1, 2, 4, or 8. In all other cases, it's 1. +- `DoubleDelta` — Calculates delta of deltas and writes it in compact binary form. Optimal compression rates are achieved for monotonic sequences with a constant stride, such as time series data. Can be used with any fixed-width type. Implements the algorithm used in Gorilla TSDB, extending it to support 64-bit types. Uses 1 extra bit for 32-byte deltas: 5-bit prefixes instead of 4-bit prefixes. For additional information, see Compressing Time Stamps in [Gorilla: A Fast, Scalable, In-Memory Time Series Database](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf). +- `Gorilla` — Calculates XOR between current and previous value and writes it in compact binary form. Efficient when storing a series of floating point values that change slowly, because the best compression rate is achieved when neighboring values are binary equal. Implements the algorithm used in Gorilla TSDB, extending it to support 64-bit types. For additional information, see Compressing Values in [Gorilla: A Fast, Scalable, In-Memory Time Series Database](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf). +- `T64` — Compression approach that crops unused high bits of values in integer data types (including `Enum`, `Date` and `DateTime`). At each step of its algorithm, codec takes a block of 64 values, puts them into 64x64 bit matrix, transposes it, crops the unused bits of values and returns the rest as a sequence. Unused bits are the bits, that don't differ between maximum and minimum values in the whole data part for which the compression is used. + +`DoubleDelta` and `Gorilla` codecs are used in Gorilla TSDB as the components of its compressing algorithm. Gorilla approach is effective in scenarios when there is a sequence of slowly changing values with their timestamps. Timestamps are effectively compressed by the `DoubleDelta` codec, and values are effectively compressed by the `Gorilla` codec. For example, to get an effectively stored table, you can create it in the following configuration: + +```sql +CREATE TABLE codec_example +( + timestamp DateTime CODEC(DoubleDelta), + slow_values Float32 CODEC(Gorilla) +) +ENGINE = MergeTree() +``` + +### Common purpose codecs {#create-query-common-purpose-codecs} + +Codecs: + +- `NONE` — No compression. +- `LZ4` — Lossless [data compression algorithm](https://github.com/lz4/lz4) used by default. Applies LZ4 fast compression. +- `LZ4HC[(level)]` — LZ4 HC (high compression) algorithm with configurable level. Default level: 9. Setting `level <= 0` applies the default level. Possible levels: [1, 12]. Recommended level range: [4, 9]. +- `ZSTD[(level)]` — [ZSTD compression algorithm](https://en.wikipedia.org/wiki/Zstandard) with configurable `level`. Possible levels: [1, 22]. Default value: 1. + +High compression levels useful for asymmetric scenarios, like compress once, decompress a lot of times. Greater levels stands for better compression and higher CPU usage. + +## 临时表 ClickHouse支持临时表，其具有以下特征： diff --git a/docs/zh/query_language/functions/hash_functions.md b/docs/zh/query_language/functions/hash_functions.md index 57af83a4a5e..9fc6c79b0b3 100644 --- a/docs/zh/query_language/functions/hash_functions.md +++ b/docs/zh/query_language/functions/hash_functions.md @@ -21,7 +21,7 @@ Hash函数可以用于将元素不可逆的伪随机打乱。 SipHash是一种加密哈希函数。它的处理性能至少比MD5快三倍。 有关详细信息，请参阅链接： -## sipHash128 +## sipHash128 {#hash_functions-siphash128} 计算字符串的SipHash。 接受String类型的参数，返回FixedString(16)。 diff --git a/docs/zh/query_language/functions/other_functions.md b/docs/zh/query_language/functions/other_functions.md index 84fbdaeb3ca..329db222b48 100644 --- a/docs/zh/query_language/functions/other_functions.md +++ b/docs/zh/query_language/functions/other_functions.md @@ -122,7 +122,7 @@ Accepts constant strings: database name, table name, and column name. Returns a The function throws an exception if the table does not exist. For elements in a nested data structure, the function checks for the existence of a column. For the nested data structure itself, the function returns 0. -## bar +## bar {#function-bar} 使用unicode构建图表。 diff --git a/libs/CMakeLists.txt b/libs/CMakeLists.txt index d526a662dc0..b5bcbd804be 100644 --- a/libs/CMakeLists.txt +++ b/libs/CMakeLists.txt @@ -15,10 +15,6 @@ if (USE_INTERNAL_MEMCPY) add_subdirectory (libmemcpy) endif() -if (GLIBC_COMPATIBILITY) - add_subdirectory (libglibc-compatibility) -endif () - if (USE_MYSQL) add_subdirectory (libmysqlxx) endif () diff --git a/libs/libcommon/CMakeLists.txt b/libs/libcommon/CMakeLists.txt index 2744714a9c4..c78473890dc 100644 --- a/libs/libcommon/CMakeLists.txt +++ b/libs/libcommon/CMakeLists.txt @@ -123,9 +123,7 @@ target_link_libraries (common PUBLIC ${Boost_SYSTEM_LIBRARY} PRIVATE - ${CMAKE_DL_LIBS} ${MALLOC_LIBRARIES} - Threads::Threads ${MEMCPY_LIBRARIES}) if (RT_LIBRARY) diff --git a/libs/libcommon/include/common/DateLUTImpl.h b/libs/libcommon/include/common/DateLUTImpl.h index 2258620eb26..ef50d6ede3f 100644 --- a/libs/libcommon/include/common/DateLUTImpl.h +++ b/libs/libcommon/include/common/DateLUTImpl.h @@ -28,7 +28,7 @@ enum class WeekModeFlag : UInt8 FIRST_WEEKDAY = 4, NEWYEAR_DAY = 8 }; -typedef std::pair YearWeek; +using YearWeek = std::pair; /** Lookup table to conversion of time to date, and to month / year / day of week / day of month and so on. * First time was implemented for OLAPServer, that needed to do billions of such transformations. diff --git a/libs/libcommon/include/common/config_common.h.in b/libs/libcommon/include/common/config_common.h.in index 1301049b24b..810cf0b87f9 100644 --- a/libs/libcommon/include/common/config_common.h.in +++ b/libs/libcommon/include/common/config_common.h.in @@ -8,5 +8,4 @@ #cmakedefine01 USE_LIBEDIT #cmakedefine01 HAVE_READLINE_HISTORY #cmakedefine01 UNBUNDLED -#cmakedefine01 USE_INTERNAL_UNWIND_LIBRARY #cmakedefine01 WITH_COVERAGE diff --git a/libs/libcommon/include/common/find_symbols.h b/libs/libcommon/include/common/find_symbols.h index 68b49397683..162c73251fa 100644 --- a/libs/libcommon/include/common/find_symbols.h +++ b/libs/libcommon/include/common/find_symbols.h @@ -17,7 +17,7 @@ * but with the following differencies: * - works with any memory ranges, including containing zero bytes; * - doesn't require terminating zero byte: end of memory range is passed explicitly; - * - if not found, returns pointer to end instead of NULL; + * - if not found, returns pointer to end instead of nullptr; * - maximum number of symbols to search is 16. * * Uses SSE 2 in case of small number of symbols for search and SSE 4.2 in the case of large number of symbols, @@ -65,115 +65,154 @@ inline __m128i mm_is_in(__m128i bytes) } #endif - -template -inline const char * find_first_symbols_sse2(const char * begin, const char * end) +template +bool maybe_negate(bool x) { + if constexpr (positive) + return x; + else + return !x; +} + +template +uint16_t maybe_negate(uint16_t x) +{ + if constexpr (positive) + return x; + else + return ~x; +} + +enum class ReturnMode +{ + End, + Nullptr, +}; + + +template +inline const char * find_first_symbols_sse2(const char * const begin, const char * const end) +{ + const char * pos = begin; + #if defined(__SSE2__) - for (; begin + 15 < end; begin += 16) + for (; pos + 15 < end; pos += 16) { - __m128i bytes = _mm_loadu_si128(reinterpret_cast(begin)); + __m128i bytes = _mm_loadu_si128(reinterpret_cast(pos)); __m128i eq = mm_is_in(bytes); - uint16_t bit_mask = _mm_movemask_epi8(eq); + uint16_t bit_mask = maybe_negate(uint16_t(_mm_movemask_epi8(eq))); if (bit_mask) - return begin + __builtin_ctz(bit_mask); + return pos + __builtin_ctz(bit_mask); } #endif - for (; begin < end; ++begin) - if (is_in(*begin)) - return begin; - return end; + for (; pos < end; ++pos) + if (maybe_negate(is_in(*pos))) + return pos; + + return return_mode == ReturnMode::End ? end : nullptr; } -template -inline const char * find_last_symbols_or_null_sse2(const char * begin, const char * end) +template +inline const char * find_last_symbols_sse2(const char * const begin, const char * const end) { + const char * pos = end; + #if defined(__SSE2__) - for (; end - 16 >= begin; end -= 16) /// Assuming the pointer cannot overflow. Assuming we can compare these pointers. + for (; pos - 16 >= begin; pos -= 16) /// Assuming the pointer cannot overflow. Assuming we can compare these pointers. { - __m128i bytes = _mm_loadu_si128(reinterpret_cast(end - 16)); + __m128i bytes = _mm_loadu_si128(reinterpret_cast(pos - 16)); __m128i eq = mm_is_in(bytes); - uint16_t bit_mask = _mm_movemask_epi8(eq); + uint16_t bit_mask = maybe_negate(uint16_t(_mm_movemask_epi8(eq))); if (bit_mask) - return end - 1 - (__builtin_clz(bit_mask) - 16); /// because __builtin_clz works with mask as uint32. + return pos - 1 - (__builtin_clz(bit_mask) - 16); /// because __builtin_clz works with mask as uint32. } #endif - --end; - for (; end >= begin; --end) - if (is_in(*end)) - return end; + --pos; + for (; pos >= begin; --pos) + if (maybe_negate(is_in(*pos))) + return pos; - return nullptr; + return return_mode == ReturnMode::End ? end : nullptr; } -template -inline const char * find_first_symbols_sse42_impl(const char * begin, const char * end) +inline const char * find_first_symbols_sse42_impl(const char * const begin, const char * const end) { + const char * pos = begin; + #if defined(__SSE4_2__) #define MODE (_SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT) __m128i set = _mm_setr_epi8(c01, c02, c03, c04, c05, c06, c07, c08, c09, c10, c11, c12, c13, c14, c15, c16); - for (; begin + 15 < end; begin += 16) + for (; pos + 15 < end; pos += 16) { - __m128i bytes = _mm_loadu_si128(reinterpret_cast(begin)); + __m128i bytes = _mm_loadu_si128(reinterpret_cast(pos)); - if (_mm_cmpestrc(set, num_chars, bytes, 16, MODE)) - return begin + _mm_cmpestri(set, num_chars, bytes, 16, MODE); + if constexpr (positive) + { + if (_mm_cmpestrc(set, num_chars, bytes, 16, MODE)) + return pos + _mm_cmpestri(set, num_chars, bytes, 16, MODE); + } + else + { + if (_mm_cmpestrc(set, num_chars, bytes, 16, MODE | _SIDD_NEGATIVE_POLARITY)) + return pos + _mm_cmpestri(set, num_chars, bytes, 16, MODE | _SIDD_NEGATIVE_POLARITY); + } } #undef MODE #endif - for (; begin < end; ++begin) - if ( (num_chars >= 1 && *begin == c01) - || (num_chars >= 2 && *begin == c02) - || (num_chars >= 3 && *begin == c03) - || (num_chars >= 4 && *begin == c04) - || (num_chars >= 5 && *begin == c05) - || (num_chars >= 6 && *begin == c06) - || (num_chars >= 7 && *begin == c07) - || (num_chars >= 8 && *begin == c08) - || (num_chars >= 9 && *begin == c09) - || (num_chars >= 10 && *begin == c10) - || (num_chars >= 11 && *begin == c11) - || (num_chars >= 12 && *begin == c12) - || (num_chars >= 13 && *begin == c13) - || (num_chars >= 14 && *begin == c14) - || (num_chars >= 15 && *begin == c15) - || (num_chars >= 16 && *begin == c16)) - return begin; - return end; + for (; pos < end; ++pos) + if ( (num_chars >= 1 && maybe_negate(*pos == c01)) + || (num_chars >= 2 && maybe_negate(*pos == c02)) + || (num_chars >= 3 && maybe_negate(*pos == c03)) + || (num_chars >= 4 && maybe_negate(*pos == c04)) + || (num_chars >= 5 && maybe_negate(*pos == c05)) + || (num_chars >= 6 && maybe_negate(*pos == c06)) + || (num_chars >= 7 && maybe_negate(*pos == c07)) + || (num_chars >= 8 && maybe_negate(*pos == c08)) + || (num_chars >= 9 && maybe_negate(*pos == c09)) + || (num_chars >= 10 && maybe_negate(*pos == c10)) + || (num_chars >= 11 && maybe_negate(*pos == c11)) + || (num_chars >= 12 && maybe_negate(*pos == c12)) + || (num_chars >= 13 && maybe_negate(*pos == c13)) + || (num_chars >= 14 && maybe_negate(*pos == c14)) + || (num_chars >= 15 && maybe_negate(*pos == c15)) + || (num_chars >= 16 && maybe_negate(*pos == c16))) + return pos; + return return_mode == ReturnMode::End ? end : nullptr; } -template +template inline const char * find_first_symbols_sse42(const char * begin, const char * end) { - return find_first_symbols_sse42_impl(begin, end); + return find_first_symbols_sse42_impl(begin, end); } /// NOTE No SSE 4.2 implementation for find_last_symbols_or_null. Not worth to do. -template +template inline const char * find_first_symbols_dispatch(const char * begin, const char * end) { #if defined(__SSE4_2__) if (sizeof...(symbols) >= 5) - return find_first_symbols_sse42(begin, end); + return find_first_symbols_sse42(begin, end); else #endif - return find_first_symbols_sse2(begin, end); + return find_first_symbols_sse2(begin, end); } } @@ -182,7 +221,7 @@ inline const char * find_first_symbols_dispatch(const char * begin, const char * template inline const char * find_first_symbols(const char * begin, const char * end) { - return detail::find_first_symbols_dispatch(begin, end); + return detail::find_first_symbols_dispatch(begin, end); } /// Returning non const result for non const arguments. @@ -190,18 +229,66 @@ inline const char * find_first_symbols(const char * begin, const char * end) template inline char * find_first_symbols(char * begin, char * end) { - return const_cast(detail::find_first_symbols_dispatch(begin, end)); + return const_cast(detail::find_first_symbols_dispatch(begin, end)); +} + +template +inline const char * find_first_not_symbols(const char * begin, const char * end) +{ + return detail::find_first_symbols_dispatch(begin, end); +} + +template +inline char * find_first_not_symbols(char * begin, char * end) +{ + return const_cast(detail::find_first_symbols_dispatch(begin, end)); +} + +template +inline const char * find_first_symbols_or_null(const char * begin, const char * end) +{ + return detail::find_first_symbols_dispatch(begin, end); +} + +template +inline char * find_first_symbols_or_null(char * begin, char * end) +{ + return const_cast(detail::find_first_symbols_dispatch(begin, end)); +} + +template +inline const char * find_first_not_symbols_or_null(const char * begin, const char * end) +{ + return detail::find_first_symbols_dispatch(begin, end); +} + +template +inline char * find_first_not_symbols_or_null(char * begin, char * end) +{ + return const_cast(detail::find_first_symbols_dispatch(begin, end)); } template inline const char * find_last_symbols_or_null(const char * begin, const char * end) { - return detail::find_last_symbols_or_null_sse2(begin, end); + return detail::find_last_symbols_sse2(begin, end); } template inline char * find_last_symbols_or_null(char * begin, char * end) { - return const_cast(detail::find_last_symbols_or_null_sse2(begin, end)); + return const_cast(detail::find_last_symbols_sse2(begin, end)); +} + +template +inline const char * find_last_not_symbols_or_null(const char * begin, const char * end) +{ + return detail::find_last_symbols_sse2(begin, end); +} + +template +inline char * find_last_not_symbols_or_null(char * begin, char * end) +{ + return const_cast(detail::find_last_symbols_sse2(begin, end)); } diff --git a/libs/libcommon/include/ext/shared_ptr_helper.h b/libs/libcommon/include/ext/shared_ptr_helper.h index f7fd7c38ace..ca7219e6261 100644 --- a/libs/libcommon/include/ext/shared_ptr_helper.h +++ b/libs/libcommon/include/ext/shared_ptr_helper.h @@ -7,32 +7,16 @@ namespace ext /** Allows to make std::shared_ptr from T with protected constructor. * - * Derive your T class from shared_ptr_helper + * Derive your T class from shared_ptr_helper and add shared_ptr_helper as a friend * and you will have static 'create' method in your class. - * - * Downsides: - * - your class cannot be final; - * - awful compilation error messages; - * - bad code navigation. - * - different dynamic type of created object, you cannot use typeid. */ template struct shared_ptr_helper { template - static auto create(TArgs &&... args) + static std::shared_ptr create(TArgs &&... args) { - /** Local struct makes protected constructor to be accessible by std::make_shared function. - * This trick is suggested by Yurii Diachenko, - * inspired by https://habrahabr.ru/company/mailru/blog/341584/ - * that is translation of http://videocortex.io/2017/Bestiary/#-voldemort-types - */ - struct Local : T - { - Local(TArgs &&... args) : T(std::forward(args)...) {} - }; - - return std::make_shared(std::forward(args)...); + return std::shared_ptr(new T(std::forward(args)...)); } }; diff --git a/libs/libcommon/src/tests/CMakeLists.txt b/libs/libcommon/src/tests/CMakeLists.txt index 2bb8afe6fa1..15d872ac49d 100644 --- a/libs/libcommon/src/tests/CMakeLists.txt +++ b/libs/libcommon/src/tests/CMakeLists.txt @@ -16,7 +16,7 @@ target_link_libraries (date_lut3 common ${PLATFORM_LIBS}) target_link_libraries (date_lut4 common ${PLATFORM_LIBS}) target_link_libraries (date_lut_default_timezone common ${PLATFORM_LIBS}) target_link_libraries (local_date_time_comparison common) -target_link_libraries (realloc-perf common Threads::Threads) +target_link_libraries (realloc-perf common) add_check(local_date_time_comparison) if(USE_GTEST) diff --git a/libs/libglibc-compatibility/CMakeLists.txt b/libs/libglibc-compatibility/CMakeLists.txt index fe98ae9bf0d..a62f5e75e17 100644 --- a/libs/libglibc-compatibility/CMakeLists.txt +++ b/libs/libglibc-compatibility/CMakeLists.txt @@ -1,25 +1,45 @@ -enable_language(ASM) -include(CheckIncludeFile) +if (GLIBC_COMPATIBILITY) + set (USE_INTERNAL_MEMCPY ON) -check_include_file("sys/random.h" HAVE_SYS_RANDOM_H) + enable_language(ASM) + include(CheckIncludeFile) -if(COMPILER_CLANG) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-builtin-requires-header") -endif() + check_include_file("sys/random.h" HAVE_SYS_RANDOM_H) -add_headers_and_sources(glibc_compatibility .) -add_headers_and_sources(glibc_compatibility musl) -list(APPEND glibc_compatibility_sources musl/syscall.s musl/longjmp.s) + if(COMPILER_CLANG) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-builtin-requires-header") + endif() -list(REMOVE_ITEM glibc_compatibility_sources musl/getentropy.c) -if(HAVE_SYS_RANDOM_H) - list(APPEND glibc_compatibility_sources musl/getentropy.c) -endif() + add_headers_and_sources(glibc_compatibility .) + add_headers_and_sources(glibc_compatibility musl) + list(APPEND glibc_compatibility_sources musl/syscall.s musl/longjmp.s) -if(MAKE_STATIC_LIBRARIES) - list(APPEND glibc_compatibility_sources libcxxabi/cxa_thread_atexit.cpp) -endif() + list(REMOVE_ITEM glibc_compatibility_sources musl/getentropy.c) + if(HAVE_SYS_RANDOM_H) + list(APPEND glibc_compatibility_sources musl/getentropy.c) + endif() -add_library(glibc-compatibility STATIC ${glibc_compatibility_sources}) + if(MAKE_STATIC_LIBRARIES) + list(APPEND glibc_compatibility_sources libcxxabi/cxa_thread_atexit.cpp) + endif() -target_include_directories(glibc-compatibility PRIVATE libcxxabi) + add_library(glibc-compatibility STATIC ${glibc_compatibility_sources}) + + target_include_directories(glibc-compatibility PRIVATE libcxxabi) + + if (USE_STATIC_LIBRARIES=0 AND MAKE_STATIC_LIBRARIES=OFF) + target_compile_options(PRIVATE -fPIC) + endif () + + target_link_libraries(global-libs INTERFACE glibc-compatibility) + + install( + TARGETS glibc-compatibility + EXPORT global + ARCHIVE DESTINATION lib + ) + + message (STATUS "Some symbols from glibc will be replaced for compatibility") +elseif (YANDEX_OFFICIAL_BUILD) + message (WARNING "Option GLIBC_COMPATIBILITY must be turned on for production builds.") +endif () diff --git a/utils/check-style/check-style b/utils/check-style/check-style index fed4b6b8670..deed481f043 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -25,3 +25,9 @@ find $ROOT_PATH/dbms -name '*.h' -or -name '*.cpp' | find $ROOT_PATH/dbms -name '*.h' -or -name '*.cpp' | grep -vP 'Compiler|build' | xargs grep $@ -P '}\s*//+\s*namespace\s*' + +# Broken symlinks +find -L $ROOT_PATH -type l | grep -v contrib && echo "^ Broken symlinks found" + +# Double whitespaces +find $ROOT_PATH/dbms -name '*.h' -or -name '*.cpp' | while read i; do $ROOT_PATH/utils/check-style/double-whitespaces.pl < $i || echo -e "^ File $i contains double whitespaces\n"; done diff --git a/utils/check-style/double-whitespaces.pl b/utils/check-style/double-whitespaces.pl new file mode 100755 index 00000000000..47b03cb74ab --- /dev/null +++ b/utils/check-style/double-whitespaces.pl @@ -0,0 +1,33 @@ +#!/usr/bin/perl + +use strict; + +# Find double whitespace such as "a, b, c" that looks very ugly and annoying. +# But skip double whitespaces if they are used as an alignment - by comparing to surrounding lines. + +my @array; + +while (<>) +{ + push @array, $_; +} + +my $ret = 0; + +for (my $i = 1; $i < $#array; ++$i) +{ + if ($array[$i] =~ ',( {2,3})[^ /]') + { + # https://stackoverflow.com/questions/87380/how-can-i-find-the-location-of-a-regex-match-in-perl + + if ((substr($array[$i - 1], $+[1] - 1, 2) !~ /^[ -][^ ]$/) # whitespaces are not part of alignment + && (substr($array[$i + 1], $+[1] - 1, 2) !~ /^[ -][^ ]$/) + && $array[$i] !~ /(-?\d+\w*,\s+){3,}/) # this is not a number table like { 10, -1, 2 } + { + print(($i + 1) . ":" . $array[$i]); + $ret = 1; + } + } +} + +exit $ret; diff --git a/utils/compressor/CMakeLists.txt b/utils/compressor/CMakeLists.txt index 3fdf8aa5eaf..c032054187b 100644 --- a/utils/compressor/CMakeLists.txt +++ b/utils/compressor/CMakeLists.txt @@ -4,7 +4,7 @@ add_executable (zstd_test zstd_test.cpp) if(ZSTD_LIBRARY) target_link_libraries(zstd_test PRIVATE ${ZSTD_LIBRARY}) endif() -target_link_libraries (zstd_test PRIVATE common Threads::Threads) +target_link_libraries (zstd_test PRIVATE common) add_executable (mutator mutator.cpp) target_link_libraries(mutator PRIVATE clickhouse_common_io)