diff --git a/CMakeLists.txt b/CMakeLists.txt index 5330c8daeb5..9513ee85e02 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,3 +1,6 @@ +project(ClickHouse) +cmake_minimum_required(VERSION 3.3) + foreach(policy CMP0023 CMP0048 # CMake 3.0 @@ -10,9 +13,6 @@ foreach(policy endif() endforeach() -project(ClickHouse) -cmake_minimum_required(VERSION 3.3) - # Ignore export() since we don't use it, # but it gets broken with a global targets via link_libraries() macro (export) @@ -150,8 +150,12 @@ endif () if (LINKER_NAME) message(STATUS "Using linker: ${LINKER_NAME} (selected from: LLD_PATH=${LLD_PATH}; GOLD_PATH=${GOLD_PATH}; COMPILER_POSTFIX=${COMPILER_POSTFIX})") set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}") + set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}") endif () +# Make sure the final executable has symbols exported +set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -rdynamic") + cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_PHYSICAL_MEMORY) # Not available under freebsd if(NOT AVAILABLE_PHYSICAL_MEMORY OR AVAILABLE_PHYSICAL_MEMORY GREATER 8000) option(COMPILER_PIPE "-pipe compiler option [less /tmp usage, more ram usage]" ON) @@ -226,8 +230,8 @@ endif () # Make this extra-checks for correct library dependencies. if (NOT SANITIZE) - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-undefined") - set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-undefined") + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-undefined") + set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-undefined") endif () include(cmake/dbms_glob_sources.cmake) @@ -343,7 +347,7 @@ include (cmake/find_hyperscan.cmake) include (cmake/find_simdjson.cmake) include (cmake/find_rapidjson.cmake) include (cmake/find_fastops.cmake) -include (cmake/find_orc.cmake) +#include (cmake/find_orc.cmake) find_contrib_lib(cityhash) find_contrib_lib(farmhash) @@ -360,6 +364,29 @@ include (libs/libcommon/cmake/find_jemalloc.cmake) include (libs/libcommon/cmake/find_cctz.cmake) include (libs/libmysqlxx/cmake/find_mysqlclient.cmake) +# When testing for memory leaks with Valgrind, dont link tcmalloc or jemalloc. + +if (USE_JEMALLOC) + message (STATUS "Link jemalloc: ${JEMALLOC_LIBRARIES}") + set (MALLOC_LIBRARIES ${JEMALLOC_LIBRARIES}) +elseif (USE_TCMALLOC) + if (DEBUG_TCMALLOC AND NOT GPERFTOOLS_TCMALLOC_MINIMAL_DEBUG) + message (FATAL_ERROR "Requested DEBUG_TCMALLOC but debug library is not found. You should install Google Perftools. Example: sudo apt-get install libgoogle-perftools-dev") + endif () + + if (DEBUG_TCMALLOC AND GPERFTOOLS_TCMALLOC_MINIMAL_DEBUG) + message (STATUS "Link libtcmalloc_minimal_debug for testing: ${GPERFTOOLS_TCMALLOC_MINIMAL_DEBUG}") + set (MALLOC_LIBRARIES ${GPERFTOOLS_TCMALLOC_MINIMAL_DEBUG}) + else () + message (STATUS "Link libtcmalloc_minimal: ${GPERFTOOLS_TCMALLOC_MINIMAL}") + set (MALLOC_LIBRARIES ${GPERFTOOLS_TCMALLOC_MINIMAL}) + endif () +elseif (SANITIZE) + message (STATUS "Will use ${SANITIZE} sanitizer.") +else () + message (WARNING "Non default allocator is disabled. This is not recommended for production Linux builds.") +endif () + include (cmake/print_flags.cmake) install (EXPORT global DESTINATION cmake) diff --git a/PreLoad.cmake b/PreLoad.cmake index c9bc96b6219..d66bcd68a10 100644 --- a/PreLoad.cmake +++ b/PreLoad.cmake @@ -12,8 +12,7 @@ # https://youtrack.jetbrains.com/issue/CPP-2659 # https://youtrack.jetbrains.com/issue/CPP-870 -string(TOLOWER "${CMAKE_COMMAND}" CMAKE_COMMAND_LOWER) -if (NOT ${CMAKE_COMMAND_LOWER} MATCHES "clion") +if (NOT DEFINED ENV{CLION_IDE}) find_program(NINJA_PATH ninja) if (NINJA_PATH) set(CMAKE_GENERATOR "Ninja" CACHE INTERNAL "" FORCE) diff --git a/README.md b/README.md index d5b0bf63165..ab96a293aeb 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,9 @@ ClickHouse is an open-source column-oriented database management system that all * You can also [fill this form](https://forms.yandex.com/surveys/meet-yandex-clickhouse-team/) to meet Yandex ClickHouse team in person. ## Upcoming Events -* [ClickHouse Meetup in Moscow](https://yandex.ru/promo/clickhouse/moscow-2019) on September 5. * [ClickHouse Meetup in Munich](https://www.meetup.com/ClickHouse-Meetup-Munich/events/264185199/) on September 17. * [ClickHouse Meetup in Paris](https://www.eventbrite.com/e/clickhouse-paris-meetup-2019-registration-68493270215) on October 3. * [ClickHouse Meetup in Hong Kong](https://www.meetup.com/Hong-Kong-Machine-Learning-Meetup/events/263580542/) on October 17. * [ClickHouse Meetup in Shenzhen](https://www.huodongxing.com/event/3483759917300) on October 20. * [ClickHouse Meetup in Shanghai](https://www.huodongxing.com/event/4483760336000) on October 27. +* [ClickHouse Meetup in Tokyo](https://clickhouse.connpass.com/event/147001/) on November 14. diff --git a/cmake/find_capnp.cmake b/cmake/find_capnp.cmake index 572fc1b3341..d59b8b3a2a9 100644 --- a/cmake/find_capnp.cmake +++ b/cmake/find_capnp.cmake @@ -1,20 +1,26 @@ -option (USE_CAPNP "Enable Cap'n Proto" ON) +option (ENABLE_CAPNP "Enable Cap'n Proto" ON) -if (USE_CAPNP) - option (USE_INTERNAL_CAPNP_LIBRARY "Set to FALSE to use system capnproto library instead of bundled" ${NOT_UNBUNDLED}) +if (ENABLE_CAPNP) - # FIXME: refactor to use `add_library(… IMPORTED)` if possible. - if (NOT USE_INTERNAL_CAPNP_LIBRARY) - find_library (KJ kj) - find_library (CAPNP capnp) - find_library (CAPNPC capnpc) +option (USE_INTERNAL_CAPNP_LIBRARY "Set to FALSE to use system capnproto library instead of bundled" ${NOT_UNBUNDLED}) - set (CAPNP_LIBRARIES ${CAPNPC} ${CAPNP} ${KJ}) - else () - add_subdirectory(contrib/capnproto-cmake) +# FIXME: refactor to use `add_library(… IMPORTED)` if possible. +if (NOT USE_INTERNAL_CAPNP_LIBRARY) + find_library (KJ kj) + find_library (CAPNP capnp) + find_library (CAPNPC capnpc) - set (CAPNP_LIBRARIES capnpc) - endif () + set (CAPNP_LIBRARIES ${CAPNPC} ${CAPNP} ${KJ}) +else () + add_subdirectory(contrib/capnproto-cmake) - message (STATUS "Using capnp: ${CAPNP_LIBRARIES}") + set (CAPNP_LIBRARIES capnpc) endif () + +if (CAPNP_LIBRARIES) + set (USE_CAPNP 1) +endif () + +endif () + +message (STATUS "Using capnp: ${CAPNP_LIBRARIES}") diff --git a/cmake/find_hdfs3.cmake b/cmake/find_hdfs3.cmake index 4c29047fc75..9c593d3266a 100644 --- a/cmake/find_hdfs3.cmake +++ b/cmake/find_hdfs3.cmake @@ -1,24 +1,29 @@ -if (NOT ARCH_ARM AND NOT OS_FREEBSD AND NOT APPLE AND USE_PROTOBUF) - option (ENABLE_HDFS "Enable HDFS" ${NOT_UNBUNDLED}) -endif () +if(NOT ARCH_ARM AND NOT OS_FREEBSD AND NOT APPLE AND USE_PROTOBUF) + option(ENABLE_HDFS "Enable HDFS" 1) +endif() -if (ENABLE_HDFS AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libhdfs3/include/hdfs/hdfs.h") - message (WARNING "submodule contrib/libhdfs3 is missing. to fix try run: \n git submodule update --init --recursive") - set (ENABLE_HDFS 0) -endif () +if(ENABLE_HDFS) +option(USE_INTERNAL_HDFS3_LIBRARY "Set to FALSE to use system HDFS3 instead of bundled" ${NOT_UNBUNDLED}) -if (ENABLE_HDFS) -option (USE_INTERNAL_HDFS3_LIBRARY "Set to FALSE to use system HDFS3 instead of bundled" ON) +if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libhdfs3/include/hdfs/hdfs.h") + if(USE_INTERNAL_HDFS3_LIBRARY) + message(WARNING "submodule contrib/libhdfs3 is missing. to fix try run: \n git submodule update --init --recursive") + endif() + set(MISSING_INTERNAL_HDFS3_LIBRARY 1) + set(USE_INTERNAL_HDFS3_LIBRARY 0) +endif() -if (NOT USE_INTERNAL_HDFS3_LIBRARY) - find_package(hdfs3) -endif () +if(NOT USE_INTERNAL_HDFS3_LIBRARY) + find_library(HDFS3_LIBRARY hdfs3) + find_path(HDFS3_INCLUDE_DIR NAMES hdfs/hdfs.h PATHS ${HDFS3_INCLUDE_PATHS}) +endif() -if (HDFS3_LIBRARY AND HDFS3_INCLUDE_DIR) +if(HDFS3_LIBRARY AND HDFS3_INCLUDE_DIR) set(USE_HDFS 1) -elseif (LIBGSASL_LIBRARY AND LIBXML2_LIBRARY) +elseif(NOT MISSING_INTERNAL_HDFS3_LIBRARY AND LIBGSASL_LIBRARY AND LIBXML2_LIBRARY) set(HDFS3_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libhdfs3/include") set(HDFS3_LIBRARY hdfs3) + set(USE_INTERNAL_HDFS3_LIBRARY 1) set(USE_HDFS 1) else() set(USE_INTERNAL_HDFS3_LIBRARY 0) @@ -26,4 +31,4 @@ endif() endif() -message (STATUS "Using hdfs3=${USE_HDFS}: ${HDFS3_INCLUDE_DIR} : ${HDFS3_LIBRARY}") +message(STATUS "Using hdfs3=${USE_HDFS}: ${HDFS3_INCLUDE_DIR} : ${HDFS3_LIBRARY}") diff --git a/cmake/find_llvm.cmake b/cmake/find_llvm.cmake index 3692a98b979..c668416c0c0 100644 --- a/cmake/find_llvm.cmake +++ b/cmake/find_llvm.cmake @@ -18,22 +18,12 @@ if (ENABLE_EMBEDDED_COMPILER) elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") find_package(LLVM ${CMAKE_CXX_COMPILER_VERSION} CONFIG PATHS ${LLVM_PATHS}) else () - #TODO: - #if(NOT LLVM_FOUND) - # find_package(LLVM 9 CONFIG PATHS ${LLVM_PATHS}) - #endif() - #if(NOT LLVM_FOUND) - # find_package(LLVM 8 CONFIG PATHS ${LLVM_PATHS}) - #endif() - if (NOT LLVM_FOUND) - find_package (LLVM 7 CONFIG PATHS ${LLVM_PATHS}) - endif () - if (NOT LLVM_FOUND) - find_package (LLVM 6 CONFIG PATHS ${LLVM_PATHS}) - endif () - if (NOT LLVM_FOUND) - find_package (LLVM 5 CONFIG PATHS ${LLVM_PATHS}) - endif () + # TODO: 9 8 + foreach(llvm_v 7.1 7 6 5) + if (NOT LLVM_FOUND) + find_package (LLVM ${llvm_v} CONFIG PATHS ${LLVM_PATHS}) + endif () + endforeach () endif () if (LLVM_FOUND) diff --git a/cmake/find_orc.cmake b/cmake/find_orc.cmake index 3676bec1b6b..50e563b04b4 100644 --- a/cmake/find_orc.cmake +++ b/cmake/find_orc.cmake @@ -1,8 +1,38 @@ -##TODO replace hardcode to find procedure +option (ENABLE_ORC "Enable ORC" 1) -set(USE_ORC 0) -set(USE_INTERNAL_ORC_LIBRARY ON) +if(ENABLE_ORC) +option (USE_INTERNAL_ORC_LIBRARY "Set to FALSE to use system ORC instead of bundled" ${NOT_UNBUNDLED}) -if (ARROW_LIBRARY) +if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/orc/c++/include/orc/OrcFile.hh") + if(USE_INTERNAL_ORC_LIBRARY) + message(WARNING "submodule contrib/orc is missing. to fix try run: \n git submodule update --init --recursive") + set(USE_INTERNAL_ORC_LIBRARY 0) + endif() + set(MISSING_INTERNAL_ORC_LIBRARY 1) +endif () + +if (NOT USE_INTERNAL_ORC_LIBRARY) + find_package(orc) +endif () + +#if (USE_INTERNAL_ORC_LIBRARY) +#find_path(CYRUS_SASL_INCLUDE_DIR sasl/sasl.h) +#find_library(CYRUS_SASL_SHARED_LIB sasl2) +#if (NOT CYRUS_SASL_INCLUDE_DIR OR NOT CYRUS_SASL_SHARED_LIB) +# set(USE_ORC 0) +#endif() +#endif() + +if (ORC_LIBRARY AND ORC_INCLUDE_DIR) set(USE_ORC 1) -endif() \ No newline at end of file +elseif(NOT MISSING_INTERNAL_ORC_LIBRARY AND ARROW_LIBRARY) # (LIBGSASL_LIBRARY AND LIBXML2_LIBRARY) + set(ORC_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/orc/c++/include") + set(ORC_LIBRARY orc) + set(USE_ORC 1) +else() + set(USE_INTERNAL_ORC_LIBRARY 0) +endif() + +endif() + +message (STATUS "Using internal=${USE_INTERNAL_ORC_LIBRARY} orc=${USE_ORC}: ${ORC_INCLUDE_DIR} : ${ORC_LIBRARY}") diff --git a/cmake/find_parquet.cmake b/cmake/find_parquet.cmake index 63f589a9ea5..5c5bc664113 100644 --- a/cmake/find_parquet.cmake +++ b/cmake/find_parquet.cmake @@ -62,6 +62,7 @@ elseif(NOT MISSING_INTERNAL_PARQUET_LIBRARY AND NOT OS_FREEBSD) endif() set(USE_PARQUET 1) + set(USE_ORC 1) endif() endif() diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 96462de0190..54fdc4d69e0 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -10,19 +10,6 @@ endif () set_property(DIRECTORY PROPERTY EXCLUDE_FROM_ALL 1) -if (USE_INTERNAL_ORC_LIBRARY) - set(BUILD_JAVA OFF) - set (ANALYZE_JAVA OFF) - set (BUILD_CPP_TESTS OFF) - set (BUILD_TOOLS OFF) - option(BUILD_JAVA OFF) - option (ANALYZE_JAVA OFF) - option (BUILD_CPP_TESTS OFF) - option (BUILD_TOOLS OFF) - set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/contrib/orc/cmake_modules") - add_subdirectory(orc) -endif() - if (USE_INTERNAL_BOOST_LIBRARY) add_subdirectory (boost-cmake) endif () @@ -327,3 +314,7 @@ endif() if (USE_FASTOPS) add_subdirectory (fastops-cmake) endif() + +#if (USE_INTERNAL_ORC_LIBRARY) +# add_subdirectory(orc-cmake) +#endif () diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index ba1ddc2414a..cfd57f2b296 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -56,11 +56,11 @@ set(ORC_SOURCE_WRAP_DIR ${ORC_SOURCE_DIR}/wrap) set(ORC_BUILD_SRC_DIR ${CMAKE_CURRENT_BINARY_DIR}/../orc/c++/src) set(ORC_BUILD_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/../orc/c++/include) -set(GOOGLE_PROTOBUF_DIR ${ClickHouse_SOURCE_DIR}/contrib/protobuf/src/) +set(GOOGLE_PROTOBUF_DIR ${Protobuf_INCLUDE_DIR}/) set(ORC_ADDITION_SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}) set(ARROW_SRC_DIR ${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src) -set(PROTOBUF_EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/../protobuf/cmake/protoc) +set(PROTOBUF_EXECUTABLE ${Protobuf_PROTOC_EXECUTABLE}) set(PROTO_DIR ${ORC_SOURCE_DIR}/../proto) @@ -70,14 +70,10 @@ add_custom_command(OUTPUT orc_proto.pb.h orc_proto.pb.cc --cpp_out="${CMAKE_CURRENT_BINARY_DIR}" "${PROTO_DIR}/orc_proto.proto") -include_directories(SYSTEM ${ORC_INCLUDE_DIR}) -include_directories(SYSTEM ${ORC_SOURCE_SRC_DIR}) -include_directories(SYSTEM ${ORC_SOURCE_WRAP_DIR}) -include_directories(SYSTEM ${GOOGLE_PROTOBUF_DIR}) -include_directories(SYSTEM ${ORC_BUILD_SRC_DIR}) -include_directories(SYSTEM ${ORC_BUILD_INCLUDE_DIR}) -include_directories(SYSTEM ${ORC_ADDITION_SOURCE_DIR}) -include_directories(SYSTEM ${ARROW_SRC_DIR}) +include(${ClickHouse_SOURCE_DIR}/contrib/orc/cmake_modules/CheckSourceCompiles.cmake) +include(orc_check.cmake) +configure_file("${ORC_INCLUDE_DIR}/orc/orc-config.hh.in" "${ORC_BUILD_INCLUDE_DIR}/orc/orc-config.hh") +configure_file("${ORC_SOURCE_SRC_DIR}/Adaptor.hh.in" "${ORC_BUILD_INCLUDE_DIR}/Adaptor.hh") set(ORC_SRCS @@ -232,6 +228,14 @@ if (ARROW_WITH_ZSTD) target_link_libraries(${ARROW_LIBRARY} PRIVATE ${ZSTD_LIBRARY}) endif() +target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ORC_INCLUDE_DIR}) +target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ORC_SOURCE_SRC_DIR}) +target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ORC_SOURCE_WRAP_DIR}) +target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${GOOGLE_PROTOBUF_DIR}) +target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ORC_BUILD_SRC_DIR}) +target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ORC_BUILD_INCLUDE_DIR}) +target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ORC_ADDITION_SOURCE_DIR}) +target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ARROW_SRC_DIR}) # === parquet diff --git a/contrib/arrow-cmake/orc_check.cmake b/contrib/arrow-cmake/orc_check.cmake new file mode 100644 index 00000000000..ad3b72e44cf --- /dev/null +++ b/contrib/arrow-cmake/orc_check.cmake @@ -0,0 +1,130 @@ +# Not changed part of contrib/orc/c++/src/CMakeLists.txt + +INCLUDE(CheckCXXSourceCompiles) + +CHECK_CXX_SOURCE_COMPILES(" + #include + #include + int main(int,char*[]){ + int f = open(\"/x/y\", O_RDONLY); + char buf[100]; + return pread(f, buf, 100, 1000) == 0; + }" + HAS_PREAD +) + +CHECK_CXX_SOURCE_COMPILES(" + #include + int main(int,char*[]){ + struct tm time2020; + return !strptime(\"2020-02-02 12:34:56\", \"%Y-%m-%d %H:%M:%S\", &time2020); + }" + HAS_STRPTIME +) + +CHECK_CXX_SOURCE_COMPILES(" + #include + int main(int,char* argv[]){ + return static_cast(std::stoll(argv[0])); + }" + HAS_STOLL +) + +CHECK_CXX_SOURCE_COMPILES(" + #include + #include + int main(int,char*[]){ + int64_t x = 1; printf(\"%lld\",x); + }" + INT64_IS_LL +) + +CHECK_CXX_SOURCE_COMPILES(" + #ifdef __clang__ + #pragma clang diagnostic push + #pragma clang diagnostic ignored \"-Wdeprecated\" + #pragma clang diagnostic pop + #elif defined(__GNUC__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored \"-Wdeprecated\" + #pragma GCC diagnostic pop + #elif defined(_MSC_VER) + #pragma warning( push ) + #pragma warning( disable : 4996 ) + #pragma warning( pop ) + #else + unknownCompiler! + #endif + int main(int, char *[]) {}" + HAS_DIAGNOSTIC_PUSH +) + +CHECK_CXX_SOURCE_COMPILES(" + #include + int main(int, char *[]) { + return std::isnan(1.0f); + }" + HAS_STD_ISNAN +) + +CHECK_CXX_SOURCE_COMPILES(" + #include + int main(int, char *[]) { + std::mutex test_mutex; + std::lock_guard lock_mutex(test_mutex); + }" + HAS_STD_MUTEX +) + +CHECK_CXX_SOURCE_COMPILES(" + #include + std::string func() { + std::string var = \"test\"; + return std::move(var); + } + int main(int, char *[]) {}" + NEEDS_REDUNDANT_MOVE +) + +INCLUDE(CheckCXXSourceRuns) + +CHECK_CXX_SOURCE_RUNS(" + #include + int main(int, char *[]) { + time_t t = -14210715; // 1969-07-20 12:34:45 + struct tm *ptm = gmtime(&t); + return !(ptm && ptm->tm_year == 69); + }" + HAS_PRE_1970 +) + +CHECK_CXX_SOURCE_RUNS(" + #include + #include + int main(int, char *[]) { + setenv(\"TZ\", \"America/Los_Angeles\", 1); + tzset(); + struct tm time2037; + struct tm time2038; + strptime(\"2037-05-05 12:34:56\", \"%Y-%m-%d %H:%M:%S\", &time2037); + strptime(\"2038-05-05 12:34:56\", \"%Y-%m-%d %H:%M:%S\", &time2038); + return mktime(&time2038) - mktime(&time2037) != 31536000; + }" + HAS_POST_2038 +) + +set(CMAKE_REQUIRED_INCLUDES ${ZLIB_INCLUDE_DIR}) +set(CMAKE_REQUIRED_LIBRARIES zlib) +CHECK_CXX_SOURCE_COMPILES(" + #define Z_PREFIX + #include + z_stream strm; + int main(int, char *[]) { + deflateReset(&strm); + }" + NEEDS_Z_PREFIX +) + +# See https://cmake.org/cmake/help/v3.14/policy/CMP0075.html. Without unsetting it breaks thrift. +set(CMAKE_REQUIRED_INCLUDES) +set(CMAKE_REQUIRED_LIBRARIES) diff --git a/contrib/libhdfs3-cmake/CMakeLists.txt b/contrib/libhdfs3-cmake/CMakeLists.txt index 8ec14f897b9..e1ba7225b0f 100644 --- a/contrib/libhdfs3-cmake/CMakeLists.txt +++ b/contrib/libhdfs3-cmake/CMakeLists.txt @@ -199,17 +199,17 @@ if (WITH_KERBEROS) endif() target_include_directories(hdfs3 PRIVATE ${LIBXML2_INCLUDE_DIR}) -target_link_libraries(hdfs3 ${LIBGSASL_LIBRARY}) +target_link_libraries(hdfs3 PRIVATE ${LIBGSASL_LIBRARY}) if (WITH_KERBEROS) - target_link_libraries(hdfs3 ${KERBEROS_LIBRARIES}) + target_link_libraries(hdfs3 PRIVATE ${KERBEROS_LIBRARIES}) endif() -target_link_libraries(hdfs3 ${LIBXML2_LIBRARY}) +target_link_libraries(hdfs3 PRIVATE ${LIBXML2_LIBRARY}) # inherit from parent cmake target_include_directories(hdfs3 PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(hdfs3 PRIVATE ${Protobuf_INCLUDE_DIR}) -target_link_libraries(hdfs3 ${Protobuf_LIBRARY}) +target_link_libraries(hdfs3 PRIVATE ${Protobuf_LIBRARY}) if(OPENSSL_INCLUDE_DIR AND OPENSSL_LIBRARIES) target_include_directories(hdfs3 PRIVATE ${OPENSSL_INCLUDE_DIR}) - target_link_libraries(hdfs3 ${OPENSSL_LIBRARIES}) + target_link_libraries(hdfs3 PRIVATE ${OPENSSL_LIBRARIES}) endif() diff --git a/contrib/orc-cmake/CMakeLists.txt b/contrib/orc-cmake/CMakeLists.txt new file mode 100644 index 00000000000..066ba00aede --- /dev/null +++ b/contrib/orc-cmake/CMakeLists.txt @@ -0,0 +1,229 @@ +# modifyed copy of contrib/orc/c++/src/CMakeLists.txt +set(LIBRARY_INCLUDE ${ClickHouse_SOURCE_DIR}/contrib/orc/c++/include) +set(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/orc/c++/src) + +set(PROTOBUF_INCLUDE_DIR ${Protobuf_INCLUDE_DIR}) +set(PROTOBUF_EXECUTABLE ${Protobuf_PROTOC_EXECUTABLE}) + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX11_FLAGS} ${WARN_FLAGS}") + +INCLUDE(CheckCXXSourceCompiles) + +CHECK_CXX_SOURCE_COMPILES(" + #include + #include + int main(int,char*[]){ + int f = open(\"/x/y\", O_RDONLY); + char buf[100]; + return pread(f, buf, 100, 1000) == 0; + }" + HAS_PREAD +) + +CHECK_CXX_SOURCE_COMPILES(" + #include + int main(int,char*[]){ + struct tm time2020; + return !strptime(\"2020-02-02 12:34:56\", \"%Y-%m-%d %H:%M:%S\", &time2020); + }" + HAS_STRPTIME +) + +CHECK_CXX_SOURCE_COMPILES(" + #include + int main(int,char* argv[]){ + return static_cast(std::stoll(argv[0])); + }" + HAS_STOLL +) + +CHECK_CXX_SOURCE_COMPILES(" + #include + #include + int main(int,char*[]){ + int64_t x = 1; printf(\"%lld\",x); + }" + INT64_IS_LL +) + +CHECK_CXX_SOURCE_COMPILES(" + #ifdef __clang__ + #pragma clang diagnostic push + #pragma clang diagnostic ignored \"-Wdeprecated\" + #pragma clang diagnostic pop + #elif defined(__GNUC__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored \"-Wdeprecated\" + #pragma GCC diagnostic pop + #elif defined(_MSC_VER) + #pragma warning( push ) + #pragma warning( disable : 4996 ) + #pragma warning( pop ) + #else + unknownCompiler! + #endif + int main(int, char *[]) {}" + HAS_DIAGNOSTIC_PUSH +) + +CHECK_CXX_SOURCE_COMPILES(" + #include + int main(int, char *[]) { + return std::isnan(1.0f); + }" + HAS_STD_ISNAN +) + +CHECK_CXX_SOURCE_COMPILES(" + #include + int main(int, char *[]) { + std::mutex test_mutex; + std::lock_guard lock_mutex(test_mutex); + }" + HAS_STD_MUTEX +) + +CHECK_CXX_SOURCE_COMPILES(" + #include + std::string func() { + std::string var = \"test\"; + return std::move(var); + } + int main(int, char *[]) {}" + NEEDS_REDUNDANT_MOVE +) + +INCLUDE(CheckCXXSourceRuns) + +CHECK_CXX_SOURCE_RUNS(" + #include + int main(int, char *[]) { + time_t t = -14210715; // 1969-07-20 12:34:45 + struct tm *ptm = gmtime(&t); + return !(ptm && ptm->tm_year == 69); + }" + HAS_PRE_1970 +) + +CHECK_CXX_SOURCE_RUNS(" + #include + #include + int main(int, char *[]) { + setenv(\"TZ\", \"America/Los_Angeles\", 1); + tzset(); + struct tm time2037; + struct tm time2038; + strptime(\"2037-05-05 12:34:56\", \"%Y-%m-%d %H:%M:%S\", &time2037); + strptime(\"2038-05-05 12:34:56\", \"%Y-%m-%d %H:%M:%S\", &time2038); + return mktime(&time2038) - mktime(&time2037) != 31536000; + }" + HAS_POST_2038 +) + +set(CMAKE_REQUIRED_INCLUDES ${ZLIB_INCLUDE_DIR}) +set(CMAKE_REQUIRED_LIBRARIES zlib) +CHECK_CXX_SOURCE_COMPILES(" + #define Z_PREFIX + #include + z_stream strm; + int main(int, char *[]) { + deflateReset(&strm); + }" + NEEDS_Z_PREFIX +) + +configure_file ( + "${LIBRARY_DIR}/Adaptor.hh.in" + "${CMAKE_CURRENT_BINARY_DIR}/Adaptor.hh" + ) + + +add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/orc_proto.pb.h ${CMAKE_CURRENT_BINARY_DIR}/orc_proto.pb.cc + COMMAND ${PROTOBUF_EXECUTABLE} + -I${ClickHouse_SOURCE_DIR}/contrib/orc/proto + --cpp_out="${CMAKE_CURRENT_BINARY_DIR}" + "${ClickHouse_SOURCE_DIR}/contrib/orc/proto/orc_proto.proto" +) + +set(SOURCE_FILES + "${CMAKE_CURRENT_BINARY_DIR}/Adaptor.hh" + ${CMAKE_CURRENT_BINARY_DIR}/orc_proto.pb.h + ${LIBRARY_DIR}/io/InputStream.cc + ${LIBRARY_DIR}/io/OutputStream.cc + ${LIBRARY_DIR}/wrap/orc-proto-wrapper.cc + ${LIBRARY_DIR}/Adaptor.cc + ${LIBRARY_DIR}/ByteRLE.cc + ${LIBRARY_DIR}/ColumnPrinter.cc + ${LIBRARY_DIR}/ColumnReader.cc + ${LIBRARY_DIR}/ColumnWriter.cc + ${LIBRARY_DIR}/Common.cc + ${LIBRARY_DIR}/Compression.cc + ${LIBRARY_DIR}/Exceptions.cc + ${LIBRARY_DIR}/Int128.cc + ${LIBRARY_DIR}/LzoDecompressor.cc + ${LIBRARY_DIR}/MemoryPool.cc + ${LIBRARY_DIR}/OrcFile.cc + ${LIBRARY_DIR}/Reader.cc + ${LIBRARY_DIR}/RLEv1.cc + ${LIBRARY_DIR}/RLEv2.cc + ${LIBRARY_DIR}/RLE.cc + ${LIBRARY_DIR}/Statistics.cc + ${LIBRARY_DIR}/StripeStream.cc + ${LIBRARY_DIR}/Timezone.cc + ${LIBRARY_DIR}/TypeImpl.cc + ${LIBRARY_DIR}/Vector.cc + ${LIBRARY_DIR}/Writer.cc + ) + +if(ORC_CXX_HAS_THREAD_LOCAL AND BUILD_LIBHDFSPP) + set(SOURCE_FILES ${SOURCE_FILES} ${LIBRARY_DIR}/OrcHdfsFile.cc) +endif(ORC_CXX_HAS_THREAD_LOCAL AND BUILD_LIBHDFSPP) + +#list(TRANSFORM SOURCE_FILES PREPEND ${LIBRARY_DIR}/) + +configure_file ( + "${LIBRARY_INCLUDE}/orc/orc-config.hh.in" + "${CMAKE_CURRENT_BINARY_DIR}/orc/orc-config.hh" + ) + +add_library (orc ${SOURCE_FILES}) + +target_include_directories (orc + PRIVATE + ${LIBRARY_INCLUDE} + ${LIBRARY_DIR} + #PUBLIC + ${CMAKE_CURRENT_BINARY_DIR} + PRIVATE + ${PROTOBUF_INCLUDE_DIR} + ${ZLIB_INCLUDE_DIR} + ${SNAPPY_INCLUDE_DIR} + ${LZ4_INCLUDE_DIR} + ${LIBHDFSPP_INCLUDE_DIR} + ) + +target_link_libraries (orc PRIVATE + ${Protobuf_LIBRARY} + ${ZLIB_LIBRARIES} + ${SNAPPY_LIBRARY} + ${LZ4_LIBRARY} + ${LIBHDFSPP_LIBRARIES} + ) + +#install(TARGETS orc DESTINATION lib) + +if(ORC_CXX_HAS_THREAD_LOCAL AND BUILD_LIBHDFSPP) + add_definitions(-DBUILD_LIBHDFSPP) +endif(ORC_CXX_HAS_THREAD_LOCAL AND BUILD_LIBHDFSPP) diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index 992b672bccc..2c8f99b9738 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -101,28 +101,6 @@ add_headers_and_sources(clickhouse_common_io src/Common) add_headers_and_sources(clickhouse_common_io src/Common/HashTable) add_headers_and_sources(clickhouse_common_io src/IO) -add_headers_and_sources(dbms src/Core) -add_headers_and_sources(dbms src/Compression/) -add_headers_and_sources(dbms src/DataStreams) -add_headers_and_sources(dbms src/DataTypes) -add_headers_and_sources(dbms src/Databases) -add_headers_and_sources(dbms src/Interpreters) -add_headers_and_sources(dbms src/Interpreters/ClusterProxy) -add_headers_and_sources(dbms src/Columns) -add_headers_and_sources(dbms src/Storages) -add_headers_and_sources(dbms src/Storages/Distributed) -add_headers_and_sources(dbms src/Storages/MergeTree) -add_headers_and_sources(dbms src/Storages/LiveView) -add_headers_and_sources(dbms src/Client) -add_headers_and_sources(dbms src/Formats) -add_headers_and_sources(dbms src/Processors) -add_headers_and_sources(dbms src/Processors/Executors) -add_headers_and_sources(dbms src/Processors/Formats) -add_headers_and_sources(dbms src/Processors/Formats/Impl) -add_headers_and_sources(dbms src/Processors/Transforms) -add_headers_and_sources(dbms src/Processors/Sources) -add_headers_only(dbms src/Server) - if(USE_RDKAFKA) add_headers_and_sources(dbms src/Storages/Kafka) endif() @@ -160,23 +138,71 @@ if (OS_FREEBSD) target_compile_definitions (clickhouse_common_io PUBLIC CLOCK_MONOTONIC_COARSE=CLOCK_MONOTONIC_FAST) endif () -if (USE_UNWIND) - target_link_libraries (clickhouse_common_io PRIVATE ${UNWIND_LIBRARIES}) -endif () - add_subdirectory(src/Common/ZooKeeper) add_subdirectory(src/Common/Config) +set (all_modules) +macro(add_object_library name common_path) + if (MAKE_STATIC_LIBRARIES OR NOT SPLIT_SHARED_LIBRARIES) + add_glob(dbms_headers RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${common_path}/*.h) + add_glob(dbms_sources ${common_path}/*.cpp ${common_path}/*.c ${common_path}/*.h) + else () + list (APPEND all_modules ${name}) + add_glob(${name}_headers RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${common_path}/*.h) + add_glob(${name}_sources ${common_path}/*.cpp ${common_path}/*.c ${common_path}/*.h) + add_library(${name} SHARED ${${name}_sources} ${${name}_headers}) + target_link_libraries (${name} PRIVATE -Wl,--unresolved-symbols=ignore-all) + endif () +endmacro() + +add_object_library(clickhouse_core src/Core) +add_object_library(clickhouse_compression src/Compression/) +add_object_library(clickhouse_datastreams src/DataStreams) +add_object_library(clickhouse_datatypes src/DataTypes) +add_object_library(clickhouse_databases src/Databases) +add_object_library(clickhouse_interpreters src/Interpreters) +add_object_library(clickhouse_interpreters_clusterproxy src/Interpreters/ClusterProxy) +add_object_library(clickhouse_columns src/Columns) +add_object_library(clickhouse_storages src/Storages) +add_object_library(clickhouse_storages_distributed src/Storages/Distributed) +add_object_library(clickhouse_storages_mergetree src/Storages/MergeTree) +add_object_library(clickhouse_storages_liveview src/Storages/LiveView) +add_object_library(clickhouse_client src/Client) +add_object_library(clickhouse_formats src/Formats) +add_object_library(clickhouse_processors src/Processors) +add_object_library(clickhouse_processors_executors src/Processors/Executors) +add_object_library(clickhouse_processors_formats src/Processors/Formats) +add_object_library(clickhouse_processors_formats_impl src/Processors/Formats/Impl) +add_object_library(clickhouse_processors_transforms src/Processors/Transforms) +add_object_library(clickhouse_processors_sources src/Processors/Sources) + if (MAKE_STATIC_LIBRARIES OR NOT SPLIT_SHARED_LIBRARIES) - add_library(dbms ${dbms_headers} ${dbms_sources}) -else () - add_library(dbms SHARED ${dbms_headers} ${dbms_sources}) + add_library (dbms STATIC ${dbms_headers} ${dbms_sources}) + set (all_modules dbms) +else() + add_library (dbms SHARED ${dbms_headers} ${dbms_sources}) + target_link_libraries (dbms PUBLIC ${all_modules}) + list (APPEND all_modules dbms) + # force all split libs to be linked + set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-as-needed") endif () +macro (dbms_target_include_directories) + foreach (module ${all_modules}) + target_include_directories (${module} ${ARGN}) + endforeach () +endmacro () + +macro (dbms_target_link_libraries) + foreach (module ${all_modules}) + target_link_libraries (${module} ${ARGN}) + endforeach () +endmacro () + if (USE_EMBEDDED_COMPILER) llvm_libs_all(REQUIRED_LLVM_LIBRARIES) - target_link_libraries (dbms PRIVATE ${REQUIRED_LLVM_LIBRARIES}) - target_include_directories (dbms SYSTEM BEFORE PUBLIC ${LLVM_INCLUDE_DIRS}) + dbms_target_link_libraries (PRIVATE ${REQUIRED_LLVM_LIBRARIES}) + dbms_target_include_directories (SYSTEM BEFORE PUBLIC ${LLVM_INCLUDE_DIRS}) endif () if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" OR CMAKE_BUILD_TYPE_UC STREQUAL "MINSIZEREL") @@ -245,9 +271,9 @@ target_link_libraries(clickhouse_common_io ) if (USE_RDKAFKA) - target_link_libraries(dbms PRIVATE ${CPPKAFKA_LIBRARY} ${RDKAFKA_LIBRARY}) + dbms_target_link_libraries(PRIVATE ${CPPKAFKA_LIBRARY} ${RDKAFKA_LIBRARY}) if(NOT USE_INTERNAL_RDKAFKA_LIBRARY) - target_include_directories(dbms SYSTEM BEFORE PRIVATE ${RDKAFKA_INCLUDE_DIR}) + dbms_target_include_directories(SYSTEM BEFORE PRIVATE ${RDKAFKA_INCLUDE_DIR}) endif() endif() @@ -264,7 +290,7 @@ if(CPUINFO_LIBRARY) target_link_libraries(clickhouse_common_io PRIVATE ${CPUINFO_LIBRARY}) endif() -target_link_libraries (dbms +dbms_target_link_libraries ( PRIVATE clickhouse_parsers clickhouse_common_config @@ -285,21 +311,24 @@ target_link_libraries (dbms ${Boost_SYSTEM_LIBRARY} ) -target_include_directories(dbms PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/src/Core/include) target_include_directories(clickhouse_common_io PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/src/Core/include) # uses some includes from core -target_include_directories(dbms SYSTEM BEFORE PUBLIC ${PDQSORT_INCLUDE_DIR}) -target_include_directories(dbms SYSTEM PUBLIC ${PCG_RANDOM_INCLUDE_DIR}) +dbms_target_include_directories(PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/src/Core/include) + +target_include_directories(clickhouse_common_io SYSTEM PUBLIC ${PCG_RANDOM_INCLUDE_DIR}) +dbms_target_include_directories(SYSTEM PUBLIC ${PCG_RANDOM_INCLUDE_DIR}) + +dbms_target_include_directories(SYSTEM BEFORE PUBLIC ${PDQSORT_INCLUDE_DIR}) if (NOT USE_INTERNAL_LZ4_LIBRARY) - target_include_directories(dbms SYSTEM BEFORE PRIVATE ${LZ4_INCLUDE_DIR}) + dbms_target_include_directories(SYSTEM BEFORE PRIVATE ${LZ4_INCLUDE_DIR}) endif () if (ZSTD_LIBRARY) - target_link_libraries(dbms PRIVATE ${ZSTD_LIBRARY}) + dbms_target_link_libraries(PRIVATE ${ZSTD_LIBRARY}) + if (NOT USE_INTERNAL_ZSTD_LIBRARY AND ZSTD_INCLUDE_DIR) + dbms_target_include_directories(SYSTEM BEFORE PRIVATE ${ZSTD_INCLUDE_DIR}) + endif () endif() -if (NOT USE_INTERNAL_ZSTD_LIBRARY AND ZSTD_INCLUDE_DIR) - target_include_directories(dbms SYSTEM BEFORE PRIVATE ${ZSTD_INCLUDE_DIR}) -endif () if (NOT USE_INTERNAL_BOOST_LIBRARY) target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS}) @@ -307,68 +336,69 @@ endif () if (Poco_SQL_FOUND AND NOT USE_INTERNAL_POCO_LIBRARY) target_include_directories (clickhouse_common_io SYSTEM PRIVATE ${Poco_SQL_INCLUDE_DIR}) - target_include_directories (dbms SYSTEM PRIVATE ${Poco_SQL_INCLUDE_DIR}) + dbms_target_include_directories (SYSTEM PRIVATE ${Poco_SQL_INCLUDE_DIR}) endif() if (USE_POCO_SQLODBC) target_link_libraries (clickhouse_common_io PRIVATE ${Poco_SQL_LIBRARY}) - target_link_libraries (dbms PRIVATE ${Poco_SQLODBC_LIBRARY} ${Poco_SQL_LIBRARY}) + dbms_target_link_libraries (PRIVATE ${Poco_SQLODBC_LIBRARY} ${Poco_SQL_LIBRARY}) if (NOT USE_INTERNAL_POCO_LIBRARY) target_include_directories (clickhouse_common_io SYSTEM PRIVATE ${ODBC_INCLUDE_DIRS} ${Poco_SQL_INCLUDE_DIR}) - target_include_directories (dbms SYSTEM PRIVATE ${ODBC_INCLUDE_DIRS} ${Poco_SQLODBC_INCLUDE_DIR} SYSTEM PUBLIC ${Poco_SQL_INCLUDE_DIR}) + dbms_target_include_directories (SYSTEM PRIVATE ${ODBC_INCLUDE_DIRS} ${Poco_SQLODBC_INCLUDE_DIR} SYSTEM PUBLIC ${Poco_SQL_INCLUDE_DIR}) endif() endif() if (Poco_Data_FOUND) target_include_directories (clickhouse_common_io SYSTEM PRIVATE ${Poco_Data_INCLUDE_DIR}) - target_include_directories (dbms SYSTEM PRIVATE ${Poco_Data_INCLUDE_DIR}) + dbms_target_include_directories (SYSTEM PRIVATE ${Poco_Data_INCLUDE_DIR}) endif() if (USE_POCO_DATAODBC) target_link_libraries (clickhouse_common_io PRIVATE ${Poco_Data_LIBRARY}) - target_link_libraries (dbms PRIVATE ${Poco_DataODBC_LIBRARY}) + dbms_target_link_libraries (PRIVATE ${Poco_DataODBC_LIBRARY}) if (NOT USE_INTERNAL_POCO_LIBRARY) - target_include_directories (dbms SYSTEM PRIVATE ${ODBC_INCLUDE_DIRS} ${Poco_DataODBC_INCLUDE_DIR}) + dbms_target_include_directories (SYSTEM PRIVATE ${ODBC_INCLUDE_DIRS} ${Poco_DataODBC_INCLUDE_DIR}) endif() endif() if (USE_POCO_MONGODB) - target_link_libraries (dbms PRIVATE ${Poco_MongoDB_LIBRARY}) + dbms_target_link_libraries (PRIVATE ${Poco_MongoDB_LIBRARY}) endif() if (USE_POCO_NETSSL) target_link_libraries (clickhouse_common_io PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY}) - target_link_libraries (dbms PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY}) + dbms_target_link_libraries (PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY}) endif() -target_link_libraries (dbms PRIVATE ${Poco_Foundation_LIBRARY}) +dbms_target_link_libraries (PRIVATE ${Poco_Foundation_LIBRARY}) if (USE_ICU) - target_link_libraries (dbms PRIVATE ${ICU_LIBRARIES}) - target_include_directories (dbms SYSTEM PRIVATE ${ICU_INCLUDE_DIRS}) + dbms_target_link_libraries (PRIVATE ${ICU_LIBRARIES}) + dbms_target_include_directories (SYSTEM PRIVATE ${ICU_INCLUDE_DIRS}) endif () if (USE_CAPNP) - target_link_libraries (dbms PRIVATE ${CAPNP_LIBRARIES}) + dbms_target_link_libraries (PRIVATE ${CAPNP_LIBRARIES}) endif () if (USE_PARQUET) - target_link_libraries(dbms PRIVATE ${PARQUET_LIBRARY}) + dbms_target_link_libraries(PRIVATE ${PARQUET_LIBRARY}) if (NOT USE_INTERNAL_PARQUET_LIBRARY OR USE_INTERNAL_PARQUET_LIBRARY_NATIVE_CMAKE) - target_include_directories (dbms SYSTEM BEFORE PRIVATE ${PARQUET_INCLUDE_DIR} ${ARROW_INCLUDE_DIR}) + dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${PARQUET_INCLUDE_DIR} ${ARROW_INCLUDE_DIR}) endif () endif () -if(OPENSSL_CRYPTO_LIBRARY) - target_link_libraries(dbms PRIVATE ${OPENSSL_CRYPTO_LIBRARY}) +if (OPENSSL_CRYPTO_LIBRARY) + dbms_target_link_libraries (PRIVATE ${OPENSSL_CRYPTO_LIBRARY}) + target_link_libraries (clickhouse_common_io PRIVATE ${OPENSSL_CRYPTO_LIBRARY}) endif () -target_include_directories (dbms SYSTEM BEFORE PRIVATE ${DIVIDE_INCLUDE_DIR}) -target_include_directories (dbms SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) +dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${DIVIDE_INCLUDE_DIR}) +dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) if (USE_PROTOBUF) - target_link_libraries (dbms PRIVATE ${Protobuf_LIBRARY}) - target_include_directories (dbms SYSTEM BEFORE PRIVATE ${Protobuf_INCLUDE_DIR}) + dbms_target_link_libraries (PRIVATE ${Protobuf_LIBRARY}) + dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${Protobuf_INCLUDE_DIR}) endif () if (USE_HDFS) @@ -382,13 +412,13 @@ if (USE_BROTLI) endif() if (USE_JEMALLOC) - target_include_directories (dbms SYSTEM BEFORE PRIVATE ${JEMALLOC_INCLUDE_DIR}) # used in Interpreters/AsynchronousMetrics.cpp + dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${JEMALLOC_INCLUDE_DIR}) # used in Interpreters/AsynchronousMetrics.cpp target_include_directories (clickhouse_common_io SYSTEM BEFORE PRIVATE ${JEMALLOC_INCLUDE_DIR}) # new_delete.cpp endif () -target_include_directories (dbms PUBLIC ${DBMS_INCLUDE_DIR} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/src/Formats/include) +dbms_target_include_directories (PUBLIC ${DBMS_INCLUDE_DIR} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/src/Formats/include) target_include_directories (clickhouse_common_io PUBLIC ${DBMS_INCLUDE_DIR}) -target_include_directories (clickhouse_common_io SYSTEM PUBLIC ${PCG_RANDOM_INCLUDE_DIR}) + target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${DOUBLE_CONVERSION_INCLUDE_DIR}) # also for copy_headers.sh: diff --git a/dbms/programs/CMakeLists.txt b/dbms/programs/CMakeLists.txt index 0dcd4d7ab91..1967aee2c09 100644 --- a/dbms/programs/CMakeLists.txt +++ b/dbms/programs/CMakeLists.txt @@ -24,9 +24,9 @@ configure_file (config_tools.h.in ${CMAKE_CURRENT_BINARY_DIR}/config_tools.h) macro(clickhouse_target_link_split_lib target name) if(NOT CLICKHOUSE_ONE_SHARED) - target_link_libraries(${target} PRIVATE clickhouse-${name}-lib) + target_link_libraries(${target} PRIVATE clickhouse-${name}-lib ${MALLOC_LIBRARIES}) else() - target_link_libraries(${target} PRIVATE clickhouse-lib) + target_link_libraries(${target} PRIVATE clickhouse-lib ${MALLOC_LIBRARIES}) endif() endmacro() @@ -111,7 +111,7 @@ if (CLICKHOUSE_SPLIT_BINARY) install(PROGRAMS clickhouse-split-helper DESTINATION ${CMAKE_INSTALL_BINDIR} RENAME clickhouse COMPONENT clickhouse) else () add_executable (clickhouse main.cpp) - target_link_libraries (clickhouse PRIVATE clickhouse_common_io string_utils) + target_link_libraries (clickhouse PRIVATE clickhouse_common_io string_utils ${MALLOC_LIBRARIES}) target_include_directories (clickhouse BEFORE PRIVATE ${COMMON_INCLUDE_DIR}) target_include_directories (clickhouse PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index e6d3f950908..ce04561453f 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -563,9 +563,17 @@ private: if (is_interactive) { std::cout << "Connected to " << server_name - << " server version " << server_version - << " revision " << server_revision - << "." << std::endl << std::endl; + << " server version " << server_version + << " revision " << server_revision + << "." << std::endl << std::endl; + + if (std::make_tuple(VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH) + < std::make_tuple(server_version_major, server_version_minor, server_version_patch)) + { + std::cout << "ClickHouse client version is older than ClickHouse server. " + << "It may lack support for new features." + << std::endl << std::endl; + } } } diff --git a/dbms/programs/client/Suggest.h b/dbms/programs/client/Suggest.h index 975d0611253..57895b38764 100644 --- a/dbms/programs/client/Suggest.h +++ b/dbms/programs/client/Suggest.h @@ -8,7 +8,6 @@ #include #include -#include #include #include @@ -25,7 +24,7 @@ namespace ErrorCodes extern const int UNKNOWN_PACKET_FROM_SERVER; } -class Suggest : public ext::singleton +class Suggest : private boost::noncopyable { private: /// The vector will be filled with completion words from the server and sorted. @@ -161,6 +160,12 @@ private: } public: + static Suggest & instance() + { + static Suggest instance; + return instance; + } + /// More old server versions cannot execute the query above. static constexpr int MIN_SERVER_REVISION = 54406; diff --git a/dbms/programs/copier/CMakeLists.txt b/dbms/programs/copier/CMakeLists.txt index 85b2819ac7a..8e13040b29d 100644 --- a/dbms/programs/copier/CMakeLists.txt +++ b/dbms/programs/copier/CMakeLists.txt @@ -1,5 +1,5 @@ set(CLICKHOUSE_COPIER_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/ClusterCopier.cpp) -set(CLICKHOUSE_COPIER_LINK PRIVATE clickhouse_common_zookeeper clickhouse_parsers clickhouse_functions clickhouse_table_functions clickhouse_aggregate_functions clickhouse_dictionaries string_utils PUBLIC daemon) +set(CLICKHOUSE_COPIER_LINK PRIVATE clickhouse_common_zookeeper clickhouse_parsers clickhouse_functions clickhouse_table_functions clickhouse_aggregate_functions clickhouse_dictionaries string_utils ${Poco_XML_LIBRARY} PUBLIC daemon) set(CLICKHOUSE_COPIER_INCLUDE SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR}) clickhouse_program_add(copier) diff --git a/dbms/programs/odbc-bridge/CMakeLists.txt b/dbms/programs/odbc-bridge/CMakeLists.txt index 444d6e8d8e0..73574f8dc2e 100644 --- a/dbms/programs/odbc-bridge/CMakeLists.txt +++ b/dbms/programs/odbc-bridge/CMakeLists.txt @@ -35,8 +35,7 @@ clickhouse_program_add_library(odbc-bridge) # clickhouse-odbc-bridge is always a separate binary. # Reason: it must not export symbols from SSL, mariadb-client, etc. to not break ABI compatibility with ODBC drivers. -# For this reason, we disabling -rdynamic linker flag. But we do it in strange way: -SET(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") +set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-export-dynamic") add_executable(clickhouse-odbc-bridge odbc-bridge.cpp) set_target_properties(clickhouse-odbc-bridge PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..) diff --git a/dbms/programs/server/MySQLHandler.cpp b/dbms/programs/server/MySQLHandler.cpp index 7e555220489..76cc7ddc632 100644 --- a/dbms/programs/server/MySQLHandler.cpp +++ b/dbms/programs/server/MySQLHandler.cpp @@ -1,3 +1,5 @@ +#include +#if USE_POCO_NETSSL #include "MySQLHandler.h" #include @@ -301,3 +303,4 @@ void MySQLHandler::comQuery(ReadBuffer & payload) } } +#endif diff --git a/dbms/programs/server/MySQLHandler.h b/dbms/programs/server/MySQLHandler.h index 8dfd178e7ed..e1e4297ea8d 100644 --- a/dbms/programs/server/MySQLHandler.h +++ b/dbms/programs/server/MySQLHandler.h @@ -1,4 +1,6 @@ #pragma once +#include +#if USE_POCO_NETSSL #include #include @@ -56,3 +58,4 @@ private: }; } +#endif diff --git a/dbms/programs/server/MySQLHandlerFactory.cpp b/dbms/programs/server/MySQLHandlerFactory.cpp index 0f949987c54..8e7ebcb8291 100644 --- a/dbms/programs/server/MySQLHandlerFactory.cpp +++ b/dbms/programs/server/MySQLHandlerFactory.cpp @@ -1,3 +1,5 @@ +#include +#if USE_POCO_NETSSL #include #include #include @@ -122,3 +124,4 @@ Poco::Net::TCPServerConnection * MySQLHandlerFactory::createConnection(const Poc } } +#endif diff --git a/dbms/programs/server/MySQLHandlerFactory.h b/dbms/programs/server/MySQLHandlerFactory.h index e8fe81ed384..2da1d0f94a9 100644 --- a/dbms/programs/server/MySQLHandlerFactory.h +++ b/dbms/programs/server/MySQLHandlerFactory.h @@ -1,5 +1,8 @@ #pragma once +#include +#if USE_POCO_NETSSL + #include #include #include @@ -37,3 +40,4 @@ public: }; } +#endif diff --git a/dbms/src/AggregateFunctions/AggregateFunctionCombinatorFactory.cpp b/dbms/src/AggregateFunctions/AggregateFunctionCombinatorFactory.cpp index 17143edf9f0..02874927061 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionCombinatorFactory.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionCombinatorFactory.cpp @@ -26,4 +26,10 @@ AggregateFunctionCombinatorPtr AggregateFunctionCombinatorFactory::tryFindSuffix return {}; } +AggregateFunctionCombinatorFactory & AggregateFunctionCombinatorFactory::instance() +{ + static AggregateFunctionCombinatorFactory ret; + return ret; +} + } diff --git a/dbms/src/AggregateFunctions/AggregateFunctionCombinatorFactory.h b/dbms/src/AggregateFunctions/AggregateFunctionCombinatorFactory.h index 579951cecb1..b535475d111 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionCombinatorFactory.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionCombinatorFactory.h @@ -2,7 +2,6 @@ #include -#include #include #include @@ -13,13 +12,16 @@ namespace DB /** Create aggregate function combinator by matching suffix in aggregate function name. */ -class AggregateFunctionCombinatorFactory final: public ext::singleton +class AggregateFunctionCombinatorFactory final: private boost::noncopyable { private: using Dict = std::unordered_map; Dict dict; public: + + static AggregateFunctionCombinatorFactory & instance(); + /// Not thread safe. You must register before using tryGet. void registerCombinator(const AggregateFunctionCombinatorPtr & value); diff --git a/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp b/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp index 83fb9f48abe..0c9a3b1ad63 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp @@ -160,4 +160,10 @@ bool AggregateFunctionFactory::isAggregateFunctionName(const String & name, int return false; } +AggregateFunctionFactory & AggregateFunctionFactory::instance() +{ + static AggregateFunctionFactory ret; + return ret; +} + } diff --git a/dbms/src/AggregateFunctions/AggregateFunctionFactory.h b/dbms/src/AggregateFunctions/AggregateFunctionFactory.h index 92598e52509..6e755cc9e8c 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionFactory.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionFactory.h @@ -3,7 +3,6 @@ #include #include -#include #include #include @@ -30,9 +29,12 @@ using AggregateFunctionCreator = std::function, public IFactoryWithAliases +class AggregateFunctionFactory final : private boost::noncopyable, public IFactoryWithAliases { public: + + static AggregateFunctionFactory & instance(); + /// Register a function by its name. /// No locking, you must register all functions before usage of get. void registerFunction( diff --git a/dbms/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h b/dbms/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h index 422d1f7a98f..c85a74af1ce 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h @@ -88,10 +88,11 @@ public: if (is_large) { - toLarge(); - UInt32 cardinality; - readBinary(cardinality, in); - db_roaring_bitmap_add_many(in, rb, cardinality); + std::string s; + readStringBinary(s,in); + rb = roaring_bitmap_portable_deserialize(s.c_str()); + for (const auto & x : small) //merge from small + roaring_bitmap_add(rb, x.getValue()); } else small.read(in); @@ -103,9 +104,10 @@ public: if (isLarge()) { - UInt32 cardinality = roaring_bitmap_get_cardinality(rb); - writePODBinary(cardinality, out); - db_ra_to_uint32_array(out, &rb->high_low_container); + uint32_t expectedsize = roaring_bitmap_portable_size_in_bytes(rb); + std::string s(expectedsize,0); + roaring_bitmap_portable_serialize(rb, const_cast(s.data())); + writeStringBinary(s,out); } else small.write(out); diff --git a/dbms/src/Common/ColumnsHashing.h b/dbms/src/Common/ColumnsHashing.h index a37a1ddd28a..c02d79d3648 100644 --- a/dbms/src/Common/ColumnsHashing.h +++ b/dbms/src/Common/ColumnsHashing.h @@ -60,12 +60,6 @@ struct HashMethodOneNumber /// Is used for default implementation in HashMethodBase. FieldType getKeyHolder(size_t row, Arena &) const { return unalignedLoad(vec + row * sizeof(FieldType)); } - - /// Get StringRef from value which can be inserted into column. - static StringRef getValueRef(const Value & value) - { - return StringRef(reinterpret_cast(&value.first), sizeof(value.first)); - } }; @@ -102,8 +96,6 @@ struct HashMethodString } } - static StringRef getValueRef(const Value & value) { return value.first; } - protected: friend class columns_hashing_impl::HashMethodBase; }; @@ -142,8 +134,6 @@ struct HashMethodFixedString } } - static StringRef getValueRef(const Value & value) { return value.first; } - protected: friend class columns_hashing_impl::HashMethodBase; }; @@ -560,11 +550,6 @@ struct HashMethodHashed { return hash128(row, key_columns.size(), key_columns); } - - static ALWAYS_INLINE StringRef getValueRef(const Value & value) - { - return StringRef(reinterpret_cast(&value.first), sizeof(value.first)); - } }; } diff --git a/dbms/src/Common/DNSResolver.cpp b/dbms/src/Common/DNSResolver.cpp index 5314068b291..88f4b286940 100644 --- a/dbms/src/Common/DNSResolver.cpp +++ b/dbms/src/Common/DNSResolver.cpp @@ -223,5 +223,10 @@ void DNSResolver::addToNewHosts(const String & host) DNSResolver::~DNSResolver() = default; +DNSResolver & DNSResolver::instance() +{ + static DNSResolver ret; + return ret; +} } diff --git a/dbms/src/Common/DNSResolver.h b/dbms/src/Common/DNSResolver.h index df646be9981..7dfbe49ab77 100644 --- a/dbms/src/Common/DNSResolver.h +++ b/dbms/src/Common/DNSResolver.h @@ -2,20 +2,21 @@ #include #include #include -#include #include #include +#include namespace DB { /// A singleton implementing DNS names resolving with optional DNS cache -/// The cache is being updated asynchronous in separate thread (see DNSCacheUpdater) +/// The cache is being updated asynchronous in separate thread (see DNSCacheUpdater) /// or it could be updated manually via drop() method. -class DNSResolver : public ext::singleton +class DNSResolver : private boost::noncopyable { public: + static DNSResolver & instance(); DNSResolver(const DNSResolver &) = delete; @@ -46,8 +47,6 @@ private: DNSResolver(); - friend class ext::singleton; - struct Impl; std::unique_ptr impl; diff --git a/dbms/src/Common/HashTable/FixedClearableHashMap.h b/dbms/src/Common/HashTable/FixedClearableHashMap.h index 7f8f739893b..ab808b56f34 100644 --- a/dbms/src/Common/HashTable/FixedClearableHashMap.h +++ b/dbms/src/Common/HashTable/FixedClearableHashMap.h @@ -36,7 +36,6 @@ struct FixedClearableHashMapCell } Key key; FixedClearableHashMapCell * ptr; - Key & getFirstMutable() { return key; } const Key & getFirst() const { return key; } Mapped & getSecond() { return ptr->mapped; } const Mapped & getSecond() const { return *ptr->mapped; } diff --git a/dbms/src/Common/HashTable/FixedClearableHashSet.h b/dbms/src/Common/HashTable/FixedClearableHashSet.h index 915875ebda4..f22e41fcd4e 100644 --- a/dbms/src/Common/HashTable/FixedClearableHashSet.h +++ b/dbms/src/Common/HashTable/FixedClearableHashSet.h @@ -23,7 +23,6 @@ struct FixedClearableHashTableCell struct CellExt { Key key; - value_type & getValueMutable() { return key; } const value_type & getValue() const { return key; } void update(Key && key_, FixedClearableHashTableCell *) { key = key_; } }; diff --git a/dbms/src/Common/HashTable/FixedHashMap.h b/dbms/src/Common/HashTable/FixedHashMap.h index d50c87a6583..4e7686002eb 100644 --- a/dbms/src/Common/HashTable/FixedHashMap.h +++ b/dbms/src/Common/HashTable/FixedHashMap.h @@ -39,7 +39,6 @@ struct FixedHashMapCell Key key; FixedHashMapCell * ptr; - Key & getFirstMutable() { return key; } const Key & getFirst() const { return key; } Mapped & getSecond() { return ptr->mapped; } const Mapped & getSecond() const { return ptr->mapped; } @@ -53,12 +52,53 @@ class FixedHashMap : public FixedHashTable, A { public: using Base = FixedHashTable, Allocator>; + using Self = FixedHashMap; using key_type = Key; using mapped_type = Mapped; - using value_type = typename Base::cell_type::value_type; + using Cell = typename Base::cell_type; + using value_type = typename Cell::value_type; using Base::Base; + template + void ALWAYS_INLINE mergeToViaEmplace(Self & that, Func && func) + { + for (auto it = this->begin(), end = this->end(); it != end; ++it) + { + decltype(it) res_it; + bool inserted; + that.emplace(it->getFirst(), res_it, inserted, it.getHash()); + func(res_it->getSecond(), it->getSecond(), inserted); + } + } + + template + void ALWAYS_INLINE mergeToViaFind(Self & that, Func && func) + { + for (auto it = this->begin(), end = this->end(); it != end; ++it) + { + decltype(it) res_it = that.find(it->getFirst(), it.getHash()); + if (res_it == that.end()) + func(it->getSecond(), it->getSecond(), false); + else + func(res_it->getSecond(), it->getSecond(), true); + } + } + + template + void forEachValue(Func && func) + { + for (auto & v : *this) + func(v.getFirst(), v.getSecond()); + } + + template + void forEachMapped(Func && func) + { + for (auto & v : *this) + func(v.getSecond()); + } + mapped_type & ALWAYS_INLINE operator[](Key x) { typename Base::iterator it; diff --git a/dbms/src/Common/HashTable/FixedHashTable.h b/dbms/src/Common/HashTable/FixedHashTable.h index 0ad56f354b9..b673dbcea8f 100644 --- a/dbms/src/Common/HashTable/FixedHashTable.h +++ b/dbms/src/Common/HashTable/FixedHashTable.h @@ -28,7 +28,6 @@ struct FixedHashTableCell { Key key; - value_type & getValueMutable() { return key; } const value_type & getValue() const { return key; } void update(Key && key_, FixedHashTableCell *) { key = key_; } }; diff --git a/dbms/src/Common/HashTable/HashMap.h b/dbms/src/Common/HashTable/HashMap.h index 98669619d3d..55a4d91dd84 100644 --- a/dbms/src/Common/HashTable/HashMap.h +++ b/dbms/src/Common/HashTable/HashMap.h @@ -49,12 +49,10 @@ struct HashMapCell HashMapCell(const Key & key_, const State &) : value(key_, NoInitTag()) {} HashMapCell(const value_type & value_, const State &) : value(value_) {} - Key & getFirstMutable() { return value.first; } const Key & getFirst() const { return value.first; } Mapped & getSecond() { return value.second; } const Mapped & getSecond() const { return value.second; } - value_type & getValueMutable() { return value; } const value_type & getValue() const { return value; } static const Key & getKey(const value_type & value) { return value.first; } @@ -137,12 +135,65 @@ template < class HashMapTable : public HashTable { public: + using Self = HashMapTable; using key_type = Key; using mapped_type = typename Cell::Mapped; using value_type = typename Cell::value_type; using HashTable::HashTable; + /// Merge every cell's value of current map into the destination map via emplace. + /// Func should have signature void(Mapped & dst, Mapped & src, bool emplaced). + /// Each filled cell in current map will invoke func once. If that map doesn't + /// have a key equals to the given cell, a new cell gets emplaced into that map, + /// and func is invoked with the third argument emplaced set to true. Otherwise + /// emplaced is set to false. + template + void ALWAYS_INLINE mergeToViaEmplace(Self & that, Func && func) + { + for (auto it = this->begin(), end = this->end(); it != end; ++it) + { + decltype(it) res_it; + bool inserted; + that.emplace(it->getFirst(), res_it, inserted, it.getHash()); + func(res_it->getSecond(), it->getSecond(), inserted); + } + } + + /// Merge every cell's value of current map into the destination map via find. + /// Func should have signature void(Mapped & dst, Mapped & src, bool exist). + /// Each filled cell in current map will invoke func once. If that map doesn't + /// have a key equals to the given cell, func is invoked with the third argument + /// exist set to false. Otherwise exist is set to true. + template + void ALWAYS_INLINE mergeToViaFind(Self & that, Func && func) + { + for (auto it = this->begin(), end = this->end(); it != end; ++it) + { + decltype(it) res_it = that.find(it->getFirst(), it.getHash()); + if (res_it == that.end()) + func(it->getSecond(), it->getSecond(), false); + else + func(res_it->getSecond(), it->getSecond(), true); + } + } + + /// Call func(const Key &, Mapped &) for each hash map element. + template + void forEachValue(Func && func) + { + for (auto & v : *this) + func(v.getFirst(), v.getSecond()); + } + + /// Call func(Mapped &) for each hash map element. + template + void forEachMapped(Func && func) + { + for (auto & v : *this) + func(v.getSecond()); + } + mapped_type & ALWAYS_INLINE operator[](Key x) { typename HashMapTable::iterator it; diff --git a/dbms/src/Common/HashTable/HashTable.h b/dbms/src/Common/HashTable/HashTable.h index 42d74210d4a..dadc73c3f44 100644 --- a/dbms/src/Common/HashTable/HashTable.h +++ b/dbms/src/Common/HashTable/HashTable.h @@ -98,7 +98,6 @@ struct HashTableCell HashTableCell(const Key & key_, const State &) : key(key_) {} /// Get what the value_type of the container will be. - value_type & getValueMutable() { return key; } const value_type & getValue() const { return key; } /// Get the key. diff --git a/dbms/src/Common/HashTable/TwoLevelHashMap.h b/dbms/src/Common/HashTable/TwoLevelHashMap.h index a5e2467b131..cd08de702d4 100644 --- a/dbms/src/Common/HashTable/TwoLevelHashMap.h +++ b/dbms/src/Common/HashTable/TwoLevelHashMap.h @@ -22,6 +22,13 @@ public: using TwoLevelHashTable>::TwoLevelHashTable; + template + void ALWAYS_INLINE forEachMapped(Func && func) + { + for (auto i = 0u; i < this->NUM_BUCKETS; ++i) + this->impls[i].forEachMapped(func); + } + mapped_type & ALWAYS_INLINE operator[](Key x) { typename TwoLevelHashMapTable::iterator it; diff --git a/dbms/src/Common/OpenSSLHelpers.cpp b/dbms/src/Common/OpenSSLHelpers.cpp index 6efee6eec17..28542f1151e 100644 --- a/dbms/src/Common/OpenSSLHelpers.cpp +++ b/dbms/src/Common/OpenSSLHelpers.cpp @@ -1,3 +1,5 @@ +#include +#if USE_POCO_NETSSL #include "OpenSSLHelpers.h" #include #include @@ -16,3 +18,4 @@ String getOpenSSLErrors() } } +#endif diff --git a/dbms/src/Common/OpenSSLHelpers.h b/dbms/src/Common/OpenSSLHelpers.h index 0056cb4d0cd..90c0d002fb4 100644 --- a/dbms/src/Common/OpenSSLHelpers.h +++ b/dbms/src/Common/OpenSSLHelpers.h @@ -1,4 +1,6 @@ #pragma once +#include +#if USE_POCO_NETSSL #include @@ -10,3 +12,4 @@ namespace DB String getOpenSSLErrors(); } +#endif diff --git a/dbms/src/Common/RWLock.cpp b/dbms/src/Common/RWLock.cpp index 1b26b275528..b17e5e32674 100644 --- a/dbms/src/Common/RWLock.cpp +++ b/dbms/src/Common/RWLock.cpp @@ -4,8 +4,6 @@ #include #include -#include - namespace ProfileEvents { @@ -35,22 +33,48 @@ namespace ErrorCodes } +/** A single-use object that represents lock's ownership + * For the purpose of exception safety guarantees LockHolder is to be used in two steps: + * 1. Create an instance (allocating all the memory needed) + * 2. Associate the instance with the lock (attach to the lock and locking request group) + */ class RWLockImpl::LockHolderImpl { + bool bound{false}; + Type lock_type; + String query_id; + CurrentMetrics::Increment active_client_increment; RWLock parent; GroupsContainer::iterator it_group; - ClientsContainer::iterator it_client; - QueryIdToHolder::key_type query_id; - CurrentMetrics::Increment active_client_increment; - - LockHolderImpl(RWLock && parent, GroupsContainer::iterator it_group, ClientsContainer::iterator it_client); public: - LockHolderImpl(const LockHolderImpl & other) = delete; + LockHolderImpl& operator=(const LockHolderImpl & other) = delete; + + /// Implicit memory allocation for query_id is done here + LockHolderImpl(const String & query_id_, Type type) + : lock_type{type}, query_id{query_id_}, + active_client_increment{ + type == Type::Read ? CurrentMetrics::RWLockActiveReaders : CurrentMetrics::RWLockActiveWriters} + { + } ~LockHolderImpl(); +private: + /// A separate method which binds the lock holder to the owned lock + /// N.B. It is very important that this method produces no allocations + bool bind_with(RWLock && parent_, GroupsContainer::iterator it_group_) noexcept + { + if (bound) + return false; + it_group = it_group_; + parent = std::move(parent_); + ++it_group->refererrs; + bound = true; + return true; + } + friend class RWLockImpl; }; @@ -62,29 +86,33 @@ namespace class QueryLockInfo { private: - std::mutex mutex; + mutable std::mutex mutex; std::map queries; public: void add(const String & query_id) { std::lock_guard lock(mutex); - ++queries[query_id]; + + const auto res = queries.emplace(query_id, 1); // may throw + if (!res.second) + ++res.first->second; } - void remove(const String & query_id) + void remove(const String & query_id) noexcept { std::lock_guard lock(mutex); - auto it = queries.find(query_id); - assert(it != queries.end()); - if (--it->second == 0) - queries.erase(it); + + const auto query_it = queries.find(query_id); + if (query_it != queries.cend() && --query_it->second == 0) + queries.erase(query_it); } - void check(const String & query_id) + void check(const String & query_id) const { std::lock_guard lock(mutex); - if (queries.count(query_id)) + + if (queries.find(query_id) != queries.cend()) throw Exception("Possible deadlock avoided. Client should retry.", ErrorCodes::DEADLOCK_AVOIDED); } }; @@ -93,8 +121,16 @@ namespace } +/** To guarantee that we do not get any piece of our data corrupted: + * 1. Perform all actions that include allocations before changing lock's internal state + * 2. Roll back any changes that make the state inconsistent + * + * Note: "SM" in the commentaries below stands for STATE MODIFICATION + */ RWLockImpl::LockHolder RWLockImpl::getLock(RWLockImpl::Type type, const String & query_id) { + const bool request_has_query_id = query_id != NO_QUERY; + Stopwatch watch(CLOCK_MONOTONIC_COARSE); CurrentMetrics::Increment waiting_client_increment((type == Read) ? CurrentMetrics::RWLockWaitingReaders : CurrentMetrics::RWLockWaitingWriters); @@ -106,29 +142,39 @@ RWLockImpl::LockHolder RWLockImpl::getLock(RWLockImpl::Type type, const String & : ProfileEvents::RWLockWritersWaitMilliseconds, watch.elapsedMilliseconds()); }; - GroupsContainer::iterator it_group; - ClientsContainer::iterator it_client; - /// This object is placed above unique_lock, because it may lock in destructor. - LockHolder res; + auto lock_holder = std::make_shared(query_id, type); std::unique_lock lock(mutex); - /// Check if the same query is acquiring previously acquired lock - if (query_id != RWLockImpl::NO_QUERY) + /// The FastPath: + /// Check if the same query_id already holds the required lock in which case we can proceed without waiting + if (request_has_query_id) { - auto it_query = query_id_to_holder.find(query_id); - if (it_query != query_id_to_holder.end()) - res = it_query->second.lock(); - } + const auto it_query = owner_queries.find(query_id); + if (it_query != owner_queries.end()) + { + const auto current_owner_group = queue.begin(); - if (res) - { - /// XXX: it means we can't upgrade lock from read to write - with proper waiting! - if (type != Read || res->it_group->type != Read) - throw Exception("Attempt to acquire exclusive lock recursively", ErrorCodes::LOGICAL_ERROR); - else - return res; + /// XXX: it means we can't upgrade lock from read to write! + if (type == Write) + throw Exception( + "RWLockImpl::getLock(): Cannot acquire exclusive lock while RWLock is already locked", + ErrorCodes::LOGICAL_ERROR); + + if (current_owner_group->type == Write) + throw Exception( + "RWLockImpl::getLock(): RWLock is already locked in exclusive mode", + ErrorCodes::LOGICAL_ERROR); + + /// N.B. Type is Read here, query_id is not empty and it_query is a valid iterator + all_read_locks.add(query_id); /// SM1: may throw on insertion (nothing to roll back) + ++it_query->second; /// SM2: nothrow + lock_holder->bind_with(shared_from_this(), current_owner_group); /// SM3: nothrow + + finalize_metrics(); + return lock_holder; + } } /** If the query already has any active read lock and tries to acquire another read lock @@ -148,86 +194,106 @@ RWLockImpl::LockHolder RWLockImpl::getLock(RWLockImpl::Type type, const String & if (type == Type::Write || queue.empty() || queue.back().type == Type::Write) { - if (type == Type::Read && !queue.empty() && queue.back().type == Type::Write && query_id != RWLockImpl::NO_QUERY) + if (type == Type::Read && request_has_query_id && !queue.empty()) all_read_locks.check(query_id); - /// Create new group of clients - it_group = queue.emplace(queue.end(), type); + /// Create a new group of locking requests + queue.emplace_back(type); /// SM1: may throw (nothing to roll back) } - else - { - /// Will append myself to last group - it_group = std::prev(queue.end()); + else if (request_has_query_id && queue.size() > 1) + all_read_locks.check(query_id); - if (it_group != queue.begin() && query_id != RWLockImpl::NO_QUERY) - all_read_locks.check(query_id); - } + GroupsContainer::iterator it_group = std::prev(queue.end()); - /// Append myself to the end of chosen group - auto & clients = it_group->clients; - try - { - it_client = clients.emplace(clients.end(), type); - } - catch (...) - { - /// Remove group if it was the first client in the group and an error occurred - if (clients.empty()) - queue.erase(it_group); - throw; - } - - res.reset(new LockHolderImpl(shared_from_this(), it_group, it_client)); + /// We need to reference the associated group before waiting to guarantee + /// that this group does not get deleted prematurely + ++it_group->refererrs; /// Wait a notification until we will be the only in the group. it_group->cv.wait(lock, [&] () { return it_group == queue.begin(); }); - /// Insert myself (weak_ptr to the holder) to queries set to implement recursive lock - if (query_id != RWLockImpl::NO_QUERY) - { - query_id_to_holder.emplace(query_id, res); + --it_group->refererrs; - if (type == Type::Read) - all_read_locks.add(query_id); + if (request_has_query_id) + { + try + { + if (type == Type::Read) + all_read_locks.add(query_id); /// SM2: may throw on insertion + /// and is safe to roll back unconditionally + const auto emplace_res = + owner_queries.emplace(query_id, 1); /// SM3: may throw on insertion + if (!emplace_res.second) + ++emplace_res.first->second; /// SM4: nothrow + } + catch (...) + { + /// Methods std::list<>::emplace_back() and std::unordered_map<>::emplace() provide strong exception safety + /// We only need to roll back the changes to these objects: all_read_locks and the locking queue + if (type == Type::Read) + all_read_locks.remove(query_id); /// Rollback(SM2): nothrow + + if (it_group->refererrs == 0) + { + const auto next = queue.erase(it_group); /// Rollback(SM1): nothrow + if (next != queue.end()) + next->cv.notify_all(); + } + + throw; + } } - res->query_id = query_id; + + lock_holder->bind_with(shared_from_this(), it_group); /// SM: nothrow finalize_metrics(); - return res; + return lock_holder; } +/** The sequence points of acquiring lock's ownership by an instance of LockHolderImpl: + * 1. all_read_locks is updated + * 2. owner_queries is updated + * 3. request group is updated by LockHolderImpl which in turn becomes "bound" + * + * If by the time when destructor of LockHolderImpl is called the instance has been "bound", + * it is guaranteed that all three steps have been executed successfully and the resulting state is consistent. + * With the mutex locked the order of steps to restore the lock's state can be arbitrary + * + * We do not employ try-catch: if something bad happens, there is nothing we can do =( + */ RWLockImpl::LockHolderImpl::~LockHolderImpl() { + if (!bound || parent == nullptr) + return; + std::lock_guard lock(parent->mutex); - /// Remove weak_ptrs to the holder, since there are no owners of the current lock - parent->query_id_to_holder.erase(query_id); + /// The associated group must exist (and be the beginning of the queue?) + if (parent->queue.empty() || it_group != parent->queue.begin()) + return; - if (*it_client == RWLockImpl::Read && query_id != RWLockImpl::NO_QUERY) - all_read_locks.remove(query_id); - - /// Removes myself from client list of our group - it_group->clients.erase(it_client); - - /// Remove the group if we were the last client and notify the next group - if (it_group->clients.empty()) + /// If query_id is not empty it must be listed in parent->owner_queries + if (query_id != RWLockImpl::NO_QUERY) { - auto & parent_queue = parent->queue; - parent_queue.erase(it_group); + const auto owner_it = parent->owner_queries.find(query_id); + if (owner_it != parent->owner_queries.end()) + { + if (--owner_it->second == 0) /// SM: nothrow + parent->owner_queries.erase(owner_it); /// SM: nothrow - if (!parent_queue.empty()) - parent_queue.front().cv.notify_all(); + if (lock_type == RWLockImpl::Read) + all_read_locks.remove(query_id); /// SM: nothrow + } + } + + /// If we are the last remaining referrer, remove the group and notify the next group + if (--it_group->refererrs == 0) /// SM: nothrow + { + const auto next = parent->queue.erase(it_group); /// SM: nothrow + if (next != parent->queue.end()) + next->cv.notify_all(); } } - -RWLockImpl::LockHolderImpl::LockHolderImpl(RWLock && parent_, RWLockImpl::GroupsContainer::iterator it_group_, - RWLockImpl::ClientsContainer::iterator it_client_) - : parent{std::move(parent_)}, it_group{it_group_}, it_client{it_client_}, - active_client_increment{(*it_client == RWLockImpl::Read) ? CurrentMetrics::RWLockActiveReaders - : CurrentMetrics::RWLockActiveWriters} -{ -} - } diff --git a/dbms/src/Common/RWLock.h b/dbms/src/Common/RWLock.h index c94dd3fe1a4..a7084720d6c 100644 --- a/dbms/src/Common/RWLock.h +++ b/dbms/src/Common/RWLock.h @@ -8,6 +8,7 @@ #include #include #include +#include namespace DB @@ -53,25 +54,24 @@ private: struct Group; using GroupsContainer = std::list; - using ClientsContainer = std::list; - using QueryIdToHolder = std::map>; + using OwnerQueryIds = std::unordered_map; - /// Group of clients that should be executed concurrently - /// i.e. a group could contain several readers, but only one writer + /// Group of locking requests that should be granted concurrently + /// i.e. a group can contain several readers, but only one writer struct Group { - // FIXME: there is only redundant |type| information inside |clients|. const Type type; - ClientsContainer clients; + size_t refererrs; - std::condition_variable cv; /// all clients of the group wait group condvar + std::condition_variable cv; /// all locking requests of the group wait on this condvar - explicit Group(Type type_) : type{type_} {} + explicit Group(Type type_) : type{type_}, refererrs{0} {} }; - mutable std::mutex mutex; GroupsContainer queue; - QueryIdToHolder query_id_to_holder; + OwnerQueryIds owner_queries; + + mutable std::mutex mutex; }; diff --git a/dbms/src/Common/SymbolIndex.cpp b/dbms/src/Common/SymbolIndex.cpp index 5cb424ef48b..ebd5c68807a 100644 --- a/dbms/src/Common/SymbolIndex.cpp +++ b/dbms/src/Common/SymbolIndex.cpp @@ -361,6 +361,12 @@ const SymbolIndex::Object * SymbolIndex::findObject(const void * address) const return find(address, data.objects); } +SymbolIndex & SymbolIndex::instance() +{ + static SymbolIndex instance; + return instance; +} + } #endif diff --git a/dbms/src/Common/SymbolIndex.h b/dbms/src/Common/SymbolIndex.h index 01afe9bf7fa..7cc72511c41 100644 --- a/dbms/src/Common/SymbolIndex.h +++ b/dbms/src/Common/SymbolIndex.h @@ -4,8 +4,8 @@ #include #include -#include #include +#include namespace DB @@ -15,13 +15,14 @@ namespace DB * Used as a replacement for "dladdr" function which is extremely slow. * It works better than "dladdr" because it also allows to search private symbols, that are not participated in shared linking. */ -class SymbolIndex : public ext::singleton +class SymbolIndex : private boost::noncopyable { protected: - friend class ext::singleton; SymbolIndex() { update(); } public: + static SymbolIndex & instance(); + struct Symbol { const void * address_begin; diff --git a/dbms/src/Common/ThreadPool.cpp b/dbms/src/Common/ThreadPool.cpp index e790ac07839..b88106fab73 100644 --- a/dbms/src/Common/ThreadPool.cpp +++ b/dbms/src/Common/ThreadPool.cpp @@ -287,3 +287,8 @@ ThreadPool::Job createExceptionHandledJob(ThreadPool::Job job, ExceptionHandler }; } +GlobalThreadPool & GlobalThreadPool::instance() +{ + static GlobalThreadPool ret; + return ret; +} diff --git a/dbms/src/Common/ThreadPool.h b/dbms/src/Common/ThreadPool.h index 4354b9194b0..2ced4626a1b 100644 --- a/dbms/src/Common/ThreadPool.h +++ b/dbms/src/Common/ThreadPool.h @@ -8,7 +8,6 @@ #include #include #include -#include #include #include @@ -121,10 +120,11 @@ using FreeThreadPool = ThreadPoolImpl; * - address sanitizer and thread sanitizer will not fail due to global limit on number of created threads. * - program will work faster in gdb; */ -class GlobalThreadPool : public FreeThreadPool, public ext::singleton +class GlobalThreadPool : public FreeThreadPool, private boost::noncopyable { public: GlobalThreadPool() : FreeThreadPool(10000, 1000, 10000) {} + static GlobalThreadPool & instance(); }; diff --git a/dbms/src/Common/tests/gtest_sensitive_data_masker.cpp b/dbms/src/Common/tests/gtest_sensitive_data_masker.cpp index 004237aa57f..9b9af39f6ea 100644 --- a/dbms/src/Common/tests/gtest_sensitive_data_masker.cpp +++ b/dbms/src/Common/tests/gtest_sensitive_data_masker.cpp @@ -11,6 +11,7 @@ #endif #include +#include namespace DB diff --git a/dbms/src/Compression/CompressionFactory.cpp b/dbms/src/Compression/CompressionFactory.cpp index b67ac78b970..5dd238d85d8 100644 --- a/dbms/src/Compression/CompressionFactory.cpp +++ b/dbms/src/Compression/CompressionFactory.cpp @@ -155,4 +155,10 @@ CompressionCodecFactory::CompressionCodecFactory() registerCodecGorilla(*this); } +CompressionCodecFactory & CompressionCodecFactory::instance() +{ + static CompressionCodecFactory ret; + return ret; +} + } diff --git a/dbms/src/Compression/CompressionFactory.h b/dbms/src/Compression/CompressionFactory.h index 8ad9ca330a9..f3018d93071 100644 --- a/dbms/src/Compression/CompressionFactory.h +++ b/dbms/src/Compression/CompressionFactory.h @@ -6,8 +6,6 @@ #include #include -#include - #include #include #include @@ -24,7 +22,7 @@ using CodecNameWithLevel = std::pair>; /** Creates a codec object by name of compression algorithm family and parameters. */ -class CompressionCodecFactory final : public ext::singleton +class CompressionCodecFactory final : private boost::noncopyable { protected: using Creator = std::function; @@ -34,6 +32,8 @@ protected: using CompressionCodecsCodeDictionary = std::unordered_map; public: + static CompressionCodecFactory & instance(); + /// Return default codec (currently LZ4) CompressionCodecPtr getDefaultCodec() const; @@ -65,8 +65,6 @@ private: CompressionCodecPtr default_codec; CompressionCodecFactory(); - - friend class ext::singleton; }; } diff --git a/dbms/src/Core/MySQLProtocol.cpp b/dbms/src/Core/MySQLProtocol.cpp index a133fc64e24..37bb8025ae2 100644 --- a/dbms/src/Core/MySQLProtocol.cpp +++ b/dbms/src/Core/MySQLProtocol.cpp @@ -1,3 +1,5 @@ +#include +#if USE_POCO_NETSSL #include #include #include @@ -100,3 +102,4 @@ size_t getLengthEncodedStringSize(const String & s) } } +#endif diff --git a/dbms/src/Core/MySQLProtocol.h b/dbms/src/Core/MySQLProtocol.h index 029d7ded18a..e810001321a 100644 --- a/dbms/src/Core/MySQLProtocol.h +++ b/dbms/src/Core/MySQLProtocol.h @@ -1,4 +1,6 @@ #pragma once +#include +#if USE_POCO_NETSSL #include #include @@ -1075,3 +1077,4 @@ private: } } +#endif diff --git a/dbms/src/Core/SettingsCommon.h b/dbms/src/Core/SettingsCommon.h index 48f5b240a7f..a5e79afed11 100644 --- a/dbms/src/Core/SettingsCommon.h +++ b/dbms/src/Core/SettingsCommon.h @@ -6,8 +6,8 @@ #include #include #include -#include #include +#include namespace DB @@ -329,14 +329,10 @@ private: bool isChanged(const Derived & collection) const { return is_changed(collection); } }; - class MemberInfos + class MemberInfos : private boost::noncopyable { public: - static const MemberInfos & instance() - { - static const MemberInfos single_instance; - return single_instance; - } + static const MemberInfos & instance(); size_t size() const { return infos.size(); } const MemberInfo & operator[](size_t index) const { return infos[index]; } @@ -630,6 +626,12 @@ public: LIST_OF_SETTINGS_MACRO(IMPLEMENT_SETTINGS_COLLECTION_DEFINE_FUNCTIONS_HELPER_) \ }; \ LIST_OF_SETTINGS_MACRO(IMPLEMENT_SETTINGS_COLLECTION_ADD_MEMBER_INFO_HELPER_) \ + } \ + template <> \ + const SettingsCollection::MemberInfos & SettingsCollection::MemberInfos::instance() \ + { \ + static const SettingsCollection::MemberInfos single_instance; \ + return single_instance; \ } diff --git a/dbms/src/DataStreams/copyData.cpp b/dbms/src/DataStreams/copyData.cpp index 5000c87be7c..9d17596fc8d 100644 --- a/dbms/src/DataStreams/copyData.cpp +++ b/dbms/src/DataStreams/copyData.cpp @@ -28,8 +28,6 @@ void copyDataImpl(IBlockInputStream & from, IBlockOutputStream & to, TCancelCall break; to.write(block); - if (!block.rows()) - to.flush(); progress(block); } diff --git a/dbms/src/DataTypes/DataTypeEnum.h b/dbms/src/DataTypes/DataTypeEnum.h index b99e2383860..0fd9a898fb0 100644 --- a/dbms/src/DataTypes/DataTypeEnum.h +++ b/dbms/src/DataTypes/DataTypeEnum.h @@ -125,5 +125,4 @@ public: using DataTypeEnum8 = DataTypeEnum; using DataTypeEnum16 = DataTypeEnum; - } diff --git a/dbms/src/DataTypes/DataTypeFactory.cpp b/dbms/src/DataTypes/DataTypeFactory.cpp index 8c4c899516a..c3f6e14fedb 100644 --- a/dbms/src/DataTypes/DataTypeFactory.cpp +++ b/dbms/src/DataTypes/DataTypeFactory.cpp @@ -202,4 +202,10 @@ DataTypeFactory::DataTypeFactory() DataTypeFactory::~DataTypeFactory() {} +DataTypeFactory & DataTypeFactory::instance() +{ + static DataTypeFactory ret; + return ret; +} + } diff --git a/dbms/src/DataTypes/DataTypeFactory.h b/dbms/src/DataTypes/DataTypeFactory.h index a6c714c1a0e..15eb0c6a6bd 100644 --- a/dbms/src/DataTypes/DataTypeFactory.h +++ b/dbms/src/DataTypes/DataTypeFactory.h @@ -4,7 +4,6 @@ #include #include -#include #include #include @@ -20,7 +19,7 @@ using DataTypePtr = std::shared_ptr; /** Creates a data type by name of data type family and parameters. */ -class DataTypeFactory final : public ext::singleton, public IFactoryWithAliases> +class DataTypeFactory final : private boost::noncopyable, public IFactoryWithAliases> { private: using SimpleCreator = std::function; @@ -29,6 +28,8 @@ private: using SimpleCreatorWithCustom = std::function()>; public: + static DataTypeFactory & instance(); + DataTypePtr get(const String & full_name) const; DataTypePtr get(const String & family_name, const ASTPtr & parameters) const; DataTypePtr get(const ASTPtr & ast) const; @@ -62,8 +63,6 @@ private: const DataTypesDictionary & getCaseInsensitiveCreatorMap() const override { return case_insensitive_data_types; } String getFactoryName() const override { return "DataTypeFactory"; } - - friend class ext::singleton; }; } diff --git a/dbms/src/Dictionaries/DictionaryFactory.cpp b/dbms/src/Dictionaries/DictionaryFactory.cpp index a6c20e38096..43ae9d5623b 100644 --- a/dbms/src/Dictionaries/DictionaryFactory.cpp +++ b/dbms/src/Dictionaries/DictionaryFactory.cpp @@ -47,4 +47,10 @@ DictionaryPtr DictionaryFactory::create( throw Exception{name + ": unknown dictionary layout type: " + layout_type, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG}; } +DictionaryFactory & DictionaryFactory::instance() +{ + static DictionaryFactory ret; + return ret; +} + } diff --git a/dbms/src/Dictionaries/DictionaryFactory.h b/dbms/src/Dictionaries/DictionaryFactory.h index e0a2dd642cb..dbfdc563aa4 100644 --- a/dbms/src/Dictionaries/DictionaryFactory.h +++ b/dbms/src/Dictionaries/DictionaryFactory.h @@ -1,6 +1,5 @@ #pragma once -#include #include "IDictionary.h" @@ -22,9 +21,12 @@ namespace DB class Context; -class DictionaryFactory : public ext::singleton +class DictionaryFactory : private boost::noncopyable { public: + + static DictionaryFactory & instance(); + DictionaryPtr create(const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Context & context) const; using Creator = std::function #include -#include namespace Poco { @@ -22,9 +21,11 @@ class Context; struct DictionaryStructure; /// creates IDictionarySource instance from config and DictionaryStructure -class DictionarySourceFactory : public ext::singleton +class DictionarySourceFactory : private boost::noncopyable { public: + static DictionarySourceFactory & instance(); + using Creator = std::function #include #include -#include #include "GeodataProviders/IHierarchiesProvider.h" diff --git a/dbms/src/Formats/FormatFactory.cpp b/dbms/src/Formats/FormatFactory.cpp index 8a1ad3d7bd2..c1798b2d84f 100644 --- a/dbms/src/Formats/FormatFactory.cpp +++ b/dbms/src/Formats/FormatFactory.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -312,7 +313,15 @@ FormatFactory::FormatFactory() registerOutputFormatProcessorODBCDriver(*this); registerOutputFormatProcessorODBCDriver2(*this); registerOutputFormatProcessorNull(*this); +#if USE_POCO_NETSSL registerOutputFormatProcessorMySQLWrite(*this); +#endif +} + +FormatFactory & FormatFactory::instance() +{ + static FormatFactory ret; + return ret; } } diff --git a/dbms/src/Formats/FormatFactory.h b/dbms/src/Formats/FormatFactory.h index 576faa9174e..04e9ce22e09 100644 --- a/dbms/src/Formats/FormatFactory.h +++ b/dbms/src/Formats/FormatFactory.h @@ -2,11 +2,11 @@ #include #include -#include #include #include #include +#include namespace DB @@ -34,7 +34,7 @@ using OutputFormatPtr = std::shared_ptr; /** Allows to create an IBlockInputStream or IBlockOutputStream by the name of the format. * Note: format and compression are independent things. */ -class FormatFactory final : public ext::singleton +class FormatFactory final : private boost::noncopyable { public: /// This callback allows to perform some additional actions after reading a single row. @@ -87,6 +87,9 @@ private: using FormatsDictionary = std::unordered_map; public: + + static FormatFactory & instance(); + BlockInputStreamPtr getInput( const String & name, ReadBuffer & buf, @@ -128,7 +131,6 @@ private: FormatsDictionary dict; FormatFactory(); - friend class ext::singleton; const Creators & getCreators(const String & name) const; }; diff --git a/dbms/src/Formats/MySQLBlockInputStream.cpp b/dbms/src/Formats/MySQLBlockInputStream.cpp index dcf609f62ce..4f8291ffebe 100644 --- a/dbms/src/Formats/MySQLBlockInputStream.cpp +++ b/dbms/src/Formats/MySQLBlockInputStream.cpp @@ -131,8 +131,6 @@ Block MySQLBlockInputStream::readImpl() row = result.fetch(); } - if (auto_close) - entry.disconnect(); return description.sample_block.cloneWithColumns(std::move(columns)); } diff --git a/dbms/src/Formats/ProtobufSchemas.cpp b/dbms/src/Formats/ProtobufSchemas.cpp index 377fd5ee0c2..11afbffe694 100644 --- a/dbms/src/Formats/ProtobufSchemas.cpp +++ b/dbms/src/Formats/ProtobufSchemas.cpp @@ -15,6 +15,11 @@ namespace ErrorCodes extern const int CANNOT_PARSE_PROTOBUF_SCHEMA; } +ProtobufSchemas & ProtobufSchemas::instance() +{ + static ProtobufSchemas instance; + return instance; +} class ProtobufSchemas::ImporterWithSourceTree : public google::protobuf::compiler::MultiFileErrorCollector { diff --git a/dbms/src/Formats/ProtobufSchemas.h b/dbms/src/Formats/ProtobufSchemas.h index 6a806de2213..590c479bcc8 100644 --- a/dbms/src/Formats/ProtobufSchemas.h +++ b/dbms/src/Formats/ProtobufSchemas.h @@ -6,7 +6,7 @@ #include #include #include -#include +#include namespace google @@ -24,9 +24,11 @@ class FormatSchemaInfo; /** Keeps parsed google protobuf schemas parsed from files. * This class is used to handle the "Protobuf" input/output formats. */ -class ProtobufSchemas : public ext::singleton +class ProtobufSchemas : private boost::noncopyable { public: + static ProtobufSchemas & instance(); + ProtobufSchemas(); ~ProtobufSchemas(); diff --git a/dbms/src/Functions/FunctionFactory.cpp b/dbms/src/Functions/FunctionFactory.cpp index 0cc9c79462b..22dc795fa61 100644 --- a/dbms/src/Functions/FunctionFactory.cpp +++ b/dbms/src/Functions/FunctionFactory.cpp @@ -73,4 +73,10 @@ FunctionBuilderPtr FunctionFactory::tryGet( return {}; } +FunctionFactory & FunctionFactory::instance() +{ + static FunctionFactory ret; + return ret; +} + } diff --git a/dbms/src/Functions/FunctionFactory.h b/dbms/src/Functions/FunctionFactory.h index 7fa0f81f475..c1eadb89e76 100644 --- a/dbms/src/Functions/FunctionFactory.h +++ b/dbms/src/Functions/FunctionFactory.h @@ -3,7 +3,6 @@ #include #include -#include #include #include @@ -21,10 +20,12 @@ class Context; * Function could use for initialization (take ownership of shared_ptr, for example) * some dictionaries from Context. */ -class FunctionFactory : public ext::singleton, public IFactoryWithAliases> +class FunctionFactory : private boost::noncopyable, public IFactoryWithAliases> { public: + static FunctionFactory & instance(); + template void registerFunction(CaseSensitiveness case_sensitiveness = CaseSensitive) { diff --git a/dbms/src/Functions/URL/CMakeLists.txt b/dbms/src/Functions/URL/CMakeLists.txt index 54221b43cf6..92bdbd8fcab 100644 --- a/dbms/src/Functions/URL/CMakeLists.txt +++ b/dbms/src/Functions/URL/CMakeLists.txt @@ -15,7 +15,6 @@ if(USE_HYPERSCAN) target_include_directories(clickhouse_functions_url SYSTEM PRIVATE ${HYPERSCAN_INCLUDE_DIR}) endif() -include(${ClickHouse_SOURCE_DIR}/cmake/find_gperf.cmake) if (USE_GPERF) # Only for regenerate add_custom_target(generate-tldlookup-gperf ./tldLookup.sh diff --git a/dbms/src/IO/AIOContextPool.cpp b/dbms/src/IO/AIOContextPool.cpp index 933d463cc38..2c3e72161bd 100644 --- a/dbms/src/IO/AIOContextPool.cpp +++ b/dbms/src/IO/AIOContextPool.cpp @@ -158,6 +158,12 @@ std::future AIOContextPool::post(struct iocb & iocb) return promises[request_id].get_future(); } +AIOContextPool & AIOContextPool::instance() +{ + static AIOContextPool instance; + return instance; +} + } #endif diff --git a/dbms/src/IO/AIOContextPool.h b/dbms/src/IO/AIOContextPool.h index d90a79ba4cb..abbfa52725a 100644 --- a/dbms/src/IO/AIOContextPool.h +++ b/dbms/src/IO/AIOContextPool.h @@ -2,7 +2,6 @@ #if defined(__linux__) || defined(__FreeBSD__) -#include #include #include #include @@ -14,10 +13,8 @@ namespace DB { -class AIOContextPool : public ext::singleton +class AIOContextPool : private boost::noncopyable { - friend class ext::singleton; - static const auto max_concurrent_events = 128; static const auto timeout_sec = 1; @@ -45,6 +42,8 @@ class AIOContextPool : public ext::singleton void reportExceptionToAnyProducer(); public: + static AIOContextPool & instance(); + /// Request AIO read operation for iocb, returns a future with number of bytes read std::future post(struct iocb & iocb); }; diff --git a/dbms/src/IO/DoubleConverter.cpp b/dbms/src/IO/DoubleConverter.cpp new file mode 100644 index 00000000000..911da5eabce --- /dev/null +++ b/dbms/src/IO/DoubleConverter.cpp @@ -0,0 +1,16 @@ +#include + +namespace DB +{ +template +const double_conversion::DoubleToStringConverter & DoubleConverter::instance() +{ + static const double_conversion::DoubleToStringConverter instance{ + DoubleToStringConverterFlags::flags, "inf", "nan", 'e', -6, 21, 6, 1}; + + return instance; +} + +template class DoubleConverter; +template class DoubleConverter; +} diff --git a/dbms/src/IO/DoubleConverter.h b/dbms/src/IO/DoubleConverter.h index 0c46024d65e..0896aca717e 100644 --- a/dbms/src/IO/DoubleConverter.h +++ b/dbms/src/IO/DoubleConverter.h @@ -6,6 +6,7 @@ #endif #include +#include #ifdef __clang__ #pragma clang diagnostic pop @@ -26,7 +27,7 @@ template <> struct DoubleToStringConverterFlags }; template -class DoubleConverter +class DoubleConverter : private boost::noncopyable { DoubleConverter(const DoubleConverter &) = delete; DoubleConverter & operator=(const DoubleConverter &) = delete; @@ -41,14 +42,7 @@ public: 1 + double_conversion::DoubleToStringConverter::kMaxFixedDigitsAfterPoint + 1; using BufferType = char[MAX_REPRESENTATION_LENGTH]; - static const auto & instance() - { - static const double_conversion::DoubleToStringConverter instance{ - DoubleToStringConverterFlags::flags, "inf", "nan", 'e', -6, 21, 6, 1 - }; - - return instance; - } + static const double_conversion::DoubleToStringConverter & instance(); }; } diff --git a/dbms/src/IO/HTTPCommon.cpp b/dbms/src/IO/HTTPCommon.cpp index ca5b5ab700b..946819c5f63 100644 --- a/dbms/src/IO/HTTPCommon.cpp +++ b/dbms/src/IO/HTTPCommon.cpp @@ -112,15 +112,13 @@ void setTimeouts(Poco::Net::HTTPClientSession & session, const ConnectionTimeout } }; - class HTTPSessionPool : public ext::singleton + class HTTPSessionPool : private boost::noncopyable { private: using Key = std::tuple; using PoolPtr = std::shared_ptr; using Entry = SingleEndpointHTTPSessionPool::Entry; - friend class ext::singleton; - struct Hasher { size_t operator()(const Key & k) const @@ -140,6 +138,12 @@ void setTimeouts(Poco::Net::HTTPClientSession & session, const ConnectionTimeout HTTPSessionPool() = default; public: + static auto & instance() + { + static HTTPSessionPool instance; + return instance; + } + Entry getSession( const Poco::URI & uri, const ConnectionTimeouts & timeouts, diff --git a/dbms/src/IO/HashingWriteBuffer.h b/dbms/src/IO/HashingWriteBuffer.h index 7a3eb6a172c..539c0390dc2 100644 --- a/dbms/src/IO/HashingWriteBuffer.h +++ b/dbms/src/IO/HashingWriteBuffer.h @@ -83,4 +83,5 @@ public: return IHashingBuffer::getHash(); } }; + } diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index f8d58a69fac..91941791afc 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -930,15 +930,15 @@ void NO_INLINE Aggregator::convertToBlockImplFinal( } } - for (const auto & value : data) + data.forEachValue([&](const auto & key, auto & mapped) { - method.insertKeyIntoColumns(value.getValue(), key_columns, key_sizes); + method.insertKeyIntoColumns(key, key_columns, key_sizes); for (size_t i = 0; i < params.aggregates_size; ++i) aggregate_functions[i]->insertResultInto( - value.getSecond() + offsets_of_aggregate_states[i], + mapped + offsets_of_aggregate_states[i], *final_aggregate_columns[i]); - } + }); destroyImpl(data); } @@ -961,16 +961,16 @@ void NO_INLINE Aggregator::convertToBlockImplNotFinal( } } - for (auto & value : data) + data.forEachValue([&](const auto & key, auto & mapped) { - method.insertKeyIntoColumns(value.getValue(), key_columns, key_sizes); + method.insertKeyIntoColumns(key, key_columns, key_sizes); /// reserved, so push_back does not throw exceptions for (size_t i = 0; i < params.aggregates_size; ++i) - aggregate_columns[i]->push_back(value.getSecond() + offsets_of_aggregate_states[i]); + aggregate_columns[i]->push_back(mapped + offsets_of_aggregate_states[i]); - value.getSecond() = nullptr; - } + mapped = nullptr; + }); } @@ -1303,32 +1303,27 @@ void NO_INLINE Aggregator::mergeDataImpl( if constexpr (Method::low_cardinality_optimization) mergeDataNullKey(table_dst, table_src, arena); - for (auto it = table_src.begin(), end = table_src.end(); it != end; ++it) + table_src.mergeToViaEmplace(table_dst, + [&](AggregateDataPtr & dst, AggregateDataPtr & src, bool inserted) { - typename Table::iterator res_it; - bool inserted; - table_dst.emplace(it->getFirst(), res_it, inserted, it.getHash()); - if (!inserted) { for (size_t i = 0; i < params.aggregates_size; ++i) aggregate_functions[i]->merge( - res_it->getSecond() + offsets_of_aggregate_states[i], - it->getSecond() + offsets_of_aggregate_states[i], + dst + offsets_of_aggregate_states[i], + src + offsets_of_aggregate_states[i], arena); for (size_t i = 0; i < params.aggregates_size; ++i) - aggregate_functions[i]->destroy( - it->getSecond() + offsets_of_aggregate_states[i]); + aggregate_functions[i]->destroy(src + offsets_of_aggregate_states[i]); } else { - res_it->getSecond() = it->getSecond(); + dst = src; } - it->getSecond() = nullptr; - } - + src = nullptr; + }); table_src.clearAndShrink(); } @@ -1344,26 +1339,21 @@ void NO_INLINE Aggregator::mergeDataNoMoreKeysImpl( if constexpr (Method::low_cardinality_optimization) mergeDataNullKey(table_dst, table_src, arena); - for (auto it = table_src.begin(), end = table_src.end(); it != end; ++it) + table_src.mergeToViaFind(table_dst, [&](AggregateDataPtr dst, AggregateDataPtr & src, bool found) { - typename Table::iterator res_it = table_dst.find(it->getFirst(), it.getHash()); - - AggregateDataPtr res_data = table_dst.end() == res_it - ? overflows - : res_it->getSecond(); + AggregateDataPtr res_data = found ? dst : overflows; for (size_t i = 0; i < params.aggregates_size; ++i) aggregate_functions[i]->merge( res_data + offsets_of_aggregate_states[i], - it->getSecond() + offsets_of_aggregate_states[i], + src + offsets_of_aggregate_states[i], arena); for (size_t i = 0; i < params.aggregates_size; ++i) - aggregate_functions[i]->destroy(it->getSecond() + offsets_of_aggregate_states[i]); - - it->getSecond() = nullptr; - } + aggregate_functions[i]->destroy(src + offsets_of_aggregate_states[i]); + src = nullptr; + }); table_src.clearAndShrink(); } @@ -1377,27 +1367,23 @@ void NO_INLINE Aggregator::mergeDataOnlyExistingKeysImpl( if constexpr (Method::low_cardinality_optimization) mergeDataNullKey(table_dst, table_src, arena); - for (auto it = table_src.begin(); it != table_src.end(); ++it) + table_src.mergeToViaFind(table_dst, + [&](AggregateDataPtr dst, AggregateDataPtr & src, bool found) { - decltype(it) res_it = table_dst.find(it->getFirst(), it.getHash()); - - if (table_dst.end() == res_it) - continue; - - AggregateDataPtr res_data = res_it->getSecond(); + if (!found) + return; for (size_t i = 0; i < params.aggregates_size; ++i) aggregate_functions[i]->merge( - res_data + offsets_of_aggregate_states[i], - it->getSecond() + offsets_of_aggregate_states[i], + dst + offsets_of_aggregate_states[i], + src + offsets_of_aggregate_states[i], arena); for (size_t i = 0; i < params.aggregates_size; ++i) - aggregate_functions[i]->destroy(it->getSecond() + offsets_of_aggregate_states[i]); - - it->getSecond() = nullptr; - } + aggregate_functions[i]->destroy(src + offsets_of_aggregate_states[i]); + src = nullptr; + }); table_src.clearAndShrink(); } @@ -2255,23 +2241,21 @@ std::vector Aggregator::convertBlockToTwoLevel(const Block & block) template void NO_INLINE Aggregator::destroyImpl(Table & table) const { - for (auto elem : table) + table.forEachMapped([&](AggregateDataPtr & data) { - AggregateDataPtr & data = elem.getSecond(); - /** If an exception (usually a lack of memory, the MemoryTracker throws) arose * after inserting the key into a hash table, but before creating all states of aggregate functions, * then data will be equal nullptr. */ if (nullptr == data) - continue; + return; for (size_t i = 0; i < params.aggregates_size; ++i) if (!aggregate_functions[i]->isState()) aggregate_functions[i]->destroy(data + offsets_of_aggregate_states[i]); data = nullptr; - } + }); } diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index ca088f66bd8..34c9b1a6bfd 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -182,9 +182,11 @@ struct AggregationMethodOneNumber static const bool low_cardinality_optimization = false; // Insert the key from the hash table into columns. - static void insertKeyIntoColumns(const typename Data::value_type & value, MutableColumns & key_columns, const Sizes & /*key_sizes*/) + static void insertKeyIntoColumns(const Key & key, MutableColumns & key_columns, const Sizes & /*key_sizes*/) { - static_cast(key_columns[0].get())->insertRawData(reinterpret_cast(&value.first)); + auto key_holder = reinterpret_cast(&key); + auto column = static_cast(key_columns[0].get()); + column->insertRawData(key_holder); } }; @@ -208,9 +210,9 @@ struct AggregationMethodString static const bool low_cardinality_optimization = false; - static void insertKeyIntoColumns(const typename Data::value_type & value, MutableColumns & key_columns, const Sizes &) + static void insertKeyIntoColumns(const StringRef & key, MutableColumns & key_columns, const Sizes &) { - key_columns[0]->insertData(value.first.data, value.first.size); + key_columns[0]->insertData(key.data, key.size); } }; @@ -234,9 +236,9 @@ struct AggregationMethodFixedString static const bool low_cardinality_optimization = false; - static void insertKeyIntoColumns(const typename Data::value_type & value, MutableColumns & key_columns, const Sizes &) + static void insertKeyIntoColumns(const StringRef & key, MutableColumns & key_columns, const Sizes &) { - key_columns[0]->insertData(value.first.data, value.first.size); + key_columns[0]->insertData(key.data, key.size); } }; @@ -262,10 +264,19 @@ struct AggregationMethodSingleLowCardinalityColumn : public SingleColumnMethod static const bool low_cardinality_optimization = true; - static void insertKeyIntoColumns(const typename Data::value_type & value, MutableColumns & key_columns_low_cardinality, const Sizes & /*key_sizes*/) + static void insertKeyIntoColumns(const Key & key, + MutableColumns & key_columns_low_cardinality, const Sizes & /*key_sizes*/) { - auto ref = BaseState::getValueRef(value); - assert_cast(key_columns_low_cardinality[0].get())->insertData(ref.data, ref.size); + auto col = assert_cast(key_columns_low_cardinality[0].get()); + + if constexpr (std::is_same_v) + { + col->insertData(key.data, key.size); + } + else + { + col->insertData(reinterpret_cast(&key), sizeof(key)); + } } }; @@ -293,7 +304,7 @@ struct AggregationMethodKeysFixed static const bool low_cardinality_optimization = false; - static void insertKeyIntoColumns(const typename Data::value_type & value, MutableColumns & key_columns, const Sizes & key_sizes) + static void insertKeyIntoColumns(const Key & key, MutableColumns & key_columns, const Sizes & key_sizes) { size_t keys_size = key_columns.size(); @@ -330,7 +341,7 @@ struct AggregationMethodKeysFixed /// corresponding key is nullable. Update the null map accordingly. size_t bucket = i / 8; size_t offset = i % 8; - UInt8 val = (reinterpret_cast(&value.first)[bucket] >> offset) & 1; + UInt8 val = (reinterpret_cast(&key)[bucket] >> offset) & 1; null_map->insertValue(val); is_null = val == 1; } @@ -340,7 +351,7 @@ struct AggregationMethodKeysFixed else { size_t size = key_sizes[i]; - observed_column->insertData(reinterpret_cast(&value.first) + pos, size); + observed_column->insertData(reinterpret_cast(&key) + pos, size); pos += size; } } @@ -371,9 +382,9 @@ struct AggregationMethodSerialized static const bool low_cardinality_optimization = false; - static void insertKeyIntoColumns(const typename Data::value_type & value, MutableColumns & key_columns, const Sizes &) + static void insertKeyIntoColumns(const StringRef & key, MutableColumns & key_columns, const Sizes &) { - auto pos = value.first.data; + auto pos = key.data; for (auto & column : key_columns) pos = column->deserializeAndInsertFromArena(pos); } diff --git a/dbms/src/Interpreters/executeQuery.cpp b/dbms/src/Interpreters/executeQuery.cpp index efe4c8ccf2f..ea27ab35968 100644 --- a/dbms/src/Interpreters/executeQuery.cpp +++ b/dbms/src/Interpreters/executeQuery.cpp @@ -38,6 +38,7 @@ #include #include #include +#include namespace ProfileEvents { @@ -660,7 +661,19 @@ void executeQuery( if (set_query_id) set_query_id(context.getClientInfo().current_query_id); - copyData(*streams.in, *out); + if (ast->as()) + { + /// For Watch query, flush data if block is empty (to send data to client). + auto flush_callback = [&out](const Block & block) + { + if (block.rows() == 0) + out->flush(); + }; + + copyData(*streams.in, *out, [](){ return false; }, std::move(flush_callback)); + } + else + copyData(*streams.in, *out); } if (pipeline.initialized()) diff --git a/dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp index 0cd5ffb03e0..edb8d5c15f4 100644 --- a/dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +++ b/dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp @@ -1,7 +1,7 @@ #include "config_formats.h" #include "ArrowColumnToCHColumn.h" -#if USE_ORC or USE_PARQUET +#if USE_ORC || USE_PARQUET #include #include #include diff --git a/dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h b/dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h index b5f4732d107..34b58a80091 100644 --- a/dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h +++ b/dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h @@ -1,6 +1,6 @@ #include "config_formats.h" -#if USE_ORC or USE_PARQUET +#if USE_ORC || USE_PARQUET #include #include diff --git a/dbms/src/Processors/Formats/Impl/MySQLOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/MySQLOutputFormat.cpp index f7ba96a63bd..8b17bd0048d 100644 --- a/dbms/src/Processors/Formats/Impl/MySQLOutputFormat.cpp +++ b/dbms/src/Processors/Formats/Impl/MySQLOutputFormat.cpp @@ -1,3 +1,5 @@ +#include +#if USE_POCO_NETSSL #include #include @@ -116,3 +118,4 @@ void registerOutputFormatProcessorMySQLWrite(FormatFactory & factory) } } +#endif diff --git a/dbms/src/Processors/Formats/Impl/MySQLOutputFormat.h b/dbms/src/Processors/Formats/Impl/MySQLOutputFormat.h index 39d04818dee..3858cc65047 100644 --- a/dbms/src/Processors/Formats/Impl/MySQLOutputFormat.h +++ b/dbms/src/Processors/Formats/Impl/MySQLOutputFormat.h @@ -1,4 +1,6 @@ #pragma once +#include +#if USE_POCO_NETSSL #include #include @@ -40,3 +42,4 @@ private: }; } +#endif diff --git a/dbms/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp b/dbms/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp index 4614e581a3c..083b471d4f1 100644 --- a/dbms/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp +++ b/dbms/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp @@ -72,9 +72,7 @@ void ReadBufferFromKafkaConsumer::commit() PrintOffsets("Polled offset", consumer->get_offsets_position(consumer->get_assignment())); - /// Since we can poll more messages than we already processed - commit only processed messages. - if (!messages.empty()) - consumer->async_commit(*std::prev(current)); + consumer->async_commit(); PrintOffsets("Committed offset", consumer->get_offsets_committed(consumer->get_assignment())); @@ -152,8 +150,6 @@ bool ReadBufferFromKafkaConsumer::nextImpl() return true; } - put_delimiter = (delimiter != 0); - if (current == messages.end()) { if (intermediate_commit) @@ -185,6 +181,10 @@ bool ReadBufferFromKafkaConsumer::nextImpl() // XXX: very fishy place with const casting. auto new_position = reinterpret_cast(const_cast(current->get_payload().get_data())); BufferBase::set(new_position, current->get_payload().get_size(), 0); + put_delimiter = (delimiter != 0); + + /// Since we can poll more messages than we already processed - commit only processed messages. + consumer->store_offset(*current); ++current; diff --git a/dbms/src/Storages/Kafka/StorageKafka.cpp b/dbms/src/Storages/Kafka/StorageKafka.cpp index 2b41fa9e772..ed067993a18 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.cpp +++ b/dbms/src/Storages/Kafka/StorageKafka.cpp @@ -261,9 +261,10 @@ ConsumerBufferPtr StorageKafka::createReadBuffer() conf.set("metadata.broker.list", brokers); conf.set("group.id", group); conf.set("client.id", VERSION_FULL); - conf.set("auto.offset.reset", "smallest"); // If no offset stored for this group, read all messages from the start - conf.set("enable.auto.commit", "false"); // We manually commit offsets after a stream successfully finished - conf.set("enable.partition.eof", "false"); // Ignore EOF messages + conf.set("auto.offset.reset", "smallest"); // If no offset stored for this group, read all messages from the start + conf.set("enable.auto.commit", "false"); // We manually commit offsets after a stream successfully finished + conf.set("enable.auto.offset.store", "false"); // Update offset automatically - to commit them all at once. + conf.set("enable.partition.eof", "false"); // Ignore EOF messages updateConfiguration(conf); // Create a consumer and subscribe to topics diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndices.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndices.cpp index a799cc5cffb..b4062f84112 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndices.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndices.cpp @@ -86,4 +86,10 @@ MergeTreeIndexFactory::MergeTreeIndexFactory() registerIndex("bloom_filter", bloomFilterIndexCreatorNew); } +MergeTreeIndexFactory & MergeTreeIndexFactory::instance() +{ + static MergeTreeIndexFactory instance; + return instance; +} + } diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndices.h b/dbms/src/Storages/MergeTree/MergeTreeIndices.h index 22a392ca9c5..1174b5c293c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndices.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIndices.h @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -129,11 +128,11 @@ public: using MergeTreeIndices = std::vector; -class MergeTreeIndexFactory : public ext::singleton +class MergeTreeIndexFactory : private boost::noncopyable { - friend class ext::singleton; - public: + static MergeTreeIndexFactory & instance(); + using Creator = std::function< std::unique_ptr( const NamesAndTypesList & columns, diff --git a/dbms/src/Storages/StorageFactory.cpp b/dbms/src/Storages/StorageFactory.cpp index dc73e8a53b8..0eafafbd41e 100644 --- a/dbms/src/Storages/StorageFactory.cpp +++ b/dbms/src/Storages/StorageFactory.cpp @@ -163,4 +163,10 @@ StoragePtr StorageFactory::get( return it->second(arguments); } +StorageFactory & StorageFactory::instance() +{ + static StorageFactory ret; + return ret; +} + } diff --git a/dbms/src/Storages/StorageFactory.h b/dbms/src/Storages/StorageFactory.h index 43ffc5a638e..ca19c55522d 100644 --- a/dbms/src/Storages/StorageFactory.h +++ b/dbms/src/Storages/StorageFactory.h @@ -5,7 +5,6 @@ #include #include #include -#include #include @@ -21,9 +20,12 @@ class ASTStorage; * In 'columns' Nested data structures must be flattened. * You should subsequently call IStorage::startup method to work with table. */ -class StorageFactory : public ext::singleton, public IHints<1, StorageFactory> +class StorageFactory : private boost::noncopyable, public IHints<1, StorageFactory> { public: + + static StorageFactory & instance(); + struct Arguments { const String & engine_name; diff --git a/dbms/src/TableFunctions/TableFunctionFactory.cpp b/dbms/src/TableFunctions/TableFunctionFactory.cpp index 7ad98599f76..09e976c7427 100644 --- a/dbms/src/TableFunctions/TableFunctionFactory.cpp +++ b/dbms/src/TableFunctions/TableFunctionFactory.cpp @@ -70,4 +70,10 @@ bool TableFunctionFactory::isTableFunctionName(const std::string & name) const return table_functions.count(name); } +TableFunctionFactory & TableFunctionFactory::instance() +{ + static TableFunctionFactory ret; + return ret; +} + } diff --git a/dbms/src/TableFunctions/TableFunctionFactory.h b/dbms/src/TableFunctions/TableFunctionFactory.h index ae1c3e997e7..cd87fa9c7f0 100644 --- a/dbms/src/TableFunctions/TableFunctionFactory.h +++ b/dbms/src/TableFunctions/TableFunctionFactory.h @@ -4,12 +4,12 @@ #include #include -#include #include #include #include #include +#include namespace DB @@ -21,10 +21,12 @@ using TableFunctionCreator = std::function; /** Lets you get a table function by its name. */ -class TableFunctionFactory final: public ext::singleton, public IFactoryWithAliases +class TableFunctionFactory final: private boost::noncopyable, public IFactoryWithAliases { public: + static TableFunctionFactory & instance(); + /// Register a function by its name. /// No locking, you must register all functions before usage of get. void registerFunction(const std::string & name, Creator creator, CaseSensitiveness case_sensitiveness = CaseSensitive); diff --git a/dbms/tests/integration/test_storage_kafka/test.py b/dbms/tests/integration/test_storage_kafka/test.py index a3aa290c98d..c629ac9f22e 100644 --- a/dbms/tests/integration/test_storage_kafka/test.py +++ b/dbms/tests/integration/test_storage_kafka/test.py @@ -248,6 +248,21 @@ def test_kafka_tsv_with_delimiter(kafka_cluster): kafka_check_result(result, True) +@pytest.mark.timeout(180) +def test_kafka_select_empty(kafka_cluster): + instance.query(''' + CREATE TABLE test.kafka (key UInt64) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'kafka1:19092', + kafka_topic_list = 'empty', + kafka_group_name = 'empty', + kafka_format = 'TSV', + kafka_row_delimiter = '\\n'; + ''') + + assert int(instance.query('SELECT count() FROM test.kafka')) == 0 + + @pytest.mark.timeout(180) def test_kafka_json_without_delimiter(kafka_cluster): instance.query(''' diff --git a/dbms/tests/queries/0_stateless/01009_insert_select_data_loss.reference b/dbms/tests/queries/0_stateless/01009_insert_select_data_loss.reference new file mode 100644 index 00000000000..25e7f55667e --- /dev/null +++ b/dbms/tests/queries/0_stateless/01009_insert_select_data_loss.reference @@ -0,0 +1,2 @@ +0 +10 diff --git a/dbms/tests/queries/0_stateless/01009_insert_select_data_loss.sql b/dbms/tests/queries/0_stateless/01009_insert_select_data_loss.sql new file mode 100644 index 00000000000..9a754d94323 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01009_insert_select_data_loss.sql @@ -0,0 +1,5 @@ +drop table if exists tab; +create table tab (x UInt64) engine = MergeTree order by tuple(); + +insert into tab select number as n from numbers(20) any inner join (select number * 10 as n from numbers(2)) using(n) settings any_join_distinct_right_table_keys = 1, max_block_size = 5; +select * from tab order by x; diff --git a/dbms/tests/queries/0_stateless/01009_insert_select_nicelulu.reference b/dbms/tests/queries/0_stateless/01009_insert_select_nicelulu.reference new file mode 100644 index 00000000000..ccdf6eb3297 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01009_insert_select_nicelulu.reference @@ -0,0 +1 @@ +1189 diff --git a/dbms/tests/queries/0_stateless/01009_insert_select_nicelulu.sql b/dbms/tests/queries/0_stateless/01009_insert_select_nicelulu.sql new file mode 100644 index 00000000000..90a902c352d --- /dev/null +++ b/dbms/tests/queries/0_stateless/01009_insert_select_nicelulu.sql @@ -0,0 +1,24 @@ +Set any_join_distinct_right_table_keys=1; +DROP TABLE IF EXISTS test_insert_t1; +DROP TABLE IF EXISTS test_insert_t2; +DROP TABLE IF EXISTS test_insert_t3; + +CREATE TABLE test_insert_t1 (`dt` Date, `uid` String, `name` String, `city` String) ENGINE = MergeTree PARTITION BY toYYYYMMDD(dt) ORDER BY name SETTINGS index_granularity = 8192; +CREATE TABLE test_insert_t2 (`dt` Date, `uid` String) ENGINE = MergeTree PARTITION BY toYYYYMMDD(dt) ORDER BY uid SETTINGS index_granularity = 8192; +CREATE TABLE test_insert_t3 (`dt` Date, `uid` String, `name` String, `city` String) ENGINE = MergeTree PARTITION BY toYYYYMMDD(dt) ORDER BY name SETTINGS index_granularity = 8192; + +INSERT INTO test_insert_t1 SELECT '2019-09-01',toString(number),toString(rand()),toString(rand()) FROM system.numbers WHERE number > 10 limit 1000000; +INSERT INTO test_insert_t2 SELECT '2019-09-01',toString(number) FROM system.numbers WHERE number >=0 limit 200; +INSERT INTO test_insert_t2 SELECT '2019-09-01',toString(number) FROM system.numbers WHERE number >=100000 limit 200; +INSERT INTO test_insert_t2 SELECT '2019-09-01',toString(number) FROM system.numbers WHERE number >=300000 limit 200; +INSERT INTO test_insert_t2 SELECT '2019-09-01',toString(number) FROM system.numbers WHERE number >=500000 limit 200; +INSERT INTO test_insert_t2 SELECT '2019-09-01',toString(number) FROM system.numbers WHERE number >=700000 limit 200; +INSERT INTO test_insert_t2 SELECT '2019-09-01',toString(number) FROM system.numbers WHERE number >=900000 limit 200; + +INSERT INTO test_insert_t3 SELECT '2019-09-01', uid, name, city FROM ( SELECT dt, uid, name, city FROM test_insert_t1 WHERE dt = '2019-09-01') t1 GLOBAL ANY INNER JOIN (SELECT uid FROM test_insert_t2 WHERE dt = '2019-09-01') t2 ON t1.uid=t2.uid; + +SELECT count(*) FROM test_insert_t3; + +DROP TABLE test_insert_t1; +DROP TABLE test_insert_t2; +DROP TABLE test_insert_t3; diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index ed36e79fbc0..67fe9762ffb 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -29,6 +29,7 @@ The supported formats are: | [Protobuf](#protobuf) | ✔ | ✔ | | [Parquet](#data-format-parquet) | ✔ | ✔ | | [RowBinary](#rowbinary) | ✔ | ✔ | +| [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes) | ✔ | ✔ | | [Native](#native) | ✔ | ✔ | | [Null](#null) | ✗ | ✔ | | [XML](#xml) | ✗ | ✔ | @@ -680,9 +681,10 @@ For [NULL](../query_language/syntax.md#null-literal) support, an additional byte ## RowBinaryWithNamesAndTypes {#rowbinarywithnamesandtypes} Similar to [RowBinary](#rowbinary), but with added header: -* [LEB128](https://en.wikipedia.org/wiki/LEB128)-encoded number of columns (N) -* N `String`s specifying column names -* N `String`s specifying column types + + * [LEB128](https://en.wikipedia.org/wiki/LEB128)-encoded number of columns (N) + * N `String`s specifying column names + * N `String`s specifying column types ## Values {#data-format-values} @@ -924,17 +926,17 @@ Data types of a ClickHouse table columns can differ from the corresponding field You can insert Parquet data from a file into ClickHouse table by the following command: -``` +```bash cat {filename} | clickhouse-client --query="INSERT INTO {some_table} FORMAT Parquet" ``` You can select data from a ClickHouse table and save them into some file in the Parquet format by the following command: -``` +```sql clickhouse-client --query="SELECT * FROM {some_table} FORMAT Parquet" > {some_file.pq} ``` -To exchange data with the Hadoop, you can use [`HDFS` table engine](../../operations/table_engines/hdfs.md). +To exchange data with the Hadoop, you can use [HDFS table engine](../operations/table_engines/hdfs.md). ## Format Schema {#formatschema} diff --git a/docs/en/interfaces/third-party/gui.md b/docs/en/interfaces/third-party/gui.md index 5f7ef803bbb..140cc8a4222 100644 --- a/docs/en/interfaces/third-party/gui.md +++ b/docs/en/interfaces/third-party/gui.md @@ -69,6 +69,10 @@ Features: - Pager support for the data output. - Custom PostgreSQL-like commands. +### clickhouse-flamegraph + +[clickhouse-flamegraph](https://github.com/Slach/clickhouse-flamegraph) is a specialized tool to visualize the `system.trace_log` as [flamegraph](http://www.brendangregg.com/flamegraphs.html). + ## Commercial ### DataGrip diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 3d67fe0e61d..fce10da8f9b 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -857,6 +857,18 @@ Possible values: Default value: 0. +## optimize_throw_if_noop {#setting-optimize_throw_if_noop} + +Enables or disables throwing an exception if the [OPTIMIZE](../../query_language/misc.md#misc_operations-optimize) query have not performed a merge. + +By default `OPTIMIZE` returns successfully even if it haven't done anything. This setting allows to distinguish this situation and get the reason in exception message. + +Possible values: + +- 1 — Throwing an exception is enabled. +- 0 — Throwing an exception is disabled. + +Default value: 0. ## distributed_replica_error_half_life {#settings-distributed_replica_error_half_life} - Type: seconds @@ -882,4 +894,19 @@ Error count of each replica is capped at this value, preventing a single replica - [Table engine Distributed](../../operations/table_engines/distributed.md) - [`distributed_replica_error_half_life`](#settings-distributed_replica_error_half_life) +## os_thread_priority {#setting-os_thread_priority} + +Sets the priority ([nice](https://en.wikipedia.org/wiki/Nice_(Unix))) for threads that execute queries. OS scheduler considers this priority when choosing the next thread to run on each available CPU core. + +!!! warning "Warning" + To use this setting, you need to set the `CAP_SYS_NICE` capability. The `clickhouse-server` package sets it up during installation. Some virtual environments don't allow to set the `CAP_SYS_NICE` capability. In this case `clickhouse-server` shows a message about it at the start. + +Possible values: + +You can set values in the `[-20, 19]` range. + +The lower value means a higher priority. Threads with low values of `nice` priority are executed more frequently than threads with high values. High values are preferable for long running non-interactive queries because it allows them to quickly give up resources in favour of short interactive queries when they arrive. + +Default value: 0. + [Original article](https://clickhouse.yandex/docs/en/operations/settings/settings/) diff --git a/docs/en/operations/system_tables.md b/docs/en/operations/system_tables.md index 4e3386764fd..85c6f755f87 100644 --- a/docs/en/operations/system_tables.md +++ b/docs/en/operations/system_tables.md @@ -62,11 +62,11 @@ Columns: Please note that `errors_count` is updated once per query to the cluster, but `estimated_recovery_time` is recalculated on-demand. So there could be a case of non-zero `errors_count` and zero `estimated_recovery_time`, that next query will zero `errors_count` and try to use replica as if it has no errors. -** See also ** +**See also** -- [Table engine Distributed](../../operations/table_engines/distributed.md) -- [distributed_replica_error_cap setting](../settings/settings.md#settings-distributed_replica_error_cap) -- [distributed_replica_error_half_life setting](../settings/settings.md#settings-distributed_replica_error_half_life) +- [Table engine Distributed](table_engines/distributed.md) +- [distributed_replica_error_cap setting](settings/settings.md#settings-distributed_replica_error_cap) +- [distributed_replica_error_half_life setting](settings/settings.md#settings-distributed_replica_error_half_life) ## system.columns diff --git a/docs/en/operations/table_engines/mergetree.md b/docs/en/operations/table_engines/mergetree.md index 7c694a1612c..c3d64395a02 100644 --- a/docs/en/operations/table_engines/mergetree.md +++ b/docs/en/operations/table_engines/mergetree.md @@ -388,10 +388,66 @@ When the values in the column expire, ClickHouse replaces them with the default The `TTL` clause can't be used for key columns. +Examples: + +Creating a table with TTL + +```sql +CREATE TABLE example_table +( + d DateTime, + a Int TTL d + INTERVAL 1 MONTH, + b Int TTL d + INTERVAL 1 MONTH, + c String +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(d) +ORDER BY d; +``` + +Adding TTL to a column of an existing table + +```sql +ALTER TABLE example_table + MODIFY COLUMN + c String TTL d + INTERVAL 1 DAY; +``` + +Altering TTL of the column + +```sql +ALTER TABLE example_table + MODIFY COLUMN + c String TTL d + INTERVAL 1 MONTH; +``` + **Table TTL** When data in a table expires, ClickHouse deletes all corresponding rows. +Examples: + +Creating a table with TTL + +```sql +CREATE TABLE example_table +( + d DateTime, + a Int +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(d) +ORDER BY d +TTL d + INTERVAL 1 MONTH; +``` + +Altering TTL of the table + +```sql +ALTER TABLE example_table + MODIFY TTL d + INTERVAL 1 DAY; +``` + **Removing Data** Data with an expired TTL is removed when ClickHouse merges data parts. diff --git a/docs/en/query_language/functions/geo.md b/docs/en/query_language/functions/geo.md index d05345da29e..79b8390d59f 100644 --- a/docs/en/query_language/functions/geo.md +++ b/docs/en/query_language/functions/geo.md @@ -38,6 +38,7 @@ SELECT greatCircleDistance(55.755831, 37.617673, -55.755831, -37.617673) ## pointInEllipses Checks whether the point belongs to at least one of the ellipses. +Coordinates are geometric in the Cartesian coordinate system. ``` pointInEllipses(x, y, x₀, y₀, a₀, b₀,...,xₙ, yₙ, aₙ, bₙ) @@ -47,7 +48,7 @@ pointInEllipses(x, y, x₀, y₀, a₀, b₀,...,xₙ, yₙ, aₙ, bₙ) - `x, y` — Coordinates of a point on the plane. - `xᵢ, yᵢ` — Coordinates of the center of the `i`-th ellipsis. -- `aᵢ, bᵢ` — Axes of the `i`-th ellipsis in meters. +- `aᵢ, bᵢ` — Axes of the `i`-th ellipsis in units of x, y coordinates. The input parameters must be `2+4⋅n`, where `n` is the number of ellipses. @@ -58,13 +59,13 @@ The input parameters must be `2+4⋅n`, where `n` is the number of ellipses. **Example** ``` sql -SELECT pointInEllipses(55.755831, 37.617673, 55.755831, 37.617673, 1.0, 2.0) +SELECT pointInEllipses(10., 10., 10., 9.1, 1., 0.9999) ``` ``` -┌─pointInEllipses(55.755831, 37.617673, 55.755831, 37.617673, 1., 2.)─┐ -│ 1 │ -└─────────────────────────────────────────────────────────────────────┘ +┌─pointInEllipses(10., 10., 10., 9.1, 1., 0.9999)─┐ +│ 1 │ +└─────────────────────────────────────────────────┘ ``` ## pointInPolygon diff --git a/docs/en/query_language/misc.md b/docs/en/query_language/misc.md index 08e8f819b8c..337049d6624 100644 --- a/docs/en/query_language/misc.md +++ b/docs/en/query_language/misc.md @@ -177,10 +177,13 @@ Changes already made by the mutation are not rolled back. OPTIMIZE TABLE [db.]name [ON CLUSTER cluster] [PARTITION partition] [FINAL] ``` -Asks the table engine to do something for optimization. -Supported only by `*MergeTree` engines, in which this query initializes a non-scheduled merge of data parts. -If you specify a `PARTITION`, only the specified partition will be optimized. -If you specify `FINAL`, optimization will be performed even when all the data is already in one part. +This query tries to initialize an unscheduled merge of data parts for tables with a table engine of [MergeTree](../operations/table_engines/mergetree.md) family. Other kinds of table engines are not supported. + +When `OPTIMIZE` is used with [ReplicatedMergeTree](../operations/table_engines/replication.md) family of table engines, ClickHouse creates a task for merging and waits for execution on all nodes (if the `replication_alter_partitions_sync` setting is enabled). + +- If `OPTIMIZE` doesn't perform merging for any reason, it doesn't notify the client about it. To enable notification use the [optimize_throw_if_noop](../operations/settings/settings.md#setting-optimize_throw_if_noop) setting. +- If you specify a `PARTITION`, only the specified partition is optimized. +- If you specify `FINAL`, optimization is performed even when all the data is already in one part. !!! warning OPTIMIZE can't fix the "Too many parts" error. diff --git a/docs/en/query_language/system.md b/docs/en/query_language/system.md index b7797df490b..3ef504e46b3 100644 --- a/docs/en/query_language/system.md +++ b/docs/en/query_language/system.md @@ -1,9 +1,58 @@ # SYSTEM Queries {#query_language-system} +- [RELOAD DICTIONARIES](#query_language-system-reload-dictionaries) +- [RELOAD DICTIONARY](#query_language-system-reload-dictionary) +- [DROP DNS CACHE](#query_language-system-drop-dns-cache) +- [DROP MARKS CACHE](#query_language-system-drop-marks-cache) +- [FLUSH LOGS](#query_language-system-flush_logs) +- [RELOAD CONFIG](#query_language-system-reload-config) +- [SHUTDOWN](#query_language-system-shutdown) +- [KILL](#query_language-system-kill) - [STOP DISTRIBUTED SENDS](#query_language-system-stop-distributed-sends) - [FLUSH DISTRIBUTED](#query_language-system-flush-distributed) - [START DISTRIBUTED SENDS](#query_language-system-start-distributed-sends) +## RELOAD DICTIONARIES {#query_language-system-reload-dictionaries} + +Reloads all dictionaries that have been successfully loaded before. +By default, dictionaries are loaded lazily (see [dictionaries_lazy_load](../operations/server_settings/settings.md#server_settings-dictionaries_lazy_load)), so instead of being loaded automatically at startup, they are initialized on first access through dictGet function or SELECT from tables with ENGINE = Dictionary. The `SYSTEM RELOAD DICTIONARIES` query reloads such dictionaries (LOADED). +Always returns `Ok.` regardless of the result of the dictionary update. + +## RELOAD DICTIONARY dictionary_name {#query_language-system-reload-dictionary} + +Completely reloads a dictionary `dictionary_name`, regardless of the state of the dictionary (LOADED / NOT_LOADED / FAILED). +Always returns `Ok.` regardless of the result of updating the dictionary. +The status of the dictionary can be checked by querying the `system.dictionaries` table. + +```sql +SELECT name, status FROM system.dictionaries; +``` + +## DROP DNS CACHE {#query_language-system-drop-dns-cache} + +Resets ClickHouse's internal DNS cache. Sometimes (for old ClickHouse versions) it is necessary to use this command when changing the infrastructure (changing the IP address of another ClickHouse server or the server used by dictionaries). + +For more convenient (automatic) cache management, see disable_internal_dns_cache, dns_cache_update_period parameters. + +## DROP MARKS CACHE {#query_language-system-drop-marks-cache} + +Resets the mark cache. Used in development of ClickHouse and performance tests. + +## FLUSH LOGS {#query_language-system-flush_logs} + +Flushes buffers of log messages to system tables (e.g. system.query_log). Allows you to not wait 7.5 seconds when debugging. + +## RELOAD CONFIG {#query_language-system-reload-config} + +Reloads ClickHouse configuration. Used when configuration is stored in ZooKeeeper. + +## SHUTDOWN {#query_language-system-shutdown} + +Normally shuts down ClickHouse (like `service clickhouse-server stop` / `kill {$pid_clickhouse-server}`) + +## KILL {#query_language-system-kill} + +Aborts ClickHouse process (like `kill -9 {$ pid_clickhouse-server}`) ## Managing Distributed Tables {#query_language-system-distributed} diff --git a/docs/fa/interfaces/third-party/gui.md b/docs/fa/interfaces/third-party/gui.md index a02d8a3cfc5..38378f450e6 100644 --- a/docs/fa/interfaces/third-party/gui.md +++ b/docs/fa/interfaces/third-party/gui.md @@ -69,6 +69,10 @@ interface تحت وب برای ClickHouse در پروژه [Tabix](https://github - پشتیبانی از Pager برای خروجی داده. - دستورات پست سفارشی مانند PostgreSQL. +### clickhouse-flamegraph + +[clickhouse-flamegraph](https://github.com/Slach/clickhouse-flamegraph) یک ابزار تخصصی برای تجسم است `system.trace_log`مانند[flamegraph](http://www.brendangregg.com/flamegraphs.html). + ## تجاری ### DataGrip diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index 15f7552f877..9acf2d67e4a 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -28,6 +28,7 @@ ClickHouse может принимать (`INSERT`) и отдавать (`SELECT | [Protobuf](#protobuf) | ✔ | ✔ | | [Parquet](#data-format-parquet) | ✔ | ✔ | | [RowBinary](#rowbinary) | ✔ | ✔ | +| [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes) | ✔ | ✔ | | [Native](#native) | ✔ | ✔ | | [Null](#null) | ✗ | ✔ | | [XML](#xml) | ✗ | ✔ | @@ -673,7 +674,15 @@ FixedString представлены просто как последовате Array представлены как длина в формате varint (unsigned [LEB128](https://en.wikipedia.org/wiki/LEB128)), а затем элементы массива, подряд. -Для поддержки [NULL](../query_language/syntax.md#null-literal) перед каждым значением типа [Nullable](../data_types/nullable.md +Для поддержки [NULL](../query_language/syntax.md#null-literal) перед каждым значением типа [Nullable](../data_types/nullable.md) следует байт содержащий 1 или 0. Если байт 1, то значение равно NULL, и этот байт интерпретируется как отдельное значение (т.е. после него следует значение следующего поля). Если байт 0, то после байта следует значение поля (не равно NULL). + +## RowBinaryWithNamesAndTypes {#rowbinarywithnamesandtypes} + +То же самое что [RowBinary](#rowbinary), но добавляется заголовок: + + * Количество колонок - N, закодированное [LEB128](https://en.wikipedia.org/wiki/LEB128), + * N строк (`String`) с именами колонок, + * N строк (`String`) с типами колонок. ## Values {#data-format-values} @@ -927,7 +936,7 @@ cat {filename} | clickhouse-client --query="INSERT INTO {some_table} FORMAT Parq clickhouse-client --query="SELECT * FROM {some_table} FORMAT Parquet" > {some_file.pq} ``` -Для обмена данными с экосистемой Hadoop можно использовать движки таблиц [`HDFS`](../../operations/table_engines/hdfs.md) и `URL`. +Для обмена данными с экосистемой Hadoop можно использовать движки таблиц [HDFS](../operations/table_engines/hdfs.md) и `URL`. ## Схема формата {#formatschema} diff --git a/docs/ru/interfaces/third-party/gui.md b/docs/ru/interfaces/third-party/gui.md index d561f54548c..3280f1f2472 100644 --- a/docs/ru/interfaces/third-party/gui.md +++ b/docs/ru/interfaces/third-party/gui.md @@ -72,6 +72,10 @@ - Поддержка постраничного просмотра для результирующих данных; - Дополнительные PostgreSQL-подобные команды. +### clickhouse-flamegraph + +[clickhouse-flamegraph](https://github.com/Slach/clickhouse-flamegraph) — специализированный инструмент для визуализации `system.trace_log` в виде [flamegraph](http://www.brendangregg.com/flamegraphs.html). + ## Коммерческие ### DataGrip diff --git a/docs/ru/operations/table_engines/mergetree.md b/docs/ru/operations/table_engines/mergetree.md index d47336c2593..6c4aa6ce2bb 100644 --- a/docs/ru/operations/table_engines/mergetree.md +++ b/docs/ru/operations/table_engines/mergetree.md @@ -327,10 +327,64 @@ TTL date_time + INTERVAL 15 HOUR Секцию `TTL` нельзя использовать для ключевых столбцов. +Примеры: + +Создание таблицы с TTL + +```sql +CREATE TABLE example_table +( + d DateTime, + a Int TTL d + INTERVAL 1 MONTH, + b Int TTL d + INTERVAL 1 MONTH, + c String +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(d) +ORDER BY d; +``` + +Добавление TTL на колонку существующей таблицы + +```sql +ALTER TABLE example_table + MODIFY COLUMN + c String TTL d + INTERVAL 1 DAY; +``` + +Изменение TTL у колонки + +```sql +ALTER TABLE example_table + MODIFY COLUMN + c String TTL d + INTERVAL 1 MONTH; +``` + **TTL таблицы** Когда некоторые данные в таблице устаревают, ClickHouse удаляет все соответствующие строки. +Примеры: + +```sql +CREATE TABLE example_table +( + d DateTime, + a Int +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(d) +ORDER BY d +TTL d + INTERVAL 1 MONTH; +``` + +Изменение TTL + +```sql +ALTER TABLE example_table + MODIFY TTL d + INTERVAL 1 DAY; +``` + **Удаление данных** Данные с истекшим TTL удаляются, когда ClickHouse мёржит куски данных. diff --git a/docs/ru/query_language/functions/geo.md b/docs/ru/query_language/functions/geo.md index b8e37c15aca..63ceae9208e 100644 --- a/docs/ru/query_language/functions/geo.md +++ b/docs/ru/query_language/functions/geo.md @@ -38,6 +38,7 @@ SELECT greatCircleDistance(55.755831, 37.617673, -55.755831, -37.617673) ## pointInEllipses Проверяет, принадлежит ли точка хотя бы одному из эллипсов. +Координаты — геометрические в декартовой системе координат. ``` pointInEllipses(x, y, x₀, y₀, a₀, b₀,...,xₙ, yₙ, aₙ, bₙ) @@ -47,7 +48,7 @@ pointInEllipses(x, y, x₀, y₀, a₀, b₀,...,xₙ, yₙ, aₙ, bₙ) - `x, y` — координаты точки на плоскости. - `xᵢ, yᵢ` — координаты центра `i`-го эллипса. -- `aᵢ, bᵢ` — полуоси `i`-го эллипса в метрах. +- `aᵢ, bᵢ` — полуоси `i`-го эллипса (в единицах измерения координат x,y). Входных параметров должно быть `2+4⋅n`, где `n` — количество эллипсов. @@ -58,13 +59,13 @@ pointInEllipses(x, y, x₀, y₀, a₀, b₀,...,xₙ, yₙ, aₙ, bₙ) **Пример** ```sql -SELECT pointInEllipses(55.755831, 37.617673, 55.755831, 37.617673, 1.0, 2.0) +SELECT pointInEllipses(10., 10., 10., 9.1, 1., 0.9999) ``` ``` -┌─pointInEllipses(55.755831, 37.617673, 55.755831, 37.617673, 1., 2.)─┐ -│ 1 │ -└─────────────────────────────────────────────────────────────────────┘ +┌─pointInEllipses(10., 10., 10., 9.1, 1., 0.9999)─┐ +│ 1 │ +└─────────────────────────────────────────────────┘ ``` ## pointInPolygon diff --git a/docs/ru/query_language/system.md b/docs/ru/query_language/system.md index f35b4a39061..2abdc5d34de 100644 --- a/docs/ru/query_language/system.md +++ b/docs/ru/query_language/system.md @@ -1,9 +1,59 @@ # Запросы SYSTEM {#query_language-system} +- [RELOAD DICTIONARIES](#query_language-system-reload-dictionaries) +- [RELOAD DICTIONARY](#query_language-system-reload-dictionary) +- [DROP DNS CACHE](#query_language-system-drop-dns-cache) +- [DROP MARKS CACHE](#query_language-system-drop-marks-cache) +- [FLUSH LOGS](#query_language-system-flush_logs) +- [RELOAD CONFIG](#query_language-system-reload-config) +- [SHUTDOWN](#query_language-system-shutdown) +- [KILL](#query_language-system-kill) - [STOP DISTRIBUTED SENDS](#query_language-system-stop-distributed-sends) - [FLUSH DISTRIBUTED](#query_language-system-flush-distributed) - [START DISTRIBUTED SENDS](#query_language-system-start-distributed-sends) +## RELOAD DICTIONARIES {#query_language-system-reload-dictionaries} + +Перегружает все словари, которые были успешно загружены до этого. +По умолчанию включена ленивая загрузка [dictionaries_lazy_load](../operations/server_settings/settings.md#dictionaries-lazy-load), поэтому словари не загружаются автоматически при старте, а только при первом обращении через dictGet или SELECT к ENGINE=Dictionary. После этого такие словари (LOADED) будут перегружаться командой `system reload dictionaries`. +Всегда возвращает `Ok.`, вне зависимости от результата обновления словарей. + +## RELOAD DICTIONARY dictionary_name {#query_language-system-reload-dictionary} + +Полностью перегружает словарь `dictionary_name`, вне зависимости от состояния словаря (LOADED/NOT_LOADED/FAILED). +Всегда возвращает `Ok.`, вне зависимости от результата обновления словаря. +Состояние словаря можно проверить запросом к `system.dictionaries`. + +```sql +SELECT name, status FROM system.dictionaries; +``` + +## DROP DNS CACHE {#query_language-system-drop-dns-cache} + +Сбрасывает внутренний DNS кеш ClickHouse. Иногда (для старых версий ClickHouse) необходимо использовать эту команду при изменении инфраструктуры (смене IP адреса у другого ClickHouse сервера или сервера, используемого словарями). + +Для более удобного (автоматического) управления кешем см. параметры disable_internal_dns_cache, dns_cache_update_period. + +## DROP MARKS CACHE {#query_language-system-drop-marks-cache} + +Сбрасывает кеш "засечек" (`mark cache`). Используется при разработке ClickHouse и тестах производительности. + +## FLUSH LOGS {#query_language-system-flush_logs} + +Записывает буферы логов в системные таблицы (например system.query_log). Позволяет не ждать 7.5 секунд при отладке. + +## RELOAD CONFIG {#query_language-system-reload-config} + +Перечитывает конфигурацию настроек ClickHouse. Используется при хранении конфигурации в zookeeeper. + +## SHUTDOWN {#query_language-system-shutdown} + +Штатно завершает работу ClickHouse (аналог `service clickhouse-server stop` / `kill {$pid_clickhouse-server}`) + +## KILL {#query_language-system-kill} + +Аварийно завершает работу ClickHouse (аналог `kill -9 {$pid_clickhouse-server}`) + ## Управление распределёнными таблицами {#query_language-system-distributed} ClickHouse может оперировать [распределёнными](../operations/table_engines/distributed.md) таблицами. Когда пользователь вставляет данные в эти таблицы, ClickHouse сначала формирует очередь из данных, которые должны быть отправлены на узлы кластера, а затем асинхронно отправляет подготовленные данные. Вы пожете управлять очередью с помощью запросов [STOP DISTRIBUTED SENDS](#query_language-system-stop-distributed-sends), [START DISTRIBUTED SENDS](#query_language-system-start-distributed-sends) и [FLUSH DISTRIBUTED](#query_language-system-flush-distributed). Также есть возможность синхронно вставлять распределенные данные с помощью настройки `insert_distributed_sync`. diff --git a/docs/toc_en.yml b/docs/toc_en.yml index b0ea90c44a1..dccd51f3cb1 100644 --- a/docs/toc_en.yml +++ b/docs/toc_en.yml @@ -87,6 +87,7 @@ nav: - 'MySQL': 'operations/table_engines/mysql.md' - 'JDBC': 'operations/table_engines/jdbc.md' - 'ODBC': 'operations/table_engines/odbc.md' + - 'HDFS': 'operations/table_engines/hdfs.md' - 'Special': - 'Distributed': 'operations/table_engines/distributed.md' - 'External data': 'operations/table_engines/external_data.md' @@ -158,6 +159,7 @@ nav: - 'mysql': 'query_language/table_functions/mysql.md' - 'jdbc': 'query_language/table_functions/jdbc.md' - 'odbc': 'query_language/table_functions/odbc.md' + - 'hdfs': 'query_language/table_functions/hdfs.md' - 'input': 'query_language/table_functions/input.md' - 'Dictionaries': - 'Introduction': 'query_language/dicts/index.md' diff --git a/docs/toc_ru.yml b/docs/toc_ru.yml index 2b7a7f156ab..b21bcc838dc 100644 --- a/docs/toc_ru.yml +++ b/docs/toc_ru.yml @@ -88,6 +88,7 @@ nav: - 'MySQL': 'operations/table_engines/mysql.md' - 'JDBC': 'operations/table_engines/jdbc.md' - 'ODBC': 'operations/table_engines/odbc.md' + - 'HDFS': 'operations/table_engines/hdfs.md' - 'Особые': - 'Distributed': 'operations/table_engines/distributed.md' - 'Внешние данные': 'operations/table_engines/external_data.md' @@ -159,6 +160,7 @@ nav: - 'mysql': 'query_language/table_functions/mysql.md' - 'jdbc': 'query_language/table_functions/jdbc.md' - 'odbc': 'query_language/table_functions/odbc.md' + - 'hdfs': 'query_language/table_functions/hdfs.md' - 'input': 'query_language/table_functions/input.md' - 'Словари': - 'Введение': 'query_language/dicts/index.md' diff --git a/docs/zh/getting_started/index.md b/docs/zh/getting_started/index.md index f51323ce7e8..5e1a5777292 100644 --- a/docs/zh/getting_started/index.md +++ b/docs/zh/getting_started/index.md @@ -67,7 +67,7 @@ sudo yum install clickhouse-server clickhouse-client ###来自Docker -要在Docker中运行ClickHouse，请遵循[Docker Hub](https://hub.docker.com/r/yandex/clickhouse-server/)上的指南。那些图像使用官方的`deb`包。 +要在Docker中运行ClickHouse，请遵循[Docker Hub](https://hub.docker.com/r/yandex/clickhouse-server/)上的指南。这些镜像使用官方的`deb`包构建。 ### 使用源码安装 diff --git a/docs/zh/interfaces/third-party/gui.md b/docs/zh/interfaces/third-party/gui.md index 457ae3af902..31a533e229d 100644 --- a/docs/zh/interfaces/third-party/gui.md +++ b/docs/zh/interfaces/third-party/gui.md @@ -68,6 +68,10 @@ ClickHouse Web 界面 [Tabix](https://github.com/tabixio/tabix). - 寻呼机支持数据输出。 - 自定义PostgreSQL类命令。 +### clickhouse-flamegraph + + [clickhouse-flamegraph](https://github.com/Slach/clickhouse-flamegraph) 是一个可视化的专业工具`system.trace_log`如[flamegraph](http://www.brendangregg.com/flamegraphs.html). + ## 商业 ### DataGrip diff --git a/libs/libcommon/CMakeLists.txt b/libs/libcommon/CMakeLists.txt index c78473890dc..79e3d1fda80 100644 --- a/libs/libcommon/CMakeLists.txt +++ b/libs/libcommon/CMakeLists.txt @@ -62,7 +62,6 @@ add_library (common include/ext/scope_guard.h include/ext/size.h include/ext/unlock_guard.h - include/ext/singleton.h ${CONFIG_COMMON}) diff --git a/libs/libcommon/include/common/DateLUT.h b/libs/libcommon/include/common/DateLUT.h index 451bfa4d991..3cb935bc553 100644 --- a/libs/libcommon/include/common/DateLUT.h +++ b/libs/libcommon/include/common/DateLUT.h @@ -1,11 +1,11 @@ #pragma once #include "DateLUTImpl.h" -#include #include #include #include #include +#include // Also defined in Core/Defines.h #if !defined(ALWAYS_INLINE) @@ -18,25 +18,20 @@ /// This class provides lazy initialization and lookup of singleton DateLUTImpl objects for a given timezone. -class DateLUT : public ext::singleton +class DateLUT : private boost::noncopyable { - friend class ext::singleton; - public: - DateLUT(const DateLUT &) = delete; - DateLUT & operator=(const DateLUT &) = delete; - /// Return singleton DateLUTImpl instance for the default time zone. static ALWAYS_INLINE const DateLUTImpl & instance() { - const auto & date_lut = ext::singleton::instance(); + const auto & date_lut = getInstance(); return *date_lut.default_impl.load(std::memory_order_acquire); } /// Return singleton DateLUTImpl instance for a given time zone. static ALWAYS_INLINE const DateLUTImpl & instance(const std::string & time_zone) { - const auto & date_lut = ext::singleton::instance(); + const auto & date_lut = getInstance(); if (time_zone.empty()) return *date_lut.default_impl.load(std::memory_order_acquire); @@ -45,7 +40,7 @@ public: static void setDefaultTimezone(const std::string & time_zone) { - auto & date_lut = ext::singleton::instance(); + auto & date_lut = getInstance(); const auto & impl = date_lut.getImplementation(time_zone); date_lut.default_impl.store(&impl, std::memory_order_release); } @@ -54,6 +49,8 @@ protected: DateLUT(); private: + static DateLUT & getInstance(); + const DateLUTImpl & getImplementation(const std::string & time_zone) const; using DateLUTImplPtr = std::unique_ptr; diff --git a/libs/libcommon/include/ext/singleton.h b/libs/libcommon/include/ext/singleton.h deleted file mode 100644 index d7918f0867e..00000000000 --- a/libs/libcommon/include/ext/singleton.h +++ /dev/null @@ -1,44 +0,0 @@ -#pragma once - - -namespace ext -{ - -/** Example (1): - * - * class Derived : public ext::singleton - * { - * friend class ext::singleton; - * ... - * protected: - * Derived() {}; - * }; - * - * Example (2): - * - * class Some - * { - * ... - * }; - * - * class SomeSingleton : public Some, public ext::singleton {} - */ -template class singleton -{ -public: - static T & instance() - { - /// C++11 has thread safe statics. GCC and Clang have thread safe statics by default even before C++11. - static T instance; - return instance; - } - -protected: - singleton() {} - -private: - singleton(const singleton &); - singleton & operator=(const singleton &); -}; - -} diff --git a/libs/libcommon/src/DateLUT.cpp b/libs/libcommon/src/DateLUT.cpp index 183d003ffe8..8db1458d00f 100644 --- a/libs/libcommon/src/DateLUT.cpp +++ b/libs/libcommon/src/DateLUT.cpp @@ -155,3 +155,9 @@ const DateLUTImpl & DateLUT::getImplementation(const std::string & time_zone) co return *it->second; } + +DateLUT & DateLUT::getInstance() +{ + static DateLUT ret; + return ret; +} diff --git a/libs/libmysqlxx/include/mysqlxx/Connection.h b/libs/libmysqlxx/include/mysqlxx/Connection.h index 46e37eaeb48..0e5a608108c 100644 --- a/libs/libmysqlxx/include/mysqlxx/Connection.h +++ b/libs/libmysqlxx/include/mysqlxx/Connection.h @@ -5,7 +5,6 @@ #include -#include #include #include @@ -26,9 +25,11 @@ namespace mysqlxx * Usage: * LibrarySingleton::instance(); */ -class LibrarySingleton : public ext::singleton +class LibrarySingleton : private boost::noncopyable { -friend class ext::singleton; +public: + static auto & instance(); + private: LibrarySingleton(); ~LibrarySingleton(); diff --git a/libs/libmysqlxx/src/Connection.cpp b/libs/libmysqlxx/src/Connection.cpp index 80971549444..0e7d7bd5d3e 100644 --- a/libs/libmysqlxx/src/Connection.cpp +++ b/libs/libmysqlxx/src/Connection.cpp @@ -26,6 +26,11 @@ LibrarySingleton::~LibrarySingleton() mysql_library_end(); } +auto & LibrarySingleton::instance() +{ + static LibrarySingleton instance; + return instance; +} Connection::Connection() : driver(std::make_unique())