diff --git a/LICENSE b/LICENSE index 1460730b3ed..540e18567c6 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2016-2018 Yandex LLC + Copyright 2016-2019 Yandex LLC Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cmake/find_llvm.cmake b/cmake/find_llvm.cmake index 81866b720c9..fa11214f1de 100644 --- a/cmake/find_llvm.cmake +++ b/cmake/find_llvm.cmake @@ -79,7 +79,11 @@ endif() function(llvm_libs_all REQUIRED_LLVM_LIBRARIES) llvm_map_components_to_libnames (result all) - list (REMOVE_ITEM result "LTO" "LLVM") + if (USE_STATIC_LIBRARIES OR NOT "LLVM" IN_LIST result) + list (REMOVE_ITEM result "LTO" "LLVM") + else() + set (result "LLVM") + endif () if (TERMCAP_LIBRARY) list (APPEND result ${TERMCAP_LIBRARY}) endif () diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 58a0f80b435..ac7b287f886 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -1,7 +1,7 @@ # Third-party libraries may have substandard code. if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation -Wno-stringop-overflow") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation -Wno-stringop-overflow -Wno-implicit-function-declaration -Wno-return-type -Wno-array-bounds -Wno-bool-compare -Wno-int-conversion -Wno-switch") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-non-virtual-dtor -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation -Wno-implicit-fallthrough -Wno-class-memaccess -Wno-sign-compare -std=c++1z") elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-format -Wno-parentheses-equality -Wno-tautological-constant-compare -Wno-tautological-constant-out-of-range-compare -Wno-implicit-function-declaration -Wno-return-type -Wno-pointer-bool-conversion -Wno-enum-conversion -Wno-int-conversion -Wno-switch") diff --git a/contrib/jemalloc b/contrib/jemalloc index cd2931ad9bb..41b7372eade 160000 --- a/contrib/jemalloc +++ b/contrib/jemalloc @@ -1 +1 @@ -Subproject commit cd2931ad9bbd78208565716ab102e86d858c2fff +Subproject commit 41b7372eadee941b9164751b8d4963f915d3ceae diff --git a/contrib/libhdfs3-cmake/CMake/CodeCoverage.cmake b/contrib/libhdfs3-cmake/CMake/CodeCoverage.cmake deleted file mode 100644 index ce997925fcc..00000000000 --- a/contrib/libhdfs3-cmake/CMake/CodeCoverage.cmake +++ /dev/null @@ -1,48 +0,0 @@ -# Check prereqs -FIND_PROGRAM(GCOV_PATH gcov) -FIND_PROGRAM(LCOV_PATH lcov) -FIND_PROGRAM(GENHTML_PATH genhtml) - -IF(NOT GCOV_PATH) - MESSAGE(FATAL_ERROR "gcov not found! Aborting...") -ENDIF(NOT GCOV_PATH) - -IF(NOT CMAKE_BUILD_TYPE STREQUAL Debug) - MESSAGE(WARNING "Code coverage results with an optimised (non-Debug) build may be misleading") -ENDIF(NOT CMAKE_BUILD_TYPE STREQUAL Debug) - -#Setup compiler options -ADD_DEFINITIONS(-fprofile-arcs -ftest-coverage) - -SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-arcs ") -SET(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fprofile-arcs ") - -IF(NOT LCOV_PATH) - MESSAGE(FATAL_ERROR "lcov not found! Aborting...") -ENDIF(NOT LCOV_PATH) - -IF(NOT GENHTML_PATH) - MESSAGE(FATAL_ERROR "genhtml not found! Aborting...") -ENDIF(NOT GENHTML_PATH) - -#Setup target -ADD_CUSTOM_TARGET(ShowCoverage - #Capturing lcov counters and generating report - COMMAND ${LCOV_PATH} --directory . --capture --output-file CodeCoverage.info - COMMAND ${LCOV_PATH} --remove CodeCoverage.info '${CMAKE_CURRENT_BINARY_DIR}/*' 'test/*' 'mock/*' '/usr/*' '/opt/*' '*ext/rhel5_x86_64*' '*ext/osx*' --output-file CodeCoverage.info.cleaned - COMMAND ${GENHTML_PATH} -o CodeCoverageReport CodeCoverage.info.cleaned -) - - -ADD_CUSTOM_TARGET(ShowAllCoverage - #Capturing lcov counters and generating report - COMMAND ${LCOV_PATH} -a CodeCoverage.info.cleaned -a CodeCoverage.info.cleaned_withoutHA -o AllCodeCoverage.info - COMMAND sed -e 's|/.*/src|${CMAKE_SOURCE_DIR}/src|' -ig AllCodeCoverage.info - COMMAND ${GENHTML_PATH} -o AllCodeCoverageReport AllCodeCoverage.info -) - -ADD_CUSTOM_TARGET(ResetCoverage - #Cleanup lcov - COMMAND ${LCOV_PATH} --directory . --zerocounters -) - diff --git a/contrib/libhdfs3-cmake/CMake/FindBoost.cmake b/contrib/libhdfs3-cmake/CMake/FindBoost.cmake deleted file mode 100644 index 914a0a5b5cd..00000000000 --- a/contrib/libhdfs3-cmake/CMake/FindBoost.cmake +++ /dev/null @@ -1,1162 +0,0 @@ -# - Find Boost include dirs and libraries -# Use this module by invoking find_package with the form: -# find_package(Boost -# [version] [EXACT] # Minimum or EXACT version e.g. 1.36.0 -# [REQUIRED] # Fail with error if Boost is not found -# [COMPONENTS ...] # Boost libraries by their canonical name -# ) # e.g. "date_time" for "libboost_date_time" -# This module finds headers and requested component libraries OR a CMake -# package configuration file provided by a "Boost CMake" build. For the -# latter case skip to the "Boost CMake" section below. For the former -# case results are reported in variables: -# Boost_FOUND - True if headers and requested libraries were found -# Boost_INCLUDE_DIRS - Boost include directories -# Boost_LIBRARY_DIRS - Link directories for Boost libraries -# Boost_LIBRARIES - Boost component libraries to be linked -# Boost__FOUND - True if component was found ( is upper-case) -# Boost__LIBRARY - Libraries to link for component (may include -# target_link_libraries debug/optimized keywords) -# Boost_VERSION - BOOST_VERSION value from boost/version.hpp -# Boost_LIB_VERSION - Version string appended to library filenames -# Boost_MAJOR_VERSION - Boost major version number (X in X.y.z) -# Boost_MINOR_VERSION - Boost minor version number (Y in x.Y.z) -# Boost_SUBMINOR_VERSION - Boost subminor version number (Z in x.y.Z) -# Boost_LIB_DIAGNOSTIC_DEFINITIONS (Windows) -# - Pass to add_definitions() to have diagnostic -# information about Boost's automatic linking -# displayed during compilation -# -# This module reads hints about search locations from variables: -# BOOST_ROOT - Preferred installation prefix -# (or BOOSTROOT) -# BOOST_INCLUDEDIR - Preferred include directory e.g. /include -# BOOST_LIBRARYDIR - Preferred library directory e.g. /lib -# Boost_NO_SYSTEM_PATHS - Set to ON to disable searching in locations not -# specified by these hint variables. Default is OFF. -# Boost_ADDITIONAL_VERSIONS -# - List of Boost versions not known to this module -# (Boost install locations may contain the version) -# and saves search results persistently in CMake cache entries: -# Boost_INCLUDE_DIR - Directory containing Boost headers -# Boost_LIBRARY_DIR - Directory containing Boost libraries -# Boost__LIBRARY_DEBUG - Component library debug variant -# Boost__LIBRARY_RELEASE - Component library release variant -# Users may set these hints or results as cache entries. Projects should -# not read these entries directly but instead use the above result variables. -# Note that some hint names start in upper-case "BOOST". One may specify -# these as environment variables if they are not specified as CMake variables -# or cache entries. -# -# This module first searches for the Boost header files using the above hint -# variables (excluding BOOST_LIBRARYDIR) and saves the result in -# Boost_INCLUDE_DIR. Then it searches for requested component libraries using -# the above hints (excluding BOOST_INCLUDEDIR and Boost_ADDITIONAL_VERSIONS), -# "lib" directories near Boost_INCLUDE_DIR, and the library name configuration -# settings below. It saves the library directory in Boost_LIBRARY_DIR and -# individual library locations in Boost__LIBRARY_DEBUG and -# Boost__LIBRARY_RELEASE. When one changes settings used by previous -# searches in the same build tree (excluding environment variables) this -# module discards previous search results affected by the changes and searches -# again. -# -# Boost libraries come in many variants encoded in their file name. Users or -# projects may tell this module which variant to find by setting variables: -# Boost_USE_MULTITHREADED - Set to OFF to use the non-multithreaded -# libraries ('mt' tag). Default is ON. -# Boost_USE_STATIC_LIBS - Set to ON to force the use of the static -# libraries. Default is OFF. -# Boost_USE_STATIC_RUNTIME - Set to ON or OFF to specify whether to use -# libraries linked statically to the C++ runtime -# ('s' tag). Default is platform dependent. -# Boost_USE_DEBUG_PYTHON - Set to ON to use libraries compiled with a -# debug Python build ('y' tag). Default is OFF. -# Boost_USE_STLPORT - Set to ON to use libraries compiled with -# STLPort ('p' tag). Default is OFF. -# Boost_USE_STLPORT_DEPRECATED_NATIVE_IOSTREAMS -# - Set to ON to use libraries compiled with -# STLPort deprecated "native iostreams" -# ('n' tag). Default is OFF. -# Boost_COMPILER - Set to the compiler-specific library suffix -# (e.g. "-gcc43"). Default is auto-computed -# for the C++ compiler in use. -# Boost_THREADAPI - Suffix for "thread" component library name, -# such as "pthread" or "win32". Names with -# and without this suffix will both be tried. -# Other variables one may set to control this module are: -# Boost_DEBUG - Set to ON to enable debug output from FindBoost. -# Please enable this before filing any bug report. -# Boost_DETAILED_FAILURE_MSG -# - Set to ON to add detailed information to the -# failure message even when the REQUIRED option -# is not given to the find_package call. -# Boost_REALPATH - Set to ON to resolve symlinks for discovered -# libraries to assist with packaging. For example, -# the "system" component library may be resolved to -# "/usr/lib/libboost_system.so.1.42.0" instead of -# "/usr/lib/libboost_system.so". This does not -# affect linking and should not be enabled unless -# the user needs this information. -# On Visual Studio and Borland compilers Boost headers request automatic -# linking to corresponding libraries. This requires matching libraries to be -# linked explicitly or available in the link library search path. In this -# case setting Boost_USE_STATIC_LIBS to OFF may not achieve dynamic linking. -# Boost automatic linking typically requests static libraries with a few -# exceptions (such as Boost.Python). Use -# add_definitions(${Boost_LIB_DIAGNOSTIC_DEFINITIONS}) -# to ask Boost to report information about automatic linking requests. -# -# Example to find Boost headers only: -# find_package(Boost 1.36.0) -# if(Boost_FOUND) -# include_directories(${Boost_INCLUDE_DIRS}) -# add_executable(foo foo.cc) -# endif() -# Example to find Boost headers and some libraries: -# set(Boost_USE_STATIC_LIBS ON) -# set(Boost_USE_MULTITHREADED ON) -# set(Boost_USE_STATIC_RUNTIME OFF) -# find_package(Boost 1.36.0 COMPONENTS date_time filesystem system ...) -# if(Boost_FOUND) -# include_directories(${Boost_INCLUDE_DIRS}) -# add_executable(foo foo.cc) -# target_link_libraries(foo ${Boost_LIBRARIES}) -# endif() -# -# Boost CMake ---------------------------------------------------------- -# -# If Boost was built using the boost-cmake project it provides a package -# configuration file for use with find_package's Config mode. This module -# looks for the package configuration file called BoostConfig.cmake or -# boost-config.cmake and stores the result in cache entry "Boost_DIR". If -# found, the package configuration file is loaded and this module returns with -# no further action. See documentation of the Boost CMake package -# configuration for details on what it provides. -# -# Set Boost_NO_BOOST_CMAKE to ON to disable the search for boost-cmake. - -#============================================================================= -# Copyright 2006-2012 Kitware, Inc. -# Copyright 2006-2008 Andreas Schneider -# Copyright 2007 Wengo -# Copyright 2007 Mike Jackson -# Copyright 2008 Andreas Pakulat -# Copyright 2008-2012 Philip Lowman -# -# Distributed under the OSI-approved BSD License (the "License"); -# see accompanying file Copyright.txt for details. -# -# This software is distributed WITHOUT ANY WARRANTY; without even the -# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the License for more information. -#============================================================================= -# (To distribute this file outside of CMake, substitute the full -# License text for the above reference.) - - -#------------------------------------------------------------------------------- -# Before we go searching, check whether boost-cmake is available, unless the -# user specifically asked NOT to search for boost-cmake. -# -# If Boost_DIR is set, this behaves as any find_package call would. If not, -# it looks at BOOST_ROOT and BOOSTROOT to find Boost. -# -if (NOT Boost_NO_BOOST_CMAKE) - # If Boost_DIR is not set, look for BOOSTROOT and BOOST_ROOT as alternatives, - # since these are more conventional for Boost. - if ("$ENV{Boost_DIR}" STREQUAL "") - if (NOT "$ENV{BOOST_ROOT}" STREQUAL "") - set(ENV{Boost_DIR} $ENV{BOOST_ROOT}) - elseif (NOT "$ENV{BOOSTROOT}" STREQUAL "") - set(ENV{Boost_DIR} $ENV{BOOSTROOT}) - endif() - endif() - - # Do the same find_package call but look specifically for the CMake version. - # Note that args are passed in the Boost_FIND_xxxxx variables, so there is no - # need to delegate them to this find_package call. - find_package(Boost QUIET NO_MODULE) - mark_as_advanced(Boost_DIR) - - # If we found boost-cmake, then we're done. Print out what we found. - # Otherwise let the rest of the module try to find it. - if (Boost_FOUND) - message("Boost ${Boost_FIND_VERSION} found.") - if (Boost_FIND_COMPONENTS) - message("Found Boost components:") - message(" ${Boost_FIND_COMPONENTS}") - endif() - return() - endif() -endif() - - -#------------------------------------------------------------------------------- -# FindBoost functions & macros -# - -############################################ -# -# Check the existence of the libraries. -# -############################################ -# This macro was taken directly from the FindQt4.cmake file that is included -# with the CMake distribution. This is NOT my work. All work was done by the -# original authors of the FindQt4.cmake file. Only minor modifications were -# made to remove references to Qt and make this file more generally applicable -# And ELSE/ENDIF pairs were removed for readability. -######################################################################### - -macro(_Boost_ADJUST_LIB_VARS basename) - if(Boost_INCLUDE_DIR ) - if(Boost_${basename}_LIBRARY_DEBUG AND Boost_${basename}_LIBRARY_RELEASE) - # if the generator supports configuration types then set - # optimized and debug libraries, or if the CMAKE_BUILD_TYPE has a value - if(CMAKE_CONFIGURATION_TYPES OR CMAKE_BUILD_TYPE) - set(Boost_${basename}_LIBRARY optimized ${Boost_${basename}_LIBRARY_RELEASE} debug ${Boost_${basename}_LIBRARY_DEBUG}) - else() - # if there are no configuration types and CMAKE_BUILD_TYPE has no value - # then just use the release libraries - set(Boost_${basename}_LIBRARY ${Boost_${basename}_LIBRARY_RELEASE} ) - endif() - # FIXME: This probably should be set for both cases - set(Boost_${basename}_LIBRARIES optimized ${Boost_${basename}_LIBRARY_RELEASE} debug ${Boost_${basename}_LIBRARY_DEBUG}) - endif() - - # if only the release version was found, set the debug variable also to the release version - if(Boost_${basename}_LIBRARY_RELEASE AND NOT Boost_${basename}_LIBRARY_DEBUG) - set(Boost_${basename}_LIBRARY_DEBUG ${Boost_${basename}_LIBRARY_RELEASE}) - set(Boost_${basename}_LIBRARY ${Boost_${basename}_LIBRARY_RELEASE}) - set(Boost_${basename}_LIBRARIES ${Boost_${basename}_LIBRARY_RELEASE}) - endif() - - # if only the debug version was found, set the release variable also to the debug version - if(Boost_${basename}_LIBRARY_DEBUG AND NOT Boost_${basename}_LIBRARY_RELEASE) - set(Boost_${basename}_LIBRARY_RELEASE ${Boost_${basename}_LIBRARY_DEBUG}) - set(Boost_${basename}_LIBRARY ${Boost_${basename}_LIBRARY_DEBUG}) - set(Boost_${basename}_LIBRARIES ${Boost_${basename}_LIBRARY_DEBUG}) - endif() - - # If the debug & release library ends up being the same, omit the keywords - if(${Boost_${basename}_LIBRARY_RELEASE} STREQUAL ${Boost_${basename}_LIBRARY_DEBUG}) - set(Boost_${basename}_LIBRARY ${Boost_${basename}_LIBRARY_RELEASE} ) - set(Boost_${basename}_LIBRARIES ${Boost_${basename}_LIBRARY_RELEASE} ) - endif() - - if(Boost_${basename}_LIBRARY) - set(Boost_${basename}_FOUND ON) - endif() - - endif() - # Make variables changeable to the advanced user - mark_as_advanced( - Boost_${basename}_LIBRARY_RELEASE - Boost_${basename}_LIBRARY_DEBUG - ) -endmacro() - -macro(_Boost_CHANGE_DETECT changed_var) - set(${changed_var} 0) - foreach(v ${ARGN}) - if(DEFINED _Boost_COMPONENTS_SEARCHED) - if(${v}) - if(_${v}_LAST) - string(COMPARE NOTEQUAL "${${v}}" "${_${v}_LAST}" _${v}_CHANGED) - else() - set(_${v}_CHANGED 1) - endif() - elseif(_${v}_LAST) - set(_${v}_CHANGED 1) - endif() - if(_${v}_CHANGED) - set(${changed_var} 1) - endif() - else() - set(_${v}_CHANGED 0) - endif() - endforeach() -endmacro() - -macro(_Boost_FIND_LIBRARY var) - find_library(${var} ${ARGN}) - - # If we found the first library save Boost_LIBRARY_DIR. - if(${var} AND NOT Boost_LIBRARY_DIR) - get_filename_component(_dir "${${var}}" PATH) - set(Boost_LIBRARY_DIR "${_dir}" CACHE PATH "Boost library directory" FORCE) - endif() - - # If Boost_LIBRARY_DIR is known then search only there. - if(Boost_LIBRARY_DIR) - set(_boost_LIBRARY_SEARCH_DIRS ${Boost_LIBRARY_DIR} NO_DEFAULT_PATH) - endif() -endmacro() - -#------------------------------------------------------------------------------- - -# -# Runs compiler with "-dumpversion" and parses major/minor -# version with a regex. -# -function(_Boost_COMPILER_DUMPVERSION _OUTPUT_VERSION) - - exec_program(${CMAKE_CXX_COMPILER} - ARGS ${CMAKE_CXX_COMPILER_ARG1} -dumpversion - OUTPUT_VARIABLE _boost_COMPILER_VERSION - ) - string(REGEX REPLACE "([0-9])\\.([0-9])(\\.[0-9])?" "\\1\\2" - _boost_COMPILER_VERSION ${_boost_COMPILER_VERSION}) - - set(${_OUTPUT_VERSION} ${_boost_COMPILER_VERSION} PARENT_SCOPE) -endfunction() - -# -# Take a list of libraries with "thread" in it -# and prepend duplicates with "thread_${Boost_THREADAPI}" -# at the front of the list -# -function(_Boost_PREPEND_LIST_WITH_THREADAPI _output) - set(_orig_libnames ${ARGN}) - string(REPLACE "thread" "thread_${Boost_THREADAPI}" _threadapi_libnames "${_orig_libnames}") - set(${_output} ${_threadapi_libnames} ${_orig_libnames} PARENT_SCOPE) -endfunction() - -# -# If a library is found, replace its cache entry with its REALPATH -# -function(_Boost_SWAP_WITH_REALPATH _library _docstring) - if(${_library}) - get_filename_component(_boost_filepathreal ${${_library}} REALPATH) - unset(${_library} CACHE) - set(${_library} ${_boost_filepathreal} CACHE FILEPATH "${_docstring}") - endif() -endfunction() - -function(_Boost_CHECK_SPELLING _var) - if(${_var}) - string(TOUPPER ${_var} _var_UC) - message(FATAL_ERROR "ERROR: ${_var} is not the correct spelling. The proper spelling is ${_var_UC}.") - endif() -endfunction() - -# Guesses Boost's compiler prefix used in built library names -# Returns the guess by setting the variable pointed to by _ret -function(_Boost_GUESS_COMPILER_PREFIX _ret) - if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel" - OR "${CMAKE_CXX_COMPILER}" MATCHES "icl" - OR "${CMAKE_CXX_COMPILER}" MATCHES "icpc") - if(WIN32) - set (_boost_COMPILER "-iw") - else() - set (_boost_COMPILER "-il") - endif() - elseif (MSVC12) - set(_boost_COMPILER "-vc120") - elseif (MSVC11) - set(_boost_COMPILER "-vc110") - elseif (MSVC10) - set(_boost_COMPILER "-vc100") - elseif (MSVC90) - set(_boost_COMPILER "-vc90") - elseif (MSVC80) - set(_boost_COMPILER "-vc80") - elseif (MSVC71) - set(_boost_COMPILER "-vc71") - elseif (MSVC70) # Good luck! - set(_boost_COMPILER "-vc7") # yes, this is correct - elseif (MSVC60) # Good luck! - set(_boost_COMPILER "-vc6") # yes, this is correct - elseif (BORLAND) - set(_boost_COMPILER "-bcb") - elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "SunPro") - set(_boost_COMPILER "-sw") - elseif (MINGW) - if(${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION} VERSION_LESS 1.34) - set(_boost_COMPILER "-mgw") # no GCC version encoding prior to 1.34 - else() - _Boost_COMPILER_DUMPVERSION(_boost_COMPILER_VERSION) - set(_boost_COMPILER "-mgw${_boost_COMPILER_VERSION}") - endif() - elseif (UNIX) - if (CMAKE_COMPILER_IS_GNUCXX) - if(${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION} VERSION_LESS 1.34) - set(_boost_COMPILER "-gcc") # no GCC version encoding prior to 1.34 - else() - _Boost_COMPILER_DUMPVERSION(_boost_COMPILER_VERSION) - # Determine which version of GCC we have. - if(APPLE) - if(Boost_MINOR_VERSION) - if(${Boost_MINOR_VERSION} GREATER 35) - # In Boost 1.36.0 and newer, the mangled compiler name used - # on Mac OS X/Darwin is "xgcc". - set(_boost_COMPILER "-xgcc${_boost_COMPILER_VERSION}") - else() - # In Boost <= 1.35.0, there is no mangled compiler name for - # the Mac OS X/Darwin version of GCC. - set(_boost_COMPILER "") - endif() - else() - # We don't know the Boost version, so assume it's - # pre-1.36.0. - set(_boost_COMPILER "") - endif() - else() - set(_boost_COMPILER "-gcc${_boost_COMPILER_VERSION}") - endif() - endif() - endif () - else() - # TODO at least Boost_DEBUG here? - set(_boost_COMPILER "") - endif() - set(${_ret} ${_boost_COMPILER} PARENT_SCOPE) -endfunction() - -# -# End functions/macros -# -#------------------------------------------------------------------------------- - -#------------------------------------------------------------------------------- -# main. -#------------------------------------------------------------------------------- - -if(NOT DEFINED Boost_USE_MULTITHREADED) - set(Boost_USE_MULTITHREADED TRUE) -endif() - -# Check the version of Boost against the requested version. -if(Boost_FIND_VERSION AND NOT Boost_FIND_VERSION_MINOR) - message(SEND_ERROR "When requesting a specific version of Boost, you must provide at least the major and minor version numbers, e.g., 1.34") -endif() - -if(Boost_FIND_VERSION_EXACT) - # The version may appear in a directory with or without the patch - # level, even when the patch level is non-zero. - set(_boost_TEST_VERSIONS - "${Boost_FIND_VERSION_MAJOR}.${Boost_FIND_VERSION_MINOR}.${Boost_FIND_VERSION_PATCH}" - "${Boost_FIND_VERSION_MAJOR}.${Boost_FIND_VERSION_MINOR}") -else() - # The user has not requested an exact version. Among known - # versions, find those that are acceptable to the user request. - set(_Boost_KNOWN_VERSIONS ${Boost_ADDITIONAL_VERSIONS} - "1.56.0" "1.56" "1.55.0" "1.55" "1.54.0" "1.54" - "1.53.0" "1.53" "1.52.0" "1.52" "1.51.0" "1.51" - "1.50.0" "1.50" "1.49.0" "1.49" "1.48.0" "1.48" "1.47.0" "1.47" "1.46.1" - "1.46.0" "1.46" "1.45.0" "1.45" "1.44.0" "1.44" "1.43.0" "1.43" "1.42.0" "1.42" - "1.41.0" "1.41" "1.40.0" "1.40" "1.39.0" "1.39" "1.38.0" "1.38" "1.37.0" "1.37" - "1.36.1" "1.36.0" "1.36" "1.35.1" "1.35.0" "1.35" "1.34.1" "1.34.0" - "1.34" "1.33.1" "1.33.0" "1.33") - set(_boost_TEST_VERSIONS) - if(Boost_FIND_VERSION) - set(_Boost_FIND_VERSION_SHORT "${Boost_FIND_VERSION_MAJOR}.${Boost_FIND_VERSION_MINOR}") - # Select acceptable versions. - foreach(version ${_Boost_KNOWN_VERSIONS}) - if(NOT "${version}" VERSION_LESS "${Boost_FIND_VERSION}") - # This version is high enough. - list(APPEND _boost_TEST_VERSIONS "${version}") - elseif("${version}.99" VERSION_EQUAL "${_Boost_FIND_VERSION_SHORT}.99") - # This version is a short-form for the requested version with - # the patch level dropped. - list(APPEND _boost_TEST_VERSIONS "${version}") - endif() - endforeach() - else() - # Any version is acceptable. - set(_boost_TEST_VERSIONS "${_Boost_KNOWN_VERSIONS}") - endif() -endif() - -# The reason that we failed to find Boost. This will be set to a -# user-friendly message when we fail to find some necessary piece of -# Boost. -set(Boost_ERROR_REASON) - -if(Boost_DEBUG) - # Output some of their choices - message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - "_boost_TEST_VERSIONS = ${_boost_TEST_VERSIONS}") - message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - "Boost_USE_MULTITHREADED = ${Boost_USE_MULTITHREADED}") - message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - "Boost_USE_STATIC_LIBS = ${Boost_USE_STATIC_LIBS}") - message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - "Boost_USE_STATIC_RUNTIME = ${Boost_USE_STATIC_RUNTIME}") - message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - "Boost_ADDITIONAL_VERSIONS = ${Boost_ADDITIONAL_VERSIONS}") - message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - "Boost_NO_SYSTEM_PATHS = ${Boost_NO_SYSTEM_PATHS}") -endif() - -if(WIN32) - # In windows, automatic linking is performed, so you do not have - # to specify the libraries. If you are linking to a dynamic - # runtime, then you can choose to link to either a static or a - # dynamic Boost library, the default is to do a static link. You - # can alter this for a specific library "whatever" by defining - # BOOST_WHATEVER_DYN_LINK to force Boost library "whatever" to be - # linked dynamically. Alternatively you can force all Boost - # libraries to dynamic link by defining BOOST_ALL_DYN_LINK. - - # This feature can be disabled for Boost library "whatever" by - # defining BOOST_WHATEVER_NO_LIB, or for all of Boost by defining - # BOOST_ALL_NO_LIB. - - # If you want to observe which libraries are being linked against - # then defining BOOST_LIB_DIAGNOSTIC will cause the auto-linking - # code to emit a #pragma message each time a library is selected - # for linking. - set(Boost_LIB_DIAGNOSTIC_DEFINITIONS "-DBOOST_LIB_DIAGNOSTIC") -endif() - -_Boost_CHECK_SPELLING(Boost_ROOT) -_Boost_CHECK_SPELLING(Boost_LIBRARYDIR) -_Boost_CHECK_SPELLING(Boost_INCLUDEDIR) - -# Collect environment variable inputs as hints. Do not consider changes. -foreach(v BOOSTROOT BOOST_ROOT BOOST_INCLUDEDIR BOOST_LIBRARYDIR) - set(_env $ENV{${v}}) - if(_env) - file(TO_CMAKE_PATH "${_env}" _ENV_${v}) - else() - set(_ENV_${v} "") - endif() -endforeach() -if(NOT _ENV_BOOST_ROOT AND _ENV_BOOSTROOT) - set(_ENV_BOOST_ROOT "${_ENV_BOOSTROOT}") -endif() - -# Collect inputs and cached results. Detect changes since the last run. -if(NOT BOOST_ROOT AND BOOSTROOT) - set(BOOST_ROOT "${BOOSTROOT}") -endif() -set(_Boost_VARS_DIR - BOOST_ROOT - Boost_NO_SYSTEM_PATHS - ) - -if(Boost_DEBUG) - message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - "Declared as CMake or Environmental Variables:") - message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - " BOOST_ROOT = ${BOOST_ROOT}") - message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - " BOOST_INCLUDEDIR = ${BOOST_INCLUDEDIR}") - message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - " BOOST_LIBRARYDIR = ${BOOST_LIBRARYDIR}") - message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - "_boost_TEST_VERSIONS = ${_boost_TEST_VERSIONS}") -endif() - -# ------------------------------------------------------------------------ -# Search for Boost include DIR -# ------------------------------------------------------------------------ - -set(_Boost_VARS_INC BOOST_INCLUDEDIR Boost_INCLUDE_DIR Boost_ADDITIONAL_VERSIONS) -_Boost_CHANGE_DETECT(_Boost_CHANGE_INCDIR ${_Boost_VARS_DIR} ${_Boost_VARS_INC}) -# Clear Boost_INCLUDE_DIR if it did not change but other input affecting the -# location did. We will find a new one based on the new inputs. -if(_Boost_CHANGE_INCDIR AND NOT _Boost_INCLUDE_DIR_CHANGED) - unset(Boost_INCLUDE_DIR CACHE) -endif() - -if(NOT Boost_INCLUDE_DIR) - set(_boost_INCLUDE_SEARCH_DIRS "") - if(BOOST_INCLUDEDIR) - list(APPEND _boost_INCLUDE_SEARCH_DIRS ${BOOST_INCLUDEDIR}) - elseif(_ENV_BOOST_INCLUDEDIR) - list(APPEND _boost_INCLUDE_SEARCH_DIRS ${_ENV_BOOST_INCLUDEDIR}) - endif() - - if( BOOST_ROOT ) - list(APPEND _boost_INCLUDE_SEARCH_DIRS ${BOOST_ROOT}/include ${BOOST_ROOT}) - elseif( _ENV_BOOST_ROOT ) - list(APPEND _boost_INCLUDE_SEARCH_DIRS ${_ENV_BOOST_ROOT}/include ${_ENV_BOOST_ROOT}) - endif() - - if( Boost_NO_SYSTEM_PATHS) - list(APPEND _boost_INCLUDE_SEARCH_DIRS NO_CMAKE_SYSTEM_PATH) - else() - list(APPEND _boost_INCLUDE_SEARCH_DIRS PATHS - C:/boost/include - C:/boost - /sw/local/include - ) - endif() - - # Try to find Boost by stepping backwards through the Boost versions - # we know about. - # Build a list of path suffixes for each version. - set(_boost_PATH_SUFFIXES) - foreach(_boost_VER ${_boost_TEST_VERSIONS}) - # Add in a path suffix, based on the required version, ideally - # we could read this from version.hpp, but for that to work we'd - # need to know the include dir already - set(_boost_BOOSTIFIED_VERSION) - - # Transform 1.35 => 1_35 and 1.36.0 => 1_36_0 - if(_boost_VER MATCHES "[0-9]+\\.[0-9]+\\.[0-9]+") - string(REGEX REPLACE "([0-9]+)\\.([0-9]+)\\.([0-9]+)" "\\1_\\2_\\3" - _boost_BOOSTIFIED_VERSION ${_boost_VER}) - elseif(_boost_VER MATCHES "[0-9]+\\.[0-9]+") - string(REGEX REPLACE "([0-9]+)\\.([0-9]+)" "\\1_\\2" - _boost_BOOSTIFIED_VERSION ${_boost_VER}) - endif() - - list(APPEND _boost_PATH_SUFFIXES - "boost-${_boost_BOOSTIFIED_VERSION}" - "boost_${_boost_BOOSTIFIED_VERSION}" - "boost/boost-${_boost_BOOSTIFIED_VERSION}" - "boost/boost_${_boost_BOOSTIFIED_VERSION}" - ) - - endforeach() - - if(Boost_DEBUG) - message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - "Include debugging info:") - message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - " _boost_INCLUDE_SEARCH_DIRS = ${_boost_INCLUDE_SEARCH_DIRS}") - message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - " _boost_PATH_SUFFIXES = ${_boost_PATH_SUFFIXES}") - endif() - - # Look for a standard boost header file. - find_path(Boost_INCLUDE_DIR - NAMES boost/config.hpp - HINTS ${_boost_INCLUDE_SEARCH_DIRS} - PATH_SUFFIXES ${_boost_PATH_SUFFIXES} - ) -endif() - -# ------------------------------------------------------------------------ -# Extract version information from version.hpp -# ------------------------------------------------------------------------ - -# Set Boost_FOUND based only on header location and version. -# It will be updated below for component libraries. -if(Boost_INCLUDE_DIR) - if(Boost_DEBUG) - message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - "location of version.hpp: ${Boost_INCLUDE_DIR}/boost/version.hpp") - endif() - - # Extract Boost_VERSION and Boost_LIB_VERSION from version.hpp - set(Boost_VERSION 0) - set(Boost_LIB_VERSION "") - file(STRINGS "${Boost_INCLUDE_DIR}/boost/version.hpp" _boost_VERSION_HPP_CONTENTS REGEX "#define BOOST_(LIB_)?VERSION ") - set(_Boost_VERSION_REGEX "([0-9]+)") - set(_Boost_LIB_VERSION_REGEX "\"([0-9_]+)\"") - foreach(v VERSION LIB_VERSION) - if("${_boost_VERSION_HPP_CONTENTS}" MATCHES ".*#define BOOST_${v} ${_Boost_${v}_REGEX}.*") - set(Boost_${v} "${CMAKE_MATCH_1}") - endif() - endforeach() - unset(_boost_VERSION_HPP_CONTENTS) - - math(EXPR Boost_MAJOR_VERSION "${Boost_VERSION} / 100000") - math(EXPR Boost_MINOR_VERSION "${Boost_VERSION} / 100 % 1000") - math(EXPR Boost_SUBMINOR_VERSION "${Boost_VERSION} % 100") - - set(Boost_ERROR_REASON - "${Boost_ERROR_REASON}Boost version: ${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION}.${Boost_SUBMINOR_VERSION}\nBoost include path: ${Boost_INCLUDE_DIR}") - if(Boost_DEBUG) - message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - "version.hpp reveals boost " - "${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION}.${Boost_SUBMINOR_VERSION}") - endif() - - if(Boost_FIND_VERSION) - # Set Boost_FOUND based on requested version. - set(_Boost_VERSION "${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION}.${Boost_SUBMINOR_VERSION}") - if("${_Boost_VERSION}" VERSION_LESS "${Boost_FIND_VERSION}") - set(Boost_FOUND 0) - set(_Boost_VERSION_AGE "old") - elseif(Boost_FIND_VERSION_EXACT AND - NOT "${_Boost_VERSION}" VERSION_EQUAL "${Boost_FIND_VERSION}") - set(Boost_FOUND 0) - set(_Boost_VERSION_AGE "new") - else() - set(Boost_FOUND 1) - endif() - if(NOT Boost_FOUND) - # State that we found a version of Boost that is too new or too old. - set(Boost_ERROR_REASON - "${Boost_ERROR_REASON}\nDetected version of Boost is too ${_Boost_VERSION_AGE}. Requested version was ${Boost_FIND_VERSION_MAJOR}.${Boost_FIND_VERSION_MINOR}") - if (Boost_FIND_VERSION_PATCH) - set(Boost_ERROR_REASON - "${Boost_ERROR_REASON}.${Boost_FIND_VERSION_PATCH}") - endif () - if (NOT Boost_FIND_VERSION_EXACT) - set(Boost_ERROR_REASON "${Boost_ERROR_REASON} (or newer)") - endif () - set(Boost_ERROR_REASON "${Boost_ERROR_REASON}.") - endif () - else() - # Caller will accept any Boost version. - set(Boost_FOUND 1) - endif() -else() - set(Boost_FOUND 0) - set(Boost_ERROR_REASON - "${Boost_ERROR_REASON}Unable to find the Boost header files. Please set BOOST_ROOT to the root directory containing Boost or BOOST_INCLUDEDIR to the directory containing Boost's headers.") -endif() - -# ------------------------------------------------------------------------ -# Suffix initialization and compiler suffix detection. -# ------------------------------------------------------------------------ - -set(_Boost_VARS_NAME - Boost_COMPILER - Boost_THREADAPI - Boost_USE_DEBUG_PYTHON - Boost_USE_MULTITHREADED - Boost_USE_STATIC_LIBS - Boost_USE_STATIC_RUNTIME - Boost_USE_STLPORT - Boost_USE_STLPORT_DEPRECATED_NATIVE_IOSTREAMS - ) -_Boost_CHANGE_DETECT(_Boost_CHANGE_LIBNAME ${_Boost_VARS_NAME}) - -# Setting some more suffixes for the library -set(Boost_LIB_PREFIX "") -if ( WIN32 AND Boost_USE_STATIC_LIBS AND NOT CYGWIN) - set(Boost_LIB_PREFIX "lib") -endif() - -if (Boost_COMPILER) - set(_boost_COMPILER ${Boost_COMPILER}) - if(Boost_DEBUG) - message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - "using user-specified Boost_COMPILER = ${_boost_COMPILER}") - endif() -else() - # Attempt to guess the compiler suffix - # NOTE: this is not perfect yet, if you experience any issues - # please report them and use the Boost_COMPILER variable - # to work around the problems. - _Boost_GUESS_COMPILER_PREFIX(_boost_COMPILER) - if(Boost_DEBUG) - message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - "guessed _boost_COMPILER = ${_boost_COMPILER}") - endif() -endif() - -set (_boost_MULTITHREADED "-mt") -if( NOT Boost_USE_MULTITHREADED ) - set (_boost_MULTITHREADED "") -endif() -if(Boost_DEBUG) - message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - "_boost_MULTITHREADED = ${_boost_MULTITHREADED}") -endif() - -#====================== -# Systematically build up the Boost ABI tag -# http://boost.org/doc/libs/1_41_0/more/getting_started/windows.html#library-naming -set( _boost_RELEASE_ABI_TAG "-") -set( _boost_DEBUG_ABI_TAG "-") -# Key Use this library when: -# s linking statically to the C++ standard library and -# compiler runtime support libraries. -if(Boost_USE_STATIC_RUNTIME) - set( _boost_RELEASE_ABI_TAG "${_boost_RELEASE_ABI_TAG}s") - set( _boost_DEBUG_ABI_TAG "${_boost_DEBUG_ABI_TAG}s") -endif() -# g using debug versions of the standard and runtime -# support libraries -if(WIN32) - if(MSVC OR "${CMAKE_CXX_COMPILER}" MATCHES "icl" - OR "${CMAKE_CXX_COMPILER}" MATCHES "icpc") - set(_boost_DEBUG_ABI_TAG "${_boost_DEBUG_ABI_TAG}g") - endif() -endif() -# y using special debug build of python -if(Boost_USE_DEBUG_PYTHON) - set(_boost_DEBUG_ABI_TAG "${_boost_DEBUG_ABI_TAG}y") -endif() -# d using a debug version of your code -set(_boost_DEBUG_ABI_TAG "${_boost_DEBUG_ABI_TAG}d") -# p using the STLport standard library rather than the -# default one supplied with your compiler -if(Boost_USE_STLPORT) - set( _boost_RELEASE_ABI_TAG "${_boost_RELEASE_ABI_TAG}p") - set( _boost_DEBUG_ABI_TAG "${_boost_DEBUG_ABI_TAG}p") -endif() -# n using the STLport deprecated "native iostreams" feature -if(Boost_USE_STLPORT_DEPRECATED_NATIVE_IOSTREAMS) - set( _boost_RELEASE_ABI_TAG "${_boost_RELEASE_ABI_TAG}n") - set( _boost_DEBUG_ABI_TAG "${_boost_DEBUG_ABI_TAG}n") -endif() - -if(Boost_DEBUG) - message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - "_boost_RELEASE_ABI_TAG = ${_boost_RELEASE_ABI_TAG}") - message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - "_boost_DEBUG_ABI_TAG = ${_boost_DEBUG_ABI_TAG}") -endif() - -# ------------------------------------------------------------------------ -# Begin finding boost libraries -# ------------------------------------------------------------------------ -set(_Boost_VARS_LIB BOOST_LIBRARYDIR Boost_LIBRARY_DIR) -_Boost_CHANGE_DETECT(_Boost_CHANGE_LIBDIR ${_Boost_VARS_DIR} ${_Boost_VARS_LIB} Boost_INCLUDE_DIR) -# Clear Boost_LIBRARY_DIR if it did not change but other input affecting the -# location did. We will find a new one based on the new inputs. -if(_Boost_CHANGE_LIBDIR AND NOT _Boost_LIBRARY_DIR_CHANGED) - unset(Boost_LIBRARY_DIR CACHE) -endif() - -if(Boost_LIBRARY_DIR) - set(_boost_LIBRARY_SEARCH_DIRS ${Boost_LIBRARY_DIR} NO_DEFAULT_PATH) -else() - set(_boost_LIBRARY_SEARCH_DIRS "") - if(BOOST_LIBRARYDIR) - list(APPEND _boost_LIBRARY_SEARCH_DIRS ${BOOST_LIBRARYDIR}) - elseif(_ENV_BOOST_LIBRARYDIR) - list(APPEND _boost_LIBRARY_SEARCH_DIRS ${_ENV_BOOST_LIBRARYDIR}) - endif() - - if(BOOST_ROOT) - list(APPEND _boost_LIBRARY_SEARCH_DIRS ${BOOST_ROOT}/lib ${BOOST_ROOT}/stage/lib) - elseif(_ENV_BOOST_ROOT) - list(APPEND _boost_LIBRARY_SEARCH_DIRS ${_ENV_BOOST_ROOT}/lib ${_ENV_BOOST_ROOT}/stage/lib) - endif() - - list(APPEND _boost_LIBRARY_SEARCH_DIRS - ${Boost_INCLUDE_DIR}/lib - ${Boost_INCLUDE_DIR}/../lib - ${Boost_INCLUDE_DIR}/stage/lib - ) - if( Boost_NO_SYSTEM_PATHS ) - list(APPEND _boost_LIBRARY_SEARCH_DIRS NO_CMAKE_SYSTEM_PATH) - else() - list(APPEND _boost_LIBRARY_SEARCH_DIRS PATHS - C:/boost/lib - C:/boost - /sw/local/lib - ) - endif() -endif() - -if(Boost_DEBUG) - message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - "_boost_LIBRARY_SEARCH_DIRS = ${_boost_LIBRARY_SEARCH_DIRS}") -endif() - -# Support preference of static libs by adjusting CMAKE_FIND_LIBRARY_SUFFIXES -if( Boost_USE_STATIC_LIBS ) - set( _boost_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) - if(WIN32) - set(CMAKE_FIND_LIBRARY_SUFFIXES .lib .a ${CMAKE_FIND_LIBRARY_SUFFIXES}) - else() - set(CMAKE_FIND_LIBRARY_SUFFIXES .a ) - endif() -endif() - -# We want to use the tag inline below without risking double dashes -if(_boost_RELEASE_ABI_TAG) - if(${_boost_RELEASE_ABI_TAG} STREQUAL "-") - set(_boost_RELEASE_ABI_TAG "") - endif() -endif() -if(_boost_DEBUG_ABI_TAG) - if(${_boost_DEBUG_ABI_TAG} STREQUAL "-") - set(_boost_DEBUG_ABI_TAG "") - endif() -endif() - -# The previous behavior of FindBoost when Boost_USE_STATIC_LIBS was enabled -# on WIN32 was to: -# 1. Search for static libs compiled against a SHARED C++ standard runtime library (use if found) -# 2. Search for static libs compiled against a STATIC C++ standard runtime library (use if found) -# We maintain this behavior since changing it could break people's builds. -# To disable the ambiguous behavior, the user need only -# set Boost_USE_STATIC_RUNTIME either ON or OFF. -set(_boost_STATIC_RUNTIME_WORKAROUND false) -if(WIN32 AND Boost_USE_STATIC_LIBS) - if(NOT DEFINED Boost_USE_STATIC_RUNTIME) - set(_boost_STATIC_RUNTIME_WORKAROUND true) - endif() -endif() - -# On versions < 1.35, remove the System library from the considered list -# since it wasn't added until 1.35. -if(Boost_VERSION AND Boost_FIND_COMPONENTS) - if(Boost_VERSION LESS 103500) - list(REMOVE_ITEM Boost_FIND_COMPONENTS system) - endif() -endif() - -# If the user changed any of our control inputs flush previous results. -if(_Boost_CHANGE_LIBDIR OR _Boost_CHANGE_LIBNAME) - foreach(COMPONENT ${_Boost_COMPONENTS_SEARCHED}) - string(TOUPPER ${COMPONENT} UPPERCOMPONENT) - foreach(c DEBUG RELEASE) - set(_var Boost_${UPPERCOMPONENT}_LIBRARY_${c}) - unset(${_var} CACHE) - set(${_var} "${_var}-NOTFOUND") - endforeach() - endforeach() - set(_Boost_COMPONENTS_SEARCHED "") -endif() - -foreach(COMPONENT ${Boost_FIND_COMPONENTS}) - string(TOUPPER ${COMPONENT} UPPERCOMPONENT) - - set( _boost_docstring_release "Boost ${COMPONENT} library (release)") - set( _boost_docstring_debug "Boost ${COMPONENT} library (debug)") - - # - # Find RELEASE libraries - # - set(_boost_RELEASE_NAMES - ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_COMPILER}${_boost_MULTITHREADED}${_boost_RELEASE_ABI_TAG}-${Boost_LIB_VERSION} - ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_COMPILER}${_boost_MULTITHREADED}${_boost_RELEASE_ABI_TAG} - ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_MULTITHREADED}${_boost_RELEASE_ABI_TAG}-${Boost_LIB_VERSION} - ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_MULTITHREADED}${_boost_RELEASE_ABI_TAG} - ${Boost_LIB_PREFIX}boost_${COMPONENT} ) - if(_boost_STATIC_RUNTIME_WORKAROUND) - set(_boost_RELEASE_STATIC_ABI_TAG "-s${_boost_RELEASE_ABI_TAG}") - list(APPEND _boost_RELEASE_NAMES - ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_COMPILER}${_boost_MULTITHREADED}${_boost_RELEASE_STATIC_ABI_TAG}-${Boost_LIB_VERSION} - ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_COMPILER}${_boost_MULTITHREADED}${_boost_RELEASE_STATIC_ABI_TAG} - ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_MULTITHREADED}${_boost_RELEASE_STATIC_ABI_TAG}-${Boost_LIB_VERSION} - ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_MULTITHREADED}${_boost_RELEASE_STATIC_ABI_TAG} ) - endif() - if(Boost_THREADAPI AND ${COMPONENT} STREQUAL "thread") - _Boost_PREPEND_LIST_WITH_THREADAPI(_boost_RELEASE_NAMES ${_boost_RELEASE_NAMES}) - endif() - if(Boost_DEBUG) - message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - "Searching for ${UPPERCOMPONENT}_LIBRARY_RELEASE: ${_boost_RELEASE_NAMES}") - endif() - - # Avoid passing backslashes to _Boost_FIND_LIBRARY due to macro re-parsing. - string(REPLACE "\\" "/" _boost_LIBRARY_SEARCH_DIRS_tmp "${_boost_LIBRARY_SEARCH_DIRS}") - - _Boost_FIND_LIBRARY(Boost_${UPPERCOMPONENT}_LIBRARY_RELEASE - NAMES ${_boost_RELEASE_NAMES} - HINTS ${_boost_LIBRARY_SEARCH_DIRS_tmp} - NAMES_PER_DIR - DOC "${_boost_docstring_release}" - ) - - # - # Find DEBUG libraries - # - set(_boost_DEBUG_NAMES - ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_COMPILER}${_boost_MULTITHREADED}${_boost_DEBUG_ABI_TAG}-${Boost_LIB_VERSION} - ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_COMPILER}${_boost_MULTITHREADED}${_boost_DEBUG_ABI_TAG} - ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_MULTITHREADED}${_boost_DEBUG_ABI_TAG}-${Boost_LIB_VERSION} - ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_MULTITHREADED}${_boost_DEBUG_ABI_TAG} - ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_MULTITHREADED} - ${Boost_LIB_PREFIX}boost_${COMPONENT} ) - if(_boost_STATIC_RUNTIME_WORKAROUND) - set(_boost_DEBUG_STATIC_ABI_TAG "-s${_boost_DEBUG_ABI_TAG}") - list(APPEND _boost_DEBUG_NAMES - ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_COMPILER}${_boost_MULTITHREADED}${_boost_DEBUG_STATIC_ABI_TAG}-${Boost_LIB_VERSION} - ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_COMPILER}${_boost_MULTITHREADED}${_boost_DEBUG_STATIC_ABI_TAG} - ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_MULTITHREADED}${_boost_DEBUG_STATIC_ABI_TAG}-${Boost_LIB_VERSION} - ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_MULTITHREADED}${_boost_DEBUG_STATIC_ABI_TAG} ) - endif() - if(Boost_THREADAPI AND ${COMPONENT} STREQUAL "thread") - _Boost_PREPEND_LIST_WITH_THREADAPI(_boost_DEBUG_NAMES ${_boost_DEBUG_NAMES}) - endif() - if(Boost_DEBUG) - message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - "Searching for ${UPPERCOMPONENT}_LIBRARY_DEBUG: ${_boost_DEBUG_NAMES}") - endif() - - # Avoid passing backslashes to _Boost_FIND_LIBRARY due to macro re-parsing. - string(REPLACE "\\" "/" _boost_LIBRARY_SEARCH_DIRS_tmp "${_boost_LIBRARY_SEARCH_DIRS}") - - _Boost_FIND_LIBRARY(Boost_${UPPERCOMPONENT}_LIBRARY_DEBUG - NAMES ${_boost_DEBUG_NAMES} - HINTS ${_boost_LIBRARY_SEARCH_DIRS_tmp} - NAMES_PER_DIR - DOC "${_boost_docstring_debug}" - ) - - if(Boost_REALPATH) - _Boost_SWAP_WITH_REALPATH(Boost_${UPPERCOMPONENT}_LIBRARY_RELEASE "${_boost_docstring_release}") - _Boost_SWAP_WITH_REALPATH(Boost_${UPPERCOMPONENT}_LIBRARY_DEBUG "${_boost_docstring_debug}" ) - endif() - - _Boost_ADJUST_LIB_VARS(${UPPERCOMPONENT}) - -endforeach() - -# Restore the original find library ordering -if( Boost_USE_STATIC_LIBS ) - set(CMAKE_FIND_LIBRARY_SUFFIXES ${_boost_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES}) -endif() - -# ------------------------------------------------------------------------ -# End finding boost libraries -# ------------------------------------------------------------------------ - -set(Boost_INCLUDE_DIRS ${Boost_INCLUDE_DIR}) -set(Boost_LIBRARY_DIRS ${Boost_LIBRARY_DIR}) - -# The above setting of Boost_FOUND was based only on the header files. -# Update it for the requested component libraries. -if(Boost_FOUND) - # The headers were found. Check for requested component libs. - set(_boost_CHECKED_COMPONENT FALSE) - set(_Boost_MISSING_COMPONENTS "") - foreach(COMPONENT ${Boost_FIND_COMPONENTS}) - string(TOUPPER ${COMPONENT} COMPONENT) - set(_boost_CHECKED_COMPONENT TRUE) - if(NOT Boost_${COMPONENT}_FOUND) - string(TOLOWER ${COMPONENT} COMPONENT) - list(APPEND _Boost_MISSING_COMPONENTS ${COMPONENT}) - endif() - endforeach() - - if(Boost_DEBUG) - message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] Boost_FOUND = ${Boost_FOUND}") - endif() - - if (_Boost_MISSING_COMPONENTS) - set(Boost_FOUND 0) - # We were unable to find some libraries, so generate a sensible - # error message that lists the libraries we were unable to find. - set(Boost_ERROR_REASON - "${Boost_ERROR_REASON}\nCould not find the following") - if(Boost_USE_STATIC_LIBS) - set(Boost_ERROR_REASON "${Boost_ERROR_REASON} static") - endif() - set(Boost_ERROR_REASON - "${Boost_ERROR_REASON} Boost libraries:\n") - foreach(COMPONENT ${_Boost_MISSING_COMPONENTS}) - set(Boost_ERROR_REASON - "${Boost_ERROR_REASON} boost_${COMPONENT}\n") - endforeach() - - list(LENGTH Boost_FIND_COMPONENTS Boost_NUM_COMPONENTS_WANTED) - list(LENGTH _Boost_MISSING_COMPONENTS Boost_NUM_MISSING_COMPONENTS) - if (${Boost_NUM_COMPONENTS_WANTED} EQUAL ${Boost_NUM_MISSING_COMPONENTS}) - set(Boost_ERROR_REASON - "${Boost_ERROR_REASON}No Boost libraries were found. You may need to set BOOST_LIBRARYDIR to the directory containing Boost libraries or BOOST_ROOT to the location of Boost.") - else () - set(Boost_ERROR_REASON - "${Boost_ERROR_REASON}Some (but not all) of the required Boost libraries were found. You may need to install these additional Boost libraries. Alternatively, set BOOST_LIBRARYDIR to the directory containing Boost libraries or BOOST_ROOT to the location of Boost.") - endif () - endif () - - if( NOT Boost_LIBRARY_DIRS AND NOT _boost_CHECKED_COMPONENT ) - # Compatibility Code for backwards compatibility with CMake - # 2.4's FindBoost module. - - # Look for the boost library path. - # Note that the user may not have installed any libraries - # so it is quite possible the Boost_LIBRARY_DIRS may not exist. - set(_boost_LIB_DIR ${Boost_INCLUDE_DIR}) - - if("${_boost_LIB_DIR}" MATCHES "boost-[0-9]+") - get_filename_component(_boost_LIB_DIR ${_boost_LIB_DIR} PATH) - endif() - - if("${_boost_LIB_DIR}" MATCHES "/include$") - # Strip off the trailing "/include" in the path. - get_filename_component(_boost_LIB_DIR ${_boost_LIB_DIR} PATH) - endif() - - if(EXISTS "${_boost_LIB_DIR}/lib") - set(_boost_LIB_DIR ${_boost_LIB_DIR}/lib) - else() - if(EXISTS "${_boost_LIB_DIR}/stage/lib") - set(_boost_LIB_DIR ${_boost_LIB_DIR}/stage/lib) - else() - set(_boost_LIB_DIR "") - endif() - endif() - - if(_boost_LIB_DIR AND EXISTS "${_boost_LIB_DIR}") - set(Boost_LIBRARY_DIRS ${_boost_LIB_DIR}) - endif() - - endif() -else() - # Boost headers were not found so no components were found. - foreach(COMPONENT ${Boost_FIND_COMPONENTS}) - string(TOUPPER ${COMPONENT} UPPERCOMPONENT) - set(Boost_${UPPERCOMPONENT}_FOUND 0) - endforeach() -endif() - -# ------------------------------------------------------------------------ -# Notification to end user about what was found -# ------------------------------------------------------------------------ - -set(Boost_LIBRARIES "") -if(Boost_FOUND) - if(NOT Boost_FIND_QUIETLY) - message(STATUS "Boost version: ${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION}.${Boost_SUBMINOR_VERSION}") - if(Boost_FIND_COMPONENTS) - message(STATUS "Found the following Boost libraries:") - endif() - endif() - foreach( COMPONENT ${Boost_FIND_COMPONENTS} ) - string( TOUPPER ${COMPONENT} UPPERCOMPONENT ) - if( Boost_${UPPERCOMPONENT}_FOUND ) - if(NOT Boost_FIND_QUIETLY) - message (STATUS " ${COMPONENT}") - endif() - list(APPEND Boost_LIBRARIES ${Boost_${UPPERCOMPONENT}_LIBRARY}) - endif() - endforeach() -else() - if(Boost_FIND_REQUIRED) - message(SEND_ERROR "Unable to find the requested Boost libraries.\n${Boost_ERROR_REASON}") - else() - if(NOT Boost_FIND_QUIETLY) - # we opt not to automatically output Boost_ERROR_REASON here as - # it could be quite lengthy and somewhat imposing in its requests - # Since Boost is not always a required dependency we'll leave this - # up to the end-user. - if(Boost_DEBUG OR Boost_DETAILED_FAILURE_MSG) - message(STATUS "Could NOT find Boost\n${Boost_ERROR_REASON}") - else() - message(STATUS "Could NOT find Boost") - endif() - endif() - endif() -endif() - -# Configure display of cache entries in GUI. -foreach(v BOOSTROOT BOOST_ROOT ${_Boost_VARS_INC} ${_Boost_VARS_LIB}) - get_property(_type CACHE ${v} PROPERTY TYPE) - if(_type) - set_property(CACHE ${v} PROPERTY ADVANCED 1) - if("x${_type}" STREQUAL "xUNINITIALIZED") - if("x${v}" STREQUAL "xBoost_ADDITIONAL_VERSIONS") - set_property(CACHE ${v} PROPERTY TYPE STRING) - else() - set_property(CACHE ${v} PROPERTY TYPE PATH) - endif() - endif() - endif() -endforeach() - -# Record last used values of input variables so we can -# detect on the next run if the user changed them. -foreach(v - ${_Boost_VARS_INC} ${_Boost_VARS_LIB} - ${_Boost_VARS_DIR} ${_Boost_VARS_NAME} - ) - if(DEFINED ${v}) - set(_${v}_LAST "${${v}}" CACHE INTERNAL "Last used ${v} value.") - else() - unset(_${v}_LAST CACHE) - endif() -endforeach() - -# Maintain a persistent list of components requested anywhere since -# the last flush. -set(_Boost_COMPONENTS_SEARCHED "${_Boost_COMPONENTS_SEARCHED}") -list(APPEND _Boost_COMPONENTS_SEARCHED ${Boost_FIND_COMPONENTS}) -list(REMOVE_DUPLICATES _Boost_COMPONENTS_SEARCHED) -list(SORT _Boost_COMPONENTS_SEARCHED) -set(_Boost_COMPONENTS_SEARCHED "${_Boost_COMPONENTS_SEARCHED}" - CACHE INTERNAL "Components requested for this build tree.") diff --git a/contrib/libhdfs3-cmake/CMake/Options.cmake b/contrib/libhdfs3-cmake/CMake/Options.cmake index 5561f3ccc1e..728aef60e17 100644 --- a/contrib/libhdfs3-cmake/CMake/Options.cmake +++ b/contrib/libhdfs3-cmake/CMake/Options.cmake @@ -1,38 +1,15 @@ -OPTION(ENABLE_COVERAGE "enable code coverage" OFF) -OPTION(ENABLE_DEBUG "enable debug build" OFF) OPTION(ENABLE_SSE "enable SSE4.2 buildin function" ON) -OPTION(ENABLE_FRAME_POINTER "enable frame pointer on 64bit system with flag -fno-omit-frame-pointer, on 32bit system, it is always enabled" ON) -OPTION(ENABLE_LIBCPP "using libc++ instead of libstdc++, only valid for clang compiler" OFF) -OPTION(ENABLE_BOOST "using boost instead of native compiler c++0x support" OFF) INCLUDE (CheckFunctionExists) CHECK_FUNCTION_EXISTS(dladdr HAVE_DLADDR) CHECK_FUNCTION_EXISTS(nanosleep HAVE_NANOSLEEP) -IF(ENABLE_DEBUG STREQUAL ON) - SET(CMAKE_BUILD_TYPE Debug CACHE - STRING "Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel." FORCE) - SET(CMAKE_CXX_FLAGS_DEBUG "-g -O0" CACHE STRING "compiler flags for debug" FORCE) - SET(CMAKE_C_FLAGS_DEBUG "-g -O0" CACHE STRING "compiler flags for debug" FORCE) -ELSE(ENABLE_DEBUG STREQUAL ON) - SET(CMAKE_BUILD_TYPE RelWithDebInfo CACHE - STRING "Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel." FORCE) -ENDIF(ENABLE_DEBUG STREQUAL ON) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-strict-aliasing") SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-strict-aliasing") -IF(ENABLE_COVERAGE STREQUAL ON) - INCLUDE(CodeCoverage) -ENDIF(ENABLE_COVERAGE STREQUAL ON) - -IF(ENABLE_FRAME_POINTER STREQUAL ON) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer") -ENDIF(ENABLE_FRAME_POINTER STREQUAL ON) - IF(ENABLE_SSE STREQUAL ON) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2") -ENDIF(ENABLE_SSE STREQUAL ON) +ENDIF(ENABLE_SSE STREQUAL ON) IF(NOT TEST_HDFS_PREFIX) SET(TEST_HDFS_PREFIX "./" CACHE STRING "default directory prefix used for test." FORCE) @@ -41,76 +18,7 @@ ENDIF(NOT TEST_HDFS_PREFIX) ADD_DEFINITIONS(-DTEST_HDFS_PREFIX="${TEST_HDFS_PREFIX}") ADD_DEFINITIONS(-D__STDC_FORMAT_MACROS) ADD_DEFINITIONS(-D_GNU_SOURCE) - -IF(OS_MACOSX AND CMAKE_COMPILER_IS_GNUCXX) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-bind_at_load") -ENDIF(OS_MACOSX AND CMAKE_COMPILER_IS_GNUCXX) - -IF(OS_LINUX) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--export-dynamic") -ENDIF(OS_LINUX) - -SET(BOOST_ROOT ${CMAKE_PREFIX_PATH}) -IF(ENABLE_BOOST STREQUAL ON) - MESSAGE(STATUS "using boost instead of native compiler c++0x support.") - FIND_PACKAGE(Boost 1.50 REQUIRED) - SET(NEED_BOOST true CACHE INTERNAL "boost is required") -ELSE(ENABLE_BOOST STREQUAL ON) - SET(NEED_BOOST false CACHE INTERNAL "boost is required") -ENDIF(ENABLE_BOOST STREQUAL ON) - -IF(CMAKE_COMPILER_IS_GNUCXX) - IF(ENABLE_LIBCPP STREQUAL ON) - MESSAGE(FATAL_ERROR "Unsupport using GCC compiler with libc++") - ENDIF(ENABLE_LIBCPP STREQUAL ON) - - IF((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR EQUAL 4) AND OS_MACOSX) - SET(NEED_GCCEH true CACHE INTERNAL "Explicitly link with gcc_eh") - MESSAGE(STATUS "link with -lgcc_eh for TLS") - ENDIF((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR EQUAL 4) AND OS_MACOSX) - - IF((GCC_COMPILER_VERSION_MAJOR LESS 4) OR ((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR LESS 4))) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall") - IF(NOT ENABLE_BOOST STREQUAL ON) - MESSAGE(STATUS "gcc version is older than 4.6.0, boost is required.") - FIND_PACKAGE(Boost 1.50 REQUIRED) - SET(NEED_BOOST true CACHE INTERNAL "boost is required") - ENDIF(NOT ENABLE_BOOST STREQUAL ON) - ELSEIF((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR LESS 7)) - IF(NOT ENABLE_BOOST STREQUAL ON) - MESSAGE(STATUS "gcc version is older than 4.6.0, boost is required.") - FIND_PACKAGE(Boost 1.50 REQUIRED) - SET(NEED_BOOST true CACHE INTERNAL "boost is required") - ENDIF(NOT ENABLE_BOOST STREQUAL ON) - MESSAGE(STATUS "adding c++0x support for gcc compiler") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x") - ELSE((GCC_COMPILER_VERSION_MAJOR LESS 4) OR ((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR LESS 4))) - MESSAGE(STATUS "adding c++0x support for gcc compiler") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x") - ENDIF((GCC_COMPILER_VERSION_MAJOR LESS 4) OR ((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR LESS 4))) - - IF(NEED_BOOST) - IF((Boost_MAJOR_VERSION LESS 1) OR ((Boost_MAJOR_VERSION EQUAL 1) AND (Boost_MINOR_VERSION LESS 50))) - MESSAGE(FATAL_ERROR "boost 1.50+ is required") - ENDIF() - ELSE(NEED_BOOST) - IF(HAVE_NANOSLEEP) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_NANOSLEEP") - ELSE(HAVE_NANOSLEEP) - MESSAGE(FATAL_ERROR "nanosleep() is required") - ENDIF(HAVE_NANOSLEEP) - ENDIF(NEED_BOOST) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall") -ELSEIF(CMAKE_COMPILER_IS_CLANG) - MESSAGE(STATUS "adding c++0x support for clang compiler") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x") - SET(CMAKE_XCODE_ATTRIBUTE_CLANG_CXX_LANGUAGE_STANDARD "c++0x") - IF(ENABLE_LIBCPP STREQUAL ON) - MESSAGE(STATUS "using libc++ instead of libstdc++") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") - SET(CMAKE_XCODE_ATTRIBUTE_CLANG_CXX_LIBRARY "libc++") - ENDIF(ENABLE_LIBCPP STREQUAL ON) -ENDIF(CMAKE_COMPILER_IS_GNUCXX) +ADD_DEFINITIONS(-D_GLIBCXX_USE_NANOSLEEP) TRY_COMPILE(STRERROR_R_RETURN_INT ${CMAKE_CURRENT_BINARY_DIR} @@ -138,32 +46,8 @@ TRY_COMPILE(HAVE_NESTED_EXCEPTION CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'" OUTPUT_VARIABLE OUTPUT) -FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/test.cpp "#include ") -TRY_COMPILE(HAVE_BOOST_CHRONO - ${CMAKE_CURRENT_BINARY_DIR} - ${CMAKE_CURRENT_BINARY_DIR}/test.cpp - CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'" - -DINCLUDE_DIRECTORIES=${Boost_INCLUDE_DIR} - OUTPUT_VARIABLE OUTPUT) +SET(HAVE_BOOST_CHRONO 0) +SET(HAVE_BOOST_ATOMIC 0) -FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/test.cpp "#include ") -TRY_COMPILE(HAVE_STD_CHRONO - ${CMAKE_CURRENT_BINARY_DIR} - ${CMAKE_CURRENT_BINARY_DIR}/test.cpp - CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'" - OUTPUT_VARIABLE OUTPUT) - -FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/test.cpp "#include ") -TRY_COMPILE(HAVE_BOOST_ATOMIC - ${CMAKE_CURRENT_BINARY_DIR} - ${CMAKE_CURRENT_BINARY_DIR}/test.cpp - CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'" - -DINCLUDE_DIRECTORIES=${Boost_INCLUDE_DIR} - OUTPUT_VARIABLE OUTPUT) - -FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/test.cpp "#include ") -TRY_COMPILE(HAVE_STD_ATOMIC - ${CMAKE_CURRENT_BINARY_DIR} - ${CMAKE_CURRENT_BINARY_DIR}/test.cpp - CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'" - OUTPUT_VARIABLE OUTPUT) +SET(HAVE_STD_CHRONO 1) +SET(HAVE_STD_ATOMIC 1) diff --git a/contrib/murmurhash/include/murmurhash2.h b/contrib/murmurhash/include/murmurhash2.h index e95cf2a4d85..0fc95ef1c42 100644 --- a/contrib/murmurhash/include/murmurhash2.h +++ b/contrib/murmurhash/include/murmurhash2.h @@ -2,8 +2,7 @@ // MurmurHash2 was written by Austin Appleby, and is placed in the public // domain. The author hereby disclaims copyright to this source code. -#ifndef _MURMURHASH2_H_ -#define _MURMURHASH2_H_ +#pragma once //----------------------------------------------------------------------------- // Platform-specific functions and macros @@ -30,6 +29,3 @@ uint64_t MurmurHash64B (const void * key, int len, uint64_t seed); uint32_t MurmurHash2A (const void * key, int len, uint32_t seed); uint32_t MurmurHashNeutral2 (const void * key, int len, uint32_t seed); uint32_t MurmurHashAligned2 (const void * key, int len, uint32_t seed); - -#endif // _MURMURHASH2_H_ - diff --git a/contrib/murmurhash/include/murmurhash3.h b/contrib/murmurhash/include/murmurhash3.h index e1c6d34976c..256da1ad9da 100644 --- a/contrib/murmurhash/include/murmurhash3.h +++ b/contrib/murmurhash/include/murmurhash3.h @@ -2,8 +2,7 @@ // MurmurHash3 was written by Austin Appleby, and is placed in the public // domain. The author hereby disclaims copyright to this source code. -#ifndef _MURMURHASH3_H_ -#define _MURMURHASH3_H_ +#pragma once //----------------------------------------------------------------------------- // Platform-specific functions and macros @@ -33,5 +32,3 @@ void MurmurHash3_x86_128 ( const void * key, int len, uint32_t seed, void * out void MurmurHash3_x64_128 ( const void * key, int len, uint32_t seed, void * out ); //----------------------------------------------------------------------------- - -#endif // _MURMURHASH3_H_ diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index e4c7887f700..84c4b76d6fb 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -16,7 +16,7 @@ set (CONFIG_VERSION ${CMAKE_CURRENT_BINARY_DIR}/src/Common/config_version.h) set (CONFIG_COMMON ${CMAKE_CURRENT_BINARY_DIR}/src/Common/config.h) include (cmake/version.cmake) -message (STATUS "Will build ${VERSION_FULL}") +message (STATUS "Will build ${VERSION_FULL} revision ${VERSION_REVISION}") configure_file (src/Common/config.h.in ${CONFIG_COMMON}) configure_file (src/Common/config_version.h.in ${CONFIG_VERSION}) @@ -29,13 +29,35 @@ if (NOT NO_WERROR) set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror") endif () -# Add some warnings that are not available even with -Wall -Wextra. +# Add some warnings that are not available even with -Wall -Wextra -Wpedantic. + +option (WEVERYTHING "Enables -Weverything option with some exceptions. This is intended for exploration of new compiler warnings that may be found to be useful. Only makes sense for clang." ON) if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wextra-semi -Wcomma -Winconsistent-missing-destructor-override -Wunused-exception-parameter -Wshadow-uncaptured-local") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wpedantic -Wno-vla-extension -Wno-zero-length-array -Wno-gnu-anonymous-struct -Wno-nested-anon-types") - if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 6) - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wredundant-parens -Wzero-as-null-pointer-constant") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wshadow -Wshadow-uncaptured-local -Wextra-semi -Wcomma -Winconsistent-missing-destructor-override -Wunused-exception-parameter -Wcovered-switch-default -Wold-style-cast -Wrange-loop-analysis -Wunused-member-function -Wunreachable-code -Wunreachable-code-return -Wnewline-eof -Wembedded-directive -Wgnu-case-range -Wunused-macros -Wconditional-uninitialized -Wdeprecated -Wundef -Wreserved-id-macro -Wredundant-parens -Wzero-as-null-pointer-constant") + + if (WEVERYTHING) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Weverything -Wno-c++98-compat -Wno-c++98-compat-pedantic -Wno-missing-noreturn -Wno-padded -Wno-switch-enum -Wno-shadow-field-in-constructor -Wno-deprecated-dynamic-exception-spec -Wno-float-equal -Wno-weak-vtables -Wno-shift-sign-overflow -Wno-sign-conversion -Wno-conversion -Wno-exit-time-destructors -Wno-undefined-func-template -Wno-documentation-unknown-command -Wno-missing-variable-declarations -Wno-unused-template -Wno-global-constructors -Wno-c99-extensions -Wno-missing-prototypes -Wno-weak-template-vtables -Wno-zero-length-array -Wno-gnu-anonymous-struct -Wno-nested-anon-types -Wno-double-promotion -Wno-disabled-macro-expansion -Wno-used-but-marked-unused -Wno-vla-extension -Wno-vla -Wno-packed") + + # TODO Enable conversion, sign-conversion, double-promotion warnings. + endif () + + if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + + if (WEVERYTHING) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-return-std-move-in-c++11") + endif () + endif () + + if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wextra-semi-stmt -Wshadow-field -Wstring-plus-int") + + if (WEVERYTHING) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + endif () endif () endif () @@ -64,7 +86,6 @@ set(dbms_sources) include(../cmake/dbms_glob_sources.cmake) add_headers_and_sources(clickhouse_common_io src/Common) -add_headers_and_sources(clickhouse_common_io src/Compression) add_headers_and_sources(clickhouse_common_io src/Common/HashTable) add_headers_and_sources(clickhouse_common_io src/IO) @@ -151,20 +172,13 @@ if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELW PROPERTIES COMPILE_FLAGS -g0) endif () -if (NOT ARCH_ARM AND CPUID_LIBRARY) - set (LINK_LIBRARIES_ONLY_ON_X86_64 ${CPUID_LIBRARY}) -endif() - target_link_libraries (clickhouse_common_io PUBLIC common PRIVATE - clickhouse_parsers string_utils widechar_width ${LINK_LIBRARIES_ONLY_ON_X86_64} - ${LZ4_LIBRARY} - ${ZSTD_LIBRARY} ${DOUBLE_CONVERSION_LIBRARIES} pocoext PUBLIC @@ -185,8 +199,13 @@ target_link_libraries (clickhouse_common_io ${CMAKE_DL_LIBS} ) +if (NOT ARCH_ARM AND CPUID_LIBRARY) + target_link_libraries (clickhouse_common_io PRIVATE ${CPUID_LIBRARY}) +endif() + target_link_libraries (dbms PRIVATE + clickhouse_compression clickhouse_parsers clickhouse_common_config PUBLIC @@ -279,13 +298,6 @@ if (USE_HDFS) target_include_directories (dbms SYSTEM BEFORE PRIVATE ${HDFS3_INCLUDE_DIR}) endif() -if (NOT USE_INTERNAL_LZ4_LIBRARY) - target_include_directories (dbms SYSTEM BEFORE PRIVATE ${LZ4_INCLUDE_DIR}) -endif () -if (NOT USE_INTERNAL_ZSTD_LIBRARY) - target_include_directories (dbms SYSTEM BEFORE PRIVATE ${ZSTD_INCLUDE_DIR}) -endif () - if (USE_JEMALLOC) target_include_directories (dbms SYSTEM BEFORE PRIVATE ${JEMALLOC_INCLUDE_DIR}) # used in Interpreters/AsynchronousMetrics.cpp endif () diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake index 66a5d69c78b..94a10028518 100644 --- a/dbms/cmake/version.cmake +++ b/dbms/cmake/version.cmake @@ -1,11 +1,11 @@ # This strings autochanged from release_lib.sh: -set(VERSION_REVISION 54412 CACHE STRING "") # changed manually for tests -set(VERSION_MAJOR 18 CACHE STRING "") -set(VERSION_MINOR 16 CACHE STRING "") -set(VERSION_PATCH 0 CACHE STRING "") -set(VERSION_GITHASH b9b48c646c253358340bd39fd57754e92f88cd8a CACHE STRING "") -set(VERSION_DESCRIBE v18.16.0-testing CACHE STRING "") -set(VERSION_STRING 18.16.0 CACHE STRING "") +set(VERSION_REVISION 54413) +set(VERSION_MAJOR 19) +set(VERSION_MINOR 1) +set(VERSION_PATCH 0) +set(VERSION_GITHASH 014e344a36bc19a58621e0add379984cf62b9067) +set(VERSION_DESCRIBE v19.1.0-testing) +set(VERSION_STRING 19.1.0) # end of autochange set(VERSION_EXTRA "" CACHE STRING "") @@ -19,8 +19,8 @@ if (VERSION_EXTRA) string(CONCAT VERSION_STRING ${VERSION_STRING} "." ${VERSION_EXTRA}) endif () -set (VERSION_NAME "${PROJECT_NAME}" CACHE STRING "") -set (VERSION_FULL "${VERSION_NAME} ${VERSION_STRING}" CACHE STRING "") -set (VERSION_SO "${VERSION_STRING}" CACHE STRING "") +set (VERSION_NAME "${PROJECT_NAME}") +set (VERSION_FULL "${VERSION_NAME} ${VERSION_STRING}") +set (VERSION_SO "${VERSION_STRING}") math (EXPR VERSION_INTEGER "${VERSION_PATCH} + ${VERSION_MINOR}*1000 + ${VERSION_MAJOR}*1000000") diff --git a/dbms/programs/benchmark/Benchmark.cpp b/dbms/programs/benchmark/Benchmark.cpp index 07c65e27f9a..9bd3bda825a 100644 --- a/dbms/programs/benchmark/Benchmark.cpp +++ b/dbms/programs/benchmark/Benchmark.cpp @@ -229,7 +229,7 @@ private: report(info_per_interval); delay_watch.restart(); } - }; + } return true; } @@ -324,7 +324,7 @@ private: double seconds = watch.elapsedSeconds(); - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); info_per_interval.add(seconds, progress.rows, progress.bytes, info.rows, info.bytes); info_total.add(seconds, progress.rows, progress.bytes, info.rows, info.bytes); } @@ -332,7 +332,7 @@ private: void report(Stats & info) { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); /// Avoid zeros, nans or exceptions if (0 == info.queries) @@ -369,7 +369,7 @@ private: { WriteBufferFromFile json_out(filename); - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); auto print_key_value = [&](auto key, auto value, bool with_comma = true) { @@ -503,6 +503,4 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv) std::cerr << getCurrentExceptionMessage(print_stacktrace, true) << std::endl; return getCurrentExceptionCode(); } - - return 0; } diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index d5a354973e2..0edffbcf4de 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -74,8 +74,8 @@ /// http://en.wikipedia.org/wiki/ANSI_escape_code /// Similar codes \e[s, \e[u don't work in VT100 and Mosh. -#define SAVE_CURSOR_POSITION "\e7" -#define RESTORE_CURSOR_POSITION "\e8" +#define SAVE_CURSOR_POSITION "\033""7" +#define RESTORE_CURSOR_POSITION "\033""8" #define CLEAR_TO_END_OF_LINE "\033[K" @@ -554,10 +554,10 @@ private: void loop() { - String query; - String prev_query; + String input; + String prev_input; - while (char * line_ = readline(query.empty() ? prompt().c_str() : ":-] ")) + while (char * line_ = readline(input.empty() ? prompt().c_str() : ":-] ")) { String line = line_; free(line_); @@ -577,17 +577,17 @@ private: if (ends_with_backslash) line = line.substr(0, ws - 1); - query += line; + input += line; if (!ends_with_backslash && (ends_with_semicolon || has_vertical_output_suffix || (!config().has("multiline") && !hasDataInSTDIN()))) { - if (query != prev_query) + if (input != prev_input) { /// Replace line breaks with spaces to prevent the following problem. /// Every line of multi-line query is saved to history file as a separate line. /// If the user restarts the client then after pressing the "up" button /// every line of the query will be displayed separately. - std::string logged_query = query; + std::string logged_query = input; std::replace(logged_query.begin(), logged_query.end(), '\n', ' '); add_history(logged_query.c_str()); @@ -596,18 +596,18 @@ private: throwFromErrno("Cannot append history to file " + history_file, ErrorCodes::CANNOT_APPEND_HISTORY); #endif - prev_query = query; + prev_input = input; } if (has_vertical_output_suffix) - query = query.substr(0, query.length() - 2); + input = input.substr(0, input.length() - 2); try { /// Determine the terminal size. ioctl(0, TIOCGWINSZ, &terminal_size); - if (!process(query)) + if (!process(input)) break; } catch (const Exception & e) @@ -633,11 +633,11 @@ private: connect(); } - query = ""; + input = ""; } else { - query += '\n'; + input += '\n'; } } } @@ -675,8 +675,6 @@ private: /// Several queries separated by ';'. /// INSERT data is ended by the end of line, not ';'. - String query; - const char * begin = text.data(); const char * end = begin + text.size(); @@ -708,19 +706,19 @@ private: insert->end = pos; } - query = text.substr(begin - text.data(), pos - begin); + String str = text.substr(begin - text.data(), pos - begin); begin = pos; while (isWhitespaceASCII(*begin) || *begin == ';') ++begin; - TestHint test_hint(test_mode, query); + TestHint test_hint(test_mode, str); expected_client_error = test_hint.clientError(); expected_server_error = test_hint.serverError(); try { - if (!processSingleQuery(query, ast) && !ignore_error) + if (!processSingleQuery(str, ast) && !ignore_error) return false; } catch (...) @@ -728,7 +726,7 @@ private: last_exception = std::make_unique(getCurrentExceptionMessage(true), getCurrentExceptionCode()); actual_client_error = last_exception->code(); if (!ignore_error && (!actual_client_error || actual_client_error != expected_client_error)) - std::cerr << "Error on processing query: " << query << std::endl << last_exception->message(); + std::cerr << "Error on processing query: " << str << std::endl << last_exception->message(); got_exception = true; } @@ -904,8 +902,6 @@ private: ParserQuery parser(end, true); ASTPtr res; - const auto ignore_error = config().getBool("ignore-error", false); - if (is_interactive || ignore_error) { String message; @@ -1616,10 +1612,10 @@ public: for (size_t i = 0; i < external_tables_arguments.size(); ++i) { /// Parse commandline options related to external tables. - po::parsed_options parsed = po::command_line_parser( + po::parsed_options parsed_tables = po::command_line_parser( external_tables_arguments[i].size(), external_tables_arguments[i].data()).options(external_description).run(); po::variables_map external_options; - po::store(parsed, external_options); + po::store(parsed_tables, external_options); try { diff --git a/dbms/programs/client/Suggest.h b/dbms/programs/client/Suggest.h index 617e2bb520e..a36abca63b4 100644 --- a/dbms/programs/client/Suggest.h +++ b/dbms/programs/client/Suggest.h @@ -56,7 +56,7 @@ private: { std::string prefix_str(prefix); std::tie(pos, end) = std::equal_range(words.begin(), words.end(), prefix_str, - [prefix_length](const std::string & s, const std::string & prefix) { return strncmp(s.c_str(), prefix.c_str(), prefix_length) < 0; }); + [prefix_length](const std::string & s, const std::string & prefix_searched) { return strncmp(s.c_str(), prefix_searched.c_str(), prefix_length) < 0; }); } /// Iterates through matched range. diff --git a/dbms/programs/client/TestHint.h b/dbms/programs/client/TestHint.h index 7ddbfbb59f8..c1ac913e7a8 100644 --- a/dbms/programs/client/TestHint.h +++ b/dbms/programs/client/TestHint.h @@ -28,25 +28,26 @@ public: if (!enabled_) return; - String full_comment; Lexer lexer(query.data(), query.data() + query.size()); for (Token token = lexer.nextToken(); !token.isEnd(); token = lexer.nextToken()) { if (token.type == TokenType::Comment) - full_comment += String(token.begin, token.begin + token.size()) + ' '; - } - - if (!full_comment.empty()) - { - size_t pos_start = full_comment.find('{', 0); - if (pos_start != String::npos) { - size_t pos_end = full_comment.find('}', pos_start); - if (pos_end != String::npos) + String comment(token.begin, token.begin + token.size()); + + if (!comment.empty()) { - String hint(full_comment.begin() + pos_start + 1, full_comment.begin() + pos_end); - parse(hint); + size_t pos_start = comment.find('{', 0); + if (pos_start != String::npos) + { + size_t pos_end = comment.find('}', pos_start); + if (pos_end != String::npos) + { + String hint(comment.begin() + pos_start + 1, comment.begin() + pos_end); + parse(hint); + } + } } } } diff --git a/dbms/programs/compressor/CMakeLists.txt b/dbms/programs/compressor/CMakeLists.txt index 7aa2cad5708..bf3accfb8af 100644 --- a/dbms/programs/compressor/CMakeLists.txt +++ b/dbms/programs/compressor/CMakeLists.txt @@ -1,5 +1,5 @@ add_library (clickhouse-compressor-lib ${LINK_MODE} Compressor.cpp) -target_link_libraries (clickhouse-compressor-lib PRIVATE clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY}) +target_link_libraries (clickhouse-compressor-lib PRIVATE clickhouse_compression clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY}) if (CLICKHOUSE_SPLIT_BINARY) # Also in utils diff --git a/dbms/programs/compressor/Compressor.cpp b/dbms/programs/compressor/Compressor.cpp index efa45b86a88..de51f16833e 100644 --- a/dbms/programs/compressor/Compressor.cpp +++ b/dbms/programs/compressor/Compressor.cpp @@ -5,8 +5,8 @@ #include #include #include -#include -#include +#include +#include #include #include diff --git a/dbms/programs/copier/ClusterCopier.cpp b/dbms/programs/copier/ClusterCopier.cpp index 91ccdc88275..203a7f0cebb 100644 --- a/dbms/programs/copier/ClusterCopier.cpp +++ b/dbms/programs/copier/ClusterCopier.cpp @@ -243,7 +243,7 @@ struct ClusterPartition UInt64 rows_copied = 0; UInt64 blocks_copied = 0; - size_t total_tries = 0; + UInt64 total_tries = 0; }; @@ -340,7 +340,7 @@ struct TaskCluster String default_local_database; /// Limits number of simultaneous workers - size_t max_workers = 0; + UInt64 max_workers = 0; /// Base settings for pull and push Settings settings_common; @@ -773,11 +773,11 @@ public: } template - decltype(auto) retry(T && func, size_t max_tries = 100) + decltype(auto) retry(T && func, UInt64 max_tries = 100) { std::exception_ptr exception; - for (size_t try_number = 1; try_number <= max_tries; ++try_number) + for (UInt64 try_number = 1; try_number <= max_tries; ++try_number) { try { @@ -880,7 +880,7 @@ public: } /// Compute set of partitions, assume set of partitions aren't changed during the processing - void discoverTablePartitions(TaskTable & task_table, size_t num_threads = 0) + void discoverTablePartitions(TaskTable & task_table, UInt64 num_threads = 0) { /// Fetch partitions list from a shard { @@ -985,7 +985,7 @@ public: /// Retry table processing bool table_is_done = false; - for (size_t num_table_tries = 0; num_table_tries < max_table_tries; ++num_table_tries) + for (UInt64 num_table_tries = 0; num_table_tries < max_table_tries; ++num_table_tries) { if (tryProcessTable(task_table)) { @@ -1044,7 +1044,7 @@ protected: String workers_path = getWorkersPath(); String current_worker_path = getCurrentWorkerNodePath(); - size_t num_bad_version_errors = 0; + UInt64 num_bad_version_errors = 0; while (true) { @@ -1055,7 +1055,7 @@ protected: auto version = stat.version; zookeeper->get(workers_path, &stat); - if (static_cast(stat.numChildren) >= task_cluster->max_workers) + if (static_cast(stat.numChildren) >= task_cluster->max_workers) { LOG_DEBUG(log, "Too many workers (" << stat.numChildren << ", maximum " << task_cluster->max_workers << ")" << ". Postpone processing " << description); @@ -1163,7 +1163,7 @@ protected: } // If all task is finished and zxid is not changed then partition could not become dirty again - for (size_t shard_num = 0; shard_num < status_paths.size(); ++shard_num) + for (UInt64 shard_num = 0; shard_num < status_paths.size(); ++shard_num) { if (zxid1[shard_num] != zxid2[shard_num]) { @@ -1280,7 +1280,7 @@ protected: LOG_DEBUG(log, "Execute distributed DROP PARTITION: " << query); /// Limit number of max executing replicas to 1 - size_t num_shards = executeQueryOnCluster(cluster_push, query, nullptr, &settings_push, PoolMode::GET_ONE, 1); + UInt64 num_shards = executeQueryOnCluster(cluster_push, query, nullptr, &settings_push, PoolMode::GET_ONE, 1); if (num_shards < cluster_push->getShardCount()) { @@ -1299,8 +1299,8 @@ protected: } - static constexpr size_t max_table_tries = 1000; - static constexpr size_t max_shard_partition_tries = 600; + static constexpr UInt64 max_table_tries = 1000; + static constexpr UInt64 max_shard_partition_tries = 600; bool tryProcessTable(TaskTable & task_table) { @@ -1317,7 +1317,7 @@ protected: Stopwatch watch; TasksShard expected_shards; - size_t num_failed_shards = 0; + UInt64 num_failed_shards = 0; ++cluster_partition.total_tries; @@ -1368,7 +1368,7 @@ protected: bool is_unprioritized_task = !previous_shard_is_instantly_finished && shard->priority.is_remote; PartitionTaskStatus task_status = PartitionTaskStatus::Error; bool was_error = false; - for (size_t try_num = 0; try_num < max_shard_partition_tries; ++try_num) + for (UInt64 try_num = 0; try_num < max_shard_partition_tries; ++try_num) { task_status = tryProcessPartitionTask(partition, is_unprioritized_task); @@ -1434,8 +1434,8 @@ protected: } } - size_t required_partitions = task_table.cluster_partitions.size(); - size_t finished_partitions = task_table.finished_cluster_partitions.size(); + UInt64 required_partitions = task_table.cluster_partitions.size(); + UInt64 finished_partitions = task_table.finished_cluster_partitions.size(); bool table_is_done = finished_partitions >= required_partitions; if (!table_is_done) @@ -1645,7 +1645,7 @@ protected: String query = queryToString(create_query_push_ast); LOG_DEBUG(log, "Create destination tables. Query: " << query); - size_t shards = executeQueryOnCluster(task_table.cluster_push, query, create_query_push_ast, &task_cluster->settings_push, + UInt64 shards = executeQueryOnCluster(task_table.cluster_push, query, create_query_push_ast, &task_cluster->settings_push, PoolMode::GET_MANY); LOG_DEBUG(log, "Destination tables " << getDatabaseDotTable(task_table.table_push) << " have been created on " << shards << " shards of " << task_table.cluster_push->getShardCount()); @@ -1699,7 +1699,7 @@ protected: std::future future_is_dirty_checker; Stopwatch watch(CLOCK_MONOTONIC_COARSE); - constexpr size_t check_period_milliseconds = 500; + constexpr UInt64 check_period_milliseconds = 500; /// Will asynchronously check that ZooKeeper connection and is_dirty flag appearing while copy data auto cancel_check = [&] () @@ -1917,16 +1917,16 @@ protected: /** Executes simple query (without output streams, for example DDL queries) on each shard of the cluster * Returns number of shards for which at least one replica executed query successfully */ - size_t executeQueryOnCluster( + UInt64 executeQueryOnCluster( const ClusterPtr & cluster, const String & query, const ASTPtr & query_ast_ = nullptr, const Settings * settings = nullptr, PoolMode pool_mode = PoolMode::GET_ALL, - size_t max_successful_executions_per_shard = 0) const + UInt64 max_successful_executions_per_shard = 0) const { auto num_shards = cluster->getShardsInfo().size(); - std::vector per_shard_num_successful_replicas(num_shards, 0); + std::vector per_shard_num_successful_replicas(num_shards, 0); ASTPtr query_ast; if (query_ast_ == nullptr) @@ -1939,10 +1939,10 @@ protected: /// We need to execute query on one replica at least - auto do_for_shard = [&] (size_t shard_index) + auto do_for_shard = [&] (UInt64 shard_index) { const Cluster::ShardInfo & shard = cluster->getShardsInfo().at(shard_index); - size_t & num_successful_executions = per_shard_num_successful_replicas.at(shard_index); + UInt64 & num_successful_executions = per_shard_num_successful_replicas.at(shard_index); num_successful_executions = 0; auto increment_and_check_exit = [&] () @@ -1951,12 +1951,12 @@ protected: return max_successful_executions_per_shard && num_successful_executions >= max_successful_executions_per_shard; }; - size_t num_replicas = cluster->getShardsAddresses().at(shard_index).size(); - size_t num_local_replicas = shard.getLocalNodeCount(); - size_t num_remote_replicas = num_replicas - num_local_replicas; + UInt64 num_replicas = cluster->getShardsAddresses().at(shard_index).size(); + UInt64 num_local_replicas = shard.getLocalNodeCount(); + UInt64 num_remote_replicas = num_replicas - num_local_replicas; /// In that case we don't have local replicas, but do it just in case - for (size_t i = 0; i < num_local_replicas; ++i) + for (UInt64 i = 0; i < num_local_replicas; ++i) { auto interpreter = InterpreterFactory::get(query_ast, context); interpreter->execute(); @@ -1997,16 +1997,16 @@ protected: }; { - ThreadPool thread_pool(std::min(num_shards, getNumberOfPhysicalCPUCores())); + ThreadPool thread_pool(std::min(num_shards, UInt64(getNumberOfPhysicalCPUCores()))); - for (size_t shard_index = 0; shard_index < num_shards; ++shard_index) + for (UInt64 shard_index = 0; shard_index < num_shards; ++shard_index) thread_pool.schedule([=] { do_for_shard(shard_index); }); thread_pool.wait(); } - size_t successful_shards = 0; - for (size_t num_replicas : per_shard_num_successful_replicas) + UInt64 successful_shards = 0; + for (UInt64 num_replicas : per_shard_num_successful_replicas) successful_shards += (num_replicas > 0); return successful_shards; diff --git a/dbms/programs/local/LocalServer.cpp b/dbms/programs/local/LocalServer.cpp index 40e26438afc..1e3bc19f360 100644 --- a/dbms/programs/local/LocalServer.cpp +++ b/dbms/programs/local/LocalServer.cpp @@ -66,11 +66,11 @@ void LocalServer::initialize(Poco::Util::Application & self) } } -void LocalServer::applyCmdSettings(Context & context) +void LocalServer::applyCmdSettings() { #define EXTRACT_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) \ if (cmd_settings.NAME.changed) \ - context.getSettingsRef().NAME = cmd_settings.NAME; + context->getSettingsRef().NAME = cmd_settings.NAME; APPLY_FOR_SETTINGS(EXTRACT_SETTING) #undef EXTRACT_SETTING } @@ -179,7 +179,7 @@ try std::string default_database = config().getString("default_database", "_local"); context->addDatabase(default_database, std::make_shared(default_database)); context->setCurrentDatabase(default_database); - applyCmdOptions(*context); + applyCmdOptions(); if (!context->getPath().empty()) { @@ -274,7 +274,7 @@ void LocalServer::processQueries() context->setUser("default", "", Poco::Net::SocketAddress{}, ""); context->setCurrentQueryId(""); - applyCmdSettings(*context); + applyCmdSettings(); /// Use the same query_id (and thread group) for all queries CurrentThread::QueryScope query_scope_holder(*context); @@ -494,10 +494,10 @@ void LocalServer::init(int argc, char ** argv) config().setBool("ignore-error", true); } -void LocalServer::applyCmdOptions(Context & context) +void LocalServer::applyCmdOptions() { - context.setDefaultFormat(config().getString("output-format", config().getString("format", "TSV"))); - applyCmdSettings(context); + context->setDefaultFormat(config().getString("output-format", config().getString("format", "TSV"))); + applyCmdSettings(); } } diff --git a/dbms/programs/local/LocalServer.h b/dbms/programs/local/LocalServer.h index 244c7bda0b9..cabd209717b 100644 --- a/dbms/programs/local/LocalServer.h +++ b/dbms/programs/local/LocalServer.h @@ -34,8 +34,8 @@ private: std::string getInitialCreateTableQuery(); void tryInitPath(); - void applyCmdOptions(Context & context); - void applyCmdSettings(Context & context); + void applyCmdOptions(); + void applyCmdSettings(); void attachSystemTables(); void processQueries(); void setupUsers(); diff --git a/dbms/programs/obfuscator/Obfuscator.cpp b/dbms/programs/obfuscator/Obfuscator.cpp index f7eb7c5222d..6edb0de82b3 100644 --- a/dbms/programs/obfuscator/Obfuscator.cpp +++ b/dbms/programs/obfuscator/Obfuscator.cpp @@ -123,7 +123,7 @@ UInt64 hash(Ts... xs) UInt64 maskBits(UInt64 x, size_t num_bits) { - return x & ((1 << num_bits) - 1); + return x & ((1ULL << num_bits) - 1); } @@ -149,7 +149,7 @@ UInt64 feistelNetwork(UInt64 x, size_t num_bits, UInt64 seed, size_t num_rounds UInt64 bits = maskBits(x, num_bits); for (size_t i = 0; i < num_rounds; ++i) bits = feistelRound(bits, num_bits, seed, i); - return (x & ~((1 << num_bits) - 1)) ^ bits; + return (x & ~((1ULL << num_bits) - 1)) ^ bits; } @@ -317,8 +317,8 @@ void transformFixedString(const UInt8 * src, UInt8 * dst, size_t size, UInt64 se if (size >= 16) { - char * dst = reinterpret_cast(std::min(pos, end - 16)); - hash.get128(dst); + char * hash_dst = reinterpret_cast(std::min(pos, end - 16)); + hash.get128(hash_dst); } else { diff --git a/dbms/programs/odbc-bridge/getIdentifierQuote.h b/dbms/programs/odbc-bridge/getIdentifierQuote.h index 1a361dedea3..09d20937ea3 100644 --- a/dbms/programs/odbc-bridge/getIdentifierQuote.h +++ b/dbms/programs/odbc-bridge/getIdentifierQuote.h @@ -19,4 +19,4 @@ namespace DB std::string getIdentifierQuote(SQLHDBC hdbc); } -#endif \ No newline at end of file +#endif diff --git a/dbms/programs/performance-test/PerformanceTest.cpp b/dbms/programs/performance-test/PerformanceTest.cpp index 6a06283a6b8..27bf986fc1b 100644 --- a/dbms/programs/performance-test/PerformanceTest.cpp +++ b/dbms/programs/performance-test/PerformanceTest.cpp @@ -112,7 +112,8 @@ public: { return asString(padding); } - String asString(size_t padding) const + + String asString(size_t cur_padding) const { String repr = "{"; @@ -121,10 +122,10 @@ public: if (it != content.begin()) repr += ','; /// construct "key": "value" string with padding - repr += "\n" + pad(padding) + '"' + it->first + '"' + ": " + it->second; + repr += "\n" + pad(cur_padding) + '"' + it->first + '"' + ": " + it->second; } - repr += "\n" + pad(padding - 1) + '}'; + repr += "\n" + pad(cur_padding - 1) + '}'; return repr; } }; @@ -762,13 +763,13 @@ private: return true; } - void processTestsConfigurations(const Paths & input_files) + void processTestsConfigurations(const Paths & paths) { - tests_configurations.resize(input_files.size()); + tests_configurations.resize(paths.size()); - for (size_t i = 0; i != input_files.size(); ++i) + for (size_t i = 0; i != paths.size(); ++i) { - const String path = input_files[i]; + const String path = paths[i]; tests_configurations[i] = XMLConfigurationPtr(new XMLConfiguration(path)); } @@ -881,8 +882,6 @@ private: } } - Query query; - if (!test_config->has("query") && !test_config->has("query_file")) { throw DB::Exception("Missing query fields in test's config: " + test_name, DB::ErrorCodes::BAD_ARGUMENTS); @@ -907,6 +906,7 @@ private: bool tsv = fs::path(filename).extension().string() == ".tsv"; ReadBufferFromFile query_file(filename); + Query query; if (tsv) { @@ -1024,7 +1024,7 @@ private: } if (lite_output) - return minOutput(main_metric); + return minOutput(); else return constructTotalInfo(metrics); } @@ -1053,11 +1053,8 @@ private: void runQueries(const QueriesWithIndexes & queries_with_indexes) { - for (const std::pair & query_and_index : queries_with_indexes) + for (const auto & [query, run_index] : queries_with_indexes) { - Query query = query_and_index.first; - const size_t run_index = query_and_index.second; - TestStopConditions & stop_conditions = stop_conditions_by_run[run_index]; Stats & statistics = statistics_by_run[run_index]; @@ -1139,7 +1136,7 @@ private: } } - void constructSubstitutions(ConfigurationPtr & substitutions_view, StringToVector & substitutions) + void constructSubstitutions(ConfigurationPtr & substitutions_view, StringToVector & out_substitutions) { Keys xml_substitutions; substitutions_view->keys(xml_substitutions); @@ -1157,21 +1154,16 @@ private: for (size_t j = 0; j != xml_values.size(); ++j) { - substitutions[name].push_back(xml_substitution->getString("values.value[" + std::to_string(j) + "]")); + out_substitutions[name].push_back(xml_substitution->getString("values.value[" + std::to_string(j) + "]")); } } } - std::vector formatQueries(const String & query, StringToVector substitutions) + std::vector formatQueries(const String & query, StringToVector substitutions_to_generate) { - std::vector queries; - - StringToVector::iterator substitutions_first = substitutions.begin(); - StringToVector::iterator substitutions_last = substitutions.end(); - - runThroughAllOptionsAndPush(substitutions_first, substitutions_last, query, queries); - - return queries; + std::vector queries_res; + runThroughAllOptionsAndPush(substitutions_to_generate.begin(), substitutions_to_generate.end(), query, queries_res); + return queries_res; } /// Recursive method which goes through all substitution blocks in xml @@ -1179,11 +1171,11 @@ private: void runThroughAllOptionsAndPush(StringToVector::iterator substitutions_left, StringToVector::iterator substitutions_right, const String & template_query, - std::vector & queries) + std::vector & out_queries) { if (substitutions_left == substitutions_right) { - queries.push_back(template_query); /// completely substituted query + out_queries.push_back(template_query); /// completely substituted query return; } @@ -1191,7 +1183,7 @@ private: if (template_query.find(substitution_mask) == String::npos) /// nothing to substitute here { - runThroughAllOptionsAndPush(std::next(substitutions_left), substitutions_right, template_query, queries); + runThroughAllOptionsAndPush(std::next(substitutions_left), substitutions_right, template_query, out_queries); return; } @@ -1209,7 +1201,7 @@ private: query.replace(substr_pos, substitution_mask.length(), value); } - runThroughAllOptionsAndPush(std::next(substitutions_left), substitutions_right, query, queries); + runThroughAllOptionsAndPush(std::next(substitutions_left), substitutions_right, query, out_queries); } } @@ -1343,7 +1335,7 @@ public: return json_output.asString(); } - String minOutput(const String & main_metric) + String minOutput() { String output; @@ -1465,7 +1457,7 @@ try input_files = options["input-files"].as(); Strings collected_files; - for (const String filename : input_files) + for (const String & filename : input_files) { fs::path file(filename); diff --git a/dbms/programs/server/HTTPHandler.cpp b/dbms/programs/server/HTTPHandler.cpp index 34037a7c7c0..d86c526784b 100644 --- a/dbms/programs/server/HTTPHandler.cpp +++ b/dbms/programs/server/HTTPHandler.cpp @@ -19,8 +19,8 @@ #include #include #include -#include -#include +#include +#include #include #include #include diff --git a/dbms/programs/server/HTTPHandlerFactory.h b/dbms/programs/server/HTTPHandlerFactory.h index 6012af9fbb5..ce65f4476c0 100644 --- a/dbms/programs/server/HTTPHandlerFactory.h +++ b/dbms/programs/server/HTTPHandlerFactory.h @@ -31,20 +31,16 @@ public: Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest & request) override { - LOG_TRACE(log, - "HTTP Request for " << name << ". " - << "Method: " - << request.getMethod() - << ", Address: " - << request.clientAddress().toString() - << ", User-Agent: " - << (request.has("User-Agent") ? request.get("User-Agent") : "none") - << (request.hasContentLength() ? (", Length: " + std::to_string(request.getContentLength())) : ("")) -#if !NDEBUG - << ", Content Type: " << request.getContentType() - << ", Transfer Encoding: " << request.getTransferEncoding() -#endif - ); + LOG_TRACE(log, "HTTP Request for " << name << ". " + << "Method: " + << request.getMethod() + << ", Address: " + << request.clientAddress().toString() + << ", User-Agent: " + << (request.has("User-Agent") ? request.get("User-Agent") : "none") + << (request.hasContentLength() ? (", Length: " + std::to_string(request.getContentLength())) : ("")) + << ", Content Type: " << request.getContentType() + << ", Transfer Encoding: " << request.getTransferEncoding()); const auto & uri = request.getURI(); diff --git a/dbms/programs/server/InterserverIOHTTPHandler.cpp b/dbms/programs/server/InterserverIOHTTPHandler.cpp index 3c93ee1989a..94095365b6a 100644 --- a/dbms/programs/server/InterserverIOHTTPHandler.cpp +++ b/dbms/programs/server/InterserverIOHTTPHandler.cpp @@ -6,7 +6,7 @@ #include #include -#include +#include #include #include #include diff --git a/dbms/programs/server/MetricsTransmitter.cpp b/dbms/programs/server/MetricsTransmitter.cpp index 3268641942e..3e214aa1fc4 100644 --- a/dbms/programs/server/MetricsTransmitter.cpp +++ b/dbms/programs/server/MetricsTransmitter.cpp @@ -21,7 +21,7 @@ MetricsTransmitter::~MetricsTransmitter() try { { - std::lock_guard lock{mutex}; + std::lock_guard lock{mutex}; quit = true; } @@ -56,7 +56,7 @@ void MetricsTransmitter::run() std::vector prev_counters(ProfileEvents::end()); - std::unique_lock lock{mutex}; + std::unique_lock lock{mutex}; while (true) { diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index 3bfdedaeb58..c3dff11146e 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -12,8 +12,8 @@ #include #include #include -#include -#include +#include +#include #include #include #include diff --git a/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp b/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp index 7c1cb537c7c..7f3dbcfaf9d 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp @@ -102,16 +102,14 @@ AggregateFunctionPtr AggregateFunctionFactory::getImpl( { String name = getAliasToOrName(name_param); /// Find by exact match. - auto it = aggregate_functions.find(name); - if (it != aggregate_functions.end()) + if (auto it = aggregate_functions.find(name); it != aggregate_functions.end()) return it->second(name, argument_types, parameters); /// Find by case-insensitive name. /// Combinators cannot apply for case insensitive (SQL-style) aggregate function names. Only for native names. if (recursion_level == 0) { - auto it = case_insensitive_aggregate_functions.find(Poco::toLower(name)); - if (it != case_insensitive_aggregate_functions.end()) + if (auto it = case_insensitive_aggregate_functions.find(Poco::toLower(name)); it != case_insensitive_aggregate_functions.end()) return it->second(name, argument_types, parameters); } diff --git a/dbms/src/AggregateFunctions/AggregateFunctionForEach.h b/dbms/src/AggregateFunctions/AggregateFunctionForEach.h index 79932c7744f..519d1911a8a 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionForEach.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionForEach.h @@ -231,7 +231,7 @@ public: nested_state += nested_size_of_data; } - offsets_to.push_back(offsets_to.empty() ? state.dynamic_array_size : offsets_to.back() + state.dynamic_array_size); + offsets_to.push_back(offsets_to.back() + state.dynamic_array_size); } bool allocatesMemoryInArena() const override diff --git a/dbms/src/AggregateFunctions/AggregateFunctionGroupArrayInsertAt.h b/dbms/src/AggregateFunctions/AggregateFunctionGroupArrayInsertAt.h index 5ad93c042c2..90b19266e4c 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionGroupArrayInsertAt.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionGroupArrayInsertAt.h @@ -203,7 +203,7 @@ public: for (size_t i = arr.size(); i < result_array_size; ++i) to_data.insert(default_value); - to_offsets.push_back((to_offsets.empty() ? 0 : to_offsets.back()) + result_array_size); + to_offsets.push_back(to_offsets.back() + result_array_size); } const char * getHeaderFilePath() const override { return __FILE__; } diff --git a/dbms/src/AggregateFunctions/AggregateFunctionHistogram.cpp b/dbms/src/AggregateFunctions/AggregateFunctionHistogram.cpp index eaacb10be01..05c4fe86320 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionHistogram.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionHistogram.cpp @@ -45,8 +45,6 @@ AggregateFunctionPtr createAggregateFunctionHistogram(const std::string & name, throw Exception("Illegal type " + arguments[0]->getName() + " of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); return res; - - return nullptr; } } diff --git a/dbms/src/AggregateFunctions/AggregateFunctionNull.h b/dbms/src/AggregateFunctions/AggregateFunctionNull.h index 5cb33ed0a59..c8676230500 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionNull.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionNull.h @@ -187,8 +187,8 @@ template class AggregateFunctionNullUnary final : public AggregateFunctionNullBase> { public: - AggregateFunctionNullUnary(AggregateFunctionPtr nested_function) - : AggregateFunctionNullBase>(nested_function) + AggregateFunctionNullUnary(AggregateFunctionPtr nested_function_) + : AggregateFunctionNullBase>(std::move(nested_function_)) { } @@ -209,8 +209,8 @@ template class AggregateFunctionNullVariadic final : public AggregateFunctionNullBase> { public: - AggregateFunctionNullVariadic(AggregateFunctionPtr nested_function, const DataTypes & arguments) - : AggregateFunctionNullBase>(nested_function), + AggregateFunctionNullVariadic(AggregateFunctionPtr nested_function_, const DataTypes & arguments) + : AggregateFunctionNullBase>(std::move(nested_function_)), number_of_arguments(arguments.size()) { if (number_of_arguments == 1) diff --git a/dbms/src/AggregateFunctions/AggregateFunctionQuantile.h b/dbms/src/AggregateFunctions/AggregateFunctionQuantile.h index 3d420d13af0..ad25ff95af3 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionQuantile.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionQuantile.h @@ -100,9 +100,12 @@ public: return res; } - void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override + void NO_SANITIZE_UNDEFINED add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override { + /// Out of range conversion may occur. This is Ok. + const auto & column = static_cast(*columns[0]); + if constexpr (has_second_arg) this->data(place).add( column.getData()[row_num], diff --git a/dbms/src/AggregateFunctions/AggregateFunctionSumMap.h b/dbms/src/AggregateFunctions/AggregateFunctionSumMap.h index c3997b37192..4a20a314789 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionSumMap.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionSumMap.h @@ -80,11 +80,11 @@ public: void add(AggregateDataPtr place, const IColumn ** columns, const size_t row_num, Arena *) const override { // Column 0 contains array of keys of known type - const ColumnArray & array_column = static_cast(*columns[0]); - const IColumn::Offsets & offsets = array_column.getOffsets(); - const auto & keys_vec = static_cast(array_column.getData()); - const size_t keys_vec_offset = offsets[row_num - 1]; - const size_t keys_vec_size = (offsets[row_num] - keys_vec_offset); + const ColumnArray & array_column0 = static_cast(*columns[0]); + const IColumn::Offsets & offsets0 = array_column0.getOffsets(); + const auto & keys_vec = static_cast(array_column0.getData()); + const size_t keys_vec_offset = offsets0[row_num - 1]; + const size_t keys_vec_size = (offsets0[row_num] - keys_vec_offset); // Columns 1..n contain arrays of numeric values to sum auto & merged_maps = this->data(place).merged_maps; @@ -226,14 +226,14 @@ public: // Advance column offsets auto & to_keys_offsets = to_keys_arr.getOffsets(); - to_keys_offsets.push_back((to_keys_offsets.empty() ? 0 : to_keys_offsets.back()) + size); + to_keys_offsets.push_back(to_keys_offsets.back() + size); to_keys_col.reserve(size); for (size_t col = 0; col < values_types.size(); ++col) { auto & to_values_arr = static_cast(to_tuple.getColumn(col + 1)); auto & to_values_offsets = to_values_arr.getOffsets(); - to_values_offsets.push_back((to_values_offsets.empty() ? 0 : to_values_offsets.back()) + size); + to_values_offsets.push_back(to_values_offsets.back() + size); to_values_arr.getData().reserve(size); } diff --git a/dbms/src/AggregateFunctions/QuantileTiming.h b/dbms/src/AggregateFunctions/QuantileTiming.h index 55d1e9c83f5..131ca91dbbf 100644 --- a/dbms/src/AggregateFunctions/QuantileTiming.h +++ b/dbms/src/AggregateFunctions/QuantileTiming.h @@ -382,13 +382,13 @@ namespace detail if (index == BIG_THRESHOLD) break; - UInt64 count = 0; - readBinary(count, buf); + UInt64 elem_count = 0; + readBinary(elem_count, buf); if (index < SMALL_THRESHOLD) - count_small[index] = count; + count_small[index] = elem_count; else - count_big[index - SMALL_THRESHOLD] = count; + count_big[index - SMALL_THRESHOLD] = elem_count; } } } diff --git a/dbms/src/Client/Connection.cpp b/dbms/src/Client/Connection.cpp index f72eadcaec4..fa637cc555d 100644 --- a/dbms/src/Client/Connection.cpp +++ b/dbms/src/Client/Connection.cpp @@ -2,8 +2,8 @@ #include #include -#include -#include +#include +#include #include #include #include @@ -732,10 +732,10 @@ std::unique_ptr Connection::receiveException() std::vector Connection::receiveMultistringMessage(UInt64 msg_type) { size_t num = Protocol::Server::stringsInMessage(msg_type); - std::vector out(num); + std::vector strings(num); for (size_t i = 0; i < num; ++i) - readStringBinary(out[i], *in); - return out; + readStringBinary(strings[i], *in); + return strings; } diff --git a/dbms/src/Client/MultiplexedConnections.cpp b/dbms/src/Client/MultiplexedConnections.cpp index 62312eb5646..4c6e8fa72df 100644 --- a/dbms/src/Client/MultiplexedConnections.cpp +++ b/dbms/src/Client/MultiplexedConnections.cpp @@ -52,7 +52,7 @@ MultiplexedConnections::MultiplexedConnections( void MultiplexedConnections::sendExternalTablesData(std::vector & data) { - std::lock_guard lock(cancel_mutex); + std::lock_guard lock(cancel_mutex); if (!sent_query) throw Exception("Cannot send external tables data: query not yet sent.", ErrorCodes::LOGICAL_ERROR); @@ -79,7 +79,7 @@ void MultiplexedConnections::sendQuery( const ClientInfo * client_info, bool with_pending_data) { - std::lock_guard lock(cancel_mutex); + std::lock_guard lock(cancel_mutex); if (sent_query) throw Exception("Query already sent.", ErrorCodes::LOGICAL_ERROR); @@ -121,14 +121,14 @@ void MultiplexedConnections::sendQuery( Connection::Packet MultiplexedConnections::receivePacket() { - std::lock_guard lock(cancel_mutex); + std::lock_guard lock(cancel_mutex); Connection::Packet packet = receivePacketUnlocked(); return packet; } void MultiplexedConnections::disconnect() { - std::lock_guard lock(cancel_mutex); + std::lock_guard lock(cancel_mutex); for (ReplicaState & state : replica_states) { @@ -143,7 +143,7 @@ void MultiplexedConnections::disconnect() void MultiplexedConnections::sendCancel() { - std::lock_guard lock(cancel_mutex); + std::lock_guard lock(cancel_mutex); if (!sent_query || cancelled) throw Exception("Cannot cancel. Either no query sent or already cancelled.", ErrorCodes::LOGICAL_ERROR); @@ -160,7 +160,7 @@ void MultiplexedConnections::sendCancel() Connection::Packet MultiplexedConnections::drain() { - std::lock_guard lock(cancel_mutex); + std::lock_guard lock(cancel_mutex); if (!cancelled) throw Exception("Cannot drain connections: cancel first.", ErrorCodes::LOGICAL_ERROR); @@ -195,7 +195,7 @@ Connection::Packet MultiplexedConnections::drain() std::string MultiplexedConnections::dumpAddresses() const { - std::lock_guard lock(cancel_mutex); + std::lock_guard lock(cancel_mutex); return dumpAddressesUnlocked(); } diff --git a/dbms/src/Columns/ColumnArray.cpp b/dbms/src/Columns/ColumnArray.cpp index a45004058e2..0b7e2903b93 100644 --- a/dbms/src/Columns/ColumnArray.cpp +++ b/dbms/src/Columns/ColumnArray.cpp @@ -233,7 +233,10 @@ void ColumnArray::insertFrom(const IColumn & src_, size_t n) void ColumnArray::insertDefault() { - getOffsets().push_back(getOffsets().back()); + /// NOTE 1: We can use back() even if the array is empty (due to zero -1th element in PODArray). + /// NOTE 2: We cannot use reference in push_back, because reference get invalidated if array is reallocated. + auto last_offset = getOffsets().back(); + getOffsets().push_back(last_offset); } diff --git a/dbms/src/Columns/ColumnFixedString.cpp b/dbms/src/Columns/ColumnFixedString.cpp index b91c27becd7..955af04ee25 100644 --- a/dbms/src/Columns/ColumnFixedString.cpp +++ b/dbms/src/Columns/ColumnFixedString.cpp @@ -9,7 +9,7 @@ #include -#if __SSE2__ +#ifdef __SSE2__ #include #endif @@ -169,7 +169,7 @@ ColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t result const UInt8 * filt_end = filt_pos + col_size; const UInt8 * data_pos = chars.data(); -#if __SSE2__ +#ifdef __SSE2__ /** A slightly more optimized version. * Based on the assumption that often pieces of consecutive values * completely pass or do not pass the filter. diff --git a/dbms/src/Columns/ColumnLowCardinality.cpp b/dbms/src/Columns/ColumnLowCardinality.cpp index ca34d0b3bb4..49a0ca7da67 100644 --- a/dbms/src/Columns/ColumnLowCardinality.cpp +++ b/dbms/src/Columns/ColumnLowCardinality.cpp @@ -236,6 +236,9 @@ void ColumnLowCardinality::gather(ColumnGathererStream & gatherer) MutableColumnPtr ColumnLowCardinality::cloneResized(size_t size) const { auto unique_ptr = dictionary.getColumnUniquePtr(); + if (size == 0) + unique_ptr = unique_ptr->cloneEmpty(); + return ColumnLowCardinality::create((*std::move(unique_ptr)).mutate(), getIndexes().cloneResized(size)); } @@ -632,11 +635,11 @@ void ColumnLowCardinality::Dictionary::checkColumn(const IColumn & column) throw Exception("ColumnUnique expected as an argument of ColumnLowCardinality.", ErrorCodes::ILLEGAL_COLUMN); } -void ColumnLowCardinality::Dictionary::setShared(const ColumnPtr & dictionary) +void ColumnLowCardinality::Dictionary::setShared(const ColumnPtr & column_unique_) { - checkColumn(*dictionary); + checkColumn(*column_unique_); - column_unique = dictionary; + column_unique = column_unique_; shared = true; } diff --git a/dbms/src/Columns/ColumnLowCardinality.h b/dbms/src/Columns/ColumnLowCardinality.h index bfca6e41123..34a3db8589e 100644 --- a/dbms/src/Columns/ColumnLowCardinality.h +++ b/dbms/src/Columns/ColumnLowCardinality.h @@ -133,7 +133,7 @@ public: } bool valuesHaveFixedSize() const override { return getDictionary().valuesHaveFixedSize(); } - bool isFixedAndContiguous() const override { return getDictionary().isFixedAndContiguous(); } + bool isFixedAndContiguous() const override { return false; } size_t sizeOfValueIfFixed() const override { return getDictionary().sizeOfValueIfFixed(); } bool isNumeric() const override { return getDictionary().isNumeric(); } bool lowCardinality() const override { return true; } diff --git a/dbms/src/Columns/ColumnNothing.h b/dbms/src/Columns/ColumnNothing.h index 777790fcf5e..c9cde4f26ec 100644 --- a/dbms/src/Columns/ColumnNothing.h +++ b/dbms/src/Columns/ColumnNothing.h @@ -20,7 +20,7 @@ private: public: const char * getFamilyName() const override { return "Nothing"; } - MutableColumnPtr cloneDummy(size_t s) const override { return ColumnNothing::create(s); } + MutableColumnPtr cloneDummy(size_t s_) const override { return ColumnNothing::create(s_); } bool canBeInsideNullable() const override { return true; } }; diff --git a/dbms/src/Columns/ColumnUnique.h b/dbms/src/Columns/ColumnUnique.h index 4e70b44481e..85a9c498a94 100644 --- a/dbms/src/Columns/ColumnUnique.h +++ b/dbms/src/Columns/ColumnUnique.h @@ -66,9 +66,9 @@ public: Int64 getInt(size_t n) const override { return getNestedColumn()->getInt(n); } bool isNullAt(size_t n) const override { return is_nullable && n == getNullValueIndex(); } StringRef serializeValueIntoArena(size_t n, Arena & arena, char const *& begin) const override; - void updateHashWithValue(size_t n, SipHash & hash) const override + void updateHashWithValue(size_t n, SipHash & hash_func) const override { - return getNestedColumn()->updateHashWithValue(n, hash); + return getNestedColumn()->updateHashWithValue(n, hash_func); } int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override; diff --git a/dbms/src/Columns/ColumnVector.cpp b/dbms/src/Columns/ColumnVector.cpp index e8aa9a756ac..64e345acfd7 100644 --- a/dbms/src/Columns/ColumnVector.cpp +++ b/dbms/src/Columns/ColumnVector.cpp @@ -13,7 +13,7 @@ #include #include -#if __SSE2__ +#ifdef __SSE2__ #include #include @@ -162,7 +162,7 @@ ColumnPtr ColumnVector::filter(const IColumn::Filter & filt, ssize_t result_s const UInt8 * filt_end = filt_pos + size; const T * data_pos = data.data(); -#if __SSE2__ +#ifdef __SSE2__ /** A slightly more optimized version. * Based on the assumption that often pieces of consecutive values * completely pass or do not pass the filter. diff --git a/dbms/src/Columns/ColumnsCommon.cpp b/dbms/src/Columns/ColumnsCommon.cpp index 7dc5edc4be7..554d3e5eb57 100644 --- a/dbms/src/Columns/ColumnsCommon.cpp +++ b/dbms/src/Columns/ColumnsCommon.cpp @@ -1,4 +1,4 @@ -#if __SSE2__ +#ifdef __SSE2__ #include #endif @@ -24,7 +24,7 @@ size_t countBytesInFilter(const IColumn::Filter & filt) const Int8 * pos = reinterpret_cast(filt.data()); const Int8 * end = pos + filt.size(); -#if __SSE2__ && __POPCNT__ +#if defined(__SSE2__) && defined(__POPCNT__) const __m128i zero16 = _mm_setzero_si128(); const Int8 * end64 = pos + filt.size() / 64 * 64; @@ -69,7 +69,7 @@ bool memoryIsZero(const void * data, size_t size) const Int8 * pos = reinterpret_cast(data); const Int8 * end = pos + size; -#if __SSE2__ +#ifdef __SSE2__ const __m128 zero16 = _mm_setzero_ps(); const Int8 * end64 = pos + size / 64 * 64; @@ -205,17 +205,17 @@ namespace /// copy array ending at *end_offset_ptr const auto copy_array = [&] (const IColumn::Offset * offset_ptr) { - const auto offset = offset_ptr == offsets_begin ? 0 : offset_ptr[-1]; - const auto size = *offset_ptr - offset; + const auto arr_offset = offset_ptr == offsets_begin ? 0 : offset_ptr[-1]; + const auto arr_size = *offset_ptr - arr_offset; - result_offsets_builder.insertOne(size); + result_offsets_builder.insertOne(arr_size); const auto elems_size_old = res_elems.size(); - res_elems.resize(elems_size_old + size); - memcpy(&res_elems[elems_size_old], &src_elems[offset], size * sizeof(T)); + res_elems.resize(elems_size_old + arr_size); + memcpy(&res_elems[elems_size_old], &src_elems[arr_offset], arr_size * sizeof(T)); }; - #if __SSE2__ + #ifdef __SSE2__ const __m128i zero_vec = _mm_setzero_si128(); static constexpr size_t SIMD_BYTES = 16; const auto filt_end_aligned = filt_pos + size / SIMD_BYTES * SIMD_BYTES; diff --git a/dbms/src/Columns/IColumn.h b/dbms/src/Columns/IColumn.h index 38df6ab3c38..954715e4ff8 100644 --- a/dbms/src/Columns/IColumn.h +++ b/dbms/src/Columns/IColumn.h @@ -330,7 +330,9 @@ public: virtual bool lowCardinality() const { return false; } - virtual ~IColumn() {} + virtual ~IColumn() = default; + IColumn() = default; + IColumn(const IColumn &) = default; /** Print column name, size, and recursively print all subcolumns. */ diff --git a/dbms/src/Columns/ReverseIndex.h b/dbms/src/Columns/ReverseIndex.h index b9bd89f20e5..6ae780fdd99 100644 --- a/dbms/src/Columns/ReverseIndex.h +++ b/dbms/src/Columns/ReverseIndex.h @@ -393,10 +393,10 @@ UInt64 ReverseIndex::insert(const StringRef & data) if constexpr (use_saved_hash) { - auto & data = saved_hash->getData(); - if (data.size() <= num_rows) - data.resize(num_rows + 1); - data[num_rows] = hash; + auto & column_data = saved_hash->getData(); + if (column_data.size() <= num_rows) + column_data.resize(num_rows + 1); + column_data[num_rows] = hash; } else column->insertData(data.data, data.size); diff --git a/dbms/src/Columns/tests/column_unique.cpp b/dbms/src/Columns/tests/column_unique.cpp index 6fd8b6b4eaf..68b9367ee86 100644 --- a/dbms/src/Columns/tests/column_unique.cpp +++ b/dbms/src/Columns/tests/column_unique.cpp @@ -10,6 +10,7 @@ #pragma GCC diagnostic ignored "-Wsign-compare" #ifdef __clang__ #pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" +#pragma clang diagnostic ignored "-Wundef" #endif #include diff --git a/dbms/src/Common/ArrayCache.h b/dbms/src/Common/ArrayCache.h index 53e92d7db83..1521b753996 100644 --- a/dbms/src/Common/ArrayCache.h +++ b/dbms/src/Common/ArrayCache.h @@ -94,7 +94,11 @@ private: Key key; Payload payload; - void * ptr; + union + { + void * ptr; + char * char_ptr; + }; size_t size; size_t refcount = 0; void * chunk; @@ -231,7 +235,7 @@ public: ~Holder() { - std::lock_guard cache_lock(cache.mutex); + std::lock_guard cache_lock(cache.mutex); if (--region.refcount == 0) cache.lru_list.push_back(region); cache.total_size_in_use -= region.size; @@ -301,12 +305,12 @@ private: if (cleaned_up) return; - std::lock_guard token_lock(token->mutex); + std::lock_guard token_lock(token->mutex); if (token->cleaned_up) return; - std::lock_guard cache_lock(token->cache.mutex); + std::lock_guard cache_lock(token->cache.mutex); --token->refcount; if (token->refcount == 0) @@ -349,7 +353,7 @@ private: if (left_it->chunk == region.chunk && left_it->isFree()) { region.size += left_it->size; - *reinterpret_cast(®ion.ptr) -= left_it->size; + region.char_ptr-= left_it->size; size_multimap.erase(size_multimap.iterator_to(*left_it)); adjacency_list.erase_and_dispose(left_it, [](RegionMetadata * elem) { elem->destroy(); }); } @@ -479,7 +483,7 @@ private: size_multimap.erase(size_multimap.iterator_to(free_region)); free_region.size -= size; - *reinterpret_cast(&free_region.ptr) += size; + free_region.char_ptr += size; size_multimap.insert(free_region); adjacency_list.insert(adjacency_list.iterator_to(free_region), *allocated_region); @@ -536,7 +540,7 @@ public: ~ArrayCache() { - std::lock_guard cache_lock(mutex); + std::lock_guard cache_lock(mutex); key_map.clear(); lru_list.clear(); @@ -563,7 +567,7 @@ public: { InsertTokenHolder token_holder; { - std::lock_guard cache_lock(mutex); + std::lock_guard cache_lock(mutex); auto it = key_map.find(key, RegionCompareByKey()); if (key_map.end() != it) @@ -584,7 +588,7 @@ public: InsertToken * token = token_holder.token.get(); - std::lock_guard token_lock(token->mutex); + std::lock_guard token_lock(token->mutex); token_holder.cleaned_up = token->cleaned_up; @@ -605,7 +609,7 @@ public: RegionMetadata * region; { - std::lock_guard cache_lock(mutex); + std::lock_guard cache_lock(mutex); region = allocate(size); } @@ -626,14 +630,14 @@ public: catch (...) { { - std::lock_guard cache_lock(mutex); + std::lock_guard cache_lock(mutex); freeRegion(*region); } throw; } } - std::lock_guard cache_lock(mutex); + std::lock_guard cache_lock(mutex); try { @@ -692,7 +696,7 @@ public: Statistics getStatistics() const { - std::lock_guard cache_lock(mutex); + std::lock_guard cache_lock(mutex); Statistics res; res.total_chunks_size = total_chunks_size; diff --git a/dbms/src/Common/CompactArray.h b/dbms/src/Common/CompactArray.h index 55ae4239760..434b475e262 100644 --- a/dbms/src/Common/CompactArray.h +++ b/dbms/src/Common/CompactArray.h @@ -22,7 +22,7 @@ namespace ErrorCodes * simulates an array of `content_width`-bit values. */ template -class __attribute__ ((packed)) CompactArray final +class CompactArray final { public: class Reader; diff --git a/dbms/src/Common/Config/ConfigProcessor.cpp b/dbms/src/Common/Config/ConfigProcessor.cpp index 36ff7a99529..33b03dff38f 100644 --- a/dbms/src/Common/Config/ConfigProcessor.cpp +++ b/dbms/src/Common/Config/ConfigProcessor.cpp @@ -600,9 +600,9 @@ void ConfigProcessor::savePreprocessedConfig(const LoadedConfig & loaded_config, } preprocessed_path = preprocessed_dir + new_path; - auto path = Poco::Path(preprocessed_path).makeParent(); - if (!path.toString().empty()) - Poco::File(path).createDirectories(); + auto preprocessed_path_parent = Poco::Path(preprocessed_path).makeParent(); + if (!preprocessed_path_parent.toString().empty()) + Poco::File(preprocessed_path_parent).createDirectories(); } try { diff --git a/dbms/src/Common/Config/ConfigReloader.cpp b/dbms/src/Common/Config/ConfigReloader.cpp index f6edd3f16fb..ed6fad4d42c 100644 --- a/dbms/src/Common/Config/ConfigReloader.cpp +++ b/dbms/src/Common/Config/ConfigReloader.cpp @@ -78,7 +78,7 @@ void ConfigReloader::run() void ConfigReloader::reloadIfNewer(bool force, bool throw_on_error, bool fallback_to_preprocessed) { - std::lock_guard lock(reload_mutex); + std::lock_guard lock(reload_mutex); FilesChangesTracker new_files = getNewFileList(); if (force || need_reload_from_zk || new_files.isDifferOrNewerThan(files)) diff --git a/dbms/src/Common/CounterInFile.h b/dbms/src/Common/CounterInFile.h index 2c831e33302..cbf7105a728 100644 --- a/dbms/src/Common/CounterInFile.h +++ b/dbms/src/Common/CounterInFile.h @@ -54,7 +54,7 @@ public: template Int64 add(Int64 delta, Callback && locked_callback, bool create_if_need = false) { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); Int64 res = -1; diff --git a/dbms/src/Common/CurrentThread.cpp b/dbms/src/Common/CurrentThread.cpp index a3919108724..b2f165e5469 100644 --- a/dbms/src/Common/CurrentThread.cpp +++ b/dbms/src/Common/CurrentThread.cpp @@ -3,7 +3,6 @@ #include "CurrentThread.h" #include #include -#include #include #include #include @@ -24,8 +23,6 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -SimpleObjectPool task_stats_info_getter_pool; - // Smoker's implementation to avoid thread_local usage: error: undefined symbol: __cxa_thread_atexit #if defined(ARCADIA_ROOT) struct ThreadStatusPtrHolder : ThreadStatusPtr diff --git a/dbms/src/Common/FileChecker.cpp b/dbms/src/Common/FileChecker.cpp index da55d469245..d196c703e36 100644 --- a/dbms/src/Common/FileChecker.cpp +++ b/dbms/src/Common/FileChecker.cpp @@ -146,7 +146,7 @@ void FileChecker::load(Map & local_map, const std::string & path) JSON json(out.str()); JSON files = json["yandex"]; - for (const auto & name_value : files) + for (const JSON name_value : files) local_map[unescapeForFileName(name_value.getName())] = name_value.getValue()["size"].toUInt(); } diff --git a/dbms/src/Common/HashTable/Hash.h b/dbms/src/Common/HashTable/Hash.h index cc3144e94f6..57fa060cb32 100644 --- a/dbms/src/Common/HashTable/Hash.h +++ b/dbms/src/Common/HashTable/Hash.h @@ -35,20 +35,20 @@ inline DB::UInt64 intHash64(DB::UInt64 x) * due to high speed (latency 3 + 1 clock cycle, throughput 1 clock cycle). * Works only with SSE 4.2 support. */ -#if __SSE4_2__ +#ifdef __SSE4_2__ #include #endif -#if __aarch64__ && __ARM_FEATURE_CRC32 +#if defined(__aarch64__) && defined(__ARM_FEATURE_CRC32) #include #include #endif inline DB::UInt64 intHashCRC32(DB::UInt64 x) { -#if __SSE4_2__ +#ifdef __SSE4_2__ return _mm_crc32_u64(-1ULL, x); -#elif __aarch64__ && __ARM_FEATURE_CRC32 +#elif defined(__aarch64__) && defined(__ARM_FEATURE_CRC32) return __crc32cd(-1U, x); #else /// On other platforms we do not have CRC32. NOTE This can be confusing. diff --git a/dbms/src/Common/HyperLogLogCounter.h b/dbms/src/Common/HyperLogLogCounter.h index d85da31e7fd..381fc2fba22 100644 --- a/dbms/src/Common/HyperLogLogCounter.h +++ b/dbms/src/Common/HyperLogLogCounter.h @@ -278,7 +278,7 @@ template < typename BiasEstimator = TrivialBiasEstimator, HyperLogLogMode mode = HyperLogLogMode::FullFeatured, DenominatorMode denominator_mode = DenominatorMode::StableIfBig> -class __attribute__ ((packed)) HyperLogLogCounter : private Hash +class HyperLogLogCounter : private Hash { private: /// Number of buckets. diff --git a/dbms/src/Common/LRUCache.h b/dbms/src/Common/LRUCache.h index 61ae3d63dd5..98e88d9c59f 100644 --- a/dbms/src/Common/LRUCache.h +++ b/dbms/src/Common/LRUCache.h @@ -48,7 +48,7 @@ public: MappedPtr get(const Key & key) { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); auto res = getImpl(key, lock); if (res) @@ -61,7 +61,7 @@ public: void set(const Key & key, const MappedPtr & mapped) { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); setImpl(key, mapped, lock); } @@ -79,7 +79,7 @@ public: { InsertTokenHolder token_holder; { - std::lock_guard cache_lock(mutex); + std::lock_guard cache_lock(mutex); auto val = getImpl(key, cache_lock); if (val) @@ -97,7 +97,7 @@ public: InsertToken * token = token_holder.token.get(); - std::lock_guard token_lock(token->mutex); + std::lock_guard token_lock(token->mutex); token_holder.cleaned_up = token->cleaned_up; @@ -111,7 +111,7 @@ public: ++misses; token->value = load_func(); - std::lock_guard cache_lock(mutex); + std::lock_guard cache_lock(mutex); /// Insert the new value only if the token is still in present in insert_tokens. /// (The token may be absent because of a concurrent reset() call). @@ -131,26 +131,26 @@ public: void getStats(size_t & out_hits, size_t & out_misses) const { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); out_hits = hits; out_misses = misses; } size_t weight() const { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); return current_size; } size_t count() const { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); return cells.size(); } void reset() { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); queue.clear(); cells.clear(); insert_tokens.clear(); @@ -234,12 +234,12 @@ private: if (cleaned_up) return; - std::lock_guard token_lock(token->mutex); + std::lock_guard token_lock(token->mutex); if (token->cleaned_up) return; - std::lock_guard cache_lock(token->cache.mutex); + std::lock_guard cache_lock(token->cache.mutex); --token->refcount; if (token->refcount == 0) diff --git a/dbms/src/Common/ObjectPool.h b/dbms/src/Common/ObjectPool.h index 590dc396e8d..0cb0714a2b9 100644 --- a/dbms/src/Common/ObjectPool.h +++ b/dbms/src/Common/ObjectPool.h @@ -38,7 +38,7 @@ protected: void operator()(T * owning_ptr) const { - std::lock_guard lock{parent->mutex}; + std::lock_guard lock{parent->mutex}; parent->stack.emplace(owning_ptr); } }; @@ -51,7 +51,7 @@ public: template Pointer get(Factory && f) { - std::unique_lock lock(mutex); + std::unique_lock lock(mutex); if (stack.empty()) { @@ -94,7 +94,7 @@ public: template Pointer get(const Key & key, Factory && f) { - std::unique_lock lock(mutex); + std::unique_lock lock(mutex); auto it = container.find(key); if (container.end() == it) diff --git a/dbms/src/Common/PODArray.h b/dbms/src/Common/PODArray.h index cda0e03c700..d74e5b5d2c1 100644 --- a/dbms/src/Common/PODArray.h +++ b/dbms/src/Common/PODArray.h @@ -127,9 +127,6 @@ protected: c_end = c_start + end_diff; c_end_of_storage = c_start + bytes - pad_right - pad_left; - - if (pad_left) - memset(c_start - ELEMENT_SIZE, 0, ELEMENT_SIZE); } bool isInitialized() const @@ -312,13 +309,13 @@ public: this->c_end = this->c_start + this->byte_size(n); } - template - void push_back(const T & x, TAllocatorParams &&... allocator_params) + template + void push_back(U && x, TAllocatorParams &&... allocator_params) { if (unlikely(this->c_end == this->c_end_of_storage)) this->reserveForNextSize(std::forward(allocator_params)...); - *t_end() = x; + new (t_end()) T(std::forward(x)); this->c_end += this->byte_size(1); } diff --git a/dbms/src/Common/PoolBase.h b/dbms/src/Common/PoolBase.h index 194d7e421ad..306abb7d381 100644 --- a/dbms/src/Common/PoolBase.h +++ b/dbms/src/Common/PoolBase.h @@ -54,7 +54,7 @@ private: PoolEntryHelper(PooledObject & data_) : data(data_) { data.in_use = true; } ~PoolEntryHelper() { - std::unique_lock lock(data.pool.mutex); + std::unique_lock lock(data.pool.mutex); data.in_use = false; data.pool.available.notify_one(); } @@ -107,7 +107,7 @@ public: /** Allocates the object. Wait for free object in pool for 'timeout'. With 'timeout' < 0, the timeout is infinite. */ Entry get(Poco::Timespan::TimeDiff timeout) { - std::unique_lock lock(mutex); + std::unique_lock lock(mutex); while (true) { @@ -133,7 +133,7 @@ public: void reserve(size_t count) { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); while (items.size() < count) items.emplace_back(std::make_shared(allocObject(), *this)); diff --git a/dbms/src/Common/PoolWithFailoverBase.h b/dbms/src/Common/PoolWithFailoverBase.h index dc986a44ffa..40f60fb838a 100644 --- a/dbms/src/Common/PoolWithFailoverBase.h +++ b/dbms/src/Common/PoolWithFailoverBase.h @@ -195,7 +195,7 @@ PoolWithFailoverBase::getMany( /// At exit update shared error counts with error counts occured during this call. SCOPE_EXIT( { - std::lock_guard lock(pool_states_mutex); + std::lock_guard lock(pool_states_mutex); for (const ShuffledPool & pool: shuffled_pools) shared_pool_states[pool.index].error_count += pool.error_count; }); @@ -300,7 +300,7 @@ void PoolWithFailoverBase::reportError(const Entry & entry) { if (nested_pools[i]->contains(entry)) { - std::lock_guard lock(pool_states_mutex); + std::lock_guard lock(pool_states_mutex); ++shared_pool_states[i].error_count; return; } @@ -338,7 +338,7 @@ PoolWithFailoverBase::updatePoolStates() result.reserve(nested_pools.size()); { - std::lock_guard lock(pool_states_mutex); + std::lock_guard lock(pool_states_mutex); for (auto & state : shared_pool_states) state.randomize(); diff --git a/dbms/src/Common/RWLock.cpp b/dbms/src/Common/RWLock.cpp index 10a15cd7027..98e2a9f2995 100644 --- a/dbms/src/Common/RWLock.cpp +++ b/dbms/src/Common/RWLock.cpp @@ -70,7 +70,7 @@ RWLockImpl::LockHandler RWLockImpl::getLock(RWLockImpl::Type type) GroupsContainer::iterator it_group; ClientsContainer::iterator it_client; - std::unique_lock lock(mutex); + std::unique_lock lock(mutex); /// Check if the same thread is acquiring previously acquired lock auto it_handler = thread_to_handler.find(this_thread_id); @@ -139,7 +139,7 @@ RWLockImpl::LockHandler RWLockImpl::getLock(RWLockImpl::Type type) RWLockImpl::LockHandlerImpl::~LockHandlerImpl() { - std::unique_lock lock(parent->mutex); + std::unique_lock lock(parent->mutex); /// Remove weak_ptr to the handler, since there are no owners of the current lock parent->thread_to_handler.erase(it_handler); diff --git a/dbms/src/Common/ShellCommand.cpp b/dbms/src/Common/ShellCommand.cpp index 2e19d442f9a..84961292d02 100644 --- a/dbms/src/Common/ShellCommand.cpp +++ b/dbms/src/Common/ShellCommand.cpp @@ -28,37 +28,29 @@ namespace { struct Pipe { - union - { - int fds[2]; - struct - { - int read_fd; - int write_fd; - }; - }; + int fds_rw[2]; Pipe() { #ifndef __APPLE__ - if (0 != pipe2(fds, O_CLOEXEC)) + if (0 != pipe2(fds_rw, O_CLOEXEC)) DB::throwFromErrno("Cannot create pipe", DB::ErrorCodes::CANNOT_PIPE); #else - if (0 != pipe(fds)) + if (0 != pipe(fds_rw)) DB::throwFromErrno("Cannot create pipe", DB::ErrorCodes::CANNOT_PIPE); - if (0 != fcntl(fds[0], F_SETFD, FD_CLOEXEC)) + if (0 != fcntl(fds_rw[0], F_SETFD, FD_CLOEXEC)) DB::throwFromErrno("Cannot create pipe", DB::ErrorCodes::CANNOT_PIPE); - if (0 != fcntl(fds[1], F_SETFD, FD_CLOEXEC)) + if (0 != fcntl(fds_rw[1], F_SETFD, FD_CLOEXEC)) DB::throwFromErrno("Cannot create pipe", DB::ErrorCodes::CANNOT_PIPE); #endif } ~Pipe() { - if (read_fd >= 0) - close(read_fd); - if (write_fd >= 0) - close(write_fd); + if (fds_rw[0] >= 0) + close(fds_rw[0]); + if (fds_rw[1] >= 0) + close(fds_rw[1]); } }; @@ -125,15 +117,15 @@ std::unique_ptr ShellCommand::executeImpl(const char * filename, c /// And there is a lot of garbage (including, for example, mutex is blocked). And this can not be done after `vfork` - deadlock happens. /// Replace the file descriptors with the ends of our pipes. - if (STDIN_FILENO != dup2(pipe_stdin.read_fd, STDIN_FILENO)) + if (STDIN_FILENO != dup2(pipe_stdin.fds_rw[0], STDIN_FILENO)) _exit(int(ReturnCodes::CANNOT_DUP_STDIN)); if (!pipe_stdin_only) { - if (STDOUT_FILENO != dup2(pipe_stdout.write_fd, STDOUT_FILENO)) + if (STDOUT_FILENO != dup2(pipe_stdout.fds_rw[1], STDOUT_FILENO)) _exit(int(ReturnCodes::CANNOT_DUP_STDOUT)); - if (STDERR_FILENO != dup2(pipe_stderr.write_fd, STDERR_FILENO)) + if (STDERR_FILENO != dup2(pipe_stderr.fds_rw[1], STDERR_FILENO)) _exit(int(ReturnCodes::CANNOT_DUP_STDERR)); } @@ -143,12 +135,12 @@ std::unique_ptr ShellCommand::executeImpl(const char * filename, c _exit(int(ReturnCodes::CANNOT_EXEC)); } - std::unique_ptr res(new ShellCommand(pid, pipe_stdin.write_fd, pipe_stdout.read_fd, pipe_stderr.read_fd, terminate_in_destructor)); + std::unique_ptr res(new ShellCommand(pid, pipe_stdin.fds_rw[1], pipe_stdout.fds_rw[0], pipe_stderr.fds_rw[0], terminate_in_destructor)); /// Now the ownership of the file descriptors is passed to the result. - pipe_stdin.write_fd = -1; - pipe_stdout.read_fd = -1; - pipe_stderr.read_fd = -1; + pipe_stdin.fds_rw[1] = -1; + pipe_stdout.fds_rw[0] = -1; + pipe_stderr.fds_rw[0] = -1; return res; } @@ -158,8 +150,8 @@ std::unique_ptr ShellCommand::execute(const std::string & command, { /// Arguments in non-constant chunks of memory (as required for `execv`). /// Moreover, their copying must be done before calling `vfork`, so after `vfork` do a minimum of things. - std::vector argv0("sh", "sh" + strlen("sh") + 1); - std::vector argv1("-c", "-c" + strlen("-c") + 1); + std::vector argv0("sh", &("sh"[3])); + std::vector argv1("-c", &("-c"[3])); std::vector argv2(command.data(), command.data() + command.size() + 1); char * const argv[] = { argv0.data(), argv1.data(), argv2.data(), nullptr }; diff --git a/dbms/src/Common/SimpleCache.h b/dbms/src/Common/SimpleCache.h index 3028673c6b5..06d29c8ba7a 100644 --- a/dbms/src/Common/SimpleCache.h +++ b/dbms/src/Common/SimpleCache.h @@ -32,7 +32,7 @@ public: Result operator() (Args &&... args) { { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); Key key{std::forward(args)...}; auto it = cache.find(key); @@ -45,7 +45,7 @@ public: Result res = f(std::forward(args)...); { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); cache.emplace(std::forward_as_tuple(args...), res); } @@ -55,7 +55,7 @@ public: void drop() { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); cache.clear(); } }; diff --git a/dbms/src/Common/StackTrace.cpp b/dbms/src/Common/StackTrace.cpp index 0fe59f52014..e8ba21a187a 100644 --- a/dbms/src/Common/StackTrace.cpp +++ b/dbms/src/Common/StackTrace.cpp @@ -32,11 +32,10 @@ StackTrace::StackTrace() std::string StackTrace::toStringImpl(const Frames & frames, size_t frames_size) { char ** symbols = backtrace_symbols(frames.data(), frames_size); - std::stringstream res; - if (!symbols) return "Cannot get symbols for stack trace.\n"; + std::stringstream res; try { for (size_t i = 0, size = frames_size; i < size; ++i) diff --git a/dbms/src/Common/StringSearcher.h b/dbms/src/Common/StringSearcher.h index e28667a26dc..da34ccd820a 100644 --- a/dbms/src/Common/StringSearcher.h +++ b/dbms/src/Common/StringSearcher.h @@ -7,11 +7,11 @@ #include #include -#if __SSE2__ +#ifdef __SSE2__ #include #endif -#if __SSE4_1__ +#ifdef __SSE4_1__ #include #endif @@ -32,7 +32,7 @@ namespace ErrorCodes struct StringSearcherBase { -#if __SSE2__ +#ifdef __SSE2__ static constexpr auto n = sizeof(__m128i); const int page_size = getpagesize(); @@ -63,7 +63,7 @@ private: UInt8 l{}; UInt8 u{}; -#if __SSE4_1__ +#ifdef __SSE4_1__ /// vectors filled with `l` and `u`, for determining leftmost position of the first symbol __m128i patl, patu; /// lower and uppercase vectors of first 16 characters of `needle` @@ -102,7 +102,7 @@ public: u = u_seq[0]; } -#if __SSE4_1__ +#ifdef __SSE4_1__ /// for detecting leftmost position of the first symbol patl = _mm_set1_epi8(l); patu = _mm_set1_epi8(u); @@ -160,7 +160,7 @@ public: { static const Poco::UTF8Encoding utf8; -#if __SSE4_1__ +#ifdef __SSE4_1__ if (pageSafe(pos)) { const auto v_haystack = _mm_loadu_si128(reinterpret_cast(pos)); @@ -227,7 +227,7 @@ public: while (haystack < haystack_end) { -#if __SSE4_1__ +#ifdef __SSE4_1__ if (haystack + n <= haystack_end && pageSafe(haystack)) { const auto v_haystack = _mm_loadu_si128(reinterpret_cast(haystack)); @@ -249,15 +249,15 @@ public: if (haystack < haystack_end && haystack + n <= haystack_end && pageSafe(haystack)) { - const auto v_haystack = _mm_loadu_si128(reinterpret_cast(haystack)); - const auto v_against_l = _mm_cmpeq_epi8(v_haystack, cachel); - const auto v_against_u = _mm_cmpeq_epi8(v_haystack, cacheu); - const auto v_against_l_or_u = _mm_or_si128(v_against_l, v_against_u); - const auto mask = _mm_movemask_epi8(v_against_l_or_u); + const auto v_haystack_offset = _mm_loadu_si128(reinterpret_cast(haystack)); + const auto v_against_l_offset = _mm_cmpeq_epi8(v_haystack_offset, cachel); + const auto v_against_u_offset = _mm_cmpeq_epi8(v_haystack_offset, cacheu); + const auto v_against_l_or_u_offset = _mm_or_si128(v_against_l_offset, v_against_u_offset); + const auto mask_offset = _mm_movemask_epi8(v_against_l_or_u_offset); if (0xffff == cachemask) { - if (mask == cachemask) + if (mask_offset == cachemask) { auto haystack_pos = haystack + cache_valid_len; auto needle_pos = needle + cache_valid_len; @@ -276,7 +276,7 @@ public: return haystack; } } - else if ((mask & cachemask) == cachemask) + else if ((mask_offset & cachemask) == cachemask) return haystack; /// first octet was ok, but not the first 16, move to start of next sequence and reapply @@ -334,7 +334,7 @@ private: UInt8 l{}; UInt8 u{}; -#if __SSE4_1__ +#ifdef __SSE4_1__ /// vectors filled with `l` and `u`, for determining leftmost position of the first symbol __m128i patl, patu; /// lower and uppercase vectors of first 16 characters of `needle` @@ -352,7 +352,7 @@ public: l = static_cast(std::tolower(*needle)); u = static_cast(std::toupper(*needle)); -#if __SSE4_1__ +#ifdef __SSE4_1__ patl = _mm_set1_epi8(l); patu = _mm_set1_epi8(u); @@ -376,7 +376,7 @@ public: bool compare(const UInt8 * pos) const { -#if __SSE4_1__ +#ifdef __SSE4_1__ if (pageSafe(pos)) { const auto v_haystack = _mm_loadu_si128(reinterpret_cast(pos)); @@ -434,7 +434,7 @@ public: while (haystack < haystack_end) { -#if __SSE4_1__ +#ifdef __SSE4_1__ if (haystack + n <= haystack_end && pageSafe(haystack)) { const auto v_haystack = _mm_loadu_si128(reinterpret_cast(haystack)); @@ -455,15 +455,15 @@ public: if (haystack < haystack_end && haystack + n <= haystack_end && pageSafe(haystack)) { - const auto v_haystack = _mm_loadu_si128(reinterpret_cast(haystack)); - const auto v_against_l = _mm_cmpeq_epi8(v_haystack, cachel); - const auto v_against_u = _mm_cmpeq_epi8(v_haystack, cacheu); - const auto v_against_l_or_u = _mm_or_si128(v_against_l, v_against_u); - const auto mask = _mm_movemask_epi8(v_against_l_or_u); + const auto v_haystack_offset = _mm_loadu_si128(reinterpret_cast(haystack)); + const auto v_against_l_offset = _mm_cmpeq_epi8(v_haystack_offset, cachel); + const auto v_against_u_offset = _mm_cmpeq_epi8(v_haystack_offset, cacheu); + const auto v_against_l_or_u_offset = _mm_or_si128(v_against_l_offset, v_against_u_offset); + const auto mask_offset = _mm_movemask_epi8(v_against_l_or_u_offset); if (0xffff == cachemask) { - if (mask == cachemask) + if (mask_offset == cachemask) { auto haystack_pos = haystack + n; auto needle_pos = needle + n; @@ -479,7 +479,7 @@ public: return haystack; } } - else if ((mask & cachemask) == cachemask) + else if ((mask_offset & cachemask) == cachemask) return haystack; ++haystack; @@ -532,7 +532,7 @@ private: /// first character in `needle` UInt8 first{}; -#if __SSE4_1__ +#ifdef __SSE4_1__ /// vector filled `first` for determining leftmost position of the first symbol __m128i pattern; /// vector of first 16 characters of `needle` @@ -549,7 +549,7 @@ public: first = *needle; -#if __SSE4_1__ +#ifdef __SSE4_1__ pattern = _mm_set1_epi8(first); auto needle_pos = needle; @@ -570,7 +570,7 @@ public: bool compare(const UInt8 * pos) const { -#if __SSE4_1__ +#ifdef __SSE4_1__ if (pageSafe(pos)) { const auto v_haystack = _mm_loadu_si128(reinterpret_cast(pos)); @@ -620,7 +620,7 @@ public: while (haystack < haystack_end) { -#if __SSE4_1__ +#ifdef __SSE4_1__ if (haystack + n <= haystack_end && pageSafe(haystack)) { /// find first character @@ -642,13 +642,13 @@ public: if (haystack < haystack_end && haystack + n <= haystack_end && pageSafe(haystack)) { /// check for first 16 octets - const auto v_haystack = _mm_loadu_si128(reinterpret_cast(haystack)); - const auto v_against_cache = _mm_cmpeq_epi8(v_haystack, cache); - const auto mask = _mm_movemask_epi8(v_against_cache); + const auto v_haystack_offset = _mm_loadu_si128(reinterpret_cast(haystack)); + const auto v_against_cache = _mm_cmpeq_epi8(v_haystack_offset, cache); + const auto mask_offset = _mm_movemask_epi8(v_against_cache); if (0xffff == cachemask) { - if (mask == cachemask) + if (mask_offset == cachemask) { auto haystack_pos = haystack + n; auto needle_pos = needle + n; @@ -661,7 +661,7 @@ public: return haystack; } } - else if ((mask & cachemask) == cachemask) + else if ((mask_offset & cachemask) == cachemask) return haystack; ++haystack; diff --git a/dbms/src/Common/StringUtils/StringUtils.h b/dbms/src/Common/StringUtils/StringUtils.h index db3d5102f76..b8aa023eb3a 100644 --- a/dbms/src/Common/StringUtils/StringUtils.h +++ b/dbms/src/Common/StringUtils/StringUtils.h @@ -56,7 +56,7 @@ std::string getOrdinalSuffix(T n) case 2: return "nd"; case 3: return "rd"; default: return "th"; - }; + } } /// More efficient than libc, because doesn't respect locale. But for some functions table implementation could be better. diff --git a/dbms/src/Common/TaskStatsInfoGetter.cpp b/dbms/src/Common/TaskStatsInfoGetter.cpp index 867d204c6fb..43ee947d338 100644 --- a/dbms/src/Common/TaskStatsInfoGetter.cpp +++ b/dbms/src/Common/TaskStatsInfoGetter.cpp @@ -80,11 +80,6 @@ struct NetlinkMessage ::nlmsgerr error; }; - size_t payload_size() const - { - return header.nlmsg_len - sizeof(header) - sizeof(generic_header); - } - const Attribute * end() const { return reinterpret_cast(reinterpret_cast(this) + header.nlmsg_len); diff --git a/dbms/src/Common/ThreadStatus.cpp b/dbms/src/Common/ThreadStatus.cpp index 7a321cdaeb7..0ee09d527ce 100644 --- a/dbms/src/Common/ThreadStatus.cpp +++ b/dbms/src/Common/ThreadStatus.cpp @@ -21,9 +21,6 @@ namespace ErrorCodes } -extern SimpleObjectPool task_stats_info_getter_pool; - - TasksStatsCounters TasksStatsCounters::current() { TasksStatsCounters res; @@ -74,7 +71,7 @@ void ThreadStatus::initPerformanceCounters() if (TaskStatsInfoGetter::checkPermissions()) { if (!taskstats_getter) - taskstats_getter = task_stats_info_getter_pool.getDefault(); + taskstats_getter = std::make_unique(); *last_taskstats = TasksStatsCounters::current(); } diff --git a/dbms/src/Common/ThreadStatus.h b/dbms/src/Common/ThreadStatus.h index 822e1931447..3f7a91a54f0 100644 --- a/dbms/src/Common/ThreadStatus.h +++ b/dbms/src/Common/ThreadStatus.h @@ -2,7 +2,6 @@ #include #include -#include #include @@ -175,8 +174,7 @@ protected: std::unique_ptr last_taskstats; /// Set to non-nullptr only if we have enough capabilities. - /// We use pool because creation and destruction of TaskStatsInfoGetter objects are expensive. - SimpleObjectPool::Pointer taskstats_getter; + std::unique_ptr taskstats_getter; }; } diff --git a/dbms/src/Common/UInt128.h b/dbms/src/Common/UInt128.h index dd3883c253f..72fb1a2503a 100644 --- a/dbms/src/Common/UInt128.h +++ b/dbms/src/Common/UInt128.h @@ -6,7 +6,7 @@ #include -#if __SSE4_2__ +#ifdef __SSE4_2__ #include #endif @@ -75,7 +75,7 @@ struct UInt128Hash } }; -#if __SSE4_2__ +#ifdef __SSE4_2__ struct UInt128HashCRC32 { @@ -153,7 +153,7 @@ struct UInt256Hash } }; -#if __SSE4_2__ +#ifdef __SSE4_2__ struct UInt256HashCRC32 { diff --git a/dbms/src/Common/UTF8Helpers.h b/dbms/src/Common/UTF8Helpers.h index e02ce23b343..a25c5787fe6 100644 --- a/dbms/src/Common/UTF8Helpers.h +++ b/dbms/src/Common/UTF8Helpers.h @@ -3,7 +3,7 @@ #include #include -#if __SSE2__ +#ifdef __SSE2__ #include #endif @@ -55,7 +55,7 @@ inline size_t countCodePoints(const UInt8 * data, size_t size) size_t res = 0; const auto end = data + size; -#if __SSE2__ +#ifdef __SSE2__ constexpr auto bytes_sse = sizeof(__m128i); const auto src_end_sse = data + size / bytes_sse * bytes_sse; diff --git a/dbms/src/Common/Volnitsky.h b/dbms/src/Common/Volnitsky.h index f1a8325d869..f4cd57d6edf 100644 --- a/dbms/src/Common/Volnitsky.h +++ b/dbms/src/Common/Volnitsky.h @@ -194,9 +194,9 @@ template struct VolnitskyImpl; /// Case sensitive comparison template struct VolnitskyImpl : VolnitskyBase> { - VolnitskyImpl(const char * const needle, const size_t needle_size, const size_t haystack_size_hint = 0) - : VolnitskyBase>{needle, needle_size, haystack_size_hint}, - fallback_searcher{needle, needle_size} + VolnitskyImpl(const char * const needle_, const size_t needle_size_, const size_t haystack_size_hint = 0) + : VolnitskyBase>{needle_, needle_size_, haystack_size_hint}, + fallback_searcher{needle_, needle_size_} { } @@ -222,8 +222,8 @@ template struct VolnitskyImpl : VolnitskyBase struct VolnitskyImpl : VolnitskyBase> { - VolnitskyImpl(const char * const needle, const size_t needle_size, const size_t haystack_size_hint = 0) - : VolnitskyBase{needle, needle_size, haystack_size_hint}, fallback_searcher{needle, needle_size} + VolnitskyImpl(const char * const needle_, const size_t needle_size_, const size_t haystack_size_hint = 0) + : VolnitskyBase{needle_, needle_size_, haystack_size_hint}, fallback_searcher{needle_, needle_size_} { } @@ -248,8 +248,8 @@ template <> struct VolnitskyImpl : VolnitskyBase struct VolnitskyImpl : VolnitskyBase> { - VolnitskyImpl(const char * const needle, const size_t needle_size, const size_t haystack_size_hint = 0) - : VolnitskyBase{needle, needle_size, haystack_size_hint}, fallback_searcher{needle, needle_size} + VolnitskyImpl(const char * const needle_, const size_t needle_size_, const size_t haystack_size_hint = 0) + : VolnitskyBase{needle_, needle_size_, haystack_size_hint}, fallback_searcher{needle_, needle_size_} { } diff --git a/dbms/src/Common/ZooKeeper/IKeeper.h b/dbms/src/Common/ZooKeeper/IKeeper.h index 2df5db2ad60..313a11ab770 100644 --- a/dbms/src/Common/ZooKeeper/IKeeper.h +++ b/dbms/src/Common/ZooKeeper/IKeeper.h @@ -59,7 +59,9 @@ using Requests = std::vector; struct Request { - virtual ~Request() {} + Request() = default; + Request(const Request &) = default; + virtual ~Request() = default; virtual String getPath() const = 0; virtual void addRootPath(const String & /* root_path */) {} }; @@ -72,7 +74,9 @@ using ResponseCallback = std::function; struct Response { int32_t error = 0; - virtual ~Response() {} + Response() = default; + Response(const Response &) = default; + virtual ~Response() = default; virtual void removeRootPath(const String & /* root_path */) {} }; diff --git a/dbms/src/Common/ZooKeeper/ZooKeeper.cpp b/dbms/src/Common/ZooKeeper/ZooKeeper.cpp index 6f6deaddfd1..522667b8055 100644 --- a/dbms/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/dbms/src/Common/ZooKeeper/ZooKeeper.cpp @@ -17,7 +17,6 @@ #define ZOOKEEPER_CONNECTION_TIMEOUT_MS 1000 -#define ZOOKEEPER_OPERATION_TIMEOUT_MS 10000 namespace DB @@ -840,7 +839,7 @@ int32_t ZooKeeper::tryMultiNoThrow(const Coordination::Requests & requests, Coor } -size_t KeeperMultiException::getFailedOpIndex(int32_t code, const Coordination::Responses & responses) +size_t KeeperMultiException::getFailedOpIndex(int32_t exception_code, const Coordination::Responses & responses) { if (responses.empty()) throw DB::Exception("Responses for multi transaction is empty", DB::ErrorCodes::LOGICAL_ERROR); @@ -849,17 +848,17 @@ size_t KeeperMultiException::getFailedOpIndex(int32_t code, const Coordination:: if (responses[index]->error) return index; - if (!Coordination::isUserError(code)) - throw DB::Exception("There are no failed OPs because '" + ZooKeeper::error2string(code) + "' is not valid response code for that", + if (!Coordination::isUserError(exception_code)) + throw DB::Exception("There are no failed OPs because '" + ZooKeeper::error2string(exception_code) + "' is not valid response code for that", DB::ErrorCodes::LOGICAL_ERROR); throw DB::Exception("There is no failed OpResult", DB::ErrorCodes::LOGICAL_ERROR); } -KeeperMultiException::KeeperMultiException(int32_t code, const Coordination::Requests & requests, const Coordination::Responses & responses) - : KeeperException("Transaction failed", code), - requests(requests), responses(responses), failed_op_index(getFailedOpIndex(code, responses)) +KeeperMultiException::KeeperMultiException(int32_t exception_code, const Coordination::Requests & requests, const Coordination::Responses & responses) + : KeeperException("Transaction failed", exception_code), + requests(requests), responses(responses), failed_op_index(getFailedOpIndex(exception_code, responses)) { addMessage("Op #" + std::to_string(failed_op_index) + ", path: " + getPathForFirstFailedOp()); } @@ -870,15 +869,15 @@ std::string KeeperMultiException::getPathForFirstFailedOp() const return requests[failed_op_index]->getPath(); } -void KeeperMultiException::check(int32_t code, const Coordination::Requests & requests, const Coordination::Responses & responses) +void KeeperMultiException::check(int32_t exception_code, const Coordination::Requests & requests, const Coordination::Responses & responses) { - if (!code) + if (!exception_code) return; - if (Coordination::isUserError(code)) - throw KeeperMultiException(code, requests, responses); + if (Coordination::isUserError(exception_code)) + throw KeeperMultiException(exception_code, requests, responses); else - throw KeeperException(code); + throw KeeperException(exception_code); } diff --git a/dbms/src/Common/ZooKeeper/ZooKeeperHolder.cpp b/dbms/src/Common/ZooKeeper/ZooKeeperHolder.cpp index 9343bb4addc..41a36a51082 100644 --- a/dbms/src/Common/ZooKeeper/ZooKeeperHolder.cpp +++ b/dbms/src/Common/ZooKeeper/ZooKeeperHolder.cpp @@ -14,7 +14,7 @@ using namespace zkutil; ZooKeeperHolder::UnstorableZookeeperHandler ZooKeeperHolder::getZooKeeper() { - std::unique_lock lock(mutex); + std::unique_lock lock(mutex); return UnstorableZookeeperHandler(ptr); } @@ -25,7 +25,7 @@ void ZooKeeperHolder::initFromInstance(const ZooKeeper::Ptr & zookeeper_ptr) bool ZooKeeperHolder::replaceZooKeeperSessionToNewOne() { - std::unique_lock lock(mutex); + std::unique_lock lock(mutex); if (ptr.unique()) { diff --git a/dbms/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/dbms/src/Common/ZooKeeper/ZooKeeperImpl.cpp index e4ad2d27ee3..9626a54aa20 100644 --- a/dbms/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/dbms/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -1039,8 +1039,8 @@ void ZooKeeper::sendThread() { /// Wait for the next request in queue. No more than operation timeout. No more than until next heartbeat time. UInt64 max_wait = std::min( - std::chrono::duration_cast(next_heartbeat_time - now).count(), - operation_timeout.totalMilliseconds()); + UInt64(std::chrono::duration_cast(next_heartbeat_time - now).count()), + UInt64(operation_timeout.totalMilliseconds())); RequestInfo info; if (requests_queue.tryPop(info, max_wait)) @@ -1181,9 +1181,9 @@ void ZooKeeper::receiveEvent() ProfileEvents::increment(ProfileEvents::ZooKeeperWatchResponse); response = std::make_shared(); - request_info.callback = [this](const Response & response) + request_info.callback = [this](const Response & response_) { - const WatchResponse & watch_response = dynamic_cast(response); + const WatchResponse & watch_response = dynamic_cast(response_); std::lock_guard lock(watches_mutex); diff --git a/dbms/src/Common/ZooKeeper/ZooKeeperImpl.h b/dbms/src/Common/ZooKeeper/ZooKeeperImpl.h index 4df36742d44..c93f13b9351 100644 --- a/dbms/src/Common/ZooKeeper/ZooKeeperImpl.h +++ b/dbms/src/Common/ZooKeeper/ZooKeeperImpl.h @@ -252,7 +252,9 @@ struct ZooKeeperRequest : virtual Request /// If the request was sent and we didn't get the response and the error happens, then we cannot be sure was it processed or not. bool probably_sent = false; - virtual ~ZooKeeperRequest() {} + ZooKeeperRequest() = default; + ZooKeeperRequest(const ZooKeeperRequest &) = default; + virtual ~ZooKeeperRequest() = default; virtual ZooKeeper::OpNum getOpNum() const = 0; diff --git a/dbms/src/Common/ZooKeeper/ZooKeeperNodeCache.cpp b/dbms/src/Common/ZooKeeper/ZooKeeperNodeCache.cpp index 71d45ca3cb7..2fdd27e83ba 100644 --- a/dbms/src/Common/ZooKeeper/ZooKeeperNodeCache.cpp +++ b/dbms/src/Common/ZooKeeper/ZooKeeperNodeCache.cpp @@ -22,7 +22,7 @@ ZooKeeperNodeCache::ZNode ZooKeeperNodeCache::get(const std::string & path, Coor { std::unordered_set invalidated_paths; { - std::lock_guard lock(context->mutex); + std::lock_guard lock(context->mutex); if (context->all_paths_invalidated) { @@ -57,7 +57,7 @@ ZooKeeperNodeCache::ZNode ZooKeeperNodeCache::get(const std::string & path, Coor bool changed = false; { - std::lock_guard lock(owned_context->mutex); + std::lock_guard lock(owned_context->mutex); if (response.type != Coordination::SESSION) changed = owned_context->invalidated_paths.emplace(response.path).second; diff --git a/dbms/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp b/dbms/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp index 90bc779a7d5..489ac04c5e4 100644 --- a/dbms/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp +++ b/dbms/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp @@ -8,6 +8,7 @@ #pragma GCC diagnostic ignored "-Wsign-compare" #ifdef __clang__ #pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" + #pragma clang diagnostic ignored "-Wundef" #endif #include diff --git a/dbms/src/Common/ZooKeeper/tests/zk_many_watches_reconnect.cpp b/dbms/src/Common/ZooKeeper/tests/zk_many_watches_reconnect.cpp index c707d91d302..cae37319c74 100644 --- a/dbms/src/Common/ZooKeeper/tests/zk_many_watches_reconnect.cpp +++ b/dbms/src/Common/ZooKeeper/tests/zk_many_watches_reconnect.cpp @@ -63,6 +63,4 @@ int main(int argc, char ** argv) std::cerr << "Some exception" << std::endl; return 2; } - - return 0; } diff --git a/dbms/src/Common/ZooKeeper/tests/zkutil_expiration_test.cpp b/dbms/src/Common/ZooKeeper/tests/zkutil_expiration_test.cpp index 5b47ef09a13..1c74985d5f9 100644 --- a/dbms/src/Common/ZooKeeper/tests/zkutil_expiration_test.cpp +++ b/dbms/src/Common/ZooKeeper/tests/zkutil_expiration_test.cpp @@ -67,6 +67,4 @@ int main(int argc, char ** argv) std::cerr << "Some exception: " << DB::getCurrentExceptionMessage(true) << std::endl; return 2; } - - return 0; } diff --git a/dbms/src/Common/config.h.in b/dbms/src/Common/config.h.in index 624c87b91b5..09c2eadde29 100644 --- a/dbms/src/Common/config.h.in +++ b/dbms/src/Common/config.h.in @@ -16,6 +16,7 @@ #cmakedefine01 USE_BASE64 #cmakedefine01 USE_HDFS #cmakedefine01 USE_XXHASH +#cmakedefine01 USE_INTERNAL_LLVM_LIBRARY #cmakedefine01 CLICKHOUSE_SPLIT_BINARY #cmakedefine01 LLVM_HAS_RTTI diff --git a/dbms/src/Common/memcpySmall.h b/dbms/src/Common/memcpySmall.h index 1ed21fa82a5..34050f3c57f 100644 --- a/dbms/src/Common/memcpySmall.h +++ b/dbms/src/Common/memcpySmall.h @@ -3,7 +3,7 @@ #include #include -#if __SSE2__ +#ifdef __SSE2__ #include diff --git a/dbms/src/Common/tests/CMakeLists.txt b/dbms/src/Common/tests/CMakeLists.txt index 05984c9d42e..ec9636ce664 100644 --- a/dbms/src/Common/tests/CMakeLists.txt +++ b/dbms/src/Common/tests/CMakeLists.txt @@ -1,5 +1,5 @@ add_executable (hashes_test hashes_test.cpp) -target_link_libraries (hashes_test PRIVATE dbms ${OPENSSL_CRYPTO_LIBRARY}) +target_link_libraries (hashes_test PRIVATE clickhouse_common_io ${OPENSSL_CRYPTO_LIBRARY} ${CITYHASH_LIBRARIES}) add_executable (sip_hash sip_hash.cpp) target_link_libraries (sip_hash PRIVATE clickhouse_common_io) @@ -20,10 +20,10 @@ add_executable (small_table small_table.cpp) target_link_libraries (small_table PRIVATE clickhouse_common_io) add_executable (parallel_aggregation parallel_aggregation.cpp) -target_link_libraries (parallel_aggregation PRIVATE clickhouse_common_io) +target_link_libraries (parallel_aggregation PRIVATE clickhouse_compression clickhouse_common_io) add_executable (parallel_aggregation2 parallel_aggregation2.cpp) -target_link_libraries (parallel_aggregation2 PRIVATE clickhouse_common_io) +target_link_libraries (parallel_aggregation2 PRIVATE clickhouse_compression clickhouse_common_io) add_executable (int_hashes_perf int_hashes_perf.cpp AvalancheTest.cpp Random.cpp) target_link_libraries (int_hashes_perf PRIVATE clickhouse_common_io) @@ -42,7 +42,7 @@ add_executable (shell_command_test shell_command_test.cpp) target_link_libraries (shell_command_test PRIVATE clickhouse_common_io) add_executable (arena_with_free_lists arena_with_free_lists.cpp) -target_link_libraries (arena_with_free_lists PRIVATE clickhouse_common_io) +target_link_libraries (arena_with_free_lists PRIVATE clickhouse_compression clickhouse_common_io) add_executable (pod_array pod_array.cpp) target_link_libraries (pod_array PRIVATE clickhouse_common_io) @@ -61,7 +61,7 @@ target_link_libraries (space_saving PRIVATE clickhouse_common_io) add_executable (integer_hash_tables_and_hashes integer_hash_tables_and_hashes.cpp) target_include_directories (integer_hash_tables_and_hashes SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) -target_link_libraries (integer_hash_tables_and_hashes PRIVATE clickhouse_common_io) +target_link_libraries (integer_hash_tables_and_hashes PRIVATE clickhouse_compression clickhouse_common_io) add_executable (allocator allocator.cpp) target_link_libraries (allocator PRIVATE clickhouse_common_io) diff --git a/dbms/src/Common/tests/arena_with_free_lists.cpp b/dbms/src/Common/tests/arena_with_free_lists.cpp index 5091551b550..4d4915f5dcc 100644 --- a/dbms/src/Common/tests/arena_with_free_lists.cpp +++ b/dbms/src/Common/tests/arena_with_free_lists.cpp @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include using namespace DB; diff --git a/dbms/src/Common/tests/array_cache.cpp b/dbms/src/Common/tests/array_cache.cpp index 9f7e60c88b3..f6bbbba4cbc 100644 --- a/dbms/src/Common/tests/array_cache.cpp +++ b/dbms/src/Common/tests/array_cache.cpp @@ -60,7 +60,7 @@ int main(int argc, char ** argv) { pcg64 generator(randomSeed()); - for (size_t i = 0; i < num_iterations; ++i) + for (size_t j = 0; j < num_iterations; ++j) { size_t size = std::uniform_int_distribution(1, region_max_size)(generator); int key = std::uniform_int_distribution(1, max_key)(generator); @@ -70,8 +70,8 @@ int main(int argc, char ** argv) [=]{ return size; }, [=](void * /*ptr*/, int & payload) { - payload = i; - // memset(ptr, i, size); + payload = j; + // memset(ptr, j, size); }, nullptr); diff --git a/dbms/src/Common/tests/cow_columns.cpp b/dbms/src/Common/tests/cow_columns.cpp index e7a95c6d799..5d123f3c8af 100644 --- a/dbms/src/Common/tests/cow_columns.cpp +++ b/dbms/src/Common/tests/cow_columns.cpp @@ -9,7 +9,9 @@ private: virtual MutablePtr clone() const = 0; public: - virtual ~IColumn() {} + IColumn() = default; + IColumn(const IColumn &) = default; + virtual ~IColumn() = default; virtual int get() const = 0; virtual void set(int value) = 0; diff --git a/dbms/src/Common/tests/gtest_rw_lock.cpp b/dbms/src/Common/tests/gtest_rw_lock.cpp index 5a10c4a0a48..6fd16be64cd 100644 --- a/dbms/src/Common/tests/gtest_rw_lock.cpp +++ b/dbms/src/Common/tests/gtest_rw_lock.cpp @@ -1,6 +1,7 @@ #pragma GCC diagnostic ignored "-Wsign-compare" #ifdef __clang__ #pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" + #pragma clang diagnostic ignored "-Wundef" #endif #include diff --git a/dbms/src/Common/tests/gtest_unescapeForFileName.cpp b/dbms/src/Common/tests/gtest_unescapeForFileName.cpp index fc9af602ce8..b7e045b4318 100644 --- a/dbms/src/Common/tests/gtest_unescapeForFileName.cpp +++ b/dbms/src/Common/tests/gtest_unescapeForFileName.cpp @@ -3,6 +3,7 @@ #pragma GCC diagnostic ignored "-Wsign-compare" #ifdef __clang__ #pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" + #pragma clang diagnostic ignored "-Wundef" #endif #include diff --git a/dbms/src/Common/tests/integer_hash_tables_and_hashes.cpp b/dbms/src/Common/tests/integer_hash_tables_and_hashes.cpp index ca8c6140d93..d6ae055cc5e 100644 --- a/dbms/src/Common/tests/integer_hash_tables_and_hashes.cpp +++ b/dbms/src/Common/tests/integer_hash_tables_and_hashes.cpp @@ -14,7 +14,7 @@ #include #include -#include +#include #include #include @@ -171,7 +171,7 @@ namespace Hashes } }; - #if __SSE4_2__ + #ifdef __SSE4_2__ #include #endif @@ -179,7 +179,7 @@ namespace Hashes { size_t operator()(Key x) const { - #if __SSE4_2__ + #ifdef __SSE4_2__ return _mm_crc32_u64(-1ULL, x); #else /// On other platforms we do not have CRC32. NOTE This can be confusing. diff --git a/dbms/src/Common/tests/parallel_aggregation.cpp b/dbms/src/Common/tests/parallel_aggregation.cpp index f6ee3b01659..15a193de3d7 100644 --- a/dbms/src/Common/tests/parallel_aggregation.cpp +++ b/dbms/src/Common/tests/parallel_aggregation.cpp @@ -13,7 +13,7 @@ //#include #include -#include +#include #include #include diff --git a/dbms/src/Common/tests/parallel_aggregation2.cpp b/dbms/src/Common/tests/parallel_aggregation2.cpp index dac95746c02..dc43442de08 100644 --- a/dbms/src/Common/tests/parallel_aggregation2.cpp +++ b/dbms/src/Common/tests/parallel_aggregation2.cpp @@ -13,7 +13,7 @@ //#include #include -#include +#include #include #include @@ -186,8 +186,8 @@ struct MergeParallelForTwoLevelTable for (size_t i = 0; i < num_maps; ++i) section[i] = &source_maps[i]->impls[bucket]; - typename Map::Impl * result_map; - ImplMerge::execute(section.data(), num_maps, result_map, merger, pool); + typename Map::Impl * res; + ImplMerge::execute(section.data(), num_maps, res, merger, pool); }); pool.wait(); diff --git a/dbms/src/Compression/CMakeLists.txt b/dbms/src/Compression/CMakeLists.txt index e69de29bb2d..288d452bebf 100644 --- a/dbms/src/Compression/CMakeLists.txt +++ b/dbms/src/Compression/CMakeLists.txt @@ -0,0 +1,16 @@ +include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake) +add_headers_and_sources(clickhouse_compression .) +add_library(clickhouse_compression ${LINK_MODE} ${clickhouse_compression_headers} ${clickhouse_compression_sources}) +target_link_libraries(clickhouse_compression PRIVATE clickhouse_parsers clickhouse_common_io ${ZSTD_LIBRARY} ${LZ4_LIBRARY}) +target_include_directories(clickhouse_compression PUBLIC ${DBMS_INCLUDE_DIR}) + +if (NOT USE_INTERNAL_LZ4_LIBRARY) + target_include_directories(clickhouse_compression SYSTEM BEFORE PRIVATE ${LZ4_INCLUDE_DIR}) +endif () +if (NOT USE_INTERNAL_ZSTD_LIBRARY) + target_include_directories(clickhouse_compression SYSTEM BEFORE PRIVATE ${ZSTD_INCLUDE_DIR}) +endif () + +if(ENABLE_TESTS) + add_subdirectory(tests) +endif() diff --git a/dbms/src/IO/CachedCompressedReadBuffer.cpp b/dbms/src/Compression/CachedCompressedReadBuffer.cpp similarity index 97% rename from dbms/src/IO/CachedCompressedReadBuffer.cpp rename to dbms/src/Compression/CachedCompressedReadBuffer.cpp index 50c97edf1a3..e87a9a45019 100644 --- a/dbms/src/IO/CachedCompressedReadBuffer.cpp +++ b/dbms/src/Compression/CachedCompressedReadBuffer.cpp @@ -1,9 +1,9 @@ +#include "CachedCompressedReadBuffer.h" + #include -#include #include #include -#include -#include "CachedCompressedReadBuffer.h" +#include namespace DB diff --git a/dbms/src/IO/CachedCompressedReadBuffer.h b/dbms/src/Compression/CachedCompressedReadBuffer.h similarity index 97% rename from dbms/src/IO/CachedCompressedReadBuffer.h rename to dbms/src/Compression/CachedCompressedReadBuffer.h index 1b5e41972f3..174ddb98587 100644 --- a/dbms/src/IO/CachedCompressedReadBuffer.h +++ b/dbms/src/Compression/CachedCompressedReadBuffer.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include "CompressedReadBufferBase.h" #include #include diff --git a/dbms/src/IO/CompressedReadBuffer.cpp b/dbms/src/Compression/CompressedReadBuffer.cpp similarity index 96% rename from dbms/src/IO/CompressedReadBuffer.cpp rename to dbms/src/Compression/CompressedReadBuffer.cpp index cc540161c92..bae56e2a283 100644 --- a/dbms/src/IO/CompressedReadBuffer.cpp +++ b/dbms/src/Compression/CompressedReadBuffer.cpp @@ -1,6 +1,6 @@ -#include +#include "CompressedReadBuffer.h" #include -#include +#include namespace DB diff --git a/dbms/src/IO/CompressedReadBuffer.h b/dbms/src/Compression/CompressedReadBuffer.h similarity index 93% rename from dbms/src/IO/CompressedReadBuffer.h rename to dbms/src/Compression/CompressedReadBuffer.h index 60ba29012b8..1e8ea4784c7 100644 --- a/dbms/src/IO/CompressedReadBuffer.h +++ b/dbms/src/Compression/CompressedReadBuffer.h @@ -1,6 +1,6 @@ #pragma once -#include +#include "CompressedReadBufferBase.h" #include #include diff --git a/dbms/src/IO/CompressedReadBufferBase.cpp b/dbms/src/Compression/CompressedReadBufferBase.cpp similarity index 99% rename from dbms/src/IO/CompressedReadBufferBase.cpp rename to dbms/src/Compression/CompressedReadBufferBase.cpp index 5ac795a82e1..1ce83134f03 100644 --- a/dbms/src/IO/CompressedReadBufferBase.cpp +++ b/dbms/src/Compression/CompressedReadBufferBase.cpp @@ -1,4 +1,4 @@ -#include +#include "CompressedReadBufferBase.h" #include diff --git a/dbms/src/IO/CompressedReadBufferBase.h b/dbms/src/Compression/CompressedReadBufferBase.h similarity index 97% rename from dbms/src/IO/CompressedReadBufferBase.h rename to dbms/src/Compression/CompressedReadBufferBase.h index 34ff798a8f1..f44140dcd04 100644 --- a/dbms/src/IO/CompressedReadBufferBase.h +++ b/dbms/src/Compression/CompressedReadBufferBase.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include diff --git a/dbms/src/IO/CompressedReadBufferFromFile.cpp b/dbms/src/Compression/CompressedReadBufferFromFile.cpp similarity index 97% rename from dbms/src/IO/CompressedReadBufferFromFile.cpp rename to dbms/src/Compression/CompressedReadBufferFromFile.cpp index 25008c205b5..759acf0b2a5 100644 --- a/dbms/src/IO/CompressedReadBufferFromFile.cpp +++ b/dbms/src/Compression/CompressedReadBufferFromFile.cpp @@ -1,8 +1,9 @@ -#include +#include "CompressedReadBufferFromFile.h" + #include #include #include -#include +#include namespace DB diff --git a/dbms/src/IO/CompressedReadBufferFromFile.h b/dbms/src/Compression/CompressedReadBufferFromFile.h similarity index 96% rename from dbms/src/IO/CompressedReadBufferFromFile.h rename to dbms/src/Compression/CompressedReadBufferFromFile.h index f1332ea4187..288a66e321a 100644 --- a/dbms/src/IO/CompressedReadBufferFromFile.h +++ b/dbms/src/Compression/CompressedReadBufferFromFile.h @@ -1,6 +1,6 @@ #pragma once -#include +#include "CompressedReadBufferBase.h" #include #include #include diff --git a/dbms/src/IO/CompressedWriteBuffer.cpp b/dbms/src/Compression/CompressedWriteBuffer.cpp similarity index 97% rename from dbms/src/IO/CompressedWriteBuffer.cpp rename to dbms/src/Compression/CompressedWriteBuffer.cpp index 24a2021555f..7fc8d5ab5f9 100644 --- a/dbms/src/IO/CompressedWriteBuffer.cpp +++ b/dbms/src/Compression/CompressedWriteBuffer.cpp @@ -8,7 +8,7 @@ #include #include -#include +#include "CompressedWriteBuffer.h" #include diff --git a/dbms/src/IO/CompressedWriteBuffer.h b/dbms/src/Compression/CompressedWriteBuffer.h similarity index 100% rename from dbms/src/IO/CompressedWriteBuffer.h rename to dbms/src/Compression/CompressedWriteBuffer.h diff --git a/dbms/src/Compression/CompressionCodecLZ4.cpp b/dbms/src/Compression/CompressionCodecLZ4.cpp index a2f9262ae06..58ede15ca50 100644 --- a/dbms/src/Compression/CompressionCodecLZ4.cpp +++ b/dbms/src/Compression/CompressionCodecLZ4.cpp @@ -1,13 +1,17 @@ -#include +#include "CompressionCodecLZ4.h" + #include #include #include #include -#include +#include #include "CompressionCodecLZ4.h" #include #include +#ifdef __clang__ + #pragma clang diagnostic ignored "-Wold-style-cast" +#endif namespace DB diff --git a/dbms/src/Compression/CompressionCodecLZ4.h b/dbms/src/Compression/CompressionCodecLZ4.h index fe3339b40a1..5ce137dba54 100644 --- a/dbms/src/Compression/CompressionCodecLZ4.h +++ b/dbms/src/Compression/CompressionCodecLZ4.h @@ -4,7 +4,7 @@ #include #include #include -#include +#include namespace DB { diff --git a/dbms/src/IO/CompressionCodecWriteBuffer.h b/dbms/src/Compression/CompressionCodecWriteBuffer.h similarity index 100% rename from dbms/src/IO/CompressionCodecWriteBuffer.h rename to dbms/src/Compression/CompressionCodecWriteBuffer.h diff --git a/dbms/src/Compression/CompressionCodecZSTD.cpp b/dbms/src/Compression/CompressionCodecZSTD.cpp index d3f96cc7e06..8c79b4439ac 100644 --- a/dbms/src/Compression/CompressionCodecZSTD.cpp +++ b/dbms/src/Compression/CompressionCodecZSTD.cpp @@ -63,7 +63,7 @@ CompressionCodecZSTD::CompressionCodecZSTD(int level_) void registerCodecZSTD(CompressionCodecFactory & factory) { - UInt8 method_code = static_cast(CompressionMethodByte::ZSTD); + UInt8 method_code = UInt8(CompressionMethodByte::ZSTD); factory.registerCompressionCodec("ZSTD", method_code, [&](const ASTPtr & arguments) -> CompressionCodecPtr { int level = CompressionCodecZSTD::ZSTD_DEFAULT_LEVEL; diff --git a/dbms/src/Compression/ICompressionCodec.cpp b/dbms/src/Compression/ICompressionCodec.cpp index 45ed8250cd7..f9707d7c9da 100644 --- a/dbms/src/Compression/ICompressionCodec.cpp +++ b/dbms/src/Compression/ICompressionCodec.cpp @@ -1,5 +1,6 @@ -#include -#include +#include "ICompressionCodec.h" + +#include #include #include #include diff --git a/dbms/src/Compression/ICompressionCodec.h b/dbms/src/Compression/ICompressionCodec.h index eaf877b479f..2602e15c600 100644 --- a/dbms/src/Compression/ICompressionCodec.h +++ b/dbms/src/Compression/ICompressionCodec.h @@ -9,8 +9,7 @@ #include #include #include -#include - +#include #include namespace DB diff --git a/dbms/src/IO/LZ4_decompress_faster.cpp b/dbms/src/Compression/LZ4_decompress_faster.cpp similarity index 98% rename from dbms/src/IO/LZ4_decompress_faster.cpp rename to dbms/src/Compression/LZ4_decompress_faster.cpp index 243bdd997ce..11e222c757a 100644 --- a/dbms/src/IO/LZ4_decompress_faster.cpp +++ b/dbms/src/Compression/LZ4_decompress_faster.cpp @@ -1,25 +1,24 @@ +#include "LZ4_decompress_faster.h" + #include #include #include #include - -#include #include #include - #include #include #include -#if __SSE2__ +#ifdef __SSE2__ #include #endif -#if __SSSE3__ +#ifdef __SSSE3__ #include #endif -#if __aarch64__ +#ifdef __aarch64__ #include #endif @@ -214,7 +213,7 @@ template <> void inline copyOverlap<8, true>(UInt8 * op, const UInt8 *& match, c inline void copy16(UInt8 * dst, const UInt8 * src) { -#if __SSE2__ +#ifdef __SSE2__ _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), _mm_loadu_si128(reinterpret_cast(src))); #else @@ -352,8 +351,8 @@ void NO_INLINE decompressImpl( char * const dest, size_t dest_size) { - const UInt8 * ip = (UInt8 *)source; - UInt8 * op = (UInt8 *)dest; + const UInt8 * ip = reinterpret_cast(source); + UInt8 * op = reinterpret_cast(dest); UInt8 * const output_end = op + dest_size; while (1) @@ -529,8 +528,8 @@ void statistics( size_t dest_size, StreamStatistics & stat) { - const UInt8 * ip = (UInt8 *)source; - UInt8 * op = (UInt8 *)dest; + const UInt8 * ip = reinterpret_cast(source); + UInt8 * op = reinterpret_cast(dest); UInt8 * const output_end = op + dest_size; while (1) diff --git a/dbms/src/IO/LZ4_decompress_faster.h b/dbms/src/Compression/LZ4_decompress_faster.h similarity index 100% rename from dbms/src/IO/LZ4_decompress_faster.h rename to dbms/src/Compression/LZ4_decompress_faster.h diff --git a/dbms/src/Compression/tests/CMakeLists.txt b/dbms/src/Compression/tests/CMakeLists.txt new file mode 100644 index 00000000000..50f212f18c3 --- /dev/null +++ b/dbms/src/Compression/tests/CMakeLists.txt @@ -0,0 +1,5 @@ +add_executable (compressed_buffer compressed_buffer.cpp) +target_link_libraries (compressed_buffer PRIVATE clickhouse_compression clickhouse_common_io) + +add_executable (cached_compressed_read_buffer cached_compressed_read_buffer.cpp) +target_link_libraries (cached_compressed_read_buffer PRIVATE clickhouse_compression clickhouse_common_io) diff --git a/dbms/src/IO/tests/cached_compressed_read_buffer.cpp b/dbms/src/Compression/tests/cached_compressed_read_buffer.cpp similarity index 96% rename from dbms/src/IO/tests/cached_compressed_read_buffer.cpp rename to dbms/src/Compression/tests/cached_compressed_read_buffer.cpp index df30a3bc064..fb30d691745 100644 --- a/dbms/src/IO/tests/cached_compressed_read_buffer.cpp +++ b/dbms/src/Compression/tests/cached_compressed_read_buffer.cpp @@ -3,7 +3,7 @@ #include #include -#include +#include #include #include diff --git a/dbms/src/IO/tests/compressed_buffer.cpp b/dbms/src/Compression/tests/compressed_buffer.cpp similarity index 95% rename from dbms/src/IO/tests/compressed_buffer.cpp rename to dbms/src/Compression/tests/compressed_buffer.cpp index bcb8d7ae9ce..346a4068a6c 100644 --- a/dbms/src/IO/tests/compressed_buffer.cpp +++ b/dbms/src/Compression/tests/compressed_buffer.cpp @@ -9,8 +9,8 @@ #include #include #include -#include -#include +#include +#include #include #include diff --git a/dbms/src/Core/AccurateComparison.h b/dbms/src/Core/AccurateComparison.h index 1e3e236b1ea..62402c339f7 100644 --- a/dbms/src/Core/AccurateComparison.h +++ b/dbms/src/Core/AccurateComparison.h @@ -263,7 +263,8 @@ inline bool_if_not_safe_conversion equalsOp(A a, B b) template inline bool_if_safe_conversion equalsOp(A a, B b) { - return a == b; + using LargestType = std::conditional_t= sizeof(B), A, B>; + return static_cast(a) == static_cast(b); } template <> diff --git a/dbms/src/Core/BlockInfo.cpp b/dbms/src/Core/BlockInfo.cpp index aae9723d0ed..b76c54cc2f2 100644 --- a/dbms/src/Core/BlockInfo.cpp +++ b/dbms/src/Core/BlockInfo.cpp @@ -25,7 +25,7 @@ void BlockInfo::write(WriteBuffer & out) const writeVarUInt(FIELD_NUM, out); \ writeBinary(NAME, out); - APPLY_FOR_BLOCK_INFO_FIELDS(WRITE_FIELD); + APPLY_FOR_BLOCK_INFO_FIELDS(WRITE_FIELD) #undef WRITE_FIELD writeVarUInt(0, out); @@ -49,7 +49,7 @@ void BlockInfo::read(ReadBuffer & in) readBinary(NAME, in); \ break; - APPLY_FOR_BLOCK_INFO_FIELDS(READ_FIELD); + APPLY_FOR_BLOCK_INFO_FIELDS(READ_FIELD) #undef READ_FIELD default: diff --git a/dbms/src/Core/Field.cpp b/dbms/src/Core/Field.cpp index 9a7fc60f48c..c195652d051 100644 --- a/dbms/src/Core/Field.cpp +++ b/dbms/src/Core/Field.cpp @@ -75,7 +75,7 @@ namespace DB x.push_back(value); break; } - }; + } } } @@ -128,7 +128,7 @@ namespace DB DB::writeBinary(get(*it), buf); break; } - }; + } } } @@ -209,7 +209,7 @@ namespace DB x.push_back(value); break; } - }; + } } } @@ -262,7 +262,7 @@ namespace DB DB::writeBinary(get(*it), buf); break; } - }; + } } } diff --git a/dbms/src/Core/Field.h b/dbms/src/Core/Field.h index b08c75df88d..47242825f86 100644 --- a/dbms/src/Core/Field.h +++ b/dbms/src/Core/Field.h @@ -449,9 +449,6 @@ private: case Types::Decimal32: f(field.template get>()); return; case Types::Decimal64: f(field.template get>()); return; case Types::Decimal128: f(field.template get>()); return; - - default: - throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD); } } diff --git a/dbms/src/Core/tests/field.cpp b/dbms/src/Core/tests/field.cpp index c10ef1267de..6198288b507 100644 --- a/dbms/src/Core/tests/field.cpp +++ b/dbms/src/Core/tests/field.cpp @@ -30,12 +30,14 @@ int main(int argc, char ** argv) field2 = field; std::cerr << applyVisitor(to_string, field2) << std::endl; - Array array; - array.push_back(UInt64(123)); - array.push_back(Int64(-123)); - array.push_back(String("Hello")); - field = array; - std::cerr << applyVisitor(to_string, field) << std::endl; + { + Array array; + array.push_back(UInt64(123)); + array.push_back(Int64(-123)); + array.push_back(String("Hello")); + field = array; + std::cerr << applyVisitor(to_string, field) << std::endl; + } get(field).push_back(field); std::cerr << applyVisitor(to_string, field) << std::endl; @@ -54,7 +56,7 @@ int main(int argc, char ** argv) Array array(n); { - Stopwatch watch; + watch.restart(); for (size_t i = 0; i < n; ++i) array[i] = String(i % 32, '!'); @@ -67,7 +69,7 @@ int main(int argc, char ** argv) } { - Stopwatch watch; + watch.restart(); size_t sum = 0; for (size_t i = 0; i < n; ++i) diff --git a/dbms/src/DataStreams/AggregatingBlockInputStream.h b/dbms/src/DataStreams/AggregatingBlockInputStream.h index a28814dcab8..19483455742 100644 --- a/dbms/src/DataStreams/AggregatingBlockInputStream.h +++ b/dbms/src/DataStreams/AggregatingBlockInputStream.h @@ -2,7 +2,7 @@ #include #include -#include +#include #include diff --git a/dbms/src/DataStreams/BlockIO.cpp b/dbms/src/DataStreams/BlockIO.cpp index 76f6aeb0b37..5666383f345 100644 --- a/dbms/src/DataStreams/BlockIO.cpp +++ b/dbms/src/DataStreams/BlockIO.cpp @@ -5,5 +5,7 @@ namespace DB { BlockIO::~BlockIO() = default; +BlockIO::BlockIO() = default; +BlockIO::BlockIO(const BlockIO &) = default; } diff --git a/dbms/src/DataStreams/BlockIO.h b/dbms/src/DataStreams/BlockIO.h index 9f69f834e5f..46db3d27a04 100644 --- a/dbms/src/DataStreams/BlockIO.h +++ b/dbms/src/DataStreams/BlockIO.h @@ -60,6 +60,8 @@ struct BlockIO } ~BlockIO(); + BlockIO(); + BlockIO(const BlockIO &); }; } diff --git a/dbms/src/DataStreams/IProfilingBlockInputStream.cpp b/dbms/src/DataStreams/IProfilingBlockInputStream.cpp index 998ea2b42db..f7b9a02ab9e 100644 --- a/dbms/src/DataStreams/IProfilingBlockInputStream.cpp +++ b/dbms/src/DataStreams/IProfilingBlockInputStream.cpp @@ -230,9 +230,6 @@ void IProfilingBlockInputStream::checkQuota(Block & block) prev_elapsed = total_elapsed; break; } - - default: - throw Exception("Logical error: unknown limits mode.", ErrorCodes::LOGICAL_ERROR); } } diff --git a/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp b/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp index 527adc4ec64..12ad34b6433 100644 --- a/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp +++ b/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include @@ -78,7 +78,7 @@ Block MergeSortingBlockInputStream::readImpl() if (max_bytes_before_external_sort && sum_bytes_in_blocks > max_bytes_before_external_sort) { Poco::File(tmp_path).createDirectories(); - temporary_files.emplace_back(new Poco::TemporaryFile(tmp_path)); + temporary_files.emplace_back(std::make_unique(tmp_path)); const std::string & path = temporary_files.back()->path(); WriteBufferFromFile file_buf(path); CompressedWriteBuffer compressed_buf(file_buf); diff --git a/dbms/src/DataStreams/MergeSortingBlockInputStream.h b/dbms/src/DataStreams/MergeSortingBlockInputStream.h index 033a695ac27..7ff6ea6cfc2 100644 --- a/dbms/src/DataStreams/MergeSortingBlockInputStream.h +++ b/dbms/src/DataStreams/MergeSortingBlockInputStream.h @@ -12,7 +12,7 @@ #include #include -#include +#include namespace DB diff --git a/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp b/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp index d85992f8e5d..7773da79008 100644 --- a/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp +++ b/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp @@ -127,7 +127,7 @@ void MergingAggregatedMemoryEfficientBlockInputStream::cancel(bool kill) if (parallel_merge_data) { { - std::unique_lock lock(parallel_merge_data->merged_blocks_mutex); + std::unique_lock lock(parallel_merge_data->merged_blocks_mutex); parallel_merge_data->finish = true; } parallel_merge_data->merged_blocks_changed.notify_one(); /// readImpl method must stop waiting and exit. @@ -219,7 +219,7 @@ Block MergingAggregatedMemoryEfficientBlockInputStream::readImpl() while (true) { - std::unique_lock lock(parallel_merge_data->merged_blocks_mutex); + std::unique_lock lock(parallel_merge_data->merged_blocks_mutex); parallel_merge_data->merged_blocks_changed.wait(lock, [this] { @@ -323,7 +323,7 @@ void MergingAggregatedMemoryEfficientBlockInputStream::mergeThread(ThreadGroupSt * - or, if no next blocks, set 'exhausted' flag. */ { - std::lock_guard lock_next_blocks(parallel_merge_data->get_next_blocks_mutex); + std::lock_guard lock_next_blocks(parallel_merge_data->get_next_blocks_mutex); if (parallel_merge_data->exhausted || parallel_merge_data->finish) break; @@ -333,7 +333,7 @@ void MergingAggregatedMemoryEfficientBlockInputStream::mergeThread(ThreadGroupSt if (!blocks_to_merge || blocks_to_merge->empty()) { { - std::unique_lock lock_merged_blocks(parallel_merge_data->merged_blocks_mutex); + std::unique_lock lock_merged_blocks(parallel_merge_data->merged_blocks_mutex); parallel_merge_data->exhausted = true; } @@ -347,7 +347,7 @@ void MergingAggregatedMemoryEfficientBlockInputStream::mergeThread(ThreadGroupSt : blocks_to_merge->front().info.bucket_num; { - std::unique_lock lock_merged_blocks(parallel_merge_data->merged_blocks_mutex); + std::unique_lock lock_merged_blocks(parallel_merge_data->merged_blocks_mutex); parallel_merge_data->have_space.wait(lock_merged_blocks, [this] { @@ -370,7 +370,7 @@ void MergingAggregatedMemoryEfficientBlockInputStream::mergeThread(ThreadGroupSt Block res = aggregator.mergeBlocks(*blocks_to_merge, final); { - std::lock_guard lock(parallel_merge_data->merged_blocks_mutex); + std::lock_guard lock(parallel_merge_data->merged_blocks_mutex); if (parallel_merge_data->finish) break; @@ -385,7 +385,7 @@ void MergingAggregatedMemoryEfficientBlockInputStream::mergeThread(ThreadGroupSt catch (...) { { - std::lock_guard lock(parallel_merge_data->merged_blocks_mutex); + std::lock_guard lock(parallel_merge_data->merged_blocks_mutex); parallel_merge_data->exception = std::current_exception(); parallel_merge_data->finish = true; } diff --git a/dbms/src/DataStreams/NativeBlockInputStream.cpp b/dbms/src/DataStreams/NativeBlockInputStream.cpp index 7eeba3b9e50..2dbedd01b38 100644 --- a/dbms/src/DataStreams/NativeBlockInputStream.cpp +++ b/dbms/src/DataStreams/NativeBlockInputStream.cpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include diff --git a/dbms/src/DataStreams/NativeBlockOutputStream.cpp b/dbms/src/DataStreams/NativeBlockOutputStream.cpp index fb5aadb2fb3..11c3944afbb 100644 --- a/dbms/src/DataStreams/NativeBlockOutputStream.cpp +++ b/dbms/src/DataStreams/NativeBlockOutputStream.cpp @@ -3,7 +3,7 @@ #include #include -#include +#include #include #include diff --git a/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.h b/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.h index 0a74557d449..5719dc68e84 100644 --- a/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.h +++ b/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.h @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include diff --git a/dbms/src/DataStreams/ParallelInputsProcessor.h b/dbms/src/DataStreams/ParallelInputsProcessor.h index 3a6ed25f5c8..ba086b98939 100644 --- a/dbms/src/DataStreams/ParallelInputsProcessor.h +++ b/dbms/src/DataStreams/ParallelInputsProcessor.h @@ -191,7 +191,7 @@ private: { InputData unprepared_input; { - std::lock_guard lock(unprepared_inputs_mutex); + std::lock_guard lock(unprepared_inputs_mutex); if (unprepared_inputs.empty()) break; @@ -203,7 +203,7 @@ private: unprepared_input.in->readPrefix(); { - std::lock_guard lock(available_inputs_mutex); + std::lock_guard lock(available_inputs_mutex); available_inputs.push(unprepared_input); } } @@ -257,7 +257,7 @@ private: /// Select the next source. { - std::lock_guard lock(available_inputs_mutex); + std::lock_guard lock(available_inputs_mutex); /// If there are no free sources, then this thread is no longer needed. (But other threads can work with their sources.) if (available_inputs.empty()) @@ -278,7 +278,7 @@ private: /// If this source is not run out yet, then put the resulting block in the ready queue. { - std::lock_guard lock(available_inputs_mutex); + std::lock_guard lock(available_inputs_mutex); if (block) { diff --git a/dbms/src/DataStreams/RemoteBlockInputStream.cpp b/dbms/src/DataStreams/RemoteBlockInputStream.cpp index 84d5a5ad000..21c7645ec45 100644 --- a/dbms/src/DataStreams/RemoteBlockInputStream.cpp +++ b/dbms/src/DataStreams/RemoteBlockInputStream.cpp @@ -104,7 +104,7 @@ void RemoteBlockInputStream::cancel(bool kill) return; { - std::lock_guard lock(external_tables_mutex); + std::lock_guard lock(external_tables_mutex); /// Stop sending external data. for (auto & vec : external_tables_data) @@ -124,7 +124,7 @@ void RemoteBlockInputStream::sendExternalTables() size_t count = multiplexed_connections->size(); { - std::lock_guard lock(external_tables_mutex); + std::lock_guard lock(external_tables_mutex); external_tables_data.reserve(count); diff --git a/dbms/src/DataTypes/DataTypeArray.cpp b/dbms/src/DataTypes/DataTypeArray.cpp index 3dd4db815f2..0b985039ccf 100644 --- a/dbms/src/DataTypes/DataTypeArray.cpp +++ b/dbms/src/DataTypes/DataTypeArray.cpp @@ -94,7 +94,7 @@ void DataTypeArray::deserializeBinary(IColumn & column, ReadBuffer & istr) const throw; } - offsets.push_back((offsets.empty() ? 0 : offsets.back()) + size); + offsets.push_back(offsets.back() + size); } @@ -255,7 +255,7 @@ void DataTypeArray::deserializeBinaryBulkWithMultipleStreams( IColumn & nested_column = column_array.getData(); /// Number of values corresponding with `offset_values` must be read. - size_t last_offset = (offset_values.empty() ? 0 : offset_values.back()); + size_t last_offset = offset_values.back(); if (last_offset < nested_column.size()) throw Exception("Nested column is longer than last offset", ErrorCodes::LOGICAL_ERROR); size_t nested_limit = last_offset - nested_column.size(); @@ -341,7 +341,7 @@ static void deserializeTextImpl(IColumn & column, ReadBuffer & istr, Reader && r throw; } - offsets.push_back((offsets.empty() ? 0 : offsets.back()) + size); + offsets.push_back(offsets.back() + size); } diff --git a/dbms/src/DataTypes/DataTypeDateTime.cpp b/dbms/src/DataTypes/DataTypeDateTime.cpp index 519f419e8c4..6ea042454e5 100644 --- a/dbms/src/DataTypes/DataTypeDateTime.cpp +++ b/dbms/src/DataTypes/DataTypeDateTime.cpp @@ -56,8 +56,6 @@ static inline void readText(time_t & x, ReadBuffer & istr, const FormatSettings case FormatSettings::DateTimeInputFormat::BestEffort: parseDateTimeBestEffort(x, istr, time_zone, utc_time_zone); return; - default: - __builtin_unreachable(); } } diff --git a/dbms/src/DataTypes/DataTypeInterval.h b/dbms/src/DataTypes/DataTypeInterval.h index 6f4f08c16c0..5c9c15c48c8 100644 --- a/dbms/src/DataTypes/DataTypeInterval.h +++ b/dbms/src/DataTypes/DataTypeInterval.h @@ -49,8 +49,9 @@ public: case Month: return "Month"; case Quarter: return "Quarter"; case Year: return "Year"; - default: __builtin_unreachable(); } + + __builtin_unreachable(); } DataTypeInterval(Kind kind) : kind(kind) {} diff --git a/dbms/src/DataTypes/DataTypeLowCardinality.cpp b/dbms/src/DataTypes/DataTypeLowCardinality.cpp index cf38941b743..b823a9257ad 100644 --- a/dbms/src/DataTypes/DataTypeLowCardinality.cpp +++ b/dbms/src/DataTypes/DataTypeLowCardinality.cpp @@ -508,6 +508,10 @@ void DataTypeLowCardinality::serializeBinaryBulkWithMultipleStreams( size_t max_limit = column.size() - offset; limit = limit ? std::min(limit, max_limit) : max_limit; + /// Do not write anything for empty column. (May happen while writing empty arrays.) + if (limit == 0) + return; + auto sub_column = low_cardinality_column.cutAndCompact(offset, limit); ColumnPtr positions = sub_column->getIndexesPtr(); ColumnPtr keys = sub_column->getDictionary().getNestedColumn(); @@ -766,7 +770,7 @@ namespace void operator()() { if (typeid_cast *>(&keys_type)) - column = creator((ColumnVector *)(nullptr)); + column = creator(static_cast *>(nullptr)); } }; } @@ -780,13 +784,13 @@ MutableColumnUniquePtr DataTypeLowCardinality::createColumnUniqueImpl(const IDat type = nullable_type->getNestedType().get(); if (isString(type)) - return creator((ColumnString *)(nullptr)); + return creator(static_cast(nullptr)); if (isFixedString(type)) - return creator((ColumnFixedString *)(nullptr)); + return creator(static_cast(nullptr)); if (typeid_cast(type)) - return creator((ColumnVector *)(nullptr)); + return creator(static_cast *>(nullptr)); if (typeid_cast(type)) - return creator((ColumnVector *)(nullptr)); + return creator(static_cast *>(nullptr)); if (isNumber(type)) { MutableColumnUniquePtr column; diff --git a/dbms/src/DataTypes/DataTypeString.cpp b/dbms/src/DataTypes/DataTypeString.cpp index e33cf5d064e..55a4ac920b6 100644 --- a/dbms/src/DataTypes/DataTypeString.cpp +++ b/dbms/src/DataTypes/DataTypeString.cpp @@ -14,7 +14,7 @@ #include #include -#if __SSE2__ +#ifdef __SSE2__ #include #endif @@ -129,7 +129,7 @@ static NO_INLINE void deserializeBinarySSE2(ColumnString::Chars & data, ColumnSt if (size) { -#if __SSE2__ +#ifdef __SSE2__ /// An optimistic branch in which more efficient copying is possible. if (offset + 16 * UNROLL_TIMES <= data.allocated_bytes() && istr.position() + size + 16 * UNROLL_TIMES <= istr.buffer().end()) { diff --git a/dbms/src/DataTypes/DataTypeTuple.cpp b/dbms/src/DataTypes/DataTypeTuple.cpp index 58d0513173a..1f0b0f6ca3e 100644 --- a/dbms/src/DataTypes/DataTypeTuple.cpp +++ b/dbms/src/DataTypes/DataTypeTuple.cpp @@ -100,7 +100,7 @@ static inline const IColumn & extractElementColumn(const IColumn & column, size_ void DataTypeTuple::serializeBinary(const Field & field, WriteBuffer & ostr) const { const auto & tuple = get(field).toUnderType(); - for (const auto & idx_elem : ext::enumerate(elems)) + for (const auto idx_elem : ext::enumerate(elems)) idx_elem.second->serializeBinary(tuple[idx_elem.first], ostr); } @@ -115,7 +115,7 @@ void DataTypeTuple::deserializeBinary(Field & field, ReadBuffer & istr) const void DataTypeTuple::serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr) const { - for (const auto & idx_elem : ext::enumerate(elems)) + for (const auto idx_elem : ext::enumerate(elems)) idx_elem.second->serializeBinary(extractElementColumn(column, idx_elem.first), row_num, ostr); } diff --git a/dbms/src/DataTypes/DataTypesDecimal.h b/dbms/src/DataTypes/DataTypesDecimal.h index 96319dccfa4..f192dfe1a75 100644 --- a/dbms/src/DataTypes/DataTypesDecimal.h +++ b/dbms/src/DataTypes/DataTypesDecimal.h @@ -50,7 +50,7 @@ inline UInt32 leastDecimalPrecisionFor(TypeIndex int_type) return 20; default: break; - }; + } return 0; } diff --git a/dbms/src/DataTypes/tests/data_type_get_common_type.cpp b/dbms/src/DataTypes/tests/data_type_get_common_type.cpp index 1b3574dfa60..1d93146ba2e 100644 --- a/dbms/src/DataTypes/tests/data_type_get_common_type.cpp +++ b/dbms/src/DataTypes/tests/data_type_get_common_type.cpp @@ -7,6 +7,7 @@ #pragma GCC diagnostic ignored "-Wsign-compare" #ifdef __clang__ #pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" + #pragma clang diagnostic ignored "-Wundef" #endif #include diff --git a/dbms/src/Databases/DatabaseOrdinary.cpp b/dbms/src/Databases/DatabaseOrdinary.cpp index 7c4adbe9f67..cb1c7587080 100644 --- a/dbms/src/Databases/DatabaseOrdinary.cpp +++ b/dbms/src/Databases/DatabaseOrdinary.cpp @@ -273,7 +273,7 @@ void DatabaseOrdinary::createTable( /// But there is protection from it - see using DDLGuard in InterpreterCreateQuery. { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); if (tables.find(table_name) != tables.end()) throw Exception("Table " + name + "." + table_name + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS); } @@ -298,7 +298,7 @@ void DatabaseOrdinary::createTable( { /// Add a table to the map of known tables. { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); if (!tables.emplace(table_name, table).second) throw Exception("Table " + name + "." + table_name + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS); } @@ -492,7 +492,7 @@ void DatabaseOrdinary::shutdown() Tables tables_snapshot; { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); tables_snapshot = tables; } @@ -501,19 +501,19 @@ void DatabaseOrdinary::shutdown() kv.second->shutdown(); } - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); tables.clear(); } void DatabaseOrdinary::alterTable( const Context & context, - const String & name, + const String & table_name, const ColumnsDescription & columns, const ASTModifier & storage_modifier) { /// Read the definition of the table and replace the necessary parts with new ones. - String table_name_escaped = escapeForFileName(name); + String table_name_escaped = escapeForFileName(table_name); String table_metadata_tmp_path = metadata_path + "/" + table_name_escaped + ".sql.tmp"; String table_metadata_path = metadata_path + "/" + table_name_escaped + ".sql"; String statement; diff --git a/dbms/src/Databases/DatabasesCommon.cpp b/dbms/src/Databases/DatabasesCommon.cpp index 2d8dbbc2392..e64851bf470 100644 --- a/dbms/src/Databases/DatabasesCommon.cpp +++ b/dbms/src/Databases/DatabasesCommon.cpp @@ -88,7 +88,7 @@ bool DatabaseWithOwnTablesBase::isTableExist( const Context & /*context*/, const String & table_name) const { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); return tables.find(table_name) != tables.end(); } @@ -96,7 +96,7 @@ StoragePtr DatabaseWithOwnTablesBase::tryGetTable( const Context & /*context*/, const String & table_name) const { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); auto it = tables.find(table_name); if (it == tables.end()) return {}; @@ -105,13 +105,13 @@ StoragePtr DatabaseWithOwnTablesBase::tryGetTable( DatabaseIteratorPtr DatabaseWithOwnTablesBase::getIterator(const Context & /*context*/) { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); return std::make_unique(tables); } bool DatabaseWithOwnTablesBase::empty(const Context & /*context*/) const { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); return tables.empty(); } @@ -119,7 +119,7 @@ StoragePtr DatabaseWithOwnTablesBase::detachTable(const String & table_name) { StoragePtr res; { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); auto it = tables.find(table_name); if (it == tables.end()) throw Exception("Table " + name + "." + table_name + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); @@ -132,7 +132,7 @@ StoragePtr DatabaseWithOwnTablesBase::detachTable(const String & table_name) void DatabaseWithOwnTablesBase::attachTable(const String & table_name, const StoragePtr & table) { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); if (!tables.emplace(table_name, table).second) throw Exception("Table " + name + "." + table_name + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS); } @@ -144,7 +144,7 @@ void DatabaseWithOwnTablesBase::shutdown() Tables tables_snapshot; { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); tables_snapshot = tables; } @@ -153,7 +153,7 @@ void DatabaseWithOwnTablesBase::shutdown() kv.second->shutdown(); } - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); tables.clear(); } diff --git a/dbms/src/Dictionaries/Embedded/RegionsNames.h b/dbms/src/Dictionaries/Embedded/RegionsNames.h index 074a41162f1..7acb23d001e 100644 --- a/dbms/src/Dictionaries/Embedded/RegionsNames.h +++ b/dbms/src/Dictionaries/Embedded/RegionsNames.h @@ -73,7 +73,7 @@ public: { size_t language_id = static_cast(language); - if (region_id > names_refs[language_id].size()) + if (region_id >= names_refs[language_id].size()) return StringRef("", 0); StringRef ref = names_refs[language_id][region_id]; diff --git a/dbms/src/Dictionaries/ExecutableDictionarySource.cpp b/dbms/src/Dictionaries/ExecutableDictionarySource.cpp index 3cde810b685..376153bd0e9 100644 --- a/dbms/src/Dictionaries/ExecutableDictionarySource.cpp +++ b/dbms/src/Dictionaries/ExecutableDictionarySource.cpp @@ -23,8 +23,8 @@ namespace class ShellCommandOwningBlockInputStream : public OwningBlockInputStream { public: - ShellCommandOwningBlockInputStream(const BlockInputStreamPtr & stream, std::unique_ptr own) - : OwningBlockInputStream(std::move(stream), std::move(own)) + ShellCommandOwningBlockInputStream(const BlockInputStreamPtr & impl, std::unique_ptr own_) + : OwningBlockInputStream(std::move(impl), std::move(own_)) { } diff --git a/dbms/src/Dictionaries/LibraryDictionarySourceExternal.cpp b/dbms/src/Dictionaries/LibraryDictionarySourceExternal.cpp index aa0884d548a..03447df339c 100644 --- a/dbms/src/Dictionaries/LibraryDictionarySourceExternal.cpp +++ b/dbms/src/Dictionaries/LibraryDictionarySourceExternal.cpp @@ -45,7 +45,5 @@ void ClickHouseLibrary::log(ClickHouseLibrary::LogLevel level, ClickHouseLibrary if (logger.fatal()) logger.fatal(msg); break; - default: - break; } } diff --git a/dbms/src/Dictionaries/TrieDictionary.cpp b/dbms/src/Dictionaries/TrieDictionary.cpp index 886e7ff8592..00bafdf62e3 100644 --- a/dbms/src/Dictionaries/TrieDictionary.cpp +++ b/dbms/src/Dictionaries/TrieDictionary.cpp @@ -1,7 +1,6 @@ #include "TrieDictionary.h" #include #include -#include #include #include #include @@ -10,11 +9,19 @@ #include #include #include +#include #include #include #include "DictionaryBlockInputStream.h" #include "DictionaryFactory.h" +#ifdef __clang__ + #pragma clang diagnostic ignored "-Wold-style-cast" + #pragma clang diagnostic ignored "-Wnewline-eof" +#endif + +#include + namespace DB { @@ -760,13 +767,13 @@ BlockInputStreamPtr TrieDictionary::getBlockInputStream(const Names & column_nam { using BlockInputStreamType = DictionaryBlockInputStream; - auto getKeys = [](const Columns & columns, const std::vector & attributes) + auto getKeys = [](const Columns & columns, const std::vector & dict_attributes) { - const auto & attr = attributes.front(); + const auto & attr = dict_attributes.front(); return ColumnsWithTypeAndName( {ColumnWithTypeAndName(columns.front(), std::make_shared(IPV6_BINARY_LENGTH), attr.name)}); }; - auto getView = [](const Columns & columns, const std::vector & attributes) + auto getView = [](const Columns & columns, const std::vector & dict_attributes) { auto column = ColumnString::create(); const auto & ip_column = static_cast(*columns.front()); @@ -778,11 +785,11 @@ BlockInputStreamPtr TrieDictionary::getBlockInputStream(const Names & column_nam char * ptr = buffer; formatIPv6(reinterpret_cast(ip_column.getDataAt(row).data), ptr); *(ptr - 1) = '/'; - auto size = detail::writeUIntText(mask, ptr); - column->insertData(buffer, size + (ptr - buffer)); + ptr = itoa(mask, ptr); + column->insertData(buffer, ptr - buffer); } return ColumnsWithTypeAndName{ - ColumnWithTypeAndName(std::move(column), std::make_shared(), attributes.front().name)}; + ColumnWithTypeAndName(std::move(column), std::make_shared(), dict_attributes.front().name)}; }; return std::make_shared( shared_from_this(), max_block_size, getKeyColumns(), column_names, std::move(getKeys), std::move(getView)); diff --git a/dbms/src/Formats/BlockInputStreamFromRowInputStream.cpp b/dbms/src/Formats/BlockInputStreamFromRowInputStream.cpp index 89dc575dcb9..8062f59e245 100644 --- a/dbms/src/Formats/BlockInputStreamFromRowInputStream.cpp +++ b/dbms/src/Formats/BlockInputStreamFromRowInputStream.cpp @@ -88,7 +88,7 @@ Block BlockInputStreamFromRowInputStream::readImpl() throw; ++num_errors; - Float64 current_error_ratio = static_cast(num_errors) / total_rows; + Float32 current_error_ratio = static_cast(num_errors) / total_rows; if (num_errors > allow_errors_num && current_error_ratio > allow_errors_ratio) diff --git a/dbms/src/Formats/BlockInputStreamFromRowInputStream.h b/dbms/src/Formats/BlockInputStreamFromRowInputStream.h index fcbec628e09..c993f62adfe 100644 --- a/dbms/src/Formats/BlockInputStreamFromRowInputStream.h +++ b/dbms/src/Formats/BlockInputStreamFromRowInputStream.h @@ -45,7 +45,7 @@ private: BlockMissingValues block_missing_values; UInt64 allow_errors_num; - Float64 allow_errors_ratio; + Float32 allow_errors_ratio; size_t total_rows = 0; size_t num_errors = 0; diff --git a/dbms/src/Formats/FormatSettings.h b/dbms/src/Formats/FormatSettings.h index 55996c3c7a0..e8aaef290b4 100644 --- a/dbms/src/Formats/FormatSettings.h +++ b/dbms/src/Formats/FormatSettings.h @@ -60,7 +60,7 @@ struct FormatSettings DateTimeInputFormat date_time_input_format = DateTimeInputFormat::Basic; UInt64 input_allow_errors_num = 0; - Float64 input_allow_errors_ratio = 0; + Float32 input_allow_errors_ratio = 0; }; } diff --git a/dbms/src/Formats/JSONCompactRowOutputStream.cpp b/dbms/src/Formats/JSONCompactRowOutputStream.cpp index 03d5c6cfe2d..04edadf019b 100644 --- a/dbms/src/Formats/JSONCompactRowOutputStream.cpp +++ b/dbms/src/Formats/JSONCompactRowOutputStream.cpp @@ -8,8 +8,8 @@ namespace DB { -JSONCompactRowOutputStream::JSONCompactRowOutputStream(WriteBuffer & ostr_, const Block & sample_, const FormatSettings & settings) - : JSONRowOutputStream(ostr_, sample_, settings) +JSONCompactRowOutputStream::JSONCompactRowOutputStream(WriteBuffer & ostr_, const Block & sample_, const FormatSettings & settings_) + : JSONRowOutputStream(ostr_, sample_, settings_) { } diff --git a/dbms/src/Formats/PrettyCompactBlockOutputStream.h b/dbms/src/Formats/PrettyCompactBlockOutputStream.h index 74298a0535c..d0e3826bece 100644 --- a/dbms/src/Formats/PrettyCompactBlockOutputStream.h +++ b/dbms/src/Formats/PrettyCompactBlockOutputStream.h @@ -11,8 +11,8 @@ namespace DB class PrettyCompactBlockOutputStream : public PrettyBlockOutputStream { public: - PrettyCompactBlockOutputStream(WriteBuffer & ostr_, const Block & header_, const FormatSettings & format_settings) - : PrettyBlockOutputStream(ostr_, header_, format_settings) {} + PrettyCompactBlockOutputStream(WriteBuffer & ostr_, const Block & header_, const FormatSettings & format_settings_) + : PrettyBlockOutputStream(ostr_, header_, format_settings_) {} void write(const Block & block) override; diff --git a/dbms/src/Formats/PrettySpaceBlockOutputStream.h b/dbms/src/Formats/PrettySpaceBlockOutputStream.h index 2f438130568..9807bf85767 100644 --- a/dbms/src/Formats/PrettySpaceBlockOutputStream.h +++ b/dbms/src/Formats/PrettySpaceBlockOutputStream.h @@ -11,8 +11,8 @@ namespace DB class PrettySpaceBlockOutputStream : public PrettyBlockOutputStream { public: - PrettySpaceBlockOutputStream(WriteBuffer & ostr_, const Block & header_, const FormatSettings & format_settings) - : PrettyBlockOutputStream(ostr_, header_, format_settings) {} + PrettySpaceBlockOutputStream(WriteBuffer & ostr_, const Block & header_, const FormatSettings & format_settings_) + : PrettyBlockOutputStream(ostr_, header_, format_settings_) {} void write(const Block & block) override; void writeSuffix() override; diff --git a/dbms/src/Formats/TSKVRowInputStream.cpp b/dbms/src/Formats/TSKVRowInputStream.cpp index 837dfb5afaa..4416fcbcbcc 100644 --- a/dbms/src/Formats/TSKVRowInputStream.cpp +++ b/dbms/src/Formats/TSKVRowInputStream.cpp @@ -88,7 +88,7 @@ static bool readName(ReadBuffer & buf, StringRef & ref, String & tmp) } -bool TSKVRowInputStream::read(MutableColumns & columns, RowReadExtension &) +bool TSKVRowInputStream::read(MutableColumns & columns, RowReadExtension & ext) { if (istr.eof()) return false; @@ -96,9 +96,7 @@ bool TSKVRowInputStream::read(MutableColumns & columns, RowReadExtension &) size_t num_columns = columns.size(); /// Set of columns for which the values were read. The rest will be filled with default values. - /// TODO Ability to provide your DEFAULTs. - bool read_columns[num_columns]; - memset(read_columns, 0, num_columns); + read_columns.assign(num_columns, false); if (unlikely(*istr.position() == '\n')) { @@ -180,6 +178,9 @@ bool TSKVRowInputStream::read(MutableColumns & columns, RowReadExtension &) if (!read_columns[i]) header.getByPosition(i).type->insertDefaultInto(*columns[i]); + /// return info about defaults set + ext.read_columns = read_columns; + return true; } diff --git a/dbms/src/Formats/TSKVRowInputStream.h b/dbms/src/Formats/TSKVRowInputStream.h index 155322e90c1..ada2ba3ed66 100644 --- a/dbms/src/Formats/TSKVRowInputStream.h +++ b/dbms/src/Formats/TSKVRowInputStream.h @@ -41,6 +41,8 @@ private: /// Hash table matching `field name -> position in the block`. NOTE You can use perfect hash map. using NameMap = HashMap; NameMap name_map; + + std::vector read_columns; }; } diff --git a/dbms/src/Formats/TSKVRowOutputStream.cpp b/dbms/src/Formats/TSKVRowOutputStream.cpp index 1e4f5f0e409..96be0054363 100644 --- a/dbms/src/Formats/TSKVRowOutputStream.cpp +++ b/dbms/src/Formats/TSKVRowOutputStream.cpp @@ -9,8 +9,8 @@ namespace DB { -TSKVRowOutputStream::TSKVRowOutputStream(WriteBuffer & ostr_, const Block & sample_, const FormatSettings & format_settings) - : TabSeparatedRowOutputStream(ostr_, sample_, false, false, format_settings) +TSKVRowOutputStream::TSKVRowOutputStream(WriteBuffer & ostr_, const Block & sample_, const FormatSettings & format_settings_) + : TabSeparatedRowOutputStream(ostr_, sample_, false, false, format_settings_) { NamesAndTypesList columns(sample_.getNamesAndTypesList()); fields.assign(columns.begin(), columns.end()); diff --git a/dbms/src/Formats/TabSeparatedRawRowOutputStream.h b/dbms/src/Formats/TabSeparatedRawRowOutputStream.h index 3f55b68f1b9..a6a8054e383 100644 --- a/dbms/src/Formats/TabSeparatedRawRowOutputStream.h +++ b/dbms/src/Formats/TabSeparatedRawRowOutputStream.h @@ -13,8 +13,8 @@ namespace DB class TabSeparatedRawRowOutputStream : public TabSeparatedRowOutputStream { public: - TabSeparatedRawRowOutputStream(WriteBuffer & ostr_, const Block & sample_, bool with_names_, bool with_types_, const FormatSettings & format_settings) - : TabSeparatedRowOutputStream(ostr_, sample_, with_names_, with_types_, format_settings) {} + TabSeparatedRawRowOutputStream(WriteBuffer & ostr_, const Block & sample_, bool with_names_, bool with_types_, const FormatSettings & format_settings_) + : TabSeparatedRowOutputStream(ostr_, sample_, with_names_, with_types_, format_settings_) {} void writeField(const IColumn & column, const IDataType & type, size_t row_num) override { diff --git a/dbms/src/Functions/FunctionDateOrDateTimeAddInterval.h b/dbms/src/Functions/FunctionDateOrDateTimeAddInterval.h index 9b27282ec19..ee280068613 100644 --- a/dbms/src/Functions/FunctionDateOrDateTimeAddInterval.h +++ b/dbms/src/Functions/FunctionDateOrDateTimeAddInterval.h @@ -225,10 +225,10 @@ struct DateTimeAddIntervalImpl block.getByPosition(result).column = std::move(col_to); } - else if (const auto * sources = checkAndGetColumnConst>(source_col.get())) + else if (const auto * sources_const = checkAndGetColumnConst>(source_col.get())) { auto col_to = ColumnVector::create(); - Op::constant_vector(sources->template getValue(), col_to->getData(), *block.getByPosition(arguments[1]).column, time_zone); + Op::constant_vector(sources_const->template getValue(), col_to->getData(), *block.getByPosition(arguments[1]).column, time_zone); block.getByPosition(result).column = std::move(col_to); } else diff --git a/dbms/src/Functions/FunctionMathBinaryFloat64.h b/dbms/src/Functions/FunctionMathBinaryFloat64.h index e4524138701..9dd462725fc 100644 --- a/dbms/src/Functions/FunctionMathBinaryFloat64.h +++ b/dbms/src/Functions/FunctionMathBinaryFloat64.h @@ -146,7 +146,7 @@ private: block.getByPosition(result).column = std::move(dst); return true; } - else if (const auto right_arg_typed = checkAndGetColumnConst>(right_arg)) + if (const auto right_arg_typed = checkAndGetColumnConst>(right_arg)) { auto dst = ColumnVector::create(); @@ -205,7 +205,7 @@ private: throw Exception{"Illegal column " + right_arg->getName() + " of second argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN}; } - else if (const auto left_arg_typed = checkAndGetColumnConst(left_arg)) + if (const auto left_arg_typed = checkAndGetColumnConst(left_arg)) { if (executeTyped(block, result, left_arg_typed, right_arg)) return true; diff --git a/dbms/src/Functions/FunctionStartsEndsWith.h b/dbms/src/Functions/FunctionStartsEndsWith.h index 4742fc47e29..661b9dfc914 100644 --- a/dbms/src/Functions/FunctionStartsEndsWith.h +++ b/dbms/src/Functions/FunctionStartsEndsWith.h @@ -75,12 +75,12 @@ public: if (const ColumnString * haystack = checkAndGetColumn(haystack_column)) dispatch(StringSource(*haystack), needle_column, vec_res); - else if (const ColumnFixedString * haystack = checkAndGetColumn(haystack_column)) - dispatch(FixedStringSource(*haystack), needle_column, vec_res); - else if (const ColumnConst * haystack = checkAndGetColumnConst(haystack_column)) - dispatch>(ConstSource(*haystack), needle_column, vec_res); - else if (const ColumnConst * haystack = checkAndGetColumnConst(haystack_column)) - dispatch>(ConstSource(*haystack), needle_column, vec_res); + else if (const ColumnFixedString * haystack_fixed = checkAndGetColumn(haystack_column)) + dispatch(FixedStringSource(*haystack_fixed), needle_column, vec_res); + else if (const ColumnConst * haystack_const = checkAndGetColumnConst(haystack_column)) + dispatch>(ConstSource(*haystack_const), needle_column, vec_res); + else if (const ColumnConst * haystack_const_fixed = checkAndGetColumnConst(haystack_column)) + dispatch>(ConstSource(*haystack_const_fixed), needle_column, vec_res); else throw Exception("Illegal combination of columns as arguments of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); @@ -93,12 +93,12 @@ private: { if (const ColumnString * needle = checkAndGetColumn(needle_column)) execute(haystack_source, StringSource(*needle), res_data); - else if (const ColumnFixedString * needle = checkAndGetColumn(needle_column)) - execute(haystack_source, FixedStringSource(*needle), res_data); - else if (const ColumnConst * needle = checkAndGetColumnConst(needle_column)) - execute>(haystack_source, ConstSource(*needle), res_data); - else if (const ColumnConst * needle = checkAndGetColumnConst(needle_column)) - execute>(haystack_source, ConstSource(*needle), res_data); + else if (const ColumnFixedString * needle_fixed = checkAndGetColumn(needle_column)) + execute(haystack_source, FixedStringSource(*needle_fixed), res_data); + else if (const ColumnConst * needle_const = checkAndGetColumnConst(needle_column)) + execute>(haystack_source, ConstSource(*needle_const), res_data); + else if (const ColumnConst * needle_const_fixed = checkAndGetColumnConst(needle_column)) + execute>(haystack_source, ConstSource(*needle_const_fixed), res_data); else throw Exception("Illegal combination of columns as arguments of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); } diff --git a/dbms/src/Functions/FunctionStringOrArrayToT.h b/dbms/src/Functions/FunctionStringOrArrayToT.h index 88f24630862..7861d7ce867 100644 --- a/dbms/src/Functions/FunctionStringOrArrayToT.h +++ b/dbms/src/Functions/FunctionStringOrArrayToT.h @@ -62,33 +62,33 @@ public: block.getByPosition(result).column = std::move(col_res); } - else if (const ColumnFixedString * col = checkAndGetColumn(column.get())) + else if (const ColumnFixedString * col_fixed = checkAndGetColumn(column.get())) { if (Impl::is_fixed_to_constant) { ResultType res = 0; - Impl::vector_fixed_to_constant(col->getChars(), col->getN(), res); + Impl::vector_fixed_to_constant(col_fixed->getChars(), col_fixed->getN(), res); - block.getByPosition(result).column = block.getByPosition(result).type->createColumnConst(col->size(), toField(res)); + block.getByPosition(result).column = block.getByPosition(result).type->createColumnConst(col_fixed->size(), toField(res)); } else { auto col_res = ColumnVector::create(); typename ColumnVector::Container & vec_res = col_res->getData(); - vec_res.resize(col->size()); - Impl::vector_fixed_to_vector(col->getChars(), col->getN(), vec_res); + vec_res.resize(col_fixed->size()); + Impl::vector_fixed_to_vector(col_fixed->getChars(), col_fixed->getN(), vec_res); block.getByPosition(result).column = std::move(col_res); } } - else if (const ColumnArray * col = checkAndGetColumn(column.get())) + else if (const ColumnArray * col_arr = checkAndGetColumn(column.get())) { auto col_res = ColumnVector::create(); typename ColumnVector::Container & vec_res = col_res->getData(); - vec_res.resize(col->size()); - Impl::array(col->getOffsets(), vec_res); + vec_res.resize(col_arr->size()); + Impl::array(col_arr->getOffsets(), vec_res); block.getByPosition(result).column = std::move(col_res); } diff --git a/dbms/src/Functions/FunctionStringToString.h b/dbms/src/Functions/FunctionStringToString.h index 65d505cd2a5..a1f04c00f33 100644 --- a/dbms/src/Functions/FunctionStringToString.h +++ b/dbms/src/Functions/FunctionStringToString.h @@ -60,10 +60,10 @@ public: Impl::vector(col->getChars(), col->getOffsets(), col_res->getChars(), col_res->getOffsets()); block.getByPosition(result).column = std::move(col_res); } - else if (const ColumnFixedString * col = checkAndGetColumn(column.get())) + else if (const ColumnFixedString * col_fixed = checkAndGetColumn(column.get())) { - auto col_res = ColumnFixedString::create(col->getN()); - Impl::vector_fixed(col->getChars(), col->getN(), col_res->getChars()); + auto col_res = ColumnFixedString::create(col_fixed->getN()); + Impl::vector_fixed(col_fixed->getChars(), col_fixed->getN(), col_res->getChars()); block.getByPosition(result).column = std::move(col_res); } else diff --git a/dbms/src/Functions/FunctionsCoding.h b/dbms/src/Functions/FunctionsCoding.h index 62ae5ee5bd5..8742c90cf6c 100644 --- a/dbms/src/Functions/FunctionsCoding.h +++ b/dbms/src/Functions/FunctionsCoding.h @@ -1056,17 +1056,17 @@ public: block.getByPosition(result).column = std::move(col_res); } - else if (const auto col_in = checkAndGetColumn(column.get())) + else if (const auto col_in_fixed = checkAndGetColumn(column.get())) { - if (col_in->getN() != uuid_text_length) + if (col_in_fixed->getN() != uuid_text_length) throw Exception("Illegal type " + col_type_name.type->getName() + - " of column " + col_in->getName() + + " of column " + col_in_fixed->getName() + " argument of function " + getName() + ", expected FixedString(" + toString(uuid_text_length) + ")", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - const auto size = col_in->size(); - const auto & vec_in = col_in->getChars(); + const auto size = col_in_fixed->size(); + const auto & vec_in = col_in_fixed->getChars(); auto col_res = ColumnFixedString::create(uuid_bytes_length); @@ -1087,8 +1087,7 @@ public: } else throw Exception("Illegal column " + block.getByPosition(arguments[0]).column->getName() - + " of argument of function " + getName(), - ErrorCodes::ILLEGAL_COLUMN); + + " of argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); } }; diff --git a/dbms/src/Functions/FunctionsComparison.h b/dbms/src/Functions/FunctionsComparison.h index b6161ad2da7..e4773d8e360 100644 --- a/dbms/src/Functions/FunctionsComparison.h +++ b/dbms/src/Functions/FunctionsComparison.h @@ -371,7 +371,7 @@ struct StringEqualsImpl && !memcmp(&a_data[a_offsets[i - 1]], b_data, b_n))); } -#if __SSE2__ +#ifdef __SSE2__ static void NO_INLINE fixed_string_vector_fixed_string_vector_16( const ColumnString::Chars & a_data, const ColumnString::Chars & b_data, @@ -428,7 +428,7 @@ struct StringEqualsImpl /** Specialization if both sizes are 16. * To more efficient comparison of IPv6 addresses stored in FixedString(16). */ -#if __SSE2__ +#ifdef __SSE2__ if (a_n == 16 && b_n == 16) { fixed_string_vector_fixed_string_vector_16(a_data, b_data, c); @@ -448,7 +448,7 @@ struct StringEqualsImpl PaddedPODArray & c) { ColumnString::Offset b_n = b.size(); -#if __SSE2__ +#ifdef __SSE2__ if (a_n == 16 && b_n == 16) { fixed_string_vector_constant_16(a_data, b, c); diff --git a/dbms/src/Functions/FunctionsConversion.h b/dbms/src/Functions/FunctionsConversion.h index 75c86eb89b2..e5a2994857f 100644 --- a/dbms/src/Functions/FunctionsConversion.h +++ b/dbms/src/Functions/FunctionsConversion.h @@ -1662,7 +1662,7 @@ private: element_wrappers.reserve(from_element_types.size()); /// Create conversion wrapper for each element in tuple - for (const auto & idx_type : ext::enumerate(from_type->getElements())) + for (const auto idx_type : ext::enumerate(from_type->getElements())) element_wrappers.push_back(prepareUnpackDictionaries(idx_type.second, to_element_types[idx_type.first])); return [element_wrappers, from_element_types, to_element_types] @@ -1688,7 +1688,7 @@ private: element_block.insert({ nullptr, std::make_shared(to_element_types), "" }); /// invoke conversion for each element - for (const auto & idx_element_wrapper : ext::enumerate(element_wrappers)) + for (const auto idx_element_wrapper : ext::enumerate(element_wrappers)) idx_element_wrapper.second(element_block, { idx_element_wrapper.first }, tuple_size + idx_element_wrapper.first, input_rows_count); @@ -2098,35 +2098,35 @@ private: { if (const auto type = checkAndGetDataType(to_type)) return monotonicityForType(type); - else if (const auto type = checkAndGetDataType(to_type)) + if (const auto type = checkAndGetDataType(to_type)) return monotonicityForType(type); - else if (const auto type = checkAndGetDataType(to_type)) + if (const auto type = checkAndGetDataType(to_type)) return monotonicityForType(type); - else if (const auto type = checkAndGetDataType(to_type)) + if (const auto type = checkAndGetDataType(to_type)) return monotonicityForType(type); - else if (const auto type = checkAndGetDataType(to_type)) + if (const auto type = checkAndGetDataType(to_type)) return monotonicityForType(type); - else if (const auto type = checkAndGetDataType(to_type)) + if (const auto type = checkAndGetDataType(to_type)) return monotonicityForType(type); - else if (const auto type = checkAndGetDataType(to_type)) + if (const auto type = checkAndGetDataType(to_type)) return monotonicityForType(type); - else if (const auto type = checkAndGetDataType(to_type)) + if (const auto type = checkAndGetDataType(to_type)) return monotonicityForType(type); - else if (const auto type = checkAndGetDataType(to_type)) + if (const auto type = checkAndGetDataType(to_type)) return monotonicityForType(type); - else if (const auto type = checkAndGetDataType(to_type)) + if (const auto type = checkAndGetDataType(to_type)) return monotonicityForType(type); - else if (const auto type = checkAndGetDataType(to_type)) + if (const auto type = checkAndGetDataType(to_type)) return monotonicityForType(type); - else if (const auto type = checkAndGetDataType(to_type)) + if (const auto type = checkAndGetDataType(to_type)) return monotonicityForType(type); - else if (const auto type = checkAndGetDataType(to_type)) + if (const auto type = checkAndGetDataType(to_type)) return monotonicityForType(type); - else if (isEnum(from_type)) + if (isEnum(from_type)) { if (const auto type = checkAndGetDataType(to_type)) return monotonicityForType(type); - else if (const auto type = checkAndGetDataType(to_type)) + if (const auto type = checkAndGetDataType(to_type)) return monotonicityForType(type); } /// other types like Null, FixedString, Array and Tuple have no monotonicity defined diff --git a/dbms/src/Functions/FunctionsExternalDictionaries.h b/dbms/src/Functions/FunctionsExternalDictionaries.h index 176cfe80eea..85a5c9cc6a5 100644 --- a/dbms/src/Functions/FunctionsExternalDictionaries.h +++ b/dbms/src/Functions/FunctionsExternalDictionaries.h @@ -489,8 +489,8 @@ private: const auto id_col_untyped = block.getByPosition(arguments[2]).column.get(); if (const auto id_col = checkAndGetColumn(id_col_untyped)) executeDispatch(block, arguments, result, dict, attr_name, id_col); - else if (const auto id_col = checkAndGetColumnConst>(id_col_untyped)) - executeDispatch(block, arguments, result, dict, attr_name, id_col); + else if (const auto id_col_const = checkAndGetColumnConst>(id_col_untyped)) + executeDispatch(block, arguments, result, dict, attr_name, id_col_const); else throw Exception{"Third argument of function " + getName() + " must be UInt64", ErrorCodes::ILLEGAL_COLUMN}; @@ -512,12 +512,12 @@ private: dictionary->getString(attr_name, ids, default_col, out.get()); block.getByPosition(result).column = std::move(out); } - else if (const auto default_col = checkAndGetColumnConstStringOrFixedString(default_col_untyped)) + else if (const auto default_col_const = checkAndGetColumnConstStringOrFixedString(default_col_untyped)) { /// vector ids, const defaults auto out = ColumnString::create(); const auto & ids = id_col->getData(); - String def = default_col->getValue(); + String def = default_col_const->getValue(); dictionary->getString(attr_name, ids, def, out.get()); block.getByPosition(result).column = std::move(out); } @@ -547,12 +547,12 @@ private: else block.getByPosition(result).column = block.getByPosition(arguments[3]).column; // reuse the default column } - else if (const auto default_col = checkAndGetColumnConstStringOrFixedString(default_col_untyped)) + else if (const auto default_col_const = checkAndGetColumnConstStringOrFixedString(default_col_untyped)) { /// const ids, const defaults const PaddedPODArray ids(1, id_col->getValue()); auto out = ColumnString::create(); - String def = default_col->getValue(); + String def = default_col_const->getValue(); dictionary->getString(attr_name, ids, def, out.get()); block.getByPosition(result).column = DataTypeString().createColumnConst(id_col->size(), out->getDataAt(0).toString()); } @@ -588,9 +588,9 @@ private: { dict->getString(attr_name, key_columns, key_types, default_col, out.get()); } - else if (const auto default_col = checkAndGetColumnConstStringOrFixedString(default_col_untyped)) + else if (const auto default_col_const = checkAndGetColumnConstStringOrFixedString(default_col_untyped)) { - String def = default_col->getValue(); + String def = default_col_const->getValue(); dict->getString(attr_name, key_columns, key_types, def, out.get()); } else @@ -775,12 +775,12 @@ private: DictGetTraits::get(dict, attr_name, ids, data); block.getByPosition(result).column = std::move(out); } - else if (const auto id_col = checkAndGetColumnConst>(id_col_untyped)) + else if (const auto id_col_const = checkAndGetColumnConst>(id_col_untyped)) { - const PaddedPODArray ids(1, id_col->getValue()); + const PaddedPODArray ids(1, id_col_const->getValue()); PaddedPODArray data(1); DictGetTraits::get(dict, attr_name, ids, data); - block.getByPosition(result).column = DataTypeNumber().createColumnConst(id_col->size(), toField(data.front())); + block.getByPosition(result).column = DataTypeNumber().createColumnConst(id_col_const->size(), toField(data.front())); } else throw Exception{"Third argument of function " + getName() + " must be UInt64", ErrorCodes::ILLEGAL_COLUMN}; @@ -982,8 +982,8 @@ private: const auto id_col_untyped = block.getByPosition(arguments[2]).column.get(); if (const auto id_col = checkAndGetColumn(id_col_untyped)) executeDispatch(block, arguments, result, dict, attr_name, id_col); - else if (const auto id_col = checkAndGetColumnConst>(id_col_untyped)) - executeDispatch(block, arguments, result, dict, attr_name, id_col); + else if (const auto id_col_const = checkAndGetColumnConst>(id_col_untyped)) + executeDispatch(block, arguments, result, dict, attr_name, id_col_const); else throw Exception{"Third argument of function " + getName() + " must be UInt64", ErrorCodes::ILLEGAL_COLUMN}; @@ -1007,13 +1007,13 @@ private: DictGetTraits::getOrDefault(dictionary, attr_name, ids, defs, data); block.getByPosition(result).column = std::move(out); } - else if (const auto default_col = checkAndGetColumnConst>(default_col_untyped)) + else if (const auto default_col_const = checkAndGetColumnConst>(default_col_untyped)) { /// vector ids, const defaults auto out = ColumnVector::create(id_col->size()); const auto & ids = id_col->getData(); auto & data = out->getData(); - const auto def = default_col->template getValue(); + const auto def = default_col_const->template getValue(); DictGetTraits::getOrDefault(dictionary, attr_name, ids, def, data); block.getByPosition(result).column = std::move(out); } @@ -1043,12 +1043,12 @@ private: else block.getByPosition(result).column = block.getByPosition(arguments[3]).column; // reuse the default column } - else if (const auto default_col = checkAndGetColumnConst>(default_col_untyped)) + else if (const auto default_col_const = checkAndGetColumnConst>(default_col_untyped)) { /// const ids, const defaults const PaddedPODArray ids(1, id_col->getValue()); PaddedPODArray data(1); - const auto & def = default_col->template getValue(); + const auto & def = default_col_const->template getValue(); DictGetTraits::getOrDefault(dictionary, attr_name, ids, def, data); block.getByPosition(result).column = DataTypeNumber().createColumnConst(id_col->size(), toField(data.front())); } @@ -1091,9 +1091,9 @@ private: DictGetTraits::getOrDefault(dict, attr_name, key_columns, key_types, defs, data); } - else if (const auto default_col = checkAndGetColumnConst>(default_col_untyped)) + else if (const auto default_col_const = checkAndGetColumnConst>(default_col_untyped)) { - const auto def = default_col->template getValue(); + const auto def = default_col_const->template getValue(); DictGetTraits::getOrDefault(dict, attr_name, key_columns, key_types, def, data); } @@ -1489,14 +1489,14 @@ private: get_hierarchies(in, backend->getData(), offsets->getData()); block.getByPosition(result).column = ColumnArray::create(std::move(backend), std::move(offsets)); } - else if (const auto id_col = checkAndGetColumnConst>(id_col_untyped)) + else if (const auto id_col_const = checkAndGetColumnConst>(id_col_untyped)) { - const PaddedPODArray in(1, id_col->getValue()); + const PaddedPODArray in(1, id_col_const->getValue()); auto backend = ColumnUInt64::create(); auto offsets = ColumnArray::ColumnOffsets::create(); get_hierarchies(in, backend->getData(), offsets->getData()); auto array = ColumnArray::create(std::move(backend), std::move(offsets)); - block.getByPosition(result).column = block.getByPosition(result).type->createColumnConst(id_col->size(), (*array)[0].get()); + block.getByPosition(result).column = block.getByPosition(result).type->createColumnConst(id_col_const->size(), (*array)[0].get()); } else throw Exception{"Second argument of function " + getName() + " must be UInt64", ErrorCodes::ILLEGAL_COLUMN}; @@ -1585,8 +1585,8 @@ private: if (const auto child_id_col = checkAndGetColumn(child_id_col_untyped)) execute(block, result, dict, child_id_col, ancestor_id_col_untyped); - else if (const auto child_id_col = checkAndGetColumnConst>(child_id_col_untyped)) - execute(block, result, dict, child_id_col, ancestor_id_col_untyped); + else if (const auto child_id_col_const = checkAndGetColumnConst>(child_id_col_untyped)) + execute(block, result, dict, child_id_col_const, ancestor_id_col_untyped); else throw Exception{"Illegal column " + child_id_col_untyped->getName() + " of second argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN}; @@ -1611,12 +1611,12 @@ private: dictionary->isInVectorVector(child_ids, ancestor_ids, data); block.getByPosition(result).column = std::move(out); } - else if (const auto ancestor_id_col = checkAndGetColumnConst>(ancestor_id_col_untyped)) + else if (const auto ancestor_id_col_const = checkAndGetColumnConst>(ancestor_id_col_untyped)) { auto out = ColumnUInt8::create(); const auto & child_ids = child_id_col->getData(); - const auto ancestor_id = ancestor_id_col->getValue(); + const auto ancestor_id = ancestor_id_col_const->getValue(); auto & data = out->getData(); const auto size = child_id_col->size(); data.resize(size); @@ -1650,10 +1650,10 @@ private: dictionary->isInConstantVector(child_id, ancestor_ids, data); block.getByPosition(result).column = std::move(out); } - else if (const auto ancestor_id_col = checkAndGetColumnConst>(ancestor_id_col_untyped)) + else if (const auto ancestor_id_col_const = checkAndGetColumnConst>(ancestor_id_col_untyped)) { const auto child_id = child_id_col->getValue(); - const auto ancestor_id = ancestor_id_col->getValue(); + const auto ancestor_id = ancestor_id_col_const->getValue(); UInt8 res = 0; dictionary->isInConstantConstant(child_id, ancestor_id, res); diff --git a/dbms/src/Functions/FunctionsExternalModels.cpp b/dbms/src/Functions/FunctionsExternalModels.cpp index 6afbad31857..6c11cb78bb5 100644 --- a/dbms/src/Functions/FunctionsExternalModels.cpp +++ b/dbms/src/Functions/FunctionsExternalModels.cpp @@ -10,6 +10,10 @@ #include #include #include +#include +#include +#include +#include namespace DB { @@ -26,17 +30,43 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; } -DataTypePtr FunctionModelEvaluate::getReturnTypeImpl(const DataTypes & arguments) const +DataTypePtr FunctionModelEvaluate::getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const { if (arguments.size() < 2) throw Exception("Function " + getName() + " expects at least 2 arguments", ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION); - if (!isString(arguments[0])) - throw Exception("Illegal type " + arguments[0]->getName() + " of first argument of function " + getName() + if (!isString(arguments[0].type)) + throw Exception("Illegal type " + arguments[0].type->getName() + " of first argument of function " + getName() + ", expected a string.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - return std::make_shared(); + const auto name_col = checkAndGetColumnConst(arguments[0].column.get()); + if (!name_col) + throw Exception("First argument of function " + getName() + " must be a constant string", + ErrorCodes::ILLEGAL_COLUMN); + + bool has_nullable = false; + for (size_t i = 1; i < arguments.size(); ++i) + has_nullable = has_nullable || arguments[i].type->isNullable(); + + auto model = models.getModel(name_col->getValue()); + auto type = model->getReturnType(); + + if (has_nullable) + { + if (auto * tuple = typeid_cast(type.get())) + { + auto elements = tuple->getElements(); + for (auto & element : elements) + element = makeNullable(element); + + type = std::make_shared(elements); + } + else + type = makeNullable(type); + } + + return type; } void FunctionModelEvaluate::executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) @@ -49,11 +79,58 @@ void FunctionModelEvaluate::executeImpl(Block & block, const ColumnNumbers & arg auto model = models.getModel(name_col->getValue()); ColumnRawPtrs columns; - columns.reserve(arguments.size()); - for (auto i : ext::range(1, arguments.size())) - columns.push_back(block.getByPosition(arguments[i]).column.get()); + Columns materialized_columns; + ColumnPtr null_map; - block.getByPosition(result).column = model->evaluate(columns); + columns.reserve(arguments.size()); + for (auto arg : ext::range(1, arguments.size())) + { + auto & column = block.getByPosition(arguments[arg]).column; + columns.push_back(column.get()); + if (auto full_column = column->convertToFullColumnIfConst()) + { + materialized_columns.push_back(full_column); + columns.back() = full_column.get(); + } + if (auto * col_nullable = typeid_cast(columns.back())) + { + if (!null_map) + null_map = col_nullable->getNullMapColumnPtr(); + else + { + auto mut_null_map = (*std::move(null_map)).mutate(); + + NullMap & result_null_map = static_cast(*mut_null_map).getData(); + const NullMap & src_null_map = col_nullable->getNullMapColumn().getData(); + + for (size_t i = 0, size = result_null_map.size(); i < size; ++i) + if (src_null_map[i]) + result_null_map[i] = 1; + + null_map = std::move(mut_null_map); + } + + columns.back() = &col_nullable->getNestedColumn(); + } + } + + auto res = model->evaluate(columns); + + if (null_map) + { + if (auto * tuple = typeid_cast(res.get())) + { + auto nested = tuple->getColumns(); + for (auto & col : nested) + col = ColumnNullable::create(col, null_map); + + res = ColumnTuple::create(nested); + } + else + res = ColumnNullable::create(res, null_map); + } + + block.getByPosition(result).column = res; } void registerFunctionsExternalModels(FunctionFactory & factory) diff --git a/dbms/src/Functions/FunctionsExternalModels.h b/dbms/src/Functions/FunctionsExternalModels.h index e32fe7f066f..ab193e0a2bc 100644 --- a/dbms/src/Functions/FunctionsExternalModels.h +++ b/dbms/src/Functions/FunctionsExternalModels.h @@ -25,9 +25,11 @@ public: bool isDeterministic() const override { return false; } + bool useDefaultImplementationForNulls() const override { return false; } + size_t getNumberOfArguments() const override { return 0; } - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override; + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override; void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override; diff --git a/dbms/src/Functions/FunctionsFormatting.h b/dbms/src/Functions/FunctionsFormatting.h index 9728a0f95ba..60a30d259e8 100644 --- a/dbms/src/Functions/FunctionsFormatting.h +++ b/dbms/src/Functions/FunctionsFormatting.h @@ -183,7 +183,7 @@ private: for (size_t i = 0; i < size; ++i) { - formatReadableSizeWithBinarySuffix(vec_from[i], buf_to); + formatReadableSizeWithBinarySuffix(static_cast(vec_from[i]), buf_to); writeChar(0, buf_to); offsets_to[i] = buf_to.count(); } diff --git a/dbms/src/Functions/FunctionsGeo.h b/dbms/src/Functions/FunctionsGeo.h index 5ac23138a88..1f351633dd7 100644 --- a/dbms/src/Functions/FunctionsGeo.h +++ b/dbms/src/Functions/FunctionsGeo.h @@ -86,9 +86,9 @@ private: out_const = false; result[arg_idx] = instr_t{instr_type::get_float_64, col}; } - else if (const auto col = checkAndGetColumnConst>(column)) + else if (const auto col_const = checkAndGetColumnConst>(column)) { - result[arg_idx] = instr_t{instr_type::get_const_float_64, col}; + result[arg_idx] = instr_t{instr_type::get_const_float_64, col_const}; } else throw Exception("Illegal column " + column->getName() + " of argument of function " + getName(), @@ -233,7 +233,7 @@ private: /// Prepare array of ellipses. size_t ellipses_count = (arguments.size() - 2) / 4; - Ellipse ellipses[ellipses_count]; + std::vector ellipses(ellipses_count); for (const auto ellipse_idx : ext::range(0, ellipses_count)) { @@ -285,7 +285,7 @@ private: size_t start_index = 0; for (const auto row : ext::range(0, size)) { - dst_data[row] = isPointInEllipses(col_vec_x->getData()[row], col_vec_y->getData()[row], ellipses, ellipses_count, start_index); + dst_data[row] = isPointInEllipses(col_vec_x->getData()[row], col_vec_y->getData()[row], ellipses.data(), ellipses_count, start_index); } block.getByPosition(result).column = std::move(dst); @@ -295,7 +295,7 @@ private: const auto col_const_x = static_cast (col_x); const auto col_const_y = static_cast (col_y); size_t start_index = 0; - UInt8 res = isPointInEllipses(col_const_x->getValue(), col_const_y->getValue(), ellipses, ellipses_count, start_index); + UInt8 res = isPointInEllipses(col_const_x->getValue(), col_const_y->getValue(), ellipses.data(), ellipses_count, start_index); block.getByPosition(result).column = DataTypeUInt8().createColumnConst(size, res); } else diff --git a/dbms/src/Functions/FunctionsHashing.cpp b/dbms/src/Functions/FunctionsHashing.cpp index 787fd80ae08..a02a841e0b4 100644 --- a/dbms/src/Functions/FunctionsHashing.cpp +++ b/dbms/src/Functions/FunctionsHashing.cpp @@ -28,6 +28,7 @@ void registerFunctionsHashing(FunctionFactory & factory) factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); + factory.registerFunction(); #if USE_XXHASH factory.registerFunction(); diff --git a/dbms/src/Functions/FunctionsHashing.h b/dbms/src/Functions/FunctionsHashing.h index bce3c27b24f..2a5f7b01fe9 100644 --- a/dbms/src/Functions/FunctionsHashing.h +++ b/dbms/src/Functions/FunctionsHashing.h @@ -257,6 +257,25 @@ struct MurmurHash2Impl64 static constexpr bool use_int_hash_for_pods = false; }; +/// To be compatible with gcc: https://github.com/gcc-mirror/gcc/blob/41d6b10e96a1de98e90a7c0378437c3255814b16/libstdc%2B%2B-v3/include/bits/functional_hash.h#L191 +struct GccMurmurHashImpl +{ + static constexpr auto name = "gccMurmurHash"; + using ReturnType = UInt64; + + static UInt64 apply(const char * data, const size_t size) + { + return MurmurHash64A(data, size, 0xc70f6907UL); + } + + static UInt64 combineHashes(UInt64 h1, UInt64 h2) + { + return IntHash64Impl::apply(h1) ^ h2; + } + + static constexpr bool use_int_hash_for_pods = false; +}; + struct MurmurHash3Impl32 { static constexpr auto name = "murmurHash3_32"; @@ -622,9 +641,9 @@ private: vec_to[i] = Impl::combineHashes(vec_to[i], h); } } - else if (auto col_from = checkAndGetColumnConst>(column)) + else if (auto col_from_const = checkAndGetColumnConst>(column)) { - auto value = col_from->template getValue(); + auto value = col_from_const->template getValue(); ToType hash; if constexpr (std::is_same_v) hash = IntHash64Impl::apply(ext::bit_cast(value)); @@ -672,10 +691,10 @@ private: current_offset = offsets[i]; } } - else if (const ColumnFixedString * col_from = checkAndGetColumn(column)) + else if (const ColumnFixedString * col_from_fixed = checkAndGetColumn(column)) { - const typename ColumnString::Chars & data = col_from->getChars(); - size_t n = col_from->getN(); + const typename ColumnString::Chars & data = col_from_fixed->getChars(); + size_t n = col_from_fixed->getN(); size_t size = data.size() / n; for (size_t i = 0; i < size; ++i) @@ -687,9 +706,9 @@ private: vec_to[i] = Impl::combineHashes(vec_to[i], h); } } - else if (const ColumnConst * col_from = checkAndGetColumnConstStringOrFixedString(column)) + else if (const ColumnConst * col_from_const = checkAndGetColumnConstStringOrFixedString(column)) { - String value = col_from->getValue().data(); + String value = col_from_const->getValue().data(); const ToType hash = Impl::apply(value.data(), value.size()); const size_t size = vec_to.size(); @@ -749,10 +768,10 @@ private: current_offset = offsets[i]; } } - else if (const ColumnConst * col_from = checkAndGetColumnConst(column)) + else if (const ColumnConst * col_from_const = checkAndGetColumnConst(column)) { /// NOTE: here, of course, you can do without the materialization of the column. - ColumnPtr full_column = col_from->convertToFullColumn(); + ColumnPtr full_column = col_from_const->convertToFullColumn(); executeArray(type, &*full_column, vec_to); } else @@ -799,9 +818,9 @@ private: for (size_t i = 0; i < tuple_size; ++i) executeForArgument(tuple_types[i].get(), tuple_columns[i].get(), vec_to, is_first); } - else if (const ColumnTuple * tuple = checkAndGetColumnConstData(column)) + else if (const ColumnTuple * tuple_const = checkAndGetColumnConstData(column)) { - const Columns & tuple_columns = tuple->getColumns(); + const Columns & tuple_columns = tuple_const->getColumns(); const DataTypes & tuple_types = typeid_cast(*type).getElements(); size_t tuple_size = tuple_columns.size(); for (size_t i = 0; i < tuple_size; ++i) @@ -1070,6 +1089,7 @@ using FunctionFarmHash64 = FunctionAnyHash; using FunctionMetroHash64 = FunctionAnyHash; using FunctionMurmurHash2_32 = FunctionAnyHash; using FunctionMurmurHash2_64 = FunctionAnyHash; +using FunctionGccMurmurHash = FunctionAnyHash; using FunctionMurmurHash3_32 = FunctionAnyHash; using FunctionMurmurHash3_64 = FunctionAnyHash; using FunctionMurmurHash3_128 = FunctionStringHashFixedString; diff --git a/dbms/src/Functions/FunctionsRandom.cpp b/dbms/src/Functions/FunctionsRandom.cpp index 8bb309561f1..ede8c332d18 100644 --- a/dbms/src/Functions/FunctionsRandom.cpp +++ b/dbms/src/Functions/FunctionsRandom.cpp @@ -25,9 +25,6 @@ namespace /// And this is from `head -c8 /dev/urandom | xxd -p` UInt64 current = 0x09826f4a081cee35ULL; - LinearCongruentialGenerator() {} - LinearCongruentialGenerator(UInt64 value) : current(value) {} - void seed(UInt64 value) { current = value; diff --git a/dbms/src/Functions/FunctionsReinterpret.h b/dbms/src/Functions/FunctionsReinterpret.h index c6e0475d4a4..db8b149337c 100644 --- a/dbms/src/Functions/FunctionsReinterpret.h +++ b/dbms/src/Functions/FunctionsReinterpret.h @@ -198,12 +198,12 @@ public: block.getByPosition(result).column = std::move(col_res); } - else if (const ColumnFixedString * col_from = typeid_cast(block.getByPosition(arguments[0]).column.get())) + else if (const ColumnFixedString * col_from_fixed = typeid_cast(block.getByPosition(arguments[0]).column.get())) { auto col_res = ColumnVector::create(); - const ColumnString::Chars & data_from = col_from->getChars(); - size_t step = col_from->getN(); + const ColumnString::Chars & data_from = col_from_fixed->getChars(); + size_t step = col_from_fixed->getN(); size_t size = data_from.size() / step; typename ColumnVector::Container & vec_res = col_res->getData(); vec_res.resize(size); diff --git a/dbms/src/Functions/FunctionsRound.h b/dbms/src/Functions/FunctionsRound.h index 3b026e970ff..3ef24597faa 100644 --- a/dbms/src/Functions/FunctionsRound.h +++ b/dbms/src/Functions/FunctionsRound.h @@ -16,7 +16,7 @@ #include #include -#if __SSE4_1__ +#ifdef __SSE4_1__ #include #endif @@ -62,7 +62,7 @@ enum class ScaleMode enum class RoundingMode { -#if __SSE4_1__ +#ifdef __SSE4_1__ Round = _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, Floor = _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC, Ceil = _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC, @@ -118,8 +118,6 @@ struct IntegerRoundingComputation x = -x; return x; } - default: - __builtin_unreachable(); } } @@ -133,8 +131,6 @@ struct IntegerRoundingComputation return x; case ScaleMode::Negative: return computeImpl(x, scale); - default: - __builtin_unreachable(); } } @@ -149,7 +145,7 @@ struct IntegerRoundingComputation }; -#if __SSE4_1__ +#ifdef __SSE4_1__ template class BaseFloatRoundingComputation; diff --git a/dbms/src/Functions/FunctionsStringSearch.cpp b/dbms/src/Functions/FunctionsStringSearch.cpp index af7ea515f4e..fa7b3379623 100644 --- a/dbms/src/Functions/FunctionsStringSearch.cpp +++ b/dbms/src/Functions/FunctionsStringSearch.cpp @@ -995,10 +995,10 @@ public: Impl::vector(col->getChars(), col->getOffsets(), needle, replacement, col_res->getChars(), col_res->getOffsets()); block.getByPosition(result).column = std::move(col_res); } - else if (const ColumnFixedString * col = checkAndGetColumn(column_src.get())) + else if (const ColumnFixedString * col_fixed = checkAndGetColumn(column_src.get())) { auto col_res = ColumnString::create(); - Impl::vector_fixed(col->getChars(), col->getN(), needle, replacement, col_res->getChars(), col_res->getOffsets()); + Impl::vector_fixed(col_fixed->getChars(), col_fixed->getN(), needle, replacement, col_res->getChars(), col_res->getOffsets()); block.getByPosition(result).column = std::move(col_res); } else diff --git a/dbms/src/Functions/FunctionsVisitParam.h b/dbms/src/Functions/FunctionsVisitParam.h index a746c7b3979..2c0c9901dae 100644 --- a/dbms/src/Functions/FunctionsVisitParam.h +++ b/dbms/src/Functions/FunctionsVisitParam.h @@ -100,20 +100,23 @@ struct ExtractRaw if (*pos == current_expect_end) { expects_end.pop_back(); - current_expect_end = (UInt8) (expects_end.empty() ? 0 : expects_end.back()); + current_expect_end = expects_end.empty() ? 0 : expects_end.back(); } else { switch (*pos) { case '[': - expects_end.push_back((current_expect_end = ']')); + current_expect_end = ']'; + expects_end.push_back(current_expect_end); break; case '{': - expects_end.push_back((current_expect_end = '}')); + current_expect_end = '}'; + expects_end.push_back(current_expect_end); break; case '"' : - expects_end.push_back((current_expect_end = '"')); + current_expect_end = '"'; + expects_end.push_back(current_expect_end); break; case '\\': /// skip backslash diff --git a/dbms/src/Functions/GatherUtils/Algorithms.h b/dbms/src/Functions/GatherUtils/Algorithms.h index 34cef4939e5..aaf1828f47d 100644 --- a/dbms/src/Functions/GatherUtils/Algorithms.h +++ b/dbms/src/Functions/GatherUtils/Algorithms.h @@ -4,9 +4,11 @@ #include #include +#include #include + namespace DB::ErrorCodes { extern const int LOGICAL_ERROR; @@ -31,7 +33,7 @@ void writeSlice(const NumericArraySlice & slice, NumericArraySink & sink) sink.elements.resize(sink.current_offset + slice.size); for (size_t i = 0; i < slice.size; ++i) { - sink.elements[sink.current_offset] = slice.data[i]; + sink.elements[sink.current_offset] = static_cast(slice.data[i]); ++sink.current_offset; } } @@ -421,17 +423,12 @@ bool sliceHasImpl(const FirstSliceType & first, const SecondSliceType & second, return all; } -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wsign-compare" - template bool sliceEqualElements(const NumericArraySlice & first, const NumericArraySlice & second, size_t first_ind, size_t second_ind) { - return first.data[first_ind] == second.data[second_ind]; + return accurate::equalsOp(first.data[first_ind], second.data[second_ind]); } -#pragma GCC diagnostic pop - template bool sliceEqualElements(const NumericArraySlice &, const GenericArraySlice &, size_t, size_t) { diff --git a/dbms/src/Functions/GatherUtils/Sources.h b/dbms/src/Functions/GatherUtils/Sources.h index e076b9989d8..40115a5a240 100644 --- a/dbms/src/Functions/GatherUtils/Sources.h +++ b/dbms/src/Functions/GatherUtils/Sources.h @@ -118,21 +118,22 @@ struct ConstSource : public Base size_t total_rows; size_t row_num = 0; - explicit ConstSource(const ColumnConst & col) - : Base(static_cast(col.getDataColumn())), total_rows(col.size()) + explicit ConstSource(const ColumnConst & col_) + : Base(static_cast(col_.getDataColumn())), total_rows(col_.size()) { } template - ConstSource(const ColumnType & col, size_t total_rows) : Base(col), total_rows(total_rows) + ConstSource(const ColumnType & col_, size_t total_rows_) : Base(col_), total_rows(total_rows_) { } template - ConstSource(const ColumnType & col, const NullMap & null_map, size_t total_rows) : Base(col, null_map), total_rows(total_rows) + ConstSource(const ColumnType & col_, const NullMap & null_map_, size_t total_rows_) : Base(col_, null_map_), total_rows(total_rows_) { } + ConstSource(const ConstSource &) = default; virtual ~ConstSource() = default; virtual void accept(ArraySourceVisitor & visitor) // override diff --git a/dbms/src/Functions/GeoUtils.h b/dbms/src/Functions/GeoUtils.h index b3463354f83..de9bb866cee 100644 --- a/dbms/src/Functions/GeoUtils.h +++ b/dbms/src/Functions/GeoUtils.h @@ -199,8 +199,8 @@ UInt64 PointInPolygonWithGrid::getAllocatedBytes() const size += polygons.capacity() * sizeof(MultiPolygon); size += getPolygonAllocatedBytes(polygon); - for (const auto & polygon : polygons) - size += getMultiPolygonAllocatedBytes(polygon); + for (const auto & elem : polygons) + size += getMultiPolygonAllocatedBytes(elem); return size; } @@ -312,24 +312,23 @@ bool PointInPolygonWithGrid::contains(CoordinateType x, Coordina return cell.half_planes[0].contains(x, y); case CellType::pairOfLinesSingleConvexPolygon: return cell.half_planes[0].contains(x, y) && cell.half_planes[1].contains(x, y); - case CellType::pairOfLinesDifferentPolygons: + case CellType::pairOfLinesDifferentPolygons: [[fallthrough]]; case CellType::pairOfLinesSingleNonConvexPolygons: return cell.half_planes[0].contains(x, y) || cell.half_planes[1].contains(x, y); case CellType::complexPolygon: return boost::geometry::within(Point(x, y), polygons[cell.index_of_inner_polygon]); - default: - return false; - } + + __builtin_unreachable(); } template typename PointInPolygonWithGrid::Distance PointInPolygonWithGrid::distance( const PointInPolygonWithGrid::Point & point, - const PointInPolygonWithGrid::Polygon & polygon) + const PointInPolygonWithGrid::Polygon & poly) { - const auto & outer = polygon.outer(); + const auto & outer = poly.outer(); Distance distance = 0; for (auto i : ext::range(0, outer.size() - 1)) { @@ -341,9 +340,9 @@ PointInPolygonWithGrid::distance( } template -bool PointInPolygonWithGrid::isConvex(const PointInPolygonWithGrid::Polygon & polygon) +bool PointInPolygonWithGrid::isConvex(const PointInPolygonWithGrid::Polygon & poly) { - const auto & outer = polygon.outer(); + const auto & outer = poly.outer(); /// Segment or point. if (outer.size() < 4) return false; diff --git a/dbms/src/Functions/IFunction.cpp b/dbms/src/Functions/IFunction.cpp index 6b6186302f7..ac5d1122e4a 100644 --- a/dbms/src/Functions/IFunction.cpp +++ b/dbms/src/Functions/IFunction.cpp @@ -512,8 +512,8 @@ static std::optional removeNullables(const DataTypes & types) if (!typeid_cast(type.get())) continue; DataTypes filtered; - for (const auto & type : types) - filtered.emplace_back(removeNullable(type)); + for (const auto & sub_type : types) + filtered.emplace_back(removeNullable(sub_type)); return filtered; } return {}; diff --git a/dbms/src/Functions/LowerUpperImpl.h b/dbms/src/Functions/LowerUpperImpl.h index 5fb58587321..a3900694251 100644 --- a/dbms/src/Functions/LowerUpperImpl.h +++ b/dbms/src/Functions/LowerUpperImpl.h @@ -28,7 +28,7 @@ private: { const auto flip_case_mask = 'A' ^ 'a'; -#if __SSE2__ +#ifdef __SSE2__ const auto bytes_sse = sizeof(__m128i); const auto src_end_sse = src_end - (src_end - src) % bytes_sse; diff --git a/dbms/src/Functions/LowerUpperUTF8Impl.h b/dbms/src/Functions/LowerUpperUTF8Impl.h index dd1e87efead..9daff1fd1cd 100644 --- a/dbms/src/Functions/LowerUpperUTF8Impl.h +++ b/dbms/src/Functions/LowerUpperUTF8Impl.h @@ -2,7 +2,7 @@ #include #include -#if __SSE2__ +#ifdef __SSE2__ #include #endif @@ -165,7 +165,7 @@ private: static void array(const UInt8 * src, const UInt8 * src_end, UInt8 * dst) { -#if __SSE2__ +#ifdef __SSE2__ static constexpr auto bytes_sse = sizeof(__m128i); auto src_end_sse = src + (src_end - src) / bytes_sse * bytes_sse; diff --git a/dbms/src/Functions/array.cpp b/dbms/src/Functions/array.cpp index 5205eff6b5f..6d641d13a69 100644 --- a/dbms/src/Functions/array.cpp +++ b/dbms/src/Functions/array.cpp @@ -58,7 +58,7 @@ public: */ Columns columns_holder(num_elements); - const IColumn * columns[num_elements]; + ColumnRawPtrs columns(num_elements); for (size_t i = 0; i < num_elements; ++i) { diff --git a/dbms/src/Functions/arrayEnumerateExtended.h b/dbms/src/Functions/arrayEnumerateExtended.h index a0ceed86b4a..9d9c52ab6ea 100644 --- a/dbms/src/Functions/arrayEnumerateExtended.h +++ b/dbms/src/Functions/arrayEnumerateExtended.h @@ -328,7 +328,7 @@ bool FunctionArrayEnumerateExtended::execute128bit( size_t rank = 0; for (size_t j = prev_off; j < off; ++j) { - auto &idx = indices[packFixed(j, count, columns, key_sizes)];; + auto & idx = indices[packFixed(j, count, columns, key_sizes)]; if (!idx) idx = ++rank; res_values[j] = idx; diff --git a/dbms/src/Functions/arrayFirst.cpp b/dbms/src/Functions/arrayFirst.cpp index bd2118b4e97..96b6109dc56 100644 --- a/dbms/src/Functions/arrayFirst.cpp +++ b/dbms/src/Functions/arrayFirst.cpp @@ -34,6 +34,7 @@ struct ArrayFirstImpl const auto & offsets = array.getOffsets(); const auto & data = array.getData(); auto out = data.cloneEmpty(); + out->reserve(data.size()); size_t pos{}; for (size_t i = 0; i < offsets.size(); ++i) @@ -60,6 +61,7 @@ struct ArrayFirstImpl const auto & offsets = array.getOffsets(); const auto & data = array.getData(); auto out = data.cloneEmpty(); + out->reserve(data.size()); size_t pos{}; for (size_t i = 0; i < offsets.size(); ++i) diff --git a/dbms/src/Functions/arrayReduce.cpp b/dbms/src/Functions/arrayReduce.cpp index 31aa4d49f18..1bee774412d 100644 --- a/dbms/src/Functions/arrayReduce.cpp +++ b/dbms/src/Functions/arrayReduce.cpp @@ -132,9 +132,9 @@ void FunctionArrayReduce::executeImpl(Block & block, const ColumnNumbers & argum else if (const ColumnConst * const_arr = checkAndGetColumnConst(col)) { materialized_columns.emplace_back(const_arr->convertToFullColumn()); - const auto & arr = typeid_cast(*materialized_columns.back().get()); - aggregate_arguments_vec[i] = &arr.getData(); - offsets_i = &arr.getOffsets(); + const auto & materialized_arr = typeid_cast(*materialized_columns.back().get()); + aggregate_arguments_vec[i] = &materialized_arr.getData(); + offsets_i = &materialized_arr.getOffsets(); } else throw Exception("Illegal column " + col->getName() + " as argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); diff --git a/dbms/src/Functions/arrayReverse.cpp b/dbms/src/Functions/arrayReverse.cpp index c6a9248c2e4..87862a5e052 100644 --- a/dbms/src/Functions/arrayReverse.cpp +++ b/dbms/src/Functions/arrayReverse.cpp @@ -165,17 +165,17 @@ bool FunctionArrayReverse::executeFixedString(const IColumn & src_data, const Co if (const ColumnFixedString * src_data_concrete = checkAndGetColumn(&src_data)) { const size_t n = src_data_concrete->getN(); - const ColumnFixedString::Chars & src_data = src_data_concrete->getChars(); + const ColumnFixedString::Chars & src_data_chars = src_data_concrete->getChars(); ColumnFixedString::Chars & res_chars = typeid_cast(res_data).getChars(); size_t size = src_offsets.size(); - res_chars.resize(src_data.size()); + res_chars.resize(src_data_chars.size()); ColumnArray::Offset src_prev_offset = 0; for (size_t i = 0; i < size; ++i) { - const UInt8 * src = &src_data[src_prev_offset * n]; - const UInt8 * src_end = &src_data[src_offsets[i] * n]; + const UInt8 * src = &src_data_chars[src_prev_offset * n]; + const UInt8 * src_end = &src_data_chars[src_offsets[i] * n]; if (src == src_end) continue; @@ -205,12 +205,12 @@ bool FunctionArrayReverse::executeString(const IColumn & src_data, const ColumnA const ColumnString::Offsets & src_string_offsets = src_data_concrete->getOffsets(); ColumnString::Offsets & res_string_offsets = typeid_cast(res_data).getOffsets(); - const ColumnString::Chars & src_data = src_data_concrete->getChars(); + const ColumnString::Chars & src_data_chars = src_data_concrete->getChars(); ColumnString::Chars & res_chars = typeid_cast(res_data).getChars(); size_t size = src_array_offsets.size(); res_string_offsets.resize(src_string_offsets.size()); - res_chars.resize(src_data.size()); + res_chars.resize(src_data_chars.size()); ColumnArray::Offset src_array_prev_offset = 0; ColumnString::Offset res_string_prev_offset = 0; @@ -228,7 +228,7 @@ bool FunctionArrayReverse::executeString(const IColumn & src_data, const ColumnA auto src_pos = src_string_offsets[src_array_prev_offset + j_reversed - 1]; size_t string_size = src_string_offsets[src_array_prev_offset + j_reversed] - src_pos; - memcpySmallAllowReadWriteOverflow15(&res_chars[res_string_prev_offset], &src_data[src_pos], string_size); + memcpySmallAllowReadWriteOverflow15(&res_chars[res_string_prev_offset], &src_data_chars[src_pos], string_size); res_string_prev_offset += string_size; res_string_offsets[src_array_prev_offset + j] = res_string_prev_offset; diff --git a/dbms/src/Functions/dateDiff.cpp b/dbms/src/Functions/dateDiff.cpp index bb309f0d8b9..b056e30c576 100644 --- a/dbms/src/Functions/dateDiff.cpp +++ b/dbms/src/Functions/dateDiff.cpp @@ -125,14 +125,14 @@ private: const DateLUTImpl & timezone_x, const DateLUTImpl & timezone_y, ColumnInt64::Container & result) { - if (auto * x_vec = checkAndGetColumn(&x)) - dispatchForSecondColumn(*x_vec, y, timezone_x, timezone_y, result); - else if (auto * x_vec = checkAndGetColumn(&x)) - dispatchForSecondColumn(*x_vec, y, timezone_x, timezone_y, result); - else if (auto * x_const = checkAndGetColumnConst(&x)) - dispatchConstForSecondColumn(x_const->getValue(), y, timezone_x, timezone_y, result); - else if (auto * x_const = checkAndGetColumnConst(&x)) - dispatchConstForSecondColumn(x_const->getValue(), y, timezone_x, timezone_y, result); + if (auto * x_vec_16 = checkAndGetColumn(&x)) + dispatchForSecondColumn(*x_vec_16, y, timezone_x, timezone_y, result); + else if (auto * x_vec_32 = checkAndGetColumn(&x)) + dispatchForSecondColumn(*x_vec_32, y, timezone_x, timezone_y, result); + else if (auto * x_const_16 = checkAndGetColumnConst(&x)) + dispatchConstForSecondColumn(x_const_16->getValue(), y, timezone_x, timezone_y, result); + else if (auto * x_const_32 = checkAndGetColumnConst(&x)) + dispatchConstForSecondColumn(x_const_32->getValue(), y, timezone_x, timezone_y, result); else throw Exception("Illegal column for first argument of function " + getName() + ", must be Date or DateTime", ErrorCodes::ILLEGAL_COLUMN); } @@ -143,14 +143,14 @@ private: const DateLUTImpl & timezone_x, const DateLUTImpl & timezone_y, ColumnInt64::Container & result) { - if (auto * y_vec = checkAndGetColumn(&y)) - vector_vector(x, *y_vec, timezone_x, timezone_y, result); - else if (auto * y_vec = checkAndGetColumn(&y)) - vector_vector(x, *y_vec, timezone_x, timezone_y, result); - else if (auto * y_const = checkAndGetColumnConst(&y)) - vector_constant(x, y_const->getValue(), timezone_x, timezone_y, result); - else if (auto * y_const = checkAndGetColumnConst(&y)) - vector_constant(x, y_const->getValue(), timezone_x, timezone_y, result); + if (auto * y_vec_16 = checkAndGetColumn(&y)) + vector_vector(x, *y_vec_16, timezone_x, timezone_y, result); + else if (auto * y_vec_32 = checkAndGetColumn(&y)) + vector_vector(x, *y_vec_32, timezone_x, timezone_y, result); + else if (auto * y_const_16 = checkAndGetColumnConst(&y)) + vector_constant(x, y_const_16->getValue(), timezone_x, timezone_y, result); + else if (auto * y_const_32 = checkAndGetColumnConst(&y)) + vector_constant(x, y_const_32->getValue(), timezone_x, timezone_y, result); else throw Exception("Illegal column for second argument of function " + getName() + ", must be Date or DateTime", ErrorCodes::ILLEGAL_COLUMN); } @@ -161,10 +161,10 @@ private: const DateLUTImpl & timezone_x, const DateLUTImpl & timezone_y, ColumnInt64::Container & result) { - if (auto * y_vec = checkAndGetColumn(&y)) - constant_vector(x, *y_vec, timezone_x, timezone_y, result); - else if (auto * y_vec = checkAndGetColumn(&y)) - constant_vector(x, *y_vec, timezone_x, timezone_y, result); + if (auto * y_vec_16 = checkAndGetColumn(&y)) + constant_vector(x, *y_vec_16, timezone_x, timezone_y, result); + else if (auto * y_vec_32 = checkAndGetColumn(&y)) + constant_vector(x, *y_vec_32, timezone_x, timezone_y, result); else throw Exception("Illegal column for second argument of function " + getName() + ", must be Date or DateTime", ErrorCodes::ILLEGAL_COLUMN); } diff --git a/dbms/src/Functions/emptyArrayToSingle.cpp b/dbms/src/Functions/emptyArrayToSingle.cpp index 4ef54f6665a..ae073935c27 100644 --- a/dbms/src/Functions/emptyArrayToSingle.cpp +++ b/dbms/src/Functions/emptyArrayToSingle.cpp @@ -82,12 +82,12 @@ namespace { if (const ColumnVector * src_data_concrete = checkAndGetColumn>(&src_data)) { - const PaddedPODArray & src_data = src_data_concrete->getData(); + const PaddedPODArray & src_data_vec = src_data_concrete->getData(); PaddedPODArray & res_data = static_cast &>(res_data_col).getData(); size_t size = src_offsets.size(); res_offsets.resize(size); - res_data.reserve(src_data.size()); + res_data.reserve(src_data_vec.size()); if (nullable) res_null_map->reserve(src_null_map->size()); @@ -101,7 +101,7 @@ namespace { size_t size_to_write = src_offsets[i] - src_prev_offset; res_data.resize(res_prev_offset + size_to_write); - memcpy(&res_data[res_prev_offset], &src_data[src_prev_offset], size_to_write * sizeof(T)); + memcpy(&res_data[res_prev_offset], &src_data_vec[src_prev_offset], size_to_write * sizeof(T)); if (nullable) { @@ -142,7 +142,7 @@ namespace if (const ColumnFixedString * src_data_concrete = checkAndGetColumn(&src_data)) { const size_t n = src_data_concrete->getN(); - const ColumnFixedString::Chars & src_data = src_data_concrete->getChars(); + const ColumnFixedString::Chars & src_data_vec = src_data_concrete->getChars(); auto concrete_res_data = typeid_cast(&res_data_col); if (!concrete_res_data) @@ -151,7 +151,7 @@ namespace ColumnFixedString::Chars & res_data = concrete_res_data->getChars(); size_t size = src_offsets.size(); res_offsets.resize(size); - res_data.reserve(src_data.size()); + res_data.reserve(src_data_vec.size()); if (nullable) res_null_map->reserve(src_null_map->size()); @@ -166,7 +166,7 @@ namespace size_t size_to_write = src_offsets[i] - src_prev_offset; size_t prev_res_data_size = res_data.size(); res_data.resize(prev_res_data_size + size_to_write * n); - memcpy(&res_data[prev_res_data_size], &src_data[src_prev_offset * n], size_to_write * n); + memcpy(&res_data[prev_res_data_size], &src_data_vec[src_prev_offset * n], size_to_write * n); if (nullable) { @@ -215,7 +215,7 @@ namespace throw Exception{"Internal error", ErrorCodes::LOGICAL_ERROR}; ColumnString::Offsets & res_string_offsets = concrete_res_string_offsets->getOffsets(); - const ColumnString::Chars & src_data = src_data_concrete->getChars(); + const ColumnString::Chars & src_data_vec = src_data_concrete->getChars(); auto concrete_res_data = typeid_cast(&res_data_col); if (!concrete_res_data) @@ -225,7 +225,7 @@ namespace size_t size = src_array_offsets.size(); res_array_offsets.resize(size); res_string_offsets.reserve(src_string_offsets.size()); - res_data.reserve(src_data.size()); + res_data.reserve(src_data_vec.size()); if (nullable) res_null_map->reserve(src_null_map->size()); @@ -257,7 +257,7 @@ namespace size_t res_data_old_size = res_data.size(); res_data.resize(res_data_old_size + bytes_to_copy); - memcpy(&res_data[res_data_old_size], &src_data[src_string_prev_offset], bytes_to_copy); + memcpy(&res_data[res_data_old_size], &src_data_vec[src_string_prev_offset], bytes_to_copy); if (nullable) { diff --git a/dbms/src/Functions/getSizeOfEnumType.cpp b/dbms/src/Functions/getSizeOfEnumType.cpp index 59f2144446f..11a22ecddfb 100644 --- a/dbms/src/Functions/getSizeOfEnumType.cpp +++ b/dbms/src/Functions/getSizeOfEnumType.cpp @@ -50,10 +50,10 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - if (auto type = checkAndGetDataType(block.getByPosition(arguments[0]).type.get())) - block.getByPosition(result).column = DataTypeUInt8().createColumnConst(input_rows_count, type->getValues().size()); - else if (auto type = checkAndGetDataType(block.getByPosition(arguments[0]).type.get())) - block.getByPosition(result).column = DataTypeUInt16().createColumnConst(input_rows_count, type->getValues().size()); + if (auto type8 = checkAndGetDataType(block.getByPosition(arguments[0]).type.get())) + block.getByPosition(result).column = DataTypeUInt8().createColumnConst(input_rows_count, type8->getValues().size()); + else if (auto type16 = checkAndGetDataType(block.getByPosition(arguments[0]).type.get())) + block.getByPosition(result).column = DataTypeUInt16().createColumnConst(input_rows_count, type16->getValues().size()); else throw Exception("The argument for function " + getName() + " must be Enum", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } diff --git a/dbms/src/Functions/hasColumnInTable.cpp b/dbms/src/Functions/hasColumnInTable.cpp index 1039cd1b70b..9c8017497e3 100644 --- a/dbms/src/Functions/hasColumnInTable.cpp +++ b/dbms/src/Functions/hasColumnInTable.cpp @@ -132,7 +132,7 @@ void FunctionHasColumnInTable::executeImpl(Block & block, const ColumnNumbers & has_column = remote_columns.hasPhysical(column_name); } - block.getByPosition(result).column = DataTypeUInt8().createColumnConst(input_rows_count, has_column); + block.getByPosition(result).column = DataTypeUInt8().createColumnConst(input_rows_count, Field(has_column)); } diff --git a/dbms/src/Functions/intDiv.cpp b/dbms/src/Functions/intDiv.cpp index 9a15b140f72..bbeb2fd6f14 100644 --- a/dbms/src/Functions/intDiv.cpp +++ b/dbms/src/Functions/intDiv.cpp @@ -2,7 +2,7 @@ #include #include -#if __SSE2__ +#ifdef __SSE2__ #define LIBDIVIDE_USE_SSE2 1 #endif @@ -45,7 +45,7 @@ struct DivideIntegralByConstantImpl const A * a_end = a_pos + size; ResultType * c_pos = c.data(); -#if __SSE2__ +#ifdef __SSE2__ static constexpr size_t values_per_sse_register = 16 / sizeof(A); const A * a_end_sse = a_pos + size / values_per_sse_register * values_per_sse_register; diff --git a/dbms/src/Functions/modulo.cpp b/dbms/src/Functions/modulo.cpp index 640c2e56194..b308a820421 100644 --- a/dbms/src/Functions/modulo.cpp +++ b/dbms/src/Functions/modulo.cpp @@ -1,7 +1,7 @@ #include #include -#if __SSE2__ +#ifdef __SSE2__ #define LIBDIVIDE_USE_SSE2 1 #endif diff --git a/dbms/src/Functions/reverse.cpp b/dbms/src/Functions/reverse.cpp index b7447a7882b..d8e6c6de194 100644 --- a/dbms/src/Functions/reverse.cpp +++ b/dbms/src/Functions/reverse.cpp @@ -97,10 +97,10 @@ public: ReverseImpl::vector(col->getChars(), col->getOffsets(), col_res->getChars(), col_res->getOffsets()); block.getByPosition(result).column = std::move(col_res); } - else if (const ColumnFixedString * col = checkAndGetColumn(column.get())) + else if (const ColumnFixedString * col_fixed = checkAndGetColumn(column.get())) { - auto col_res = ColumnFixedString::create(col->getN()); - ReverseImpl::vector_fixed(col->getChars(), col->getN(), col_res->getChars()); + auto col_res = ColumnFixedString::create(col_fixed->getN()); + ReverseImpl::vector_fixed(col_fixed->getChars(), col_fixed->getN(), col_res->getChars()); block.getByPosition(result).column = std::move(col_res); } else diff --git a/dbms/src/Functions/runningDifference.h b/dbms/src/Functions/runningDifference.h index 4059d6f3ada..5a2e8051a21 100644 --- a/dbms/src/Functions/runningDifference.h +++ b/dbms/src/Functions/runningDifference.h @@ -53,13 +53,16 @@ private: /// It is possible to SIMD optimize this loop. By no need for that in practice. - Src prev; + Src prev{}; bool has_prev_value = false; for (size_t i = 0; i < size; ++i) { if (null_map && (*null_map)[i]) + { + dst[i] = Dst{}; continue; + } if (!has_prev_value) { diff --git a/dbms/src/Functions/substring.cpp b/dbms/src/Functions/substring.cpp index 7263f3ec595..5fdfb3e6870 100644 --- a/dbms/src/Functions/substring.cpp +++ b/dbms/src/Functions/substring.cpp @@ -148,15 +148,15 @@ public: if (const ColumnString * col = checkAndGetColumn(column_string.get())) executeForSource(column_start, column_length, column_start_const, column_length_const, start_value, length_value, block, result, StringSource(*col), input_rows_count); - else if (const ColumnFixedString * col = checkAndGetColumn(column_string.get())) + else if (const ColumnFixedString * col_fixed = checkAndGetColumn(column_string.get())) executeForSource(column_start, column_length, column_start_const, column_length_const, start_value, - length_value, block, result, FixedStringSource(*col), input_rows_count); - else if (const ColumnConst * col = checkAndGetColumnConst(column_string.get())) + length_value, block, result, FixedStringSource(*col_fixed), input_rows_count); + else if (const ColumnConst * col_const = checkAndGetColumnConst(column_string.get())) executeForSource(column_start, column_length, column_start_const, column_length_const, start_value, - length_value, block, result, ConstSource(*col), input_rows_count); - else if (const ColumnConst * col = checkAndGetColumnConst(column_string.get())) + length_value, block, result, ConstSource(*col_const), input_rows_count); + else if (const ColumnConst * col_const_fixed = checkAndGetColumnConst(column_string.get())) executeForSource(column_start, column_length, column_start_const, column_length_const, start_value, - length_value, block, result, ConstSource(*col), input_rows_count); + length_value, block, result, ConstSource(*col_const_fixed), input_rows_count); else throw Exception( "Illegal column " + block.getByPosition(arguments[0]).column->getName() + " of first argument of function " + getName(), diff --git a/dbms/src/Functions/transform.cpp b/dbms/src/Functions/transform.cpp index 83aa379a3ba..6ec5729e1be 100644 --- a/dbms/src/Functions/transform.cpp +++ b/dbms/src/Functions/transform.cpp @@ -742,7 +742,7 @@ private: if (0 == size) throw Exception{"Empty arrays are illegal in function " + getName(), ErrorCodes::BAD_ARGUMENTS}; - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); if (initialized) return; diff --git a/dbms/src/Functions/trim.cpp b/dbms/src/Functions/trim.cpp index ff8c7e536ce..f2e2543cc90 100644 --- a/dbms/src/Functions/trim.cpp +++ b/dbms/src/Functions/trim.cpp @@ -2,7 +2,7 @@ #include #include -#if __SSE4_2__ +#ifdef __SSE4_2__ #include #endif @@ -80,7 +80,7 @@ private: size_t chars_to_trim_left = 0; size_t chars_to_trim_right = 0; char whitespace = ' '; -#if __SSE4_2__ +#ifdef __SSE4_2__ const auto bytes_sse = sizeof(__m128i); const auto size_sse = size - (size % bytes_sse); const auto whitespace_mask = _mm_set1_epi8(whitespace); @@ -90,7 +90,7 @@ private: if constexpr (mode::trim_left) { -#if __SSE4_2__ +#ifdef __SSE4_2__ /// skip whitespace from left in blocks of up to 16 characters /// Avoid gcc bug: _mm_cmpistri: error: the third argument must be an 8-bit immediate @@ -110,7 +110,7 @@ private: if constexpr (mode::trim_right) { const auto trim_right_size = size - chars_to_trim_left; -#if __SSE4_2__ +#ifdef __SSE4_2__ /// try to skip whitespace from right in blocks of up to 16 characters /// Avoid gcc bug: _mm_cmpistri: error: the third argument must be an 8-bit immediate diff --git a/dbms/src/IO/AIOContextPool.cpp b/dbms/src/IO/AIOContextPool.cpp index 3437f53ff81..65161fcc600 100644 --- a/dbms/src/IO/AIOContextPool.cpp +++ b/dbms/src/IO/AIOContextPool.cpp @@ -78,7 +78,7 @@ void AIOContextPool::fulfillPromises(const io_event events[], const int num_even if (num_events == 0) return; - const std::lock_guard lock{mutex}; + const std::lock_guard lock{mutex}; /// look at returned events and find corresponding promise, set result and erase promise from map for (const auto & event : boost::make_iterator_range(events, events + num_events)) @@ -114,7 +114,7 @@ void AIOContextPool::notifyProducers(const int num_producers) const void AIOContextPool::reportExceptionToAnyProducer() { - const std::lock_guard lock{mutex}; + const std::lock_guard lock{mutex}; const auto any_promise_it = std::begin(promises); any_promise_it->second.set_exception(std::current_exception()); @@ -123,7 +123,7 @@ void AIOContextPool::reportExceptionToAnyProducer() std::future AIOContextPool::post(struct iocb & iocb) { - std::unique_lock lock{mutex}; + std::unique_lock lock{mutex}; /// get current id and increment it by one const auto request_id = next_id; diff --git a/dbms/src/IO/HTTPCommon.cpp b/dbms/src/IO/HTTPCommon.cpp index 8e6d52738ea..fba217ae44a 100644 --- a/dbms/src/IO/HTTPCommon.cpp +++ b/dbms/src/IO/HTTPCommon.cpp @@ -141,7 +141,7 @@ namespace public: Entry getSession(const Poco::URI & uri, const ConnectionTimeouts & timeouts, size_t max_connections_per_endpoint) { - std::unique_lock lock(mutex); + std::unique_lock lock(mutex); const std::string & host = uri.getHost(); UInt16 port = uri.getPort(); bool https = isHTTPS(uri); diff --git a/dbms/src/IO/HashingReadBuffer.h b/dbms/src/IO/HashingReadBuffer.h index 0bf1614f884..9ea646206cd 100644 --- a/dbms/src/IO/HashingReadBuffer.h +++ b/dbms/src/IO/HashingReadBuffer.h @@ -12,8 +12,8 @@ namespace DB class HashingReadBuffer : public IHashingBuffer { public: - HashingReadBuffer(ReadBuffer & in_, size_t block_size = DBMS_DEFAULT_HASHING_BLOCK_SIZE) : - IHashingBuffer(block_size), in(in_) + HashingReadBuffer(ReadBuffer & in_, size_t block_size_ = DBMS_DEFAULT_HASHING_BLOCK_SIZE) : + IHashingBuffer(block_size_), in(in_) { working_buffer = in.buffer(); pos = in.position(); diff --git a/dbms/src/IO/Progress.h b/dbms/src/IO/Progress.h index c21196befac..7dca03f03c2 100644 --- a/dbms/src/IO/Progress.h +++ b/dbms/src/IO/Progress.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include diff --git a/dbms/src/IO/ReadBufferFromFile.cpp b/dbms/src/IO/ReadBufferFromFile.cpp index f3b4ddb7d23..b94fce8e033 100644 --- a/dbms/src/IO/ReadBufferFromFile.cpp +++ b/dbms/src/IO/ReadBufferFromFile.cpp @@ -53,14 +53,14 @@ ReadBufferFromFile::ReadBufferFromFile( ReadBufferFromFile::ReadBufferFromFile( - int fd, + int fd_, const std::string & original_file_name, size_t buf_size, char * existing_memory, size_t alignment) : - ReadBufferFromFileDescriptor(fd, buf_size, existing_memory, alignment), - file_name(original_file_name.empty() ? "(fd = " + toString(fd) + ")" : original_file_name) + ReadBufferFromFileDescriptor(fd_, buf_size, existing_memory, alignment), + file_name(original_file_name.empty() ? "(fd = " + toString(fd_) + ")" : original_file_name) { } diff --git a/dbms/src/IO/ReadHelpers.cpp b/dbms/src/IO/ReadHelpers.cpp index c1d8754c1b1..ee6f163bbe6 100644 --- a/dbms/src/IO/ReadHelpers.cpp +++ b/dbms/src/IO/ReadHelpers.cpp @@ -11,7 +11,7 @@ #include #include -#if __SSE2__ +#ifdef __SSE2__ #include #endif @@ -558,7 +558,7 @@ void readCSVStringInto(Vector & s, ReadBuffer & buf, const FormatSettings::CSV & [&]() { -#if __SSE2__ +#ifdef __SSE2__ auto rc = _mm_set1_epi8('\r'); auto nc = _mm_set1_epi8('\n'); auto dc = _mm_set1_epi8(delimiter); diff --git a/dbms/src/IO/ReadWriteBufferFromHTTP.h b/dbms/src/IO/ReadWriteBufferFromHTTP.h index 215aa29a3e0..092d6829107 100644 --- a/dbms/src/IO/ReadWriteBufferFromHTTP.h +++ b/dbms/src/IO/ReadWriteBufferFromHTTP.h @@ -96,13 +96,13 @@ class ReadWriteBufferFromHTTP : public detail::ReadWriteBufferFromHTTPBase; public: - explicit ReadWriteBufferFromHTTP(Poco::URI uri, - const std::string & method = {}, + explicit ReadWriteBufferFromHTTP(Poco::URI uri_, + const std::string & method_ = {}, OutStreamCallback out_stream_callback = {}, const ConnectionTimeouts & timeouts = {}, const Poco::Net::HTTPBasicCredentials & credentials = {}, size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE) - : Parent(makeHTTPSession(uri, timeouts), uri, method, out_stream_callback, credentials, buffer_size_) + : Parent(makeHTTPSession(uri_, timeouts), uri_, method_, out_stream_callback, credentials, buffer_size_) { } }; @@ -111,16 +111,16 @@ class PooledReadWriteBufferFromHTTP : public detail::ReadWriteBufferFromHTTPBase using Parent = detail::ReadWriteBufferFromHTTPBase; public: - explicit PooledReadWriteBufferFromHTTP(Poco::URI uri, - const std::string & method = {}, + explicit PooledReadWriteBufferFromHTTP(Poco::URI uri_, + const std::string & method_ = {}, OutStreamCallback out_stream_callback = {}, const ConnectionTimeouts & timeouts = {}, const Poco::Net::HTTPBasicCredentials & credentials = {}, size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE, size_t max_connections_per_endpoint = DEFAULT_COUNT_OF_HTTP_CONNECTIONS_PER_ENDPOINT) - : Parent(makePooledHTTPSession(uri, timeouts, max_connections_per_endpoint), - uri, - method, + : Parent(makePooledHTTPSession(uri_, timeouts, max_connections_per_endpoint), + uri_, + method_, out_stream_callback, credentials, buffer_size_) diff --git a/dbms/src/IO/WriteBufferFromFile.cpp b/dbms/src/IO/WriteBufferFromFile.cpp index 7299d651acc..3082f674fff 100644 --- a/dbms/src/IO/WriteBufferFromFile.cpp +++ b/dbms/src/IO/WriteBufferFromFile.cpp @@ -58,14 +58,14 @@ WriteBufferFromFile::WriteBufferFromFile( /// Use pre-opened file descriptor. WriteBufferFromFile::WriteBufferFromFile( - int fd, + int fd_, const std::string & original_file_name, size_t buf_size, char * existing_memory, size_t alignment) : - WriteBufferFromFileDescriptor(fd, buf_size, existing_memory, alignment), - file_name(original_file_name.empty() ? "(fd = " + toString(fd) + ")" : original_file_name) + WriteBufferFromFileDescriptor(fd_, buf_size, existing_memory, alignment), + file_name(original_file_name.empty() ? "(fd = " + toString(fd_) + ")" : original_file_name) { } diff --git a/dbms/src/IO/WriteBufferFromHTTPServerResponse.cpp b/dbms/src/IO/WriteBufferFromHTTPServerResponse.cpp index 9e8f3faed74..b655dff40bb 100644 --- a/dbms/src/IO/WriteBufferFromHTTPServerResponse.cpp +++ b/dbms/src/IO/WriteBufferFromHTTPServerResponse.cpp @@ -68,7 +68,7 @@ void WriteBufferFromHTTPServerResponse::finishSendHeaders() void WriteBufferFromHTTPServerResponse::nextImpl() { { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); startSendHeaders(); @@ -147,7 +147,7 @@ WriteBufferFromHTTPServerResponse::WriteBufferFromHTTPServerResponse( void WriteBufferFromHTTPServerResponse::onProgress(const Progress & progress) { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); /// Cannot add new headers if body was started to send. if (headers_finished_sending) @@ -181,7 +181,7 @@ void WriteBufferFromHTTPServerResponse::finalize() else { /// If no remaining data, just send headers. - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); startSendHeaders(); finishSendHeaders(); } diff --git a/dbms/src/IO/WriteBufferFromTemporaryFile.cpp b/dbms/src/IO/WriteBufferFromTemporaryFile.cpp index 9a47760b8aa..e1250c58097 100644 --- a/dbms/src/IO/WriteBufferFromTemporaryFile.cpp +++ b/dbms/src/IO/WriteBufferFromTemporaryFile.cpp @@ -16,9 +16,7 @@ namespace ErrorCodes WriteBufferFromTemporaryFile::WriteBufferFromTemporaryFile(std::unique_ptr && tmp_file_) -: -WriteBufferFromFile(tmp_file_->path(), DBMS_DEFAULT_BUFFER_SIZE, O_RDWR | O_TRUNC | O_CREAT, 0600), -tmp_file(std::move(tmp_file_)) + : WriteBufferFromFile(tmp_file_->path(), DBMS_DEFAULT_BUFFER_SIZE, O_RDWR | O_TRUNC | O_CREAT, 0600), tmp_file(std::move(tmp_file_)) {} @@ -34,7 +32,6 @@ WriteBufferFromTemporaryFile::Ptr WriteBufferFromTemporaryFile::create(const std class ReadBufferFromTemporaryWriteBuffer : public ReadBufferFromFile { public: - static ReadBufferPtr createFrom(WriteBufferFromTemporaryFile * origin) { int fd = origin->getFD(); @@ -47,8 +44,8 @@ public: return std::make_shared(fd, file_name, std::move(origin->tmp_file)); } - ReadBufferFromTemporaryWriteBuffer(int fd, const std::string & file_name, std::unique_ptr && tmp_file_) - : ReadBufferFromFile(fd, file_name), tmp_file(std::move(tmp_file_)) + ReadBufferFromTemporaryWriteBuffer(int fd_, const std::string & file_name_, std::unique_ptr && tmp_file_) + : ReadBufferFromFile(fd_, file_name_), tmp_file(std::move(tmp_file_)) {} std::unique_ptr tmp_file; diff --git a/dbms/src/IO/WriteBufferValidUTF8.cpp b/dbms/src/IO/WriteBufferValidUTF8.cpp index 0567b4842a9..00673986e8a 100644 --- a/dbms/src/IO/WriteBufferValidUTF8.cpp +++ b/dbms/src/IO/WriteBufferValidUTF8.cpp @@ -2,7 +2,7 @@ #include #include -#if __SSE2__ +#ifdef __SSE2__ #include #endif @@ -69,7 +69,7 @@ void WriteBufferValidUTF8::nextImpl() while (p < pos) { -#if __SSE2__ +#ifdef __SSE2__ /// Fast skip of ASCII static constexpr size_t SIMD_BYTES = 16; const char * simd_end = p + (pos - p) / SIMD_BYTES * SIMD_BYTES; diff --git a/dbms/src/IO/WriteIntText.h b/dbms/src/IO/WriteIntText.h index 80449091f50..c947f331442 100644 --- a/dbms/src/IO/WriteIntText.h +++ b/dbms/src/IO/WriteIntText.h @@ -2,221 +2,33 @@ #include #include - -#include - #include #include +#include -/// 20 digits or 19 digits and a sign -#define WRITE_HELPERS_MAX_INT_WIDTH 20U - +/// 40 digits or 39 digits and a sign +#define WRITE_HELPERS_MAX_INT_WIDTH 40U namespace DB { - namespace detail { - /** See: - * https://github.com/localvoid/cxx-benchmark-itoa - * http://www.slideshare.net/andreialexandrescu1/three-optimization-tips-for-c-15708507 - * http://vimeo.com/55639112 - */ - - - /// The usual way, if there is not enough space in the buffer for any number to fit there. template - void writeUIntTextFallback(T x, WriteBuffer & buf) + void NO_INLINE writeUIntTextFallback(T x, WriteBuffer & buf) { - if (x == 0) - { - buf.nextIfAtEnd(); - *buf.position() = '0'; - ++buf.position(); - - return; - } - char tmp[WRITE_HELPERS_MAX_INT_WIDTH]; - - char * pos; - for (pos = tmp + WRITE_HELPERS_MAX_INT_WIDTH - 1; x != 0; --pos) - { - *pos = '0' + x % 10; - x /= 10; - } - - ++pos; - - buf.write(pos, tmp + WRITE_HELPERS_MAX_INT_WIDTH - pos); + int len = itoa(x, tmp) - tmp; + buf.write(tmp, len); } - - - /** Count the number of decimal digits in the number. - * Works well for nonuniform distribution of numbers, which usually happens. - * If most of the numbers are long, then a "branchless" code with a `bsr` instruction and a smart conversion would work better. - */ - template - UInt32 digits10(T x) - { - if (x < 10ULL) - return 1; - if (x < 100ULL) - return 2; - if (x < 1000ULL) - return 3; - - if (x < 1000000000000ULL) - { - if (x < 100000000ULL) - { - if (x < 1000000ULL) - { - if (x < 10000ULL) - return 4; - else - return 5 + (x >= 100000ULL); - } - - return 7 + (x >= 10000000ULL); - } - - if (x < 10000000000ULL) - return 9 + (x >= 1000000000ULL); - - return 11 + (x >= 100000000000ULL); - } - - return 12 + digits10(x / 1000000000000ULL); - } - - - /** Converts two digits per iteration. - * Works well for the nonuniform distribution of numbers, which usually happens. - * If most of the numbers are long, and if you do care about the cache, then the variant - * with a large table and four digits per iteration. - */ - template - UInt32 writeUIntText(T x, char * dst) - { - static const char digits[201] = - "00010203040506070809" - "10111213141516171819" - "20212223242526272829" - "30313233343536373839" - "40414243444546474849" - "50515253545556575859" - "60616263646566676869" - "70717273747576777879" - "80818283848586878889" - "90919293949596979899"; - - const UInt32 length = digits10(x); - UInt32 next = length - 1; - - while (x >= 100) - { - const auto i = (x % 100) * 2; - x /= 100; - dst[next] = digits[i + 1]; - dst[next - 1] = digits[i]; - next -= 2; - } - - if (x < 10) - { - dst[next] = '0' + x; - } - else - { - const auto i = x * 2; - dst[next] = digits[i + 1]; - dst[next - 1] = digits[i]; - } - - return length; - } - - - /** If there is enough space in the buffer - calls an optimized version, otherwise - the normal version. - */ - template - void writeUIntText(T x, WriteBuffer & buf) - { - if (likely(buf.position() + WRITE_HELPERS_MAX_INT_WIDTH < buf.buffer().end())) - buf.position() += writeUIntText(x, buf.position()); - else - writeUIntTextFallback(x, buf); - } - - - inline void writeLeadingMinus(WriteBuffer & buf) - { - buf.nextIfAtEnd(); - *buf.position() = '-'; - ++buf.position(); - } - - /** Wrapper for signed numbers. - */ - template - void writeSIntText(T x, WriteBuffer & buf) - { - /// A special case for the smallest negative number - if (unlikely(x == std::numeric_limits::min())) - { - if (sizeof(x) == 1) - buf.write("-128", 4); - else if (sizeof(x) == 2) - buf.write("-32768", 6); - else if (sizeof(x) == 4) - buf.write("-2147483648", 11); - else - buf.write("-9223372036854775808", 20); - return; - } - - if (x < 0) - { - x = -x; - writeLeadingMinus(buf); - } - - writeUIntText(static_cast>(x), buf); - } - - inline void writeSIntText(__int128 x, WriteBuffer & buf) - { - static const __int128 min_int128 = __int128(0x8000000000000000ll) << 64; - - if (unlikely(x == min_int128)) - { - buf.write("-170141183460469231731687303715884105728", 40); - return; - } - - if (x < 0) - { - x = -x; - writeLeadingMinus(buf); - } - - writeUIntText(static_cast(x), buf); - } -} - - -template -std::enable_if_t || std::is_same_v, void> writeIntText(T x, WriteBuffer & buf) -{ - detail::writeSIntText(x, buf); - } template -std::enable_if_t, void> writeIntText(T x, WriteBuffer & buf) +void writeIntText(T x, WriteBuffer & buf) { - detail::writeUIntText(x, buf); + if (likely(buf.position() + WRITE_HELPERS_MAX_INT_WIDTH < buf.buffer().end())) + buf.position() = itoa(x, buf.position()); + else + detail::writeUIntTextFallback(x, buf); } } diff --git a/dbms/src/IO/ZlibDeflatingWriteBuffer.cpp b/dbms/src/IO/ZlibDeflatingWriteBuffer.cpp index 086c9e53a8d..0c994974ad1 100644 --- a/dbms/src/IO/ZlibDeflatingWriteBuffer.cpp +++ b/dbms/src/IO/ZlibDeflatingWriteBuffer.cpp @@ -93,12 +93,14 @@ void ZlibDeflatingWriteBuffer::finish() out.position() = out.buffer().end() - zstr.avail_out; if (rc == Z_STREAM_END) + { + finished = true; return; + } + if (rc != Z_OK) throw Exception(std::string("deflate finish failed: ") + zError(rc), ErrorCodes::ZLIB_DEFLATE_FAILED); } - - finished = true; } } diff --git a/dbms/src/IO/tests/CMakeLists.txt b/dbms/src/IO/tests/CMakeLists.txt index 59a1e0088c2..73497def442 100644 --- a/dbms/src/IO/tests/CMakeLists.txt +++ b/dbms/src/IO/tests/CMakeLists.txt @@ -19,9 +19,6 @@ target_link_libraries (valid_utf8_perf PRIVATE clickhouse_common_io) add_executable (valid_utf8 valid_utf8.cpp) target_link_libraries (valid_utf8 PRIVATE clickhouse_common_io) -add_executable (compressed_buffer compressed_buffer.cpp) -target_link_libraries (compressed_buffer PRIVATE clickhouse_common_io) - add_executable (var_uint var_uint.cpp) target_link_libraries (var_uint PRIVATE clickhouse_common_io) @@ -29,7 +26,7 @@ add_executable (read_escaped_string read_escaped_string.cpp) target_link_libraries (read_escaped_string PRIVATE clickhouse_common_io) add_executable (async_write async_write.cpp) -target_link_libraries (async_write PRIVATE clickhouse_common_io) +target_link_libraries (async_write PRIVATE clickhouse_compression clickhouse_common_io) add_executable (parse_int_perf parse_int_perf.cpp) target_link_libraries (parse_int_perf PRIVATE clickhouse_common_io) @@ -43,9 +40,6 @@ target_link_libraries (read_write_int PRIVATE clickhouse_common_io) add_executable (mempbrk mempbrk.cpp) target_link_libraries (mempbrk PRIVATE clickhouse_common_io) -add_executable (cached_compressed_read_buffer cached_compressed_read_buffer.cpp) -target_link_libraries (cached_compressed_read_buffer PRIVATE clickhouse_common_io dbms ) - add_executable (o_direct_and_dirty_pages o_direct_and_dirty_pages.cpp) target_link_libraries (o_direct_and_dirty_pages PRIVATE clickhouse_common_io) diff --git a/dbms/src/IO/tests/async_write.cpp b/dbms/src/IO/tests/async_write.cpp index 9cf1b2721cf..29361f2a8ab 100644 --- a/dbms/src/IO/tests/async_write.cpp +++ b/dbms/src/IO/tests/async_write.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include diff --git a/dbms/src/IO/tests/gtest_cascade_and_memory_write_buffer.cpp b/dbms/src/IO/tests/gtest_cascade_and_memory_write_buffer.cpp index 603c7f3f92c..ecf4054e122 100644 --- a/dbms/src/IO/tests/gtest_cascade_and_memory_write_buffer.cpp +++ b/dbms/src/IO/tests/gtest_cascade_and_memory_write_buffer.cpp @@ -1,6 +1,7 @@ #pragma GCC diagnostic ignored "-Wsign-compare" #ifdef __clang__ #pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" + #pragma clang diagnostic ignored "-Wundef" #endif #include diff --git a/dbms/src/IO/tests/hashing_buffer.h b/dbms/src/IO/tests/hashing_buffer.h index 82efc5a8760..a15bfbd73e7 100644 --- a/dbms/src/IO/tests/hashing_buffer.h +++ b/dbms/src/IO/tests/hashing_buffer.h @@ -1,7 +1,7 @@ #include #include -#define FAIL(msg) { std::cout << msg; exit(1); } +#define FAIL(msg) do { std::cout << msg; exit(1); } while (0) CityHash_v1_0_2::uint128 referenceHash(const char * data, size_t len) @@ -11,9 +11,7 @@ CityHash_v1_0_2::uint128 referenceHash(const char * data, size_t len) size_t pos; for (pos = 0; pos + block_size <= len; pos += block_size) - { state = CityHash_v1_0_2::CityHash128WithSeed(data + pos, block_size, state); - } if (pos < len) state = CityHash_v1_0_2::CityHash128WithSeed(data + pos, len - pos, state); diff --git a/dbms/src/IO/tests/hashing_read_buffer.cpp b/dbms/src/IO/tests/hashing_read_buffer.cpp index a9332707c4d..3ad85a8bc85 100644 --- a/dbms/src/IO/tests/hashing_read_buffer.cpp +++ b/dbms/src/IO/tests/hashing_read_buffer.cpp @@ -33,9 +33,7 @@ void test(size_t data_size) bool failed_to_read = false; for (size_t i = 0; i < data_size; ++i) if (read_buf[i] != vec[i]) - { failed_to_read = true; - } if (failed_to_read) { @@ -47,9 +45,7 @@ void test(size_t data_size) } if (buf.getHash() != reference) - { FAIL("failed on data size " << data_size << " reading by blocks of size " << read_buffer_block_size); - } if (buf.getHash() != out.getHash()) FAIL("Hash of HashingReadBuffer doesn't match with hash of HashingWriteBuffer on data size " << data_size << " reading by blocks of size " << read_buffer_block_size); } diff --git a/dbms/src/IO/tests/parse_int_perf.cpp b/dbms/src/IO/tests/parse_int_perf.cpp index e3b40927a85..8b3c516d6aa 100644 --- a/dbms/src/IO/tests/parse_int_perf.cpp +++ b/dbms/src/IO/tests/parse_int_perf.cpp @@ -8,8 +8,8 @@ #include #include #include -#include -#include +#include +#include #include #include diff --git a/dbms/src/IO/tests/read_float_perf.cpp b/dbms/src/IO/tests/read_float_perf.cpp index cb4c5e90a06..8a870d4a960 100644 --- a/dbms/src/IO/tests/read_float_perf.cpp +++ b/dbms/src/IO/tests/read_float_perf.cpp @@ -11,7 +11,7 @@ #include #include #include -#include +#include /** How to test: diff --git a/dbms/src/Interpreters/ActionLocksManager.cpp b/dbms/src/Interpreters/ActionLocksManager.cpp index ee64eb1e0e9..6fa44741925 100644 --- a/dbms/src/Interpreters/ActionLocksManager.cpp +++ b/dbms/src/Interpreters/ActionLocksManager.cpp @@ -33,7 +33,7 @@ void ActionLocksManager::add(StorageActionBlockType action_type) if (!action_lock.expired()) { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); storage_locks[table.get()][action_type] = std::move(action_lock); } }); @@ -47,7 +47,7 @@ void ActionLocksManager::add(const String & database_name, const String & table_ if (!action_lock.expired()) { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); storage_locks[table.get()][action_type] = std::move(action_lock); } } @@ -55,7 +55,7 @@ void ActionLocksManager::add(const String & database_name, const String & table_ void ActionLocksManager::remove(StorageActionBlockType action_type) { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); for (auto & storage_elem : storage_locks) storage_elem.second.erase(action_type); @@ -65,7 +65,7 @@ void ActionLocksManager::remove(const String & database_name, const String & tab { if (auto table = global_context.tryGetTable(database_name, table_name)) { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); if (storage_locks.count(table.get())) storage_locks[table.get()].erase(action_type); @@ -74,7 +74,7 @@ void ActionLocksManager::remove(const String & database_name, const String & tab void ActionLocksManager::cleanExpired() { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); for (auto it_storage = storage_locks.begin(); it_storage != storage_locks.end();) { diff --git a/dbms/src/Interpreters/ActionsVisitor.cpp b/dbms/src/Interpreters/ActionsVisitor.cpp index ef9f73ac194..0b36a572377 100644 --- a/dbms/src/Interpreters/ActionsVisitor.cpp +++ b/dbms/src/Interpreters/ActionsVisitor.cpp @@ -446,11 +446,11 @@ void ActionsVisitor::visit(const ASTPtr & ast) for (size_t j = 0; j < lambda_arg_asts.size(); ++j) { - ASTIdentifier * identifier = typeid_cast(lambda_arg_asts[j].get()); - if (!identifier) + ASTIdentifier * lambda_identifier = typeid_cast(lambda_arg_asts[j].get()); + if (!lambda_identifier) throw Exception("lambda argument declarations must be identifiers", ErrorCodes::TYPE_MISMATCH); - String arg_name = identifier->name; + String arg_name = lambda_identifier->name; lambda_arguments.emplace_back(arg_name, lambda_type->getArgumentTypes()[j]); } diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index d90da905c2e..145ce98dbbc 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include #include @@ -196,7 +196,7 @@ Aggregator::Aggregator(const Params & params_) void Aggregator::compileIfPossible(AggregatedDataVariants::Type type) { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); if (compiled_if_possible) return; @@ -598,10 +598,6 @@ void NO_INLINE Aggregator::executeImpl( executeImplCase(method, state, aggregates_pool, rows, key_columns, aggregate_instructions, keys, overflow_row); } -#ifndef __clang__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif template void NO_INLINE Aggregator::executeImplCase( @@ -617,7 +613,7 @@ void NO_INLINE Aggregator::executeImplCase( /// NOTE When editing this code, also pay attention to SpecializedAggregator.h. /// For all rows. - typename Method::Key prev_key; + typename Method::Key prev_key{}; AggregateDataPtr value = nullptr; for (size_t i = 0; i < rows; ++i) { @@ -707,9 +703,6 @@ void NO_INLINE Aggregator::executeImplCase( } } -#ifndef __clang__ -#pragma GCC diagnostic pop -#endif void NO_INLINE Aggregator::executeWithoutKeyImpl( AggregatedDataWithoutKey & res, @@ -966,7 +959,7 @@ void Aggregator::writeToTemporaryFile(AggregatedDataVariants & data_variants) double uncompressed_bytes = compressed_buf.count(); { - std::lock_guard lock(temporary_files.mutex); + std::lock_guard lock(temporary_files.mutex); temporary_files.files.emplace_back(std::move(file)); temporary_files.sum_size_uncompressed += uncompressed_bytes; temporary_files.sum_size_compressed += compressed_bytes; @@ -1819,7 +1812,7 @@ protected: while (true) { - std::unique_lock lock(parallel_merge_data->mutex); + std::unique_lock lock(parallel_merge_data->mutex); if (parallel_merge_data->exception) std::rethrow_exception(parallel_merge_data->exception); @@ -1909,12 +1902,12 @@ private: APPLY_FOR_VARIANTS_TWO_LEVEL(M) #undef M - std::lock_guard lock(parallel_merge_data->mutex); + std::lock_guard lock(parallel_merge_data->mutex); parallel_merge_data->ready_blocks[bucket_num] = std::move(block); } catch (...) { - std::lock_guard lock(parallel_merge_data->mutex); + std::lock_guard lock(parallel_merge_data->mutex); if (!parallel_merge_data->exception) parallel_merge_data->exception = std::current_exception(); } diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index b8c31c9f346..f51f620064f 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -444,7 +444,7 @@ struct AggregationMethodSingleLowCardinalityColumn : public SingleColumnMethod struct State : public BaseState { - ColumnRawPtrs key; + ColumnRawPtrs key_columns; const IColumn * positions = nullptr; size_t size_of_index_type = 0; @@ -464,12 +464,12 @@ struct AggregationMethodSingleLowCardinalityColumn : public SingleColumnMethod throw Exception("Expected cache for AggregationMethodSingleLowCardinalityColumn::init", ErrorCodes::LOGICAL_ERROR); } - void init(ColumnRawPtrs & key_columns, const AggregationStateCachePtr & cache_ptr) + void init(ColumnRawPtrs & key_columns_low_cardinality, const AggregationStateCachePtr & cache_ptr) { - auto column = typeid_cast(key_columns[0]); + auto column = typeid_cast(key_columns_low_cardinality[0]); if (!column) throw Exception("Invalid aggregation key type for AggregationMethodSingleLowCardinalityColumn method. " - "Excepted LowCardinality, got " + key_columns[0]->getName(), ErrorCodes::LOGICAL_ERROR); + "Excepted LowCardinality, got " + key_columns_low_cardinality[0]->getName(), ErrorCodes::LOGICAL_ERROR); if (!cache_ptr) throw Exception("Cache wasn't created for AggregationMethodSingleLowCardinalityColumn", ErrorCodes::LOGICAL_ERROR); @@ -484,7 +484,7 @@ struct AggregationMethodSingleLowCardinalityColumn : public SingleColumnMethod auto * dict = column->getDictionary().getNestedNotNullableColumn().get(); is_nullable = column->getDictionary().nestedColumnIsNullable(); - key = {dict}; + key_columns = {dict}; bool is_shared_dict = column->isSharedDictionary(); typename LowCardinalityDictionaryCache::DictionaryKey dictionary_key; @@ -517,12 +517,12 @@ struct AggregationMethodSingleLowCardinalityColumn : public SingleColumnMethod } AggregateDataPtr default_data = nullptr; - aggregate_data_cache.assign(key[0]->size(), default_data); + aggregate_data_cache.assign(key_columns[0]->size(), default_data); size_of_index_type = column->getSizeOfIndexType(); positions = column->getIndexesPtr().get(); - BaseState::init(key); + BaseState::init(key_columns); } ALWAYS_INLINE size_t getIndexAt(size_t row) const @@ -547,7 +547,7 @@ struct AggregationMethodSingleLowCardinalityColumn : public SingleColumnMethod Arena & pool) const { size_t row = getIndexAt(i); - return BaseState::getKey(key, 1, row, key_sizes, keys, pool); + return BaseState::getKey(key_columns, 1, row, key_sizes, keys, pool); } template @@ -575,9 +575,8 @@ struct AggregationMethodSingleLowCardinalityColumn : public SingleColumnMethod } else { - ColumnRawPtrs key_columns; Sizes key_sizes; - auto key = getKey(key_columns, 0, i, key_sizes, keys, pool); + auto key = getKey({}, 0, i, key_sizes, keys, pool); typename D::iterator it; if (saved_hash) @@ -618,11 +617,10 @@ struct AggregationMethodSingleLowCardinalityColumn : public SingleColumnMethod if (!aggregate_data_cache[row]) { - ColumnRawPtrs key_columns; Sizes key_sizes; StringRefs keys; Arena pool; - auto key = getKey(key_columns, 0, i, key_sizes, keys, pool); + auto key = getKey({}, 0, i, key_sizes, keys, pool); typename D::iterator it; if (saved_hash) @@ -653,10 +651,10 @@ struct AggregationMethodSingleLowCardinalityColumn : public SingleColumnMethod static const bool no_consecutive_keys_optimization = true; static const bool low_cardinality_optimization = true; - static void insertKeyIntoColumns(const typename Data::value_type & value, MutableColumns & key_columns, const Sizes & /*key_sizes*/) + static void insertKeyIntoColumns(const typename Data::value_type & value, MutableColumns & key_columns_low_cardinality, const Sizes & /*key_sizes*/) { auto ref = Base::getValueRef(value); - static_cast(key_columns[0].get())->insertData(ref.data, ref.size); + static_cast(key_columns_low_cardinality[0].get())->insertData(ref.data, ref.size); } }; @@ -1132,9 +1130,6 @@ struct AggregatedDataVariants : private boost::noncopyable case Type::NAME: NAME = std::make_unique(); break; APPLY_FOR_AGGREGATED_VARIANTS(M) #undef M - - default: - throw Exception("Unknown aggregated data variant.", ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT); } type = type_; @@ -1152,10 +1147,9 @@ struct AggregatedDataVariants : private boost::noncopyable case Type::NAME: return NAME->data.size() + (without_key != nullptr); APPLY_FOR_AGGREGATED_VARIANTS(M) #undef M - - default: - throw Exception("Unknown aggregated data variant.", ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT); } + + __builtin_unreachable(); } /// The size without taking into account the row in which data is written for the calculation of TOTALS. @@ -1170,10 +1164,9 @@ struct AggregatedDataVariants : private boost::noncopyable case Type::NAME: return NAME->data.size(); APPLY_FOR_AGGREGATED_VARIANTS(M) #undef M - - default: - throw Exception("Unknown aggregated data variant.", ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT); } + + __builtin_unreachable(); } const char * getMethodName() const @@ -1187,10 +1180,9 @@ struct AggregatedDataVariants : private boost::noncopyable case Type::NAME: return #NAME; APPLY_FOR_AGGREGATED_VARIANTS(M) #undef M - - default: - throw Exception("Unknown aggregated data variant.", ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT); } + + __builtin_unreachable(); } bool isTwoLevel() const @@ -1204,10 +1196,9 @@ struct AggregatedDataVariants : private boost::noncopyable case Type::NAME: return IS_TWO_LEVEL; APPLY_FOR_AGGREGATED_VARIANTS(M) #undef M - - default: - throw Exception("Unknown aggregated data variant.", ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT); } + + __builtin_unreachable(); } #define APPLY_FOR_VARIANTS_CONVERTIBLE_TO_TWO_LEVEL(M) \ @@ -1486,7 +1477,7 @@ public: bool empty() const { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); return files.empty(); } }; diff --git a/dbms/src/Interpreters/AnalyzedJoin.cpp b/dbms/src/Interpreters/AnalyzedJoin.cpp index 83dfebcd852..c39ea9c9495 100644 --- a/dbms/src/Interpreters/AnalyzedJoin.cpp +++ b/dbms/src/Interpreters/AnalyzedJoin.cpp @@ -48,7 +48,7 @@ ExpressionActionsPtr AnalyzedJoin::createJoinedBlockActions( source_column_names.emplace_back(column.name_and_type); ASTPtr query = expression_list; - auto syntax_result = SyntaxAnalyzer(context, {}).analyze(query, source_column_names, required_columns); + auto syntax_result = SyntaxAnalyzer(context).analyze(query, source_column_names, required_columns); ExpressionAnalyzer analyzer(query, syntax_result, context, {}, required_columns); auto joined_block_actions = analyzer.getActions(false); diff --git a/dbms/src/Interpreters/AsynchronousMetrics.cpp b/dbms/src/Interpreters/AsynchronousMetrics.cpp index 38b3142e94f..9fdcf1d4e13 100644 --- a/dbms/src/Interpreters/AsynchronousMetrics.cpp +++ b/dbms/src/Interpreters/AsynchronousMetrics.cpp @@ -42,7 +42,7 @@ AsynchronousMetrics::~AsynchronousMetrics() try { { - std::lock_guard lock{wait_mutex}; + std::lock_guard lock{wait_mutex}; quit = true; } @@ -58,14 +58,14 @@ AsynchronousMetrics::~AsynchronousMetrics() AsynchronousMetrics::Container AsynchronousMetrics::getValues() const { - std::lock_guard lock{container_mutex}; + std::lock_guard lock{container_mutex}; return container; } void AsynchronousMetrics::set(const std::string & name, Value value) { - std::lock_guard lock{container_mutex}; + std::lock_guard lock{container_mutex}; container[name] = value; } @@ -74,7 +74,7 @@ void AsynchronousMetrics::run() { setThreadName("AsyncMetrics"); - std::unique_lock lock{wait_mutex}; + std::unique_lock lock{wait_mutex}; /// Next minute + 30 seconds. To be distant with moment of transmission of metrics, see MetricsTransmitter. const auto get_next_minute = [] @@ -268,7 +268,7 @@ void AsynchronousMetrics::update() set("jemalloc." NAME, value); \ } while (0); - FOR_EACH_METRIC(GET_METRIC); + FOR_EACH_METRIC(GET_METRIC) #undef GET_METRIC #undef FOR_EACH_METRIC diff --git a/dbms/src/Interpreters/CatBoostModel.cpp b/dbms/src/Interpreters/CatBoostModel.cpp index bd7c07813d0..b9f6d9beaa0 100644 --- a/dbms/src/Interpreters/CatBoostModel.cpp +++ b/dbms/src/Interpreters/CatBoostModel.cpp @@ -5,11 +5,14 @@ #include #include #include +#include #include #include #include #include #include +#include +#include namespace DB { @@ -51,12 +54,16 @@ struct CatBoostWrapperAPI double * result, size_t resultSize); int (* GetStringCatFeatureHash)(const char * data, size_t size); - int (* GetIntegerCatFeatureHash)(long long val); size_t (* GetFloatFeaturesCount)(ModelCalcerHandle* calcer); - size_t (* GetCatFeaturesCount)(ModelCalcerHandle* calcer); + size_t (* GetTreeCount)(ModelCalcerHandle* modelHandle); + size_t (* GetDimensionsCount)(ModelCalcerHandle* modelHandle); + + bool (* CheckModelMetadataHasKey)(ModelCalcerHandle* modelHandle, const char* keyPtr, size_t keySize); + size_t (*GetModelInfoValueSize)(ModelCalcerHandle* modelHandle, const char* keyPtr, size_t keySize); + const char* (*GetModelInfoValue)(ModelCalcerHandle* modelHandle, const char* keyPtr, size_t keySize); }; @@ -95,6 +102,9 @@ public: float_features_count = api->GetFloatFeaturesCount(handle_->get()); cat_features_count = api->GetCatFeaturesCount(handle_->get()); + tree_count = 1; + if (api->GetDimensionsCount) + tree_count = api->GetDimensionsCount(handle_->get()); handle = std::move(handle_); } @@ -146,17 +156,48 @@ public: } } - return evalImpl(columns, float_features_count, cat_features_count, cat_features_are_strings); + auto result = evalImpl(columns, cat_features_are_strings); + + if (tree_count == 1) + return result; + + size_t column_size = columns.front()->size(); + auto result_buf = result->getData().data(); + + /// Multiple trees case. Copy data to several columns. + MutableColumns mutable_columns(tree_count); + std::vector column_ptrs(tree_count); + for (size_t i = 0; i < tree_count; ++i) + { + auto col = ColumnFloat64::create(column_size); + column_ptrs[i] = col->getData().data(); + mutable_columns[i] = std::move(col); + } + + Float64 * data = result_buf; + for (size_t row = 0; row < column_size; ++row) + { + for (size_t i = 0; i < tree_count; ++i) + { + *column_ptrs[i] = *data; + ++column_ptrs[i]; + ++data; + } + } + + return ColumnTuple::create(std::move(mutable_columns)); } size_t getFloatFeaturesCount() const override { return float_features_count; } size_t getCatFeaturesCount() const override { return cat_features_count; } + size_t getTreeCount() const override { return tree_count; } private: std::unique_ptr handle; const CatBoostWrapperAPI * api; size_t float_features_count; size_t cat_features_count; + size_t tree_count; /// Buffer should be allocated with features_count * column->size() elements. /// Place column elements in positions buffer[0], buffer[features_count], ... , buffer[size * features_count] @@ -308,13 +349,13 @@ private: /// buffer[column_size * cat_features_count] -> char * => cat_features[column_size][cat_features_count] -> char * void fillCatFeaturesBuffer(const char *** cat_features, const char ** buffer, - size_t column_size, size_t cat_features_count_current) const + size_t column_size) const { for (size_t i = 0; i < column_size; ++i) { *cat_features = buffer; ++cat_features; - buffer += cat_features_count_current; + buffer += cat_features_count; } } @@ -322,13 +363,14 @@ private: /// * CalcModelPredictionFlat if no cat features /// * CalcModelPrediction if all cat features are strings /// * CalcModelPredictionWithHashedCatFeatures if has int cat features. - ColumnPtr evalImpl(const ColumnRawPtrs & columns, size_t float_features_count_current, size_t cat_features_count_current, - bool cat_features_are_strings) const + ColumnFloat64::MutablePtr evalImpl( + const ColumnRawPtrs & columns, + bool cat_features_are_strings) const { std::string error_msg = "Error occurred while applying CatBoost model: "; size_t column_size = columns.front()->size(); - auto result = ColumnFloat64::create(column_size); + auto result = ColumnFloat64::create(column_size * tree_count); auto result_buf = result->getData().data(); if (!column_size) @@ -338,13 +380,13 @@ private: PODArray float_features(column_size); auto float_features_buf = float_features.data(); /// Store all float data into single column. float_features is a list of pointers to it. - auto float_features_col = placeNumericColumns(columns, 0, float_features_count_current, float_features_buf); + auto float_features_col = placeNumericColumns(columns, 0, float_features_count, float_features_buf); - if (cat_features_count_current == 0) + if (cat_features_count == 0) { if (!api->CalcModelPredictionFlat(handle->get(), column_size, - float_features_buf, float_features_count_current, - result_buf, column_size)) + float_features_buf, float_features_count, + result_buf, column_size * tree_count)) { throw Exception(error_msg + api->GetErrorString(), ErrorCodes::CANNOT_APPLY_CATBOOST_MODEL); @@ -356,19 +398,19 @@ private: if (cat_features_are_strings) { /// cat_features_holder stores pointers to ColumnString data or fixed_strings_data. - PODArray cat_features_holder(cat_features_count_current * column_size); + PODArray cat_features_holder(cat_features_count * column_size); PODArray cat_features(column_size); auto cat_features_buf = cat_features.data(); - fillCatFeaturesBuffer(cat_features_buf, cat_features_holder.data(), column_size, cat_features_count_current); + fillCatFeaturesBuffer(cat_features_buf, cat_features_holder.data(), column_size); /// Fixed strings are stored without termination zero, so have to copy data into fixed_strings_data. - auto fixed_strings_data = placeStringColumns(columns, float_features_count_current, - cat_features_count_current, cat_features_holder.data()); + auto fixed_strings_data = placeStringColumns(columns, float_features_count, + cat_features_count, cat_features_holder.data()); if (!api->CalcModelPrediction(handle->get(), column_size, - float_features_buf, float_features_count_current, - cat_features_buf, cat_features_count_current, - result_buf, column_size)) + float_features_buf, float_features_count, + cat_features_buf, cat_features_count, + result_buf, column_size * tree_count)) { throw Exception(error_msg + api->GetErrorString(), ErrorCodes::CANNOT_APPLY_CATBOOST_MODEL); } @@ -377,14 +419,14 @@ private: { PODArray cat_features(column_size); auto cat_features_buf = cat_features.data(); - auto cat_features_col = placeNumericColumns(columns, float_features_count_current, - cat_features_count_current, cat_features_buf); - calcHashes(columns, float_features_count_current, cat_features_count_current, cat_features_buf); + auto cat_features_col = placeNumericColumns(columns, float_features_count, + cat_features_count, cat_features_buf); + calcHashes(columns, float_features_count, cat_features_count, cat_features_buf); if (!api->CalcModelPredictionWithHashedCatFeatures( handle->get(), column_size, - float_features_buf, float_features_count_current, - cat_features_buf, cat_features_count_current, - result_buf, column_size)) + float_features_buf, float_features_count, + cat_features_buf, cat_features_count, + result_buf, column_size * tree_count)) { throw Exception(error_msg + api->GetErrorString(), ErrorCodes::CANNOT_APPLY_CATBOOST_MODEL); } @@ -413,6 +455,9 @@ private: template void load(T& func, const std::string & name) { func = lib.get(name); } + + template + void tryLoad(T& func, const std::string & name) { func = lib.tryGet(name); } }; void CatBoostLibHolder::initAPI() @@ -428,6 +473,11 @@ void CatBoostLibHolder::initAPI() load(api.GetIntegerCatFeatureHash, "GetIntegerCatFeatureHash"); load(api.GetFloatFeaturesCount, "GetFloatFeaturesCount"); load(api.GetCatFeaturesCount, "GetCatFeaturesCount"); + tryLoad(api.CheckModelMetadataHasKey, "CheckModelMetadataHasKey"); + tryLoad(api.GetModelInfoValueSize, "GetModelInfoValueSize"); + tryLoad(api.GetModelInfoValue, "GetModelInfoValue"); + tryLoad(api.GetTreeCount, "GetTreeCount"); + tryLoad(api.GetDimensionsCount, "GetDimensionsCount"); } std::shared_ptr getCatBoostWrapperHolder(const std::string & lib_path) @@ -435,7 +485,7 @@ std::shared_ptr getCatBoostWrapperHolder(const std::string & static std::weak_ptr ptr; static std::mutex mutex; - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); auto result = ptr.lock(); if (!result || result->getCurrentPath() != lib_path) @@ -474,6 +524,7 @@ void CatBoostModel::init() model = std::make_unique(api, model_path); float_features_count = model->getFloatFeaturesCount(); cat_features_count = model->getCatFeaturesCount(); + tree_count = model->getTreeCount(); } const ExternalLoadableLifetime & CatBoostModel::getLifetime() const @@ -501,6 +552,22 @@ size_t CatBoostModel::getCatFeaturesCount() const return cat_features_count; } +size_t CatBoostModel::getTreeCount() const +{ + return tree_count; +} + +DataTypePtr CatBoostModel::getReturnType() const +{ + auto type = std::make_shared(); + if (tree_count == 1) + return type; + + DataTypes types(tree_count, type); + + return std::make_shared(types); +} + ColumnPtr CatBoostModel::evaluate(const ColumnRawPtrs & columns) const { if (!model) diff --git a/dbms/src/Interpreters/CatBoostModel.h b/dbms/src/Interpreters/CatBoostModel.h index b6a937fe048..0fc379d8d00 100644 --- a/dbms/src/Interpreters/CatBoostModel.h +++ b/dbms/src/Interpreters/CatBoostModel.h @@ -27,14 +27,19 @@ public: virtual size_t getFloatFeaturesCount() const = 0; virtual size_t getCatFeaturesCount() const = 0; + virtual size_t getTreeCount() const = 0; }; +class IDataType; +using DataTypePtr = std::shared_ptr; + /// General ML model evaluator interface. class IModel : public IExternalLoadable { public: virtual ColumnPtr evaluate(const ColumnRawPtrs & columns) const = 0; virtual std::string getTypeName() const = 0; + virtual DataTypePtr getReturnType() const = 0; }; class CatBoostModel : public IModel @@ -48,6 +53,8 @@ public: size_t getFloatFeaturesCount() const; size_t getCatFeaturesCount() const; + size_t getTreeCount() const; + DataTypePtr getReturnType() const override; /// IExternalLoadable interface. @@ -76,6 +83,7 @@ private: size_t float_features_count; size_t cat_features_count; + size_t tree_count; std::chrono::time_point creation_time; std::exception_ptr creation_exception; diff --git a/dbms/src/Interpreters/Cluster.cpp b/dbms/src/Interpreters/Cluster.cpp index db81bc58061..834a27f26a2 100644 --- a/dbms/src/Interpreters/Cluster.cpp +++ b/dbms/src/Interpreters/Cluster.cpp @@ -54,6 +54,9 @@ Cluster::Address::Address(const Poco::Util::AbstractConfiguration & config, cons host_name = config.getString(config_prefix + ".host"); port = static_cast(config.getInt(config_prefix + ".port")); + if (config.has(config_prefix + ".user")) + user_specified = true; + user = config.getString(config_prefix + ".user", "default"); password = config.getString(config_prefix + ".password", ""); default_database = config.getString(config_prefix + ".default_database", ""); diff --git a/dbms/src/Interpreters/Cluster.h b/dbms/src/Interpreters/Cluster.h index f998ad8f912..aa78f77b6a7 100644 --- a/dbms/src/Interpreters/Cluster.h +++ b/dbms/src/Interpreters/Cluster.h @@ -62,6 +62,8 @@ public: UInt32 replica_num; /// The locality is determined at the initialization, and is not changed even if DNS is changed bool is_local; + bool user_specified = false; + Protocol::Compression compression = Protocol::Compression::Enable; Protocol::Secure secure = Protocol::Secure::Disable; diff --git a/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp b/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp index 27b7d8338af..4b9aa713f07 100644 --- a/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -14,14 +14,8 @@ namespace DB namespace ClusterProxy { -BlockInputStreams executeQuery( - IStreamFactory & stream_factory, const ClusterPtr & cluster, - const ASTPtr & query_ast, const Context & context, const Settings & settings) +Context removeUserRestrictionsFromSettings(const Context & context, const Settings & settings) { - BlockInputStreams res; - - const std::string query = queryToString(query_ast); - Settings new_settings = settings; new_settings.queue_max_wait_ms = Cluster::saturate(new_settings.queue_max_wait_ms, settings.max_execution_time); @@ -39,6 +33,19 @@ BlockInputStreams executeQuery( Context new_context(context); new_context.setSettings(new_settings); + return new_context; +} + +BlockInputStreams executeQuery( + IStreamFactory & stream_factory, const ClusterPtr & cluster, + const ASTPtr & query_ast, const Context & context, const Settings & settings) +{ + BlockInputStreams res; + + const std::string query = queryToString(query_ast); + + Context new_context = removeUserRestrictionsFromSettings(context, settings); + ThrottlerPtr user_level_throttler; if (auto process_list_element = context.getProcessListElement()) user_level_throttler = process_list_element->getUserNetworkThrottler(); diff --git a/dbms/src/Interpreters/ClusterProxy/executeQuery.h b/dbms/src/Interpreters/ClusterProxy/executeQuery.h index b12fc2b4646..5c07c287954 100644 --- a/dbms/src/Interpreters/ClusterProxy/executeQuery.h +++ b/dbms/src/Interpreters/ClusterProxy/executeQuery.h @@ -16,6 +16,10 @@ namespace ClusterProxy class IStreamFactory; +/// removes different restrictions (like max_concurrent_queries_for_user, max_memory_usage_for_user, etc.) +/// from settings and creates new context with them +Context removeUserRestrictionsFromSettings(const Context & context, const Settings & settings); + /// Execute a distributed query, creating a vector of BlockInputStreams, from which the result can be read. /// `stream_factory` object encapsulates the logic of creating streams for a different type of query /// (currently SELECT, DESCRIBE). diff --git a/dbms/src/Interpreters/Compiler.cpp b/dbms/src/Interpreters/Compiler.cpp index e3150082834..8a60b24a24b 100644 --- a/dbms/src/Interpreters/Compiler.cpp +++ b/dbms/src/Interpreters/Compiler.cpp @@ -87,7 +87,7 @@ SharedLibraryPtr Compiler::getOrCount( { HashedKey hashed_key = getHash(key); - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); UInt32 count = ++counts[hashed_key]; @@ -273,7 +273,7 @@ void Compiler::compile( << " 2>&1" ") || echo Return code: $?"; -#if !NDEBUG +#ifndef NDEBUG LOG_TRACE(log, "Compile command: " << command.str()); #endif @@ -306,7 +306,7 @@ void Compiler::compile( SharedLibraryPtr lib(new SharedLibrary(so_file_path)); { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); libraries[hashed_key] = lib; } diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index c890da8e8fe..4ded6b4d216 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -908,7 +908,7 @@ DDLGuard::DDLGuard(Map & map_, std::unique_lock guards_lock_, const it = map.emplace(elem, Entry{std::make_unique(), 0}).first; ++it->second.counter; guards_lock.unlock(); - table_lock = std::unique_lock(*it->second.mutex); + table_lock = std::unique_lock(*it->second.mutex); } DDLGuard::~DDLGuard() @@ -924,7 +924,7 @@ DDLGuard::~DDLGuard() std::unique_ptr Context::getDDLGuard(const String & database, const String & table) const { - std::unique_lock lock(shared->ddl_guards_mutex); + std::unique_lock lock(shared->ddl_guards_mutex); return std::make_unique(shared->ddl_guards[database], std::move(lock), table); } @@ -1054,14 +1054,14 @@ void Context::setCurrentQueryId(const String & query_id) { UInt64 a; UInt64 b; - }; + } words; } random; { auto lock = getLock(); - random.a = shared->rng(); - random.b = shared->rng(); + random.words.a = shared->rng(); + random.words.b = shared->rng(); } /// Use protected constructor. @@ -1177,7 +1177,7 @@ ExternalModels & Context::getExternalModels() EmbeddedDictionaries & Context::getEmbeddedDictionariesImpl(const bool throw_on_error) const { - std::lock_guard lock(shared->embedded_dictionaries_mutex); + std::lock_guard lock(shared->embedded_dictionaries_mutex); if (!shared->embedded_dictionaries) { @@ -1195,7 +1195,7 @@ EmbeddedDictionaries & Context::getEmbeddedDictionariesImpl(const bool throw_on_ ExternalDictionaries & Context::getExternalDictionariesImpl(const bool throw_on_error) const { - std::lock_guard lock(shared->external_dictionaries_mutex); + std::lock_guard lock(shared->external_dictionaries_mutex); if (!shared->external_dictionaries) { @@ -1215,7 +1215,7 @@ ExternalDictionaries & Context::getExternalDictionariesImpl(const bool throw_on_ ExternalModels & Context::getExternalModelsImpl(bool throw_on_error) const { - std::lock_guard lock(shared->external_models_mutex); + std::lock_guard lock(shared->external_models_mutex); if (!shared->external_models) { @@ -1372,7 +1372,7 @@ DDLWorker & Context::getDDLWorker() const zkutil::ZooKeeperPtr Context::getZooKeeper() const { - std::lock_guard lock(shared->zookeeper_mutex); + std::lock_guard lock(shared->zookeeper_mutex); if (!shared->zookeeper) shared->zookeeper = std::make_shared(getConfigRef(), "zookeeper"); @@ -1465,7 +1465,7 @@ void Context::reloadClusterConfig() { ConfigurationPtr cluster_config; { - std::lock_guard lock(shared->clusters_mutex); + std::lock_guard lock(shared->clusters_mutex); cluster_config = shared->clusters_config; } @@ -1473,7 +1473,7 @@ void Context::reloadClusterConfig() auto new_clusters = std::make_unique(config, settings); { - std::lock_guard lock(shared->clusters_mutex); + std::lock_guard lock(shared->clusters_mutex); if (shared->clusters_config.get() == cluster_config.get()) { shared->clusters = std::move(new_clusters); @@ -1488,7 +1488,7 @@ void Context::reloadClusterConfig() Clusters & Context::getClusters() const { - std::lock_guard lock(shared->clusters_mutex); + std::lock_guard lock(shared->clusters_mutex); if (!shared->clusters) { auto & config = shared->clusters_config ? *shared->clusters_config : getConfigRef(); @@ -1502,7 +1502,7 @@ Clusters & Context::getClusters() const /// On repeating calls updates existing clusters and adds new clusters, doesn't delete old clusters void Context::setClustersConfig(const ConfigurationPtr & config, const String & config_name) { - std::lock_guard lock(shared->clusters_mutex); + std::lock_guard lock(shared->clusters_mutex); shared->clusters_config = config; @@ -1515,7 +1515,7 @@ void Context::setClustersConfig(const ConfigurationPtr & config, const String & void Context::setCluster(const String & cluster_name, const std::shared_ptr & cluster) { - std::lock_guard lock(shared->clusters_mutex); + std::lock_guard lock(shared->clusters_mutex); if (!shared->clusters) throw Exception("Clusters are not set", ErrorCodes::LOGICAL_ERROR); @@ -1620,7 +1620,7 @@ const MergeTreeSettings & Context::getMergeTreeSettings() const } -void Context::checkCanBeDropped(const String & database, const String & table, const size_t & size, const size_t & max_size_to_drop) +void Context::checkCanBeDropped(const String & database, const String & table, const size_t & size, const size_t & max_size_to_drop) const { if (!max_size_to_drop || size <= max_size_to_drop) return; @@ -1668,7 +1668,7 @@ void Context::setMaxTableSizeToDrop(size_t max_size) } -void Context::checkTableCanBeDropped(const String & database, const String & table, const size_t & table_size) +void Context::checkTableCanBeDropped(const String & database, const String & table, const size_t & table_size) const { size_t max_table_size_to_drop = shared->max_table_size_to_drop; @@ -1683,7 +1683,7 @@ void Context::setMaxPartitionSizeToDrop(size_t max_size) } -void Context::checkPartitionCanBeDropped(const String & database, const String & table, const size_t & partition_size) +void Context::checkPartitionCanBeDropped(const String & database, const String & table, const size_t & partition_size) const { size_t max_partition_size_to_drop = shared->max_partition_size_to_drop; @@ -1846,7 +1846,7 @@ SessionCleaner::~SessionCleaner() try { { - std::lock_guard lock{mutex}; + std::lock_guard lock{mutex}; quit = true; } @@ -1864,7 +1864,7 @@ void SessionCleaner::run() { setThreadName("HTTPSessionCleaner"); - std::unique_lock lock{mutex}; + std::unique_lock lock{mutex}; while (true) { diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 2609c4ff3e6..72354753e36 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -152,6 +152,7 @@ public: static Context createGlobal(); Context(const Context &) = default; + Context & operator=(const Context &) = default; ~Context(); String getPath() const; @@ -400,11 +401,11 @@ public: /// Prevents DROP TABLE if its size is greater than max_size (50GB by default, max_size=0 turn off this check) void setMaxTableSizeToDrop(size_t max_size); - void checkTableCanBeDropped(const String & database, const String & table, const size_t & table_size); + void checkTableCanBeDropped(const String & database, const String & table, const size_t & table_size) const; /// Prevents DROP PARTITION if its size is greater than max_size (50GB by default, max_size=0 turn off this check) void setMaxPartitionSizeToDrop(size_t max_size); - void checkPartitionCanBeDropped(const String & database, const String & table, const size_t & partition_size); + void checkPartitionCanBeDropped(const String & database, const String & table, const size_t & partition_size) const; /// Lets you select the compression codec according to the conditions described in the configuration file. std::shared_ptr chooseCompressionCodec(size_t part_size, double part_size_ratio) const; @@ -471,7 +472,7 @@ private: /// Session will be closed after specified timeout. void scheduleCloseSession(const SessionKey & key, std::chrono::steady_clock::duration timeout); - void checkCanBeDropped(const String & database, const String & table, const size_t & size, const size_t & max_size_to_drop); + void checkCanBeDropped(const String & database, const String & table, const size_t & size, const size_t & max_size_to_drop) const; }; diff --git a/dbms/src/Interpreters/EmbeddedDictionaries.cpp b/dbms/src/Interpreters/EmbeddedDictionaries.cpp index 441bb7c92d8..10f5692f6e6 100644 --- a/dbms/src/Interpreters/EmbeddedDictionaries.cpp +++ b/dbms/src/Interpreters/EmbeddedDictionaries.cpp @@ -64,7 +64,7 @@ bool EmbeddedDictionaries::reloadDictionary( bool EmbeddedDictionaries::reloadImpl(const bool throw_on_error, const bool force_reload) { - std::unique_lock lock(mutex); + std::unique_lock lock(mutex); /** If you can not update the directories, then despite this, do not throw an exception (use the old directories). * If there are no old correct directories, then when using functions that depend on them, diff --git a/dbms/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp b/dbms/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp index 59a8c084b9e..dcf2f0b051d 100644 --- a/dbms/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp +++ b/dbms/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp @@ -37,7 +37,7 @@ static ASTPtr addTypeConversion(std::unique_ptr && ast, const String return res; } -bool ExecuteScalarSubqueriesMatcher::needChildVisit(ASTPtr & node, const ASTPtr &) +bool ExecuteScalarSubqueriesMatcher::needChildVisit(ASTPtr & node, const ASTPtr & child) { /// Processed if (typeid_cast(node.get()) || @@ -48,6 +48,14 @@ bool ExecuteScalarSubqueriesMatcher::needChildVisit(ASTPtr & node, const ASTPtr if (typeid_cast(node.get())) return false; + if (typeid_cast(node.get())) + { + /// Do not go to FROM, JOIN, UNION. + if (typeid_cast(child.get()) || + typeid_cast(child.get())) + return false; + } + return true; } diff --git a/dbms/src/Interpreters/ExpressionActions.cpp b/dbms/src/Interpreters/ExpressionActions.cpp index 0714c8a954a..0393e86ddf3 100644 --- a/dbms/src/Interpreters/ExpressionActions.cpp +++ b/dbms/src/Interpreters/ExpressionActions.cpp @@ -328,9 +328,6 @@ void ExpressionAction::prepare(Block & sample_block, const Settings & settings) break; } - - default: - throw Exception("Unknown action type", ErrorCodes::UNKNOWN_ACTION); } } @@ -521,9 +518,6 @@ void ExpressionAction::execute(Block & block, bool dry_run) const block.insert({ block.getByName(source_name).column, result_type, result_name }); break; - - default: - throw Exception("Unknown action type", ErrorCodes::UNKNOWN_ACTION); } } @@ -603,9 +597,6 @@ std::string ExpressionAction::toString() const ss << " AS " << projection[i].second; } break; - - default: - throw Exception("Unexpected Action type", ErrorCodes::LOGICAL_ERROR); } return ss.str(); diff --git a/dbms/src/Interpreters/ExpressionActions.h b/dbms/src/Interpreters/ExpressionActions.h index ec7a7602f02..2b6034ba899 100644 --- a/dbms/src/Interpreters/ExpressionActions.h +++ b/dbms/src/Interpreters/ExpressionActions.h @@ -246,10 +246,10 @@ public: struct ActionsHash { - UInt128 operator()(const ExpressionActions::Actions & actions) const + UInt128 operator()(const ExpressionActions::Actions & elems) const { SipHash hash; - for (const ExpressionAction & act : actions) + for (const ExpressionAction & act : elems) hash.update(ExpressionAction::ActionHash{}(act)); UInt128 result; hash.get128(result.low, result.high); diff --git a/dbms/src/Interpreters/ExpressionJIT.cpp b/dbms/src/Interpreters/ExpressionJIT.cpp index f12cfedc7c9..9ac95e3a107 100644 --- a/dbms/src/Interpreters/ExpressionJIT.cpp +++ b/dbms/src/Interpreters/ExpressionJIT.cpp @@ -161,21 +161,21 @@ auto wrapJITSymbolResolver(llvm::JITSymbolResolver & jsr) // Actually this should work for 7.0.0 but now we have OLDER 7.0.0svn in contrib auto flags = [&](const llvm::orc::SymbolNameSet & symbols) { - llvm::orc::SymbolFlagsMap flags; + llvm::orc::SymbolFlagsMap flags_map; for (const auto & symbol : symbols) { auto resolved = jsr.lookupFlags({*symbol}); if (resolved && resolved->size()) - flags.emplace(symbol, resolved->begin()->second); + flags_map.emplace(symbol, resolved->begin()->second); } - return flags; + return flags_map; }; #endif - auto symbols = [&](std::shared_ptr query, llvm::orc::SymbolNameSet symbols) + auto symbols = [&](std::shared_ptr query, llvm::orc::SymbolNameSet symbols_set) { llvm::orc::SymbolNameSet missing; - for (const auto & symbol : symbols) + for (const auto & symbol : symbols_set) { auto resolved = jsr.lookup({*symbol}); if (resolved && resolved->size()) @@ -275,20 +275,20 @@ struct LLVMContext { if (!module->size()) return 0; - llvm::PassManagerBuilder builder; + llvm::PassManagerBuilder pass_manager_builder; llvm::legacy::PassManager mpm; llvm::legacy::FunctionPassManager fpm(module.get()); - builder.OptLevel = 3; - builder.SLPVectorize = true; - builder.LoopVectorize = true; - builder.RerollLoops = true; - builder.VerifyInput = true; - builder.VerifyOutput = true; - machine->adjustPassManager(builder); + pass_manager_builder.OptLevel = 3; + pass_manager_builder.SLPVectorize = true; + pass_manager_builder.LoopVectorize = true; + pass_manager_builder.RerollLoops = true; + pass_manager_builder.VerifyInput = true; + pass_manager_builder.VerifyOutput = true; + machine->adjustPassManager(pass_manager_builder); fpm.add(llvm::createTargetTransformInfoWrapperPass(machine->getTargetIRAnalysis())); mpm.add(llvm::createTargetTransformInfoWrapperPass(machine->getTargetIRAnalysis())); - builder.populateFunctionPassManager(fpm); - builder.populateModulePassManager(mpm); + pass_manager_builder.populateFunctionPassManager(fpm); + pass_manager_builder.populateModulePassManager(mpm); fpm.doInitialization(); for (auto & function : *module) fpm.run(function); @@ -626,7 +626,7 @@ size_t CompiledExpressionCache::weight() const { #if LLVM_VERSION_MAJOR >= 6 - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); size_t result{0}; std::unordered_set seen; for (const auto & cell : cells) diff --git a/dbms/src/Interpreters/ExternalLoader.cpp b/dbms/src/Interpreters/ExternalLoader.cpp index 8e0fe1436d9..e4ccd9962c6 100644 --- a/dbms/src/Interpreters/ExternalLoader.cpp +++ b/dbms/src/Interpreters/ExternalLoader.cpp @@ -48,12 +48,12 @@ ExternalLoader::ExternalLoader(const Poco::Util::AbstractConfiguration & config_ const ExternalLoaderConfigSettings & config_settings, std::unique_ptr config_repository, Logger * log, const std::string & loadable_object_name) - : config_main(config_main) - , update_settings(update_settings) - , config_settings(config_settings) - , config_repository(std::move(config_repository)) - , log(log) - , object_name(loadable_object_name) + : config_main(config_main) + , update_settings(update_settings) + , config_settings(config_settings) + , config_repository(std::move(config_repository)) + , log(log) + , object_name(loadable_object_name) { } @@ -92,7 +92,7 @@ void ExternalLoader::reloadAndUpdate(bool throw_on_error) /// list of recreated loadable objects to perform delayed removal from unordered_map std::list recreated_failed_loadable_objects; - std::unique_lock all_lock(all_mutex); + std::lock_guard all_lock(all_mutex); /// retry loading failed loadable objects for (auto & failed_loadable_object : failed_loadable_objects) @@ -109,11 +109,11 @@ void ExternalLoader::reloadAndUpdate(bool throw_on_error) { /// recalculate next attempt time std::uniform_int_distribution distribution( - 0, static_cast(std::exp2(failed_loadable_object.second.error_count))); + 0, static_cast(std::exp2(failed_loadable_object.second.error_count))); std::chrono::seconds delay(std::min( - update_settings.backoff_max_sec, - update_settings.backoff_initial_sec + distribution(rnd_engine))); + update_settings.backoff_max_sec, + update_settings.backoff_initial_sec + distribution(rnd_engine))); failed_loadable_object.second.next_attempt_time = std::chrono::system_clock::now() + delay; ++failed_loadable_object.second.error_count; @@ -122,7 +122,7 @@ void ExternalLoader::reloadAndUpdate(bool throw_on_error) } else { - const std::lock_guard lock{map_mutex}; + std::lock_guard lock{map_mutex}; const auto & lifetime = loadable_ptr->getLifetime(); std::uniform_int_distribution distribution{lifetime.min_sec, lifetime.max_sec}; @@ -153,62 +153,80 @@ void ExternalLoader::reloadAndUpdate(bool throw_on_error) failed_loadable_objects.erase(name); /// periodic update - for (auto & loadable_object : loadable_objects) - { - const auto & name = loadable_object.first; + std::vector> objects_to_update; - try + /// Collect objects that needs to be updated under lock. Then create new versions without lock, and assign under lock. + { + std::lock_guard lock{map_mutex}; + + for (auto & loadable_object : loadable_objects) { /// If the loadable objects failed to load or even failed to initialize from the config. if (!loadable_object.second.loadable) continue; - auto current = loadable_object.second.loadable; + const LoadablePtr & current = loadable_object.second.loadable; const auto & lifetime = current->getLifetime(); /// do not update loadable objects with zero as lifetime if (lifetime.min_sec == 0 || lifetime.max_sec == 0) continue; - if (current->supportUpdates()) - { - auto & update_time = update_times[current->getName()]; + if (!current->supportUpdates()) + continue; - /// check that timeout has passed - if (std::chrono::system_clock::now() < update_time) - continue; + auto update_time = update_times[current->getName()]; - SCOPE_EXIT({ - /// calculate next update time - std::uniform_int_distribution distribution{lifetime.min_sec, lifetime.max_sec}; - update_time = std::chrono::system_clock::now() + std::chrono::seconds{distribution(rnd_engine)}; - }); + /// check that timeout has passed + if (std::chrono::system_clock::now() < update_time) + continue; - /// check source modified - if (current->isModified()) - { - /// create new version of loadable object - auto new_version = current->clone(); + if (!current->isModified()) + continue; - if (const auto exception_ptr = new_version->getCreationException()) - std::rethrow_exception(exception_ptr); + objects_to_update.emplace_back(loadable_object.first, current); + } + } - loadable_object.second.loadable.reset(); - loadable_object.second.loadable = std::move(new_version); - } - } + for (auto & [name, current] : objects_to_update) + { + LoadablePtr new_version; + std::exception_ptr exception; - /// erase stored exception on success - loadable_object.second.exception = std::exception_ptr{}; + try + { + /// create new version of loadable object + new_version = current->clone(); + exception = new_version->getCreationException(); } catch (...) { - loadable_object.second.exception = std::current_exception(); + exception = std::current_exception(); + } - tryLogCurrentException(log, "Cannot update " + object_name + " '" + name + "', leaving old version"); + { + std::lock_guard lock{map_mutex}; - if (throw_on_error) - throw; + if (auto it = loadable_objects.find(name); it != loadable_objects.end()) + { + /// calculate next update time + const auto & lifetime = current->getLifetime(); + std::uniform_int_distribution distribution{lifetime.min_sec, lifetime.max_sec}; + update_times[name] = std::chrono::system_clock::now() + std::chrono::seconds{distribution(rnd_engine)}; + + it->second.exception = exception; + if (!exception) + { + it->second.loadable.reset(); + it->second.loadable = std::move(new_version); + } + else + { + tryLogCurrentException(log, "Cannot update " + object_name + " '" + name + "', leaving old version"); + if (throw_on_error) + throw; + } + } } } } @@ -233,6 +251,8 @@ void ExternalLoader::reloadFromConfigFiles(const bool throw_on_error, const bool } /// erase removed from config loadable objects + std::lock_guard lock{map_mutex}; + std::list removed_loadable_objects; for (const auto & loadable : loadable_objects) { @@ -253,7 +273,7 @@ void ExternalLoader::reloadFromConfigFile(const std::string & config_path, const } else { - std::unique_lock all_lock(all_mutex); + std::lock_guard all_lock(all_mutex); auto modification_time_it = last_modification_times.find(config_path); if (modification_time_it == std::end(last_modification_times)) @@ -305,15 +325,15 @@ void ExternalLoader::reloadFromConfigFile(const std::string & config_path, const decltype(loadable_objects.begin()) object_it; { - std::lock_guard lock{map_mutex}; + std::lock_guard lock{map_mutex}; object_it = loadable_objects.find(name); - } - /// Object with the same name was declared in other config file. - if (object_it != std::end(loadable_objects) && object_it->second.origin != config_path) - throw Exception(object_name + " '" + name + "' from file " + config_path - + " already declared in file " + object_it->second.origin, - ErrorCodes::EXTERNAL_LOADABLE_ALREADY_EXISTS); + /// Object with the same name was declared in other config file. + if (object_it != std::end(loadable_objects) && object_it->second.origin != config_path) + throw Exception(object_name + " '" + name + "' from file " + config_path + + " already declared in file " + object_it->second.origin, + ErrorCodes::EXTERNAL_LOADABLE_ALREADY_EXISTS); + } auto object_ptr = create(name, *loaded_config, key); @@ -342,7 +362,7 @@ void ExternalLoader::reloadFromConfigFile(const std::string & config_path, const } } - const std::lock_guard lock{map_mutex}; + std::lock_guard lock{map_mutex}; /// add new loadable object or update an existing version if (object_it == std::end(loadable_objects)) @@ -365,7 +385,7 @@ void ExternalLoader::reloadFromConfigFile(const std::string & config_path, const /// If the loadable object could not load data or even failed to initialize from the config. /// - all the same we insert information into the `loadable_objects`, with the zero pointer `loadable`. - const std::lock_guard lock{map_mutex}; + std::lock_guard lock{map_mutex}; const auto exception_ptr = std::current_exception(); const auto loadable_it = loadable_objects.find(name); @@ -397,14 +417,14 @@ void ExternalLoader::reload(const std::string & name) reloadFromConfigFiles(true, true, name); /// Check that specified object was loaded - const std::lock_guard lock{map_mutex}; + std::lock_guard lock{map_mutex}; if (!loadable_objects.count(name)) throw Exception("Failed to load " + object_name + " '" + name + "' during the reload process", ErrorCodes::BAD_ARGUMENTS); } ExternalLoader::LoadablePtr ExternalLoader::getLoadableImpl(const std::string & name, bool throw_on_error) const { - const std::lock_guard lock{map_mutex}; + std::lock_guard lock{map_mutex}; const auto it = loadable_objects.find(name); if (it == std::end(loadable_objects)) @@ -415,8 +435,12 @@ ExternalLoader::LoadablePtr ExternalLoader::getLoadableImpl(const std::string & } if (!it->second.loadable && throw_on_error) - it->second.exception ? std::rethrow_exception(it->second.exception) - : throw Exception{object_name + " '" + name + "' is not loaded", ErrorCodes::LOGICAL_ERROR}; + { + if (it->second.exception) + std::rethrow_exception(it->second.exception); + else + throw Exception{object_name + " '" + name + "' is not loaded", ErrorCodes::LOGICAL_ERROR}; + } return it->second.loadable; } diff --git a/dbms/src/Interpreters/ExternalLoader.h b/dbms/src/Interpreters/ExternalLoader.h index 7a27f8a81da..ac672f925e3 100644 --- a/dbms/src/Interpreters/ExternalLoader.h +++ b/dbms/src/Interpreters/ExternalLoader.h @@ -132,6 +132,9 @@ private: bool is_initialized = false; /// Protects only objects map. + /** Reading and assignment of "loadable" should be done under mutex. + * Creating new versions of "loadable" should not be done under mutex. + */ mutable std::mutex map_mutex; /// Protects all data, currently used to avoid races between updating thread and SYSTEM queries diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index c5d546d11ca..8cc3d1b88c1 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -242,7 +242,7 @@ static ColumnsDeclarationAndModifiers parseColumns(const ASTExpressionList & col /// set missing types and wrap default_expression's in a conversion-function if necessary if (!defaulted_columns.empty()) { - auto syntax_analyzer_result = SyntaxAnalyzer(context, {}).analyze(default_expr_list, columns); + auto syntax_analyzer_result = SyntaxAnalyzer(context).analyze(default_expr_list, columns); const auto actions = ExpressionAnalyzer(default_expr_list, syntax_analyzer_result, context).getActions(true); const auto block = actions->getSampleBlock(); @@ -362,11 +362,11 @@ ASTPtr InterpreterCreateQuery::formatColumns(const ColumnsDescription & columns) column_declaration->name = column.name; StringPtr type_name = std::make_shared(column.type->getName()); - auto pos = type_name->data(); - const auto end = pos + type_name->size(); + auto type_name_pos = type_name->data(); + const auto type_name_end = type_name_pos + type_name->size(); ParserIdentifierWithOptionalParameters storage_p; - column_declaration->type = parseQuery(storage_p, pos, end, "data type", 0); + column_declaration->type = parseQuery(storage_p, type_name_pos, type_name_end, "data type", 0); column_declaration->type->owned_string = type_name; const auto defaults_it = columns.defaults.find(column.name); @@ -387,10 +387,10 @@ ASTPtr InterpreterCreateQuery::formatColumns(const ColumnsDescription & columns) { String codec_desc = ct->second->getCodecDesc(); codec_desc = "CODEC(" + codec_desc + ")"; - auto pos = codec_desc.data(); - const auto end = pos + codec_desc.size(); + auto codec_desc_pos = codec_desc.data(); + const auto codec_desc_end = codec_desc_pos + codec_desc.size(); ParserIdentifierWithParameters codec_p; - column_declaration->codec = parseQuery(codec_p, pos, end, "column codec", 0); + column_declaration->codec = parseQuery(codec_p, codec_desc_pos, codec_desc_end, "column codec", 0); } columns_list->children.push_back(column_declaration_ptr); diff --git a/dbms/src/Interpreters/InterpreterDropQuery.cpp b/dbms/src/Interpreters/InterpreterDropQuery.cpp index 2c386718f98..c3af6d7fa0a 100644 --- a/dbms/src/Interpreters/InterpreterDropQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDropQuery.cpp @@ -80,7 +80,7 @@ BlockIO InterpreterDropQuery::executeToTable(String & database_name_, String & t /// If table was already dropped by anyone, an exception will be thrown auto table_lock = database_and_table.second->lockForAlter(); /// Drop table data, don't touch metadata - database_and_table.second->truncate(query_ptr); + database_and_table.second->truncate(query_ptr, context); } else if (kind == ASTDropQuery::Kind::Drop) { @@ -128,7 +128,7 @@ BlockIO InterpreterDropQuery::executeToTemporaryTable(String & table_name, ASTDr /// If table was already dropped by anyone, an exception will be thrown auto table_lock = table->lockForAlter(); /// Drop table data, don't touch metadata - table->truncate(query_ptr); + table->truncate(query_ptr, context); } else if (kind == ASTDropQuery::Kind::Drop) { diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index e9d4e3f1440..1af0210bc27 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -99,7 +99,6 @@ BlockIO InterpreterInsertQuery::execute() out = std::make_shared(query.database, query.table, table, context, query_ptr, query.no_destination); - /// Do not squash blocks if it is a sync INSERT into Distributed, since it lead to double bufferization on client and server side. /// Client-side bufferization might cause excessive timeouts (especially in case of big blocks). if (!(context.getSettingsRef().insert_distributed_sync && table->isRemote())) diff --git a/dbms/src/Interpreters/InterpreterKillQueryQuery.cpp b/dbms/src/Interpreters/InterpreterKillQueryQuery.cpp index 31535bc4bbe..fcb7b74bdbd 100644 --- a/dbms/src/Interpreters/InterpreterKillQueryQuery.cpp +++ b/dbms/src/Interpreters/InterpreterKillQueryQuery.cpp @@ -43,7 +43,7 @@ static const char * cancellationCodeToStatus(CancellationCode code) return "waiting"; default: return "unknown_status"; - }; + } } diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index 718f88d1967..991f31afcdc 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -184,8 +184,8 @@ InterpreterSelectQuery::InterpreterSelectQuery( if (storage) table_lock = storage->lockStructure(false); - syntax_analyzer_result = SyntaxAnalyzer(context, storage) - .analyze(query_ptr, source_header.getNamesAndTypesList(), required_result_column_names, subquery_depth); + syntax_analyzer_result = SyntaxAnalyzer(context, subquery_depth).analyze( + query_ptr, source_header.getNamesAndTypesList(), required_result_column_names, storage); query_analyzer = std::make_unique( query_ptr, syntax_analyzer_result, context, NamesAndTypesList(), required_result_column_names, subquery_depth, !only_analyze); @@ -744,8 +744,8 @@ void InterpreterSelectQuery::executeFetchColumns( Block prewhere_actions_result; if (prewhere_info) { - auto required_columns = prewhere_info->prewhere_actions->getRequiredColumns(); - required_prewhere_columns.insert(required_columns.begin(), required_columns.end()); + auto prewhere_required_columns = prewhere_info->prewhere_actions->getRequiredColumns(); + required_prewhere_columns.insert(prewhere_required_columns.begin(), prewhere_required_columns.end()); prewhere_actions_result = prewhere_info->prewhere_actions->getSampleBlock(); } @@ -792,9 +792,8 @@ void InterpreterSelectQuery::executeFetchColumns( } auto additional_source_columns_set = ext::map(additional_source_columns, [] (const auto & it) { return it.name; }); - ASTPtr query = required_columns_expr_list; - auto syntax_result = SyntaxAnalyzer(context, storage).analyze(query, additional_source_columns); - alias_actions = ExpressionAnalyzer(query, syntax_result, context).getActions(true); + auto syntax_result = SyntaxAnalyzer(context).analyze(required_columns_expr_list, additional_source_columns, {}, storage); + alias_actions = ExpressionAnalyzer(required_columns_expr_list, syntax_result, context).getActions(true); /// The set of required columns could be added as a result of adding an action to calculate ALIAS. required_columns = alias_actions->getRequiredColumns(); @@ -830,8 +829,7 @@ void InterpreterSelectQuery::executeFetchColumns( } prewhere_info->prewhere_actions = std::move(new_actions); - auto source_columns = storage->getColumns().getAllPhysical(); - auto analyzed_result = SyntaxAnalyzer(context, {}).analyze(required_prewhere_columns_expr_list, source_columns); + auto analyzed_result = SyntaxAnalyzer(context).analyze(required_prewhere_columns_expr_list, storage->getColumns().getAllPhysical()); prewhere_info->alias_actions = ExpressionAnalyzer(required_prewhere_columns_expr_list, analyzed_result, context) .getActions(true, false); diff --git a/dbms/src/Interpreters/InterpreterSystemQuery.cpp b/dbms/src/Interpreters/InterpreterSystemQuery.cpp index feb351180c7..fc472ad8a9e 100644 --- a/dbms/src/Interpreters/InterpreterSystemQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSystemQuery.cpp @@ -21,6 +21,7 @@ #include #include #include +#include namespace DB @@ -289,7 +290,7 @@ void InterpreterSystemQuery::restartReplicas(Context & system_context) if (replica_names.empty()) return; - ThreadPool pool(std::min(getNumberOfPhysicalCPUCores(), replica_names.size())); + ThreadPool pool(std::min(size_t(getNumberOfPhysicalCPUCores()), replica_names.size())); for (auto & table : replica_names) pool.schedule([&] () { tryRestartReplica(table.first, table.second, system_context); }); pool.wait(); diff --git a/dbms/src/Interpreters/InterserverIOHandler.h b/dbms/src/Interpreters/InterserverIOHandler.h index e1321037d2c..7cef5df9866 100644 --- a/dbms/src/Interpreters/InterserverIOHandler.h +++ b/dbms/src/Interpreters/InterserverIOHandler.h @@ -84,7 +84,7 @@ class InterserverIOHandler public: void addEndpoint(const String & name, InterserverIOEndpointPtr endpoint) { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); bool inserted = endpoint_map.try_emplace(name, std::move(endpoint)).second; if (!inserted) throw Exception("Duplicate interserver IO endpoint: " + name, ErrorCodes::DUPLICATE_INTERSERVER_IO_ENDPOINT); @@ -92,7 +92,7 @@ public: void removeEndpoint(const String & name) { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); if (!endpoint_map.erase(name)) throw Exception("No interserver IO endpoint named " + name, ErrorCodes::NO_SUCH_INTERSERVER_IO_ENDPOINT); } @@ -100,7 +100,7 @@ public: InterserverIOEndpointPtr getEndpoint(const String & name) try { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); return endpoint_map.at(name); } catch (...) diff --git a/dbms/src/Interpreters/Join.cpp b/dbms/src/Interpreters/Join.cpp index 626e5a2530f..8783d16c3c1 100644 --- a/dbms/src/Interpreters/Join.cpp +++ b/dbms/src/Interpreters/Join.cpp @@ -112,9 +112,6 @@ static void initImpl(Maps & maps, Join::Type type) case Join::Type::TYPE: maps.TYPE = std::make_unique(); break; APPLY_FOR_JOIN_VARIANTS(M) #undef M - - default: - throw Exception("Unknown JOIN keys variant.", ErrorCodes::UNKNOWN_SET_DATA_VARIANT); } } @@ -130,10 +127,9 @@ static size_t getTotalRowCountImpl(const Maps & maps, Join::Type type) case Join::Type::NAME: return maps.NAME ? maps.NAME->size() : 0; APPLY_FOR_JOIN_VARIANTS(M) #undef M - - default: - throw Exception("Unknown JOIN keys variant.", ErrorCodes::UNKNOWN_SET_DATA_VARIANT); } + + __builtin_unreachable(); } template @@ -148,10 +144,9 @@ static size_t getTotalByteCountImpl(const Maps & maps, Join::Type type) case Join::Type::NAME: return maps.NAME ? maps.NAME->getBufferSizeInBytes() : 0; APPLY_FOR_JOIN_VARIANTS(M) #undef M - - default: - throw Exception("Unknown JOIN keys variant.", ErrorCodes::UNKNOWN_SET_DATA_VARIANT); } + + __builtin_unreachable(); } @@ -412,9 +407,6 @@ namespace break; APPLY_FOR_JOIN_VARIANTS(M) #undef M - - default: - throw Exception("Unknown JOIN keys variant.", ErrorCodes::UNKNOWN_SET_DATA_VARIANT); } } } @@ -1170,7 +1162,7 @@ private: { #define M(TYPE) \ case Join::Type::TYPE: \ - rows_added = fillColumns(*maps.TYPE, num_columns_left, columns_left, num_columns_right, columns_keys_and_right); \ + rows_added = fillColumns(*maps.TYPE); \ break; APPLY_FOR_JOIN_VARIANTS(M) #undef M @@ -1193,10 +1185,11 @@ private: template - size_t fillColumns(const Map & map, - size_t num_columns_left, MutableColumns & columns_left, - size_t num_columns_right, MutableColumns & columns_right) + size_t fillColumns(const Map & map) { + size_t num_columns_left = column_indices_left.size(); + size_t num_columns_right = column_indices_keys_and_right.size(); + size_t rows_added = 0; if (!position) @@ -1212,7 +1205,7 @@ private: if (it->second.getUsed()) continue; - AdderNonJoined::add(it->second, rows_added, num_columns_left, columns_left, num_columns_right, columns_right); + AdderNonJoined::add(it->second, rows_added, num_columns_left, columns_left, num_columns_right, columns_keys_and_right); if (rows_added >= max_block_size) { diff --git a/dbms/src/Interpreters/JoinToSubqueryTransformVisitor.cpp b/dbms/src/Interpreters/JoinToSubqueryTransformVisitor.cpp index 1520711aa68..80c670c8794 100644 --- a/dbms/src/Interpreters/JoinToSubqueryTransformVisitor.cpp +++ b/dbms/src/Interpreters/JoinToSubqueryTransformVisitor.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -57,7 +58,7 @@ struct RewriteTablesVisitorData static bool needRewrite(ASTSelectQuery & select) { - auto tables = static_cast(select.tables.get()); + auto tables = typeid_cast(select.tables.get()); if (!tables) return false; @@ -70,11 +71,11 @@ static bool needRewrite(ASTSelectQuery & select) static void appendTableNameAndAlias(std::vector & hidden, const ASTPtr & table_element) { - auto element = static_cast(table_element.get()); + auto element = typeid_cast(table_element.get()); if (!element || element->children.empty()) throw Exception("Expected TablesInSelectQueryElement with at least one child", ErrorCodes::LOGICAL_ERROR); - auto table_expression = static_cast(element->children[0].get()); + auto table_expression = typeid_cast(element->children[0].get()); if (!table_expression || table_expression->children.empty()) throw Exception("Expected TableExpression with at least one child", ErrorCodes::LOGICAL_ERROR); @@ -82,10 +83,10 @@ static void appendTableNameAndAlias(std::vector & hidden, const ASTPtr & if (!alias.empty()) hidden.push_back(alias); - auto identifier = static_cast(table_expression->children[0].get()); - if (!identifier && alias.empty()) + if (auto * identifier = typeid_cast(table_expression->children[0].get())) + hidden.push_back(identifier->name); + else if (alias.empty()) throw Exception("Expected Identifier or subquery with alias", ErrorCodes::LOGICAL_ERROR); - hidden.push_back(identifier->name); } @@ -103,7 +104,7 @@ void JoinToSubqueryTransformMatcher::visit(ASTSelectQuery & select, ASTPtr & ast if (!needRewrite(select)) return; - auto tables = static_cast(select.tables.get()); + auto tables = typeid_cast(select.tables.get()); if (!tables) throw Exception("TablesInSelectQuery expected", ErrorCodes::LOGICAL_ERROR); @@ -141,15 +142,15 @@ ASTPtr JoinToSubqueryTransformMatcher::replaceJoin(ASTSelectQuery & select, ASTP OneTypeMatcher>; using RewriteVisitor = InDepthNodeVisitor; - auto left = static_cast(ast_left.get()); - auto right = static_cast(ast_right.get()); + auto left = typeid_cast(ast_left.get()); + auto right = typeid_cast(ast_right.get()); if (!left || !right) throw Exception("Two TablesInSelectQueryElements expected", ErrorCodes::LOGICAL_ERROR); if (!right->table_join || right->array_join) return {}; - auto table_join = static_cast(right->table_join.get()); + auto table_join = typeid_cast(right->table_join.get()); if (table_join->kind != ASTTableJoin::Kind::Inner) return {}; diff --git a/dbms/src/Interpreters/MutationsInterpreter.cpp b/dbms/src/Interpreters/MutationsInterpreter.cpp index 22e5800fca4..64f30bdbc8a 100644 --- a/dbms/src/Interpreters/MutationsInterpreter.cpp +++ b/dbms/src/Interpreters/MutationsInterpreter.cpp @@ -72,8 +72,7 @@ bool MutationsInterpreter::isStorageTouchedByMutations() const context_copy.getSettingsRef().merge_tree_uniform_read_distribution = 0; context_copy.getSettingsRef().max_threads = 1; - InterpreterSelectQuery interpreter_select(select, context_copy, storage, QueryProcessingStage::Complete); - BlockInputStreamPtr in = interpreter_select.execute().in; + BlockInputStreamPtr in = InterpreterSelectQuery(select, context_copy, storage, QueryProcessingStage::Complete).execute().in; Block block = in->read(); if (!block.rows()) @@ -195,7 +194,7 @@ void MutationsInterpreter::prepare(bool dry_run) if (col_default.kind == ColumnDefaultKind::Materialized) { auto query = col_default.expression->clone(); - auto syntax_result = SyntaxAnalyzer(context, {}).analyze(query, all_columns); + auto syntax_result = SyntaxAnalyzer(context).analyze(query, all_columns); ExpressionAnalyzer analyzer(query, syntax_result, context); for (const String & dependency : analyzer.getRequiredSourceColumns()) { @@ -204,10 +203,9 @@ void MutationsInterpreter::prepare(bool dry_run) } } } - } - if (!updated_columns.empty()) validateUpdateColumns(storage, updated_columns, column_to_affected_materialized); + } /// First, break a sequence of commands into stages. stages.emplace_back(context); @@ -302,7 +300,7 @@ void MutationsInterpreter::prepare(bool dry_run) for (const String & column : stage.output_columns) all_asts->children.push_back(std::make_shared(column)); - auto syntax_result = SyntaxAnalyzer(context, {}).analyze(all_asts, all_columns); + auto syntax_result = SyntaxAnalyzer(context).analyze(all_asts, all_columns); stage.analyzer = std::make_unique(all_asts, syntax_result, context); ExpressionActionsChain & actions_chain = stage.expressions_chain; diff --git a/dbms/src/Interpreters/OptimizeIfWithConstantConditionVisitor.cpp b/dbms/src/Interpreters/OptimizeIfWithConstantConditionVisitor.cpp new file mode 100644 index 00000000000..e73a734ab16 --- /dev/null +++ b/dbms/src/Interpreters/OptimizeIfWithConstantConditionVisitor.cpp @@ -0,0 +1,108 @@ +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + +static bool tryExtractConstValueFromCondition(const ASTPtr & condition, bool & value) +{ + /// numeric constant in condition + if (const ASTLiteral * literal = typeid_cast(condition.get())) + { + if (literal->value.getType() == Field::Types::Int64 || + literal->value.getType() == Field::Types::UInt64) + { + value = literal->value.get(); + return true; + } + } + + /// cast of numeric constant in condition to UInt8 + if (const ASTFunction * function = typeid_cast(condition.get())) + { + if (function->name == "CAST") + { + if (ASTExpressionList * expr_list = typeid_cast(function->arguments.get())) + { + const ASTPtr & type_ast = expr_list->children.at(1); + if (const ASTLiteral * type_literal = typeid_cast(type_ast.get())) + { + if (type_literal->value.getType() == Field::Types::String && + type_literal->value.get() == "UInt8") + return tryExtractConstValueFromCondition(expr_list->children.at(0), value); + } + } + } + } + + return false; +} + +void OptimizeIfWithConstantConditionVisitor::visit(ASTPtr & current_ast) +{ + if (!current_ast) + return; + + for (ASTPtr & child : current_ast->children) + { + auto * function_node = typeid_cast(child.get()); + if (!function_node || function_node->name != "if") + { + visit(child); + continue; + } + + visit(function_node->arguments); + auto * args = typeid_cast(function_node->arguments.get()); + + if (args->children.size() != 3) + throw Exception("Wrong number of arguments for function 'if' (" + toString(args->children.size()) + " instead of 3)", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + ASTPtr condition_expr = args->children[0]; + ASTPtr then_expr = args->children[1]; + ASTPtr else_expr = args->children[2]; + + bool condition; + if (tryExtractConstValueFromCondition(condition_expr, condition)) + { + ASTPtr replace_ast = condition ? then_expr : else_expr; + ASTPtr child_copy = child; + String replace_alias = replace_ast->tryGetAlias(); + String if_alias = child->tryGetAlias(); + + if (replace_alias.empty()) + { + replace_ast->setAlias(if_alias); + child = replace_ast; + } + else + { + /// Only copy of one node is required here. + /// But IAST has only method for deep copy of subtree. + /// This can be a reason of performance degradation in case of deep queries. + ASTPtr replace_ast_deep_copy = replace_ast->clone(); + replace_ast_deep_copy->setAlias(if_alias); + child = replace_ast_deep_copy; + } + + if (!if_alias.empty()) + { + auto alias_it = aliases.find(if_alias); + if (alias_it != aliases.end() && alias_it->second.get() == child_copy.get()) + alias_it->second = child; + } + } + } +} + +} diff --git a/dbms/src/Interpreters/OptimizeIfWithConstantConditionVisitor.h b/dbms/src/Interpreters/OptimizeIfWithConstantConditionVisitor.h new file mode 100644 index 00000000000..ee738ec05e2 --- /dev/null +++ b/dbms/src/Interpreters/OptimizeIfWithConstantConditionVisitor.h @@ -0,0 +1,27 @@ +#pragma once + +#include + +#include + +namespace DB +{ + +/// It removes Function_if node from AST if condition is constant. +/// TODO: rewrite with InDepthNodeVisitor +class OptimizeIfWithConstantConditionVisitor +{ +public: + using Aliases = std::unordered_map; + + OptimizeIfWithConstantConditionVisitor(Aliases & aliases_) + : aliases(aliases_) + {} + + void visit(ASTPtr & ast); + +private: + Aliases & aliases; +}; + +} diff --git a/dbms/src/Interpreters/PartLog.cpp b/dbms/src/Interpreters/PartLog.cpp index d5ae95b5ea3..4ca9c130ef5 100644 --- a/dbms/src/Interpreters/PartLog.cpp +++ b/dbms/src/Interpreters/PartLog.cpp @@ -93,12 +93,12 @@ void PartLogElement::appendToBlock(Block & block) const } -bool PartLog::addNewPart(Context & context, const MutableDataPartPtr & part, UInt64 elapsed_ns, const ExecutionStatus & execution_status) +bool PartLog::addNewPart(Context & current_context, const MutableDataPartPtr & part, UInt64 elapsed_ns, const ExecutionStatus & execution_status) { - return addNewParts(context, {part}, elapsed_ns, execution_status); + return addNewParts(current_context, {part}, elapsed_ns, execution_status); } -bool PartLog::addNewParts(Context & context, const PartLog::MutableDataPartsVector & parts, UInt64 elapsed_ns, +bool PartLog::addNewParts(Context & current_context, const PartLog::MutableDataPartsVector & parts, UInt64 elapsed_ns, const ExecutionStatus & execution_status) { if (parts.empty()) @@ -108,7 +108,7 @@ bool PartLog::addNewParts(Context & context, const PartLog::MutableDataPartsVect try { - part_log = context.getPartLog(parts.front()->storage.getDatabaseName()); // assume parts belong to the same table + part_log = current_context.getPartLog(parts.front()->storage.getDatabaseName()); // assume parts belong to the same table if (!part_log) return false; diff --git a/dbms/src/Interpreters/PredicateExpressionsOptimizer.cpp b/dbms/src/Interpreters/PredicateExpressionsOptimizer.cpp index af84eac7f91..408b827adae 100644 --- a/dbms/src/Interpreters/PredicateExpressionsOptimizer.cpp +++ b/dbms/src/Interpreters/PredicateExpressionsOptimizer.cpp @@ -14,6 +14,12 @@ namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; + extern const int UNKNOWN_ELEMENT_IN_AST; +} + static constexpr auto and_function_name = "and"; PredicateExpressionsOptimizer::PredicateExpressionsOptimizer( @@ -400,6 +406,8 @@ ASTs PredicateExpressionsOptimizer::evaluateAsterisk(ASTSelectQuery * select_que DatabaseAndTableWithAlias database_and_table_name(*database_and_table_ast); storage = context.getTable(database_and_table_name.database, database_and_table_name.table); } + else + throw Exception("Logical error: unexpected table expression", ErrorCodes::LOGICAL_ERROR); const auto block = storage->getSampleBlock(); for (size_t idx = 0; idx < block.columns(); idx++) diff --git a/dbms/src/Interpreters/PredicateExpressionsOptimizer.h b/dbms/src/Interpreters/PredicateExpressionsOptimizer.h index e999489475c..65148e0682a 100644 --- a/dbms/src/Interpreters/PredicateExpressionsOptimizer.h +++ b/dbms/src/Interpreters/PredicateExpressionsOptimizer.h @@ -14,12 +14,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; - extern const int NUMBER_OF_COLUMNS_DOESNT_MATCH; -} - using PredicateExpressions = std::vector; using ProjectionWithAlias = std::pair; using ProjectionsWithAliases = std::vector; diff --git a/dbms/src/Interpreters/ProcessList.cpp b/dbms/src/Interpreters/ProcessList.cpp index 20beac13cbf..4c254731277 100644 --- a/dbms/src/Interpreters/ProcessList.cpp +++ b/dbms/src/Interpreters/ProcessList.cpp @@ -283,7 +283,7 @@ QueryStatus::~QueryStatus() = default; void QueryStatus::setQueryStreams(const BlockIO & io) { - std::lock_guard lock(query_streams_mutex); + std::lock_guard lock(query_streams_mutex); query_stream_in = io.in; query_stream_out = io.out; @@ -296,7 +296,7 @@ void QueryStatus::releaseQueryStreams() BlockOutputStreamPtr out; { - std::lock_guard lock(query_streams_mutex); + std::lock_guard lock(query_streams_mutex); query_streams_status = QueryStreamsStatus::Released; in = std::move(query_stream_in); @@ -308,14 +308,14 @@ void QueryStatus::releaseQueryStreams() bool QueryStatus::streamsAreReleased() { - std::lock_guard lock(query_streams_mutex); + std::lock_guard lock(query_streams_mutex); return query_streams_status == QueryStreamsStatus::Released; } bool QueryStatus::tryGetQueryStreams(BlockInputStreamPtr & in, BlockOutputStreamPtr & out) const { - std::lock_guard lock(query_streams_mutex); + std::lock_guard lock(query_streams_mutex); if (query_streams_status != QueryStreamsStatus::Initialized) return false; @@ -358,7 +358,7 @@ QueryStatus * ProcessList::tryGetProcessListElement(const String & current_query ProcessList::CancellationCode ProcessList::sendCancelToQuery(const String & current_query_id, const String & current_user, bool kill) { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); QueryStatus * elem = tryGetProcessListElement(current_query_id, current_user); @@ -431,7 +431,7 @@ ProcessList::Info ProcessList::getInfo(bool get_thread_list, bool get_profile_ev { Info per_query_infos; - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); per_query_infos.reserve(processes.size()); for (const auto & process : processes) diff --git a/dbms/src/Interpreters/ProcessList.h b/dbms/src/Interpreters/ProcessList.h index d96209d885f..5d2b6db95d0 100644 --- a/dbms/src/Interpreters/ProcessList.h +++ b/dbms/src/Interpreters/ProcessList.h @@ -308,7 +308,7 @@ public: void setMaxSize(size_t max_size_) { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); max_size = max_size_; } diff --git a/dbms/src/Interpreters/ProfileEventsExt.cpp b/dbms/src/Interpreters/ProfileEventsExt.cpp index 75dff6daa82..e5c0a3013d5 100644 --- a/dbms/src/Interpreters/ProfileEventsExt.cpp +++ b/dbms/src/Interpreters/ProfileEventsExt.cpp @@ -41,7 +41,7 @@ void dumpToArrayColumns(const Counters & counters, DB::IColumn * column_names_, if (column_names) { auto & offsets = column_names->getOffsets(); - offsets.push_back((offsets.empty() ? 0 : offsets.back()) + size); + offsets.push_back(offsets.back() + size); } if (column_values) @@ -51,7 +51,7 @@ void dumpToArrayColumns(const Counters & counters, DB::IColumn * column_names_, if (!the_same_offsets) { auto & offsets = column_values->getOffsets(); - offsets.push_back((offsets.empty() ? 0 : offsets.back()) + size); + offsets.push_back(offsets.back() + size); } } } diff --git a/dbms/src/Interpreters/QueryPriorities.h b/dbms/src/Interpreters/QueryPriorities.h index 04d3e79825c..5ded236e074 100644 --- a/dbms/src/Interpreters/QueryPriorities.h +++ b/dbms/src/Interpreters/QueryPriorities.h @@ -60,7 +60,7 @@ private: std::chrono::nanoseconds cur_timeout = timeout; Stopwatch watch(CLOCK_MONOTONIC_COARSE); - std::unique_lock lock(mutex); + std::unique_lock lock(mutex); while (true) { @@ -109,7 +109,7 @@ public: ~HandleImpl() { { - std::lock_guard lock(parent.mutex); + std::lock_guard lock(parent.mutex); --value.second; } parent.condvar.notify_all(); @@ -132,7 +132,7 @@ public: if (0 == priority) return {}; - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); auto it = container.emplace(priority, 0).first; ++it->second; return std::make_shared(*this, *it); diff --git a/dbms/src/Interpreters/Quota.cpp b/dbms/src/Interpreters/Quota.cpp index 15655f54b6b..5123f4fd3e8 100644 --- a/dbms/src/Interpreters/Quota.cpp +++ b/dbms/src/Interpreters/Quota.cpp @@ -298,7 +298,7 @@ QuotaForIntervalsPtr Quota::get(const String & quota_key, const String & user_na ? quota_key : user_name)); - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); Container::iterator it = quota_for_keys.find(quota_key_hashed); if (quota_for_keys.end() == it) diff --git a/dbms/src/Interpreters/Quota.h b/dbms/src/Interpreters/Quota.h index 4418bbb4c1b..c1fb3f143fb 100644 --- a/dbms/src/Interpreters/Quota.h +++ b/dbms/src/Interpreters/Quota.h @@ -183,7 +183,8 @@ public: } QuotaForIntervals() = default; - QuotaForIntervals(const QuotaForIntervals & other) = default; + QuotaForIntervals(const QuotaForIntervals &) = default; + QuotaForIntervals & operator=(const QuotaForIntervals &) = default; /// Is there at least one interval for counting quota? bool empty() const diff --git a/dbms/src/Interpreters/Set.cpp b/dbms/src/Interpreters/Set.cpp index 022cd9bd404..b41593acb4d 100644 --- a/dbms/src/Interpreters/Set.cpp +++ b/dbms/src/Interpreters/Set.cpp @@ -148,7 +148,6 @@ void Set::setHeader(const Block & block) set_elements.emplace_back(removeNullable(type)->createColumn()); } - /// Choose data structure to use for the set. data.init(data.chooseMethod(key_columns, key_sizes)); } diff --git a/dbms/src/Interpreters/SetVariants.cpp b/dbms/src/Interpreters/SetVariants.cpp index f0d9bbb2af8..57d78d77526 100644 --- a/dbms/src/Interpreters/SetVariants.cpp +++ b/dbms/src/Interpreters/SetVariants.cpp @@ -25,9 +25,6 @@ void SetVariantsTemplate::init(Type type_) case Type::NAME: NAME = std::make_unique(); break; APPLY_FOR_SET_VARIANTS(M) #undef M - - default: - throw Exception("Unknown Set variant.", ErrorCodes::UNKNOWN_SET_DATA_VARIANT); } } @@ -42,10 +39,9 @@ size_t SetVariantsTemplate::getTotalRowCount() const case Type::NAME: return NAME->data.size(); APPLY_FOR_SET_VARIANTS(M) #undef M - - default: - throw Exception("Unknown Set variant.", ErrorCodes::UNKNOWN_SET_DATA_VARIANT); } + + __builtin_unreachable(); } template @@ -59,10 +55,9 @@ size_t SetVariantsTemplate::getTotalByteCount() const case Type::NAME: return NAME->data.getBufferSizeInBytes(); APPLY_FOR_SET_VARIANTS(M) #undef M - - default: - throw Exception("Unknown Set variant.", ErrorCodes::UNKNOWN_SET_DATA_VARIANT); } + + __builtin_unreachable(); } template @@ -137,7 +132,7 @@ typename SetVariantsTemplate::Type SetVariantsTemplate::choose } /// If there is one numeric key that fits into 64 bits - if (keys_size == 1 && nested_key_columns[0]->isNumeric()) + if (keys_size == 1 && nested_key_columns[0]->isNumeric() && !nested_key_columns[0]->lowCardinality()) { size_t size_of_field = nested_key_columns[0]->sizeOfValueIfFixed(); if (size_of_field == 1) diff --git a/dbms/src/Interpreters/Settings.cpp b/dbms/src/Interpreters/Settings.cpp index f749d10c139..a15c0123e6f 100644 --- a/dbms/src/Interpreters/Settings.cpp +++ b/dbms/src/Interpreters/Settings.cpp @@ -202,7 +202,7 @@ void Settings::dumpToArrayColumns(IColumn * column_names_, IColumn * column_valu if (column_names) { auto & offsets = column_names->getOffsets(); - offsets.push_back((offsets.empty() ? 0 : offsets.back()) + size); + offsets.push_back(offsets.back() + size); } /// Nested columns case @@ -211,7 +211,7 @@ void Settings::dumpToArrayColumns(IColumn * column_names_, IColumn * column_valu if (column_values && !the_same_offsets) { auto & offsets = column_values->getOffsets(); - offsets.push_back((offsets.empty() ? 0 : offsets.back()) + size); + offsets.push_back(offsets.back() + size); } } diff --git a/dbms/src/Interpreters/SettingsCommon.cpp b/dbms/src/Interpreters/SettingsCommon.cpp index b79d8650da0..45f9d92d0c7 100644 --- a/dbms/src/Interpreters/SettingsCommon.cpp +++ b/dbms/src/Interpreters/SettingsCommon.cpp @@ -355,10 +355,9 @@ String SettingTotalsMode::toString() const case TotalsMode::AFTER_HAVING_EXCLUSIVE: return "after_having_exclusive"; case TotalsMode::AFTER_HAVING_INCLUSIVE: return "after_having_inclusive"; case TotalsMode::AFTER_HAVING_AUTO: return "after_having_auto"; - - default: - throw Exception("Unknown TotalsMode enum value", ErrorCodes::UNKNOWN_TOTALS_MODE); } + + __builtin_unreachable(); } void SettingTotalsMode::set(TotalsMode x) diff --git a/dbms/src/Interpreters/SyntaxAnalyzer.cpp b/dbms/src/Interpreters/SyntaxAnalyzer.cpp index 3d9a7f55df3..364cf221f35 100644 --- a/dbms/src/Interpreters/SyntaxAnalyzer.cpp +++ b/dbms/src/Interpreters/SyntaxAnalyzer.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -34,7 +35,6 @@ namespace DB namespace ErrorCodes { - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int ALIAS_REQUIRED; extern const int MULTIPLE_EXPRESSIONS_FOR_ALIAS; extern const int EMPTY_NESTED_TABLE; @@ -42,141 +42,6 @@ namespace ErrorCodes extern const int INVALID_JOIN_ON_EXPRESSION; } -namespace -{ - -using LogAST = DebugASTLog; /// set to true to enable logs -using Aliases = SyntaxAnalyzerResult::Aliases; - -/// Add columns from storage to source_columns list. -void collectSourceColumns(ASTSelectQuery * select_query, const Context & context, - StoragePtr & storage, NamesAndTypesList & source_columns); - -/// Translate qualified names such as db.table.column, table.column, table_alias.column to unqualified names. -void translateQualifiedNames(ASTPtr & query, ASTSelectQuery * select_query, - const NameSet & source_columns, const Context & context); - -/// For star nodes(`*`), expand them to a list of all columns. For literal nodes, substitute aliases. -void normalizeTree( - ASTPtr & query, - SyntaxAnalyzerResult & result, - const Names & source_columns, - const NameSet & source_columns_set, - const StoragePtr & storage, - const Context & context, - const ASTSelectQuery * select_query, - bool asterisk_left_columns_only); - -/// Sometimes we have to calculate more columns in SELECT clause than will be returned from query. -/// This is the case when we have DISTINCT or arrayJoin: we require more columns in SELECT even if we need less columns in result. -void removeUnneededColumnsFromSelectClause(const ASTSelectQuery * select_query, const Names & required_result_columns); - -/// Replacing scalar subqueries with constant values. -void executeScalarSubqueries(ASTPtr & query, const ASTSelectQuery * select_query, - const Context & context, size_t subquery_depth); - -/// Remove Function_if AST if condition is constant. -void optimizeIfWithConstantCondition(ASTPtr & current_ast, Aliases & aliases); - -/// Eliminates injective function calls and constant expressions from group by statement. -void optimizeGroupBy(ASTSelectQuery * select_query, const NameSet & source_columns, const Context & context); - -/// Remove duplicate items from ORDER BY. -void optimizeOrderBy(const ASTSelectQuery * select_query); - -/// Remove duplicate items from LIMIT BY. -void optimizeLimitBy(const ASTSelectQuery * select_query); - -/// Remove duplicated columns from USING(...). -void optimizeUsing(const ASTSelectQuery * select_query); - -void getArrayJoinedColumns(ASTPtr & query, SyntaxAnalyzerResult & result, const ASTSelectQuery * select_query, - const Names & source_columns, const NameSet & source_columns_set); - -/// Parse JOIN ON expression and collect ASTs for joined columns. -void collectJoinedColumnsFromJoinOnExpr(AnalyzedJoin & analyzed_join, const ASTSelectQuery * select_query, - const NameSet & source_columns, const Context & context); - -/// Find the columns that are obtained by JOIN. -void collectJoinedColumns(AnalyzedJoin & analyzed_join, const ASTSelectQuery * select_query, - const NameSet & source_columns, const Context & context); -} - -SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyze( - ASTPtr & query, - const NamesAndTypesList & source_columns_, - const Names & required_result_columns, - size_t subquery_depth) const -{ - SyntaxAnalyzerResult result; - result.storage = storage; - result.source_columns = source_columns_; - auto * select_query = typeid_cast(query.get()); - collectSourceColumns(select_query, context, result.storage, result.source_columns); - - const auto & settings = context.getSettingsRef(); - - Names source_columns_list; - source_columns_list.reserve(result.source_columns.size()); - for (const auto & type_name : result.source_columns) - source_columns_list.emplace_back(type_name.name); - NameSet source_columns_set(source_columns_list.begin(), source_columns_list.end()); - - translateQualifiedNames(query, select_query, source_columns_set, context); - - /// Depending on the user's profile, check for the execution rights - /// distributed subqueries inside the IN or JOIN sections and process these subqueries. - InJoinSubqueriesPreprocessor(context).process(select_query); - - /// Optimizes logical expressions. - LogicalExpressionsOptimizer(select_query, settings.optimize_min_equality_disjunction_chain_length.value).perform(); - - /// Creates a dictionary `aliases`: alias -> ASTPtr - { - LogAST log; - QueryAliasesVisitor::Data query_aliases_data{result.aliases}; - QueryAliasesVisitor(query_aliases_data, log.stream()).visit(query); - } - - /// Common subexpression elimination. Rewrite rules. - normalizeTree(query, result, source_columns_list, source_columns_set, result.storage, - context, select_query, settings.asterisk_left_columns_only != 0); - - /// Remove unneeded columns according to 'required_result_columns'. - /// Leave all selected columns in case of DISTINCT; columns that contain arrayJoin function inside. - /// Must be after 'normalizeTree' (after expanding aliases, for aliases not get lost) - /// and before 'executeScalarSubqueries', 'analyzeAggregation', etc. to avoid excessive calculations. - removeUnneededColumnsFromSelectClause(select_query, required_result_columns); - - /// Executing scalar subqueries - replacing them with constant values. - executeScalarSubqueries(query, select_query, context, subquery_depth); - - /// Optimize if with constant condition after constants was substituted instead of sclalar subqueries. - optimizeIfWithConstantCondition(query, result.aliases); - - /// GROUP BY injective function elimination. - optimizeGroupBy(select_query, source_columns_set, context); - - /// Remove duplicate items from ORDER BY. - optimizeOrderBy(select_query); - - // Remove duplicated elements from LIMIT BY clause. - optimizeLimitBy(select_query); - - /// Remove duplicated columns from USING(...). - optimizeUsing(select_query); - - /// array_join_alias_to_name, array_join_result_to_source. - getArrayJoinedColumns(query, result, select_query, source_columns_list, source_columns_set); - - /// Push the predicate expression down to the subqueries. - result.rewrite_subqueries = PredicateExpressionsOptimizer(select_query, settings, context).optimize(); - - collectJoinedColumns(result.analyzed_join, select_query, source_columns_set, context); - - return std::make_shared(result); -} - void removeDuplicateColumns(NamesAndTypesList & columns) { std::set names; @@ -192,15 +57,12 @@ void removeDuplicateColumns(NamesAndTypesList & columns) namespace { -void collectSourceColumns(ASTSelectQuery * select_query, const Context & context, - StoragePtr & storage, NamesAndTypesList & source_columns) -{ - if (!storage && select_query) - { - if (auto db_and_table = getDatabaseAndTable(*select_query, 0)) - storage = context.tryGetTable(db_and_table->database, db_and_table->table); - } +using LogAST = DebugASTLog; /// set to true to enable logs + +/// Add columns from storage to source_columns list. +void collectSourceColumns(ASTSelectQuery * select_query, StoragePtr storage, NamesAndTypesList & source_columns) +{ if (storage) { auto physical_columns = storage->getColumns().getAllPhysical(); @@ -219,10 +81,11 @@ void collectSourceColumns(ASTSelectQuery * select_query, const Context & context removeDuplicateColumns(source_columns); } +/// Translate qualified names such as db.table.column, table.column, table_alias.column to unqualified names. void translateQualifiedNames(ASTPtr & query, ASTSelectQuery * select_query, const NameSet & source_columns, const Context & context) { - if (!select_query || !select_query->tables || select_query->tables->children.empty()) + if (!select_query->tables || select_query->tables->children.empty()) return; std::vector tables = getDatabaseAndTables(*select_query, context.getCurrentDatabase()); @@ -233,6 +96,7 @@ void translateQualifiedNames(ASTPtr & query, ASTSelectQuery * select_query, visitor.visit(query); } +/// For star nodes(`*`), expand them to a list of all columns. For literal nodes, substitute aliases. void normalizeTree( ASTPtr & query, SyntaxAnalyzerResult & result, @@ -297,11 +161,10 @@ bool hasArrayJoin(const ASTPtr & ast) return false; } +/// Sometimes we have to calculate more columns in SELECT clause than will be returned from query. +/// This is the case when we have DISTINCT or arrayJoin: we require more columns in SELECT even if we need less columns in result. void removeUnneededColumnsFromSelectClause(const ASTSelectQuery * select_query, const Names & required_result_columns) { - if (!select_query) - return; - if (required_result_columns.empty()) return; @@ -335,121 +198,12 @@ void removeUnneededColumnsFromSelectClause(const ASTSelectQuery * select_query, elements = std::move(new_elements); } -void executeScalarSubqueries(ASTPtr & query, const ASTSelectQuery * select_query, - const Context & context, size_t subquery_depth) +/// Replacing scalar subqueries with constant values. +void executeScalarSubqueries(ASTPtr & query, const Context & context, size_t subquery_depth) { LogAST log; - - if (!select_query) - { - ExecuteScalarSubqueriesVisitor::Data visitor_data{context, subquery_depth}; - ExecuteScalarSubqueriesVisitor(visitor_data, log.stream()).visit(query); - } - else - { - for (auto & child : query->children) - { - /// Do not go to FROM, JOIN, UNION. - if (!typeid_cast(child.get()) - && !typeid_cast(child.get())) - { - ExecuteScalarSubqueriesVisitor::Data visitor_data{context, subquery_depth}; - ExecuteScalarSubqueriesVisitor(visitor_data, log.stream()).visit(child); - } - } - } -} - -bool tryExtractConstValueFromCondition(const ASTPtr & condition, bool & value) -{ - /// numeric constant in condition - if (const ASTLiteral * literal = typeid_cast(condition.get())) - { - if (literal->value.getType() == Field::Types::Int64 || - literal->value.getType() == Field::Types::UInt64) - { - value = literal->value.get(); - return true; - } - } - - /// cast of numeric constant in condition to UInt8 - if (const ASTFunction * function = typeid_cast(condition.get())) - { - if (function->name == "CAST") - { - if (ASTExpressionList * expr_list = typeid_cast(function->arguments.get())) - { - const ASTPtr & type_ast = expr_list->children.at(1); - if (const ASTLiteral * type_literal = typeid_cast(type_ast.get())) - { - if (type_literal->value.getType() == Field::Types::String && - type_literal->value.get() == "UInt8") - return tryExtractConstValueFromCondition(expr_list->children.at(0), value); - } - } - } - } - - return false; -} - -void optimizeIfWithConstantCondition(ASTPtr & current_ast, Aliases & aliases) -{ - if (!current_ast) - return; - - for (ASTPtr & child : current_ast->children) - { - auto * function_node = typeid_cast(child.get()); - if (!function_node || function_node->name != "if") - { - optimizeIfWithConstantCondition(child, aliases); - continue; - } - - optimizeIfWithConstantCondition(function_node->arguments, aliases); - auto * args = typeid_cast(function_node->arguments.get()); - - if (args->children.size() != 3) - throw Exception("Wrong number of arguments for function 'if' (" + toString(args->children.size()) + " instead of 3)", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - - ASTPtr condition_expr = args->children[0]; - ASTPtr then_expr = args->children[1]; - ASTPtr else_expr = args->children[2]; - - bool condition; - if (tryExtractConstValueFromCondition(condition_expr, condition)) - { - ASTPtr replace_ast = condition ? then_expr : else_expr; - ASTPtr child_copy = child; - String replace_alias = replace_ast->tryGetAlias(); - String if_alias = child->tryGetAlias(); - - if (replace_alias.empty()) - { - replace_ast->setAlias(if_alias); - child = replace_ast; - } - else - { - /// Only copy of one node is required here. - /// But IAST has only method for deep copy of subtree. - /// This can be a reason of performance degradation in case of deep queries. - ASTPtr replace_ast_deep_copy = replace_ast->clone(); - replace_ast_deep_copy->setAlias(if_alias); - child = replace_ast_deep_copy; - } - - if (!if_alias.empty()) - { - auto alias_it = aliases.find(if_alias); - if (alias_it != aliases.end() && alias_it->second.get() == child_copy.get()) - alias_it->second = child; - } - } - } + ExecuteScalarSubqueriesVisitor::Data visitor_data{context, subquery_depth}; + ExecuteScalarSubqueriesVisitor(visitor_data, log.stream()).visit(query); } /** Calls to these functions in the GROUP BY statement would be @@ -491,9 +245,10 @@ const std::unordered_set possibly_injective_function_names "dictGetDateTime" }; +/// Eliminates injective function calls and constant expressions from group by statement. void optimizeGroupBy(ASTSelectQuery * select_query, const NameSet & source_columns, const Context & context) { - if (!(select_query && select_query->group_expression_list)) + if (!select_query->group_expression_list) return; const auto is_literal = [] (const ASTPtr & ast) @@ -594,9 +349,10 @@ void optimizeGroupBy(ASTSelectQuery * select_query, const NameSet & source_colum } } +/// Remove duplicate items from ORDER BY. void optimizeOrderBy(const ASTSelectQuery * select_query) { - if (!(select_query && select_query->order_expression_list)) + if (!select_query->order_expression_list) return; /// Make unique sorting conditions. @@ -620,9 +376,10 @@ void optimizeOrderBy(const ASTSelectQuery * select_query) elems = unique_elems; } +/// Remove duplicate items from LIMIT BY. void optimizeLimitBy(const ASTSelectQuery * select_query) { - if (!(select_query && select_query->limit_by_expression_list)) + if (!select_query->limit_by_expression_list) return; std::set elems_set; @@ -641,11 +398,9 @@ void optimizeLimitBy(const ASTSelectQuery * select_query) elems = unique_elems; } +/// Remove duplicated columns from USING(...). void optimizeUsing(const ASTSelectQuery * select_query) { - if (!select_query) - return; - auto node = const_cast(select_query->join()); if (!node) return; @@ -676,9 +431,6 @@ void optimizeUsing(const ASTSelectQuery * select_query) void getArrayJoinedColumns(ASTPtr & query, SyntaxAnalyzerResult & result, const ASTSelectQuery * select_query, const Names & source_columns, const NameSet & source_columns_set) { - if (!select_query) - return; - ASTPtr array_join_expression_list = select_query->array_join_expression_list(); if (array_join_expression_list) { @@ -740,6 +492,7 @@ void getArrayJoinedColumns(ASTPtr & query, SyntaxAnalyzerResult & result, const } } +/// Parse JOIN ON expression and collect ASTs for joined columns. void collectJoinedColumnsFromJoinOnExpr(AnalyzedJoin & analyzed_join, const ASTSelectQuery * select_query, const NameSet & source_columns, const Context & context) { @@ -899,12 +652,10 @@ void collectJoinedColumnsFromJoinOnExpr(AnalyzedJoin & analyzed_join, const ASTS add_columns_from_equals_expr(table_join.on_expression); } +/// Find the columns that are obtained by JOIN. void collectJoinedColumns(AnalyzedJoin & analyzed_join, const ASTSelectQuery * select_query, const NameSet & source_columns, const Context & context) { - if (!select_query) - return; - const ASTTablesInSelectQueryElement * node = select_query->join(); if (!node) @@ -969,4 +720,94 @@ void collectJoinedColumns(AnalyzedJoin & analyzed_join, const ASTSelectQuery * s } + +SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyze( + ASTPtr & query, + const NamesAndTypesList & source_columns_, + const Names & required_result_columns, + StoragePtr storage) const +{ + auto * select_query = typeid_cast(query.get()); + if (!storage && select_query) + { + if (auto db_and_table = getDatabaseAndTable(*select_query, 0)) + storage = context.tryGetTable(db_and_table->database, db_and_table->table); + } + + SyntaxAnalyzerResult result; + result.storage = storage; + result.source_columns = source_columns_; + + collectSourceColumns(select_query, result.storage, result.source_columns); + + const auto & settings = context.getSettingsRef(); + + Names source_columns_list; + source_columns_list.reserve(result.source_columns.size()); + for (const auto & type_name : result.source_columns) + source_columns_list.emplace_back(type_name.name); + NameSet source_columns_set(source_columns_list.begin(), source_columns_list.end()); + + if (select_query) + { + translateQualifiedNames(query, select_query, source_columns_set, context); + + /// Depending on the user's profile, check for the execution rights + /// distributed subqueries inside the IN or JOIN sections and process these subqueries. + InJoinSubqueriesPreprocessor(context).process(select_query); + + /// Optimizes logical expressions. + LogicalExpressionsOptimizer(select_query, settings.optimize_min_equality_disjunction_chain_length.value).perform(); + } + + /// Creates a dictionary `aliases`: alias -> ASTPtr + { + LogAST log; + QueryAliasesVisitor::Data query_aliases_data{result.aliases}; + QueryAliasesVisitor(query_aliases_data, log.stream()).visit(query); + } + + /// Common subexpression elimination. Rewrite rules. + normalizeTree(query, result, source_columns_list, source_columns_set, result.storage, + context, select_query, settings.asterisk_left_columns_only != 0); + + /// Remove unneeded columns according to 'required_result_columns'. + /// Leave all selected columns in case of DISTINCT; columns that contain arrayJoin function inside. + /// Must be after 'normalizeTree' (after expanding aliases, for aliases not get lost) + /// and before 'executeScalarSubqueries', 'analyzeAggregation', etc. to avoid excessive calculations. + if (select_query) + removeUnneededColumnsFromSelectClause(select_query, required_result_columns); + + /// Executing scalar subqueries - replacing them with constant values. + executeScalarSubqueries(query, context, subquery_depth); + + /// Optimize if with constant condition after constants was substituted instead of sclalar subqueries. + OptimizeIfWithConstantConditionVisitor(result.aliases).visit(query); + + if (select_query) + { + /// GROUP BY injective function elimination. + optimizeGroupBy(select_query, source_columns_set, context); + + /// Remove duplicate items from ORDER BY. + optimizeOrderBy(select_query); + + /// Remove duplicated elements from LIMIT BY clause. + optimizeLimitBy(select_query); + + /// Remove duplicated columns from USING(...). + optimizeUsing(select_query); + + /// array_join_alias_to_name, array_join_result_to_source. + getArrayJoinedColumns(query, result, select_query, source_columns_list, source_columns_set); + + /// Push the predicate expression down to the subqueries. + result.rewrite_subqueries = PredicateExpressionsOptimizer(select_query, settings, context).optimize(); + + collectJoinedColumns(result.analyzed_join, select_query, source_columns_set, context); + } + + return std::make_shared(result); +} + } diff --git a/dbms/src/Interpreters/SyntaxAnalyzer.h b/dbms/src/Interpreters/SyntaxAnalyzer.h index 38595917917..5500823b3c2 100644 --- a/dbms/src/Interpreters/SyntaxAnalyzer.h +++ b/dbms/src/Interpreters/SyntaxAnalyzer.h @@ -54,16 +54,20 @@ using SyntaxAnalyzerResultPtr = std::shared_ptr; class SyntaxAnalyzer { public: - SyntaxAnalyzer(const Context & context, StoragePtr storage) : context(context), storage(std::move(storage)) {} + SyntaxAnalyzer(const Context & context_, size_t subquery_depth_ = 0) + : context(context_) + , subquery_depth(subquery_depth_) + {} SyntaxAnalyzerResultPtr analyze( ASTPtr & query, const NamesAndTypesList & source_columns_, const Names & required_result_columns = {}, - size_t subquery_depth = 0) const; + StoragePtr storage = {}) const; +private: const Context & context; - StoragePtr storage; + size_t subquery_depth; }; } diff --git a/dbms/src/Interpreters/createBlockSelector.cpp b/dbms/src/Interpreters/createBlockSelector.cpp index eee08259d5b..192e559cbb1 100644 --- a/dbms/src/Interpreters/createBlockSelector.cpp +++ b/dbms/src/Interpreters/createBlockSelector.cpp @@ -4,7 +4,7 @@ #include -#if __SSE2__ +#ifdef __SSE2__ #define LIBDIVIDE_USE_SSE2 1 #endif diff --git a/dbms/src/Interpreters/evaluateConstantExpression.cpp b/dbms/src/Interpreters/evaluateConstantExpression.cpp index 29753a4c637..8f96160186d 100644 --- a/dbms/src/Interpreters/evaluateConstantExpression.cpp +++ b/dbms/src/Interpreters/evaluateConstantExpression.cpp @@ -31,7 +31,7 @@ std::pair> evaluateConstantExpression(co { NamesAndTypesList source_columns = {{ "_dummy", std::make_shared() }}; auto ast = node->clone(); - auto syntax_result = SyntaxAnalyzer(context, {}).analyze(ast, source_columns); + auto syntax_result = SyntaxAnalyzer(context).analyze(ast, source_columns); ExpressionActionsPtr expr_for_constant_folding = ExpressionAnalyzer(ast, syntax_result, context).getConstActions(); /// There must be at least one column in the block so that it knows the number of rows. diff --git a/dbms/src/Interpreters/evaluateMissingDefaults.cpp b/dbms/src/Interpreters/evaluateMissingDefaults.cpp index 33dce42ab8e..9a6884b25e3 100644 --- a/dbms/src/Interpreters/evaluateMissingDefaults.cpp +++ b/dbms/src/Interpreters/evaluateMissingDefaults.cpp @@ -48,7 +48,7 @@ void evaluateMissingDefaults(Block & block, if (!save_unneeded_columns) { - auto syntax_result = SyntaxAnalyzer(context, {}).analyze(default_expr_list, block.getNamesAndTypesList()); + auto syntax_result = SyntaxAnalyzer(context).analyze(default_expr_list, block.getNamesAndTypesList()); ExpressionAnalyzer{default_expr_list, syntax_result, context}.getActions(true)->execute(block); return; } @@ -57,7 +57,7 @@ void evaluateMissingDefaults(Block & block, * we are going to operate on a copy instead of the original block */ Block copy_block{block}; - auto syntax_result = SyntaxAnalyzer(context, {}).analyze(default_expr_list, block.getNamesAndTypesList()); + auto syntax_result = SyntaxAnalyzer(context).analyze(default_expr_list, block.getNamesAndTypesList()); ExpressionAnalyzer{default_expr_list, syntax_result, context}.getActions(true)->execute(copy_block); /// move evaluated columns to the original block, materializing them at the same time diff --git a/dbms/src/Interpreters/tests/CMakeLists.txt b/dbms/src/Interpreters/tests/CMakeLists.txt index 7660527cd87..2f814c5a6a0 100644 --- a/dbms/src/Interpreters/tests/CMakeLists.txt +++ b/dbms/src/Interpreters/tests/CMakeLists.txt @@ -12,28 +12,28 @@ target_link_libraries (aggregate PRIVATE dbms) add_executable (hash_map hash_map.cpp) target_include_directories (hash_map SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) -target_link_libraries (hash_map PRIVATE dbms) +target_link_libraries (hash_map PRIVATE dbms clickhouse_compression) add_executable (hash_map3 hash_map3.cpp) target_link_libraries (hash_map3 PRIVATE dbms ${FARMHASH_LIBRARIES} ${METROHASH_LIBRARIES}) add_executable (hash_map_string hash_map_string.cpp) target_include_directories (hash_map_string SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) -target_link_libraries (hash_map_string PRIVATE dbms) +target_link_libraries (hash_map_string PRIVATE dbms clickhouse_compression) add_executable (hash_map_string_2 hash_map_string_2.cpp) -target_link_libraries (hash_map_string_2 PRIVATE dbms) +target_link_libraries (hash_map_string_2 PRIVATE dbms clickhouse_compression) add_executable (hash_map_string_3 hash_map_string_3.cpp) -target_link_libraries (hash_map_string_3 PRIVATE dbms ${FARMHASH_LIBRARIES} ${METROHASH_LIBRARIES}) +target_link_libraries (hash_map_string_3 PRIVATE dbms clickhouse_compression ${FARMHASH_LIBRARIES} ${METROHASH_LIBRARIES}) add_executable (hash_map_string_small hash_map_string_small.cpp) target_include_directories (hash_map_string_small SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) -target_link_libraries (hash_map_string_small PRIVATE dbms) +target_link_libraries (hash_map_string_small PRIVATE dbms clickhouse_compression) add_executable (two_level_hash_map two_level_hash_map.cpp) target_include_directories (two_level_hash_map SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) -target_link_libraries (two_level_hash_map PRIVATE dbms) +target_link_libraries (two_level_hash_map PRIVATE dbms clickhouse_compression) add_executable (compiler_test compiler_test.cpp) target_link_libraries (compiler_test PRIVATE dbms) diff --git a/dbms/src/Interpreters/tests/hash_map.cpp b/dbms/src/Interpreters/tests/hash_map.cpp index d7e90e5e49b..a3e1cad8d12 100644 --- a/dbms/src/Interpreters/tests/hash_map.cpp +++ b/dbms/src/Interpreters/tests/hash_map.cpp @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include #include @@ -107,16 +107,16 @@ int main(int argc, char ** argv) AggregateFunctionPtr func_avg = factory.get("avg", data_types_uint64); AggregateFunctionPtr func_uniq = factory.get("uniq", data_types_uint64); - #define INIT \ - { \ - value.resize(3); \ - \ - value[0] = func_count.get();\ - value[1] = func_avg.get(); \ - value[2] = func_uniq.get(); \ + #define INIT \ + { \ + value.resize(3); \ + \ + value[0] = func_count.get(); \ + value[1] = func_avg.get(); \ + value[2] = func_uniq.get(); \ } - INIT; + INIT #ifndef USE_AUTO_ARRAY #undef INIT @@ -162,8 +162,9 @@ int main(int argc, char ** argv) map.emplace(data[i], it, inserted); if (inserted) { - new(&it->second) Value(std::move(value)); - INIT; + new(&it->second) Value; + std::swap(it->second, value); + INIT } } @@ -192,8 +193,9 @@ int main(int argc, char ** argv) map.emplace(data[i], it, inserted); if (inserted) { - new(&it->second) Value(std::move(value)); - INIT; + new(&it->second) Value; + std::swap(it->second, value); + INIT } } @@ -223,8 +225,9 @@ int main(int argc, char ** argv) map.emplace(data[i], it, inserted); if (inserted) { - new(&it->second) Value(std::move(value)); - INIT; + new(&it->second) Value; + std::swap(it->second, value); + INIT } } @@ -248,8 +251,8 @@ int main(int argc, char ** argv) std::unordered_map>::iterator it; for (size_t i = 0; i < n; ++i) { - it = map.insert(std::make_pair(data[i], std::move(value))).first; - INIT; + it = map.insert(std::make_pair(data[i], value)).first; + INIT } watch.stop(); @@ -269,8 +272,8 @@ int main(int argc, char ** argv) map.set_empty_key(-1ULL); for (size_t i = 0; i < n; ++i) { - it = map.insert(std::make_pair(data[i], std::move(value))).first; - INIT; + it = map.insert(std::make_pair(data[i], value)).first; + INIT } watch.stop(); @@ -289,8 +292,8 @@ int main(int argc, char ** argv) GOOGLE_NAMESPACE::sparse_hash_map>::iterator it; for (size_t i = 0; i < n; ++i) { - map.insert(std::make_pair(data[i], std::move(value))); - INIT; + map.insert(std::make_pair(data[i], value)); + INIT } watch.stop(); diff --git a/dbms/src/Interpreters/tests/hash_map_string.cpp b/dbms/src/Interpreters/tests/hash_map_string.cpp index 5d2a8c305da..3ccd1d710a3 100644 --- a/dbms/src/Interpreters/tests/hash_map_string.cpp +++ b/dbms/src/Interpreters/tests/hash_map_string.cpp @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include #include @@ -81,10 +81,13 @@ struct DefaultHash }; -#define mix(h) ({ \ - (h) ^= (h) >> 23; \ - (h) *= 0x2127599bf4325c37ULL; \ - (h) ^= (h) >> 47; }) +static inline UInt64 mix(UInt64 h) +{ + h ^= h >> 23; + h *= 0x2127599bf4325c37ULL; + h ^= h >> 47; + return h; +} struct FastHash64 { diff --git a/dbms/src/Interpreters/tests/hash_map_string_2.cpp b/dbms/src/Interpreters/tests/hash_map_string_2.cpp index aba8d502270..330d80af8af 100644 --- a/dbms/src/Interpreters/tests/hash_map_string_2.cpp +++ b/dbms/src/Interpreters/tests/hash_map_string_2.cpp @@ -10,12 +10,12 @@ #include #include #include -#include +#include #include #include #include -#if __SSE4_1__ +#ifdef __SSE4_1__ #include #endif @@ -77,7 +77,7 @@ DefineStringRef(StringRef_Compare16_1_byMemcmp) DefineStringRef(StringRef_Compare16_1_byUInt64_logicAnd) DefineStringRef(StringRef_Compare16_1_byUInt64_bitAnd) -#if __SSE4_1__ +#ifdef __SSE4_1__ DefineStringRef(StringRef_Compare16_1_byIntSSE) DefineStringRef(StringRef_Compare16_1_byFloatSSE) DefineStringRef(StringRef_Compare16_1_bySSE4) @@ -196,7 +196,7 @@ inline bool compare_byUInt64_bitAnd(const char * p1, const char * p2) & (reinterpret_cast(p1)[1] == reinterpret_cast(p2)[1]); } -#if __SSE4_1__ +#ifdef __SSE4_1__ inline bool compare_byIntSSE(const char * p1, const char * p2) { @@ -264,7 +264,7 @@ inline bool memequal(const char * p1, const char * p2, size_t size) } -#if __SSE4_1__ +#ifdef __SSE4_1__ inline bool memequal_sse41(const char * p1, const char * p2, size_t size) { @@ -520,7 +520,7 @@ Op(byMemcmp) Op(byUInt64_logicAnd) Op(byUInt64_bitAnd) -#if __SSE4_1__ +#ifdef __SSE4_1__ Op(byIntSSE) Op(byFloatSSE) @@ -654,7 +654,7 @@ int main(int argc, char ** argv) if (!m || m == 5) bench (data, "StringRef_Compare16_1_byMemcmp"); if (!m || m == 6) bench(data, "StringRef_Compare16_1_byUInt64_logicAnd"); if (!m || m == 7) bench (data, "StringRef_Compare16_1_byUInt64_bitAnd"); -#if __SSE4_1__ +#ifdef __SSE4_1__ if (!m || m == 8) bench (data, "StringRef_Compare16_1_byIntSSE"); if (!m || m == 9) bench (data, "StringRef_Compare16_1_byFloatSSE"); if (!m || m == 10) bench (data, "StringRef_Compare16_1_bySSE4"); diff --git a/dbms/src/Interpreters/tests/hash_map_string_3.cpp b/dbms/src/Interpreters/tests/hash_map_string_3.cpp index 24c923db8ad..5d8d8e4f7c8 100644 --- a/dbms/src/Interpreters/tests/hash_map_string_3.cpp +++ b/dbms/src/Interpreters/tests/hash_map_string_3.cpp @@ -13,12 +13,12 @@ #include #include #include -#include +#include #include #include #include -#if __SSE4_1__ +#ifdef __SSE4_1__ #include #endif @@ -164,7 +164,7 @@ struct FNV1a }; -#if __SSE4_1__ +#ifdef __SSE4_1__ struct CrapWow { @@ -254,7 +254,7 @@ struct SimpleHash if (size < 8) { -#if __SSE4_1__ +#ifdef __SSE4_1__ return hashLessThan8(x.data, x.size); #endif } @@ -291,7 +291,7 @@ struct VerySimpleHash if (size < 8) { -#if __SSE4_1__ +#ifdef __SSE4_1__ return hashLessThan8(x.data, x.size); #endif } @@ -342,7 +342,7 @@ struct MetroHash64 }; -#if __SSE4_1__ +#ifdef __SSE4_1__ /*struct CRC32Hash { @@ -499,7 +499,7 @@ int main(int argc, char ** argv) if (!m || m == 3) bench (data, "StringRef_SimpleHash"); if (!m || m == 4) bench (data, "StringRef_FNV1a"); -#if __SSE4_1__ +#ifdef __SSE4_1__ if (!m || m == 5) bench (data, "StringRef_CrapWow"); if (!m || m == 6) bench (data, "StringRef_CRC32Hash"); if (!m || m == 7) bench (data, "StringRef_CRC32ILPHash"); diff --git a/dbms/src/Interpreters/tests/hash_map_string_small.cpp b/dbms/src/Interpreters/tests/hash_map_string_small.cpp index f3b48207cc1..c50e7f68a2d 100644 --- a/dbms/src/Interpreters/tests/hash_map_string_small.cpp +++ b/dbms/src/Interpreters/tests/hash_map_string_small.cpp @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include #include @@ -64,7 +64,7 @@ inline bool operator==(SmallStringRef lhs, SmallStringRef rhs) if (lhs.size == 0) return true; -#if __SSE2__ +#ifdef __SSE2__ return memequalSSE2Wide(lhs.data(), rhs.data(), lhs.size); #else return false; diff --git a/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp b/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp index e237249e1d3..1afdd644dd1 100644 --- a/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp +++ b/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp @@ -1131,7 +1131,7 @@ TestEntries entries = }; -bool performTests(const TestEntries & entries) +bool run() { unsigned int count = 0; unsigned int i = 1; @@ -1156,11 +1156,6 @@ bool performTests(const TestEntries & entries) return count == entries.size(); } -bool run() -{ - return performTests(entries); -} - TestResult check(const TestEntry & entry) { diff --git a/dbms/src/Interpreters/tests/internal_iotop.cpp b/dbms/src/Interpreters/tests/internal_iotop.cpp index 0ca49f94512..c1088819f85 100644 --- a/dbms/src/Interpreters/tests/internal_iotop.cpp +++ b/dbms/src/Interpreters/tests/internal_iotop.cpp @@ -12,7 +12,6 @@ #include - std::mutex mutex; @@ -52,8 +51,10 @@ void do_io(size_t id) TaskStatsInfoGetter get_info; get_info.getStat(stat, tid); - std::lock_guard lock(mutex); - std::cerr << "#" << id << ", tid " << tid << ", intitial\n" << stat << "\n"; + { + std::lock_guard lock(mutex); + std::cerr << "#" << id << ", tid " << tid << ", intitial\n" << stat << "\n"; + } size_t copy_size = 1048576 * (1 + id); std::string path_dst = "test_out_" + std::to_string(id); @@ -67,7 +68,7 @@ void do_io(size_t id) get_info.getStat(stat, tid); { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); std::cerr << "#" << id << ", tid " << tid << ", step1\n" << stat << "\n"; } @@ -79,7 +80,7 @@ void do_io(size_t id) get_info.getStat(stat, tid); { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); std::cerr << "#" << id << ", tid " << tid << ", step2\n" << stat << "\n"; } @@ -91,7 +92,7 @@ void do_io(size_t id) get_info.getStat(stat, tid); { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); std::cerr << "#" << id << ", tid " << tid << ", step3\n" << stat << "\n"; } diff --git a/dbms/src/Interpreters/tests/two_level_hash_map.cpp b/dbms/src/Interpreters/tests/two_level_hash_map.cpp index d7fe0d8b4b3..7b793d4f33a 100644 --- a/dbms/src/Interpreters/tests/two_level_hash_map.cpp +++ b/dbms/src/Interpreters/tests/two_level_hash_map.cpp @@ -14,7 +14,7 @@ #include #include -#include +#include #include #include diff --git a/dbms/src/Interpreters/tests/users.cpp b/dbms/src/Interpreters/tests/users.cpp index af07629ef62..4ef406fb2cd 100644 --- a/dbms/src/Interpreters/tests/users.cpp +++ b/dbms/src/Interpreters/tests/users.cpp @@ -161,11 +161,6 @@ TestSet test_set = } }; -std::string createTmpPath(const std::string & filename); -void createFile(const std::string & filename, const char * data); -void runOneTest(const TestDescriptor & test_descriptor); -auto runTestSet(const TestSet & test_set); - std::string createTmpPath(const std::string & filename) { char pattern[] = "/tmp/fileXXXXXX"; @@ -241,7 +236,7 @@ void runOneTest(const TestDescriptor & test_descriptor) fs::remove_all(fs::path(path_name).parent_path().string()); } -auto runTestSet(const TestSet & test_set) +auto runTestSet() { size_t test_num = 1; size_t failure_count = 0; @@ -277,7 +272,7 @@ int main() size_t test_count; size_t failure_count; - std::tie(test_count, failure_count) = runTestSet(test_set); + std::tie(test_count, failure_count) = runTestSet(); std::cout << (test_count - failure_count) << " test(s) passed out of " << test_count << "\n"; diff --git a/dbms/src/Parsers/ASTQueryWithOnCluster.h b/dbms/src/Parsers/ASTQueryWithOnCluster.h index 373307700aa..e053b50ffb2 100644 --- a/dbms/src/Parsers/ASTQueryWithOnCluster.h +++ b/dbms/src/Parsers/ASTQueryWithOnCluster.h @@ -28,6 +28,9 @@ public: static bool parse(Pos & pos, std::string & cluster_str, Expected & expected); virtual ~ASTQueryWithOnCluster() = default; + ASTQueryWithOnCluster() = default; + ASTQueryWithOnCluster(const ASTQueryWithOnCluster &) = default; + ASTQueryWithOnCluster & operator=(const ASTQueryWithOnCluster &) = default; protected: template diff --git a/dbms/src/Parsers/ASTSelectQuery.cpp b/dbms/src/Parsers/ASTSelectQuery.cpp index d204ab78a6e..eb4ee270a3a 100644 --- a/dbms/src/Parsers/ASTSelectQuery.cpp +++ b/dbms/src/Parsers/ASTSelectQuery.cpp @@ -308,6 +308,17 @@ const ASTTablesInSelectQueryElement * ASTSelectQuery::join() const return getFirstTableJoin(*this); } +static String getTableExpressionAlias(const ASTTableExpression * table_expression) +{ + if (table_expression->subquery) + return table_expression->subquery->tryGetAlias(); + else if (table_expression->table_function) + return table_expression->table_function->tryGetAlias(); + else if (table_expression->database_and_table_name) + return table_expression->database_and_table_name->tryGetAlias(); + + return String(); +} void ASTSelectQuery::replaceDatabaseAndTable(const String & database_name, const String & table_name) { @@ -326,7 +337,11 @@ void ASTSelectQuery::replaceDatabaseAndTable(const String & database_name, const table_expression = table_expr.get(); } + String table_alias = getTableExpressionAlias(table_expression); table_expression->database_and_table_name = createDatabaseAndTableNode(database_name, table_name); + + if (!table_alias.empty()) + table_expression->database_and_table_name->setAlias(table_alias); } @@ -347,8 +362,13 @@ void ASTSelectQuery::addTableFunction(ASTPtr & table_function_ptr) table_expression = table_expr.get(); } - table_expression->table_function = table_function_ptr; + String table_alias = getTableExpressionAlias(table_expression); + /// Maybe need to modify the alias, so we should clone new table_function node + table_expression->table_function = table_function_ptr->clone(); table_expression->database_and_table_name = nullptr; + + if (table_alias.empty()) + table_expression->table_function->setAlias(table_alias); } } diff --git a/dbms/src/Parsers/IAST.h b/dbms/src/Parsers/IAST.h index 48bb99c7d6e..703fdfc2765 100644 --- a/dbms/src/Parsers/IAST.h +++ b/dbms/src/Parsers/IAST.h @@ -40,6 +40,8 @@ class ISemantic { public: virtual ~ISemantic() = default; + ISemantic() = default; + ISemantic(const ISemantic &) = default; virtual SemanticPtr clone() const = 0; }; @@ -59,6 +61,9 @@ public: SemanticPtr semantic; virtual ~IAST() = default; + IAST() = default; + IAST(const IAST &) = default; + IAST & operator=(const IAST &) = default; /** Get the canonical name of the column if the element is a column */ String getColumnName() const; diff --git a/dbms/src/Parsers/Lexer.cpp b/dbms/src/Parsers/Lexer.cpp index b3779512ef7..6b8afaa76b7 100644 --- a/dbms/src/Parsers/Lexer.cpp +++ b/dbms/src/Parsers/Lexer.cpp @@ -73,11 +73,11 @@ Token Lexer::nextTokenImpl() switch (*pos) { - case ' ': - case '\t': - case '\n': - case '\r': - case '\f': + case ' ': [[fallthrough]]; + case '\t': [[fallthrough]]; + case '\n': [[fallthrough]]; + case '\r': [[fallthrough]]; + case '\f': [[fallthrough]]; case '\v': { ++pos; @@ -86,17 +86,16 @@ Token Lexer::nextTokenImpl() return Token(TokenType::Whitespace, token_begin, pos); } - case 'a'...'z': - case 'A'...'Z': - case '_': - { - ++pos; - while (pos < end && isWordCharASCII(*pos)) - ++pos; - return Token(TokenType::BareWord, token_begin, pos); - } - - case '0'...'9': + case '0': [[fallthrough]]; + case '1': [[fallthrough]]; + case '2': [[fallthrough]]; + case '3': [[fallthrough]]; + case '4': [[fallthrough]]; + case '5': [[fallthrough]]; + case '6': [[fallthrough]]; + case '7': [[fallthrough]]; + case '8': [[fallthrough]]; + case '9': { /// The task is not to parse a number or check correctness, but only to skip it. @@ -304,7 +303,15 @@ Token Lexer::nextTokenImpl() } default: - return Token(TokenType::Error, token_begin, ++pos); + if (isWordCharASCII(*pos)) + { + ++pos; + while (pos < end && isWordCharASCII(*pos)) + ++pos; + return Token(TokenType::BareWord, token_begin, pos); + } + else + return Token(TokenType::Error, token_begin, ++pos); } } @@ -317,9 +324,9 @@ const char * getTokenName(TokenType type) case TokenType::TOKEN: return #TOKEN; APPLY_FOR_TOKENS(M) #undef M - default: - __builtin_unreachable(); } + + __builtin_unreachable(); } diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index 332ccfde3f0..b5fbe0f3314 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -398,7 +398,7 @@ void AlterCommands::validate(const IStorage & table, const Context & context) { const auto & default_expression = default_column.second.expression; ASTPtr query = default_expression; - auto syntax_result = SyntaxAnalyzer(context, {}).analyze(query, all_columns); + auto syntax_result = SyntaxAnalyzer(context).analyze(query, all_columns); const auto actions = ExpressionAnalyzer(query, syntax_result, context).getActions(true); const auto required_columns = actions->getRequiredColumns(); @@ -473,7 +473,7 @@ void AlterCommands::validate(const IStorage & table, const Context & context) } ASTPtr query = default_expr_list; - auto syntax_result = SyntaxAnalyzer(context, {}).analyze(query, all_columns); + auto syntax_result = SyntaxAnalyzer(context).analyze(query, all_columns); const auto actions = ExpressionAnalyzer(query, syntax_result, context).getActions(true); const auto block = actions->getSampleBlock(); diff --git a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp index 74821f432e7..1c468425aac 100644 --- a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include @@ -92,11 +92,11 @@ namespace StorageDistributedDirectoryMonitor::StorageDistributedDirectoryMonitor(StorageDistributed & storage, const std::string & name, const ConnectionPoolPtr & pool) : storage(storage), pool{pool}, path{storage.path + name + '/'} , current_batch_file_path{path + "current_batch.txt"} - , default_sleep_time{storage.context.getSettingsRef().distributed_directory_monitor_sleep_time_ms.totalMilliseconds()} + , default_sleep_time{storage.global_context.getSettingsRef().distributed_directory_monitor_sleep_time_ms.totalMilliseconds()} , sleep_time{default_sleep_time} , log{&Logger::get(getLoggerName())} { - const Settings & settings = storage.context.getSettingsRef(); + const Settings & settings = storage.global_context.getSettingsRef(); should_batch_inserts = settings.distributed_directory_monitor_batch_inserts; min_batched_block_size_rows = settings.min_insert_block_size_rows; min_batched_block_size_bytes = settings.min_insert_block_size_bytes; @@ -109,7 +109,7 @@ StorageDistributedDirectoryMonitor::~StorageDistributedDirectoryMonitor() { { quit = true; - std::lock_guard lock{mutex}; + std::lock_guard lock{mutex}; } cond.notify_one(); thread.join(); @@ -123,7 +123,7 @@ void StorageDistributedDirectoryMonitor::shutdownAndDropAllData() { { quit = true; - std::lock_guard lock{mutex}; + std::lock_guard lock{mutex}; } cond.notify_one(); thread.join(); @@ -137,7 +137,7 @@ void StorageDistributedDirectoryMonitor::run() { setThreadName("DistrDirMonitor"); - std::unique_lock lock{mutex}; + std::unique_lock lock{mutex}; const auto quit_requested = [this] { return quit.load(std::memory_order_relaxed); }; @@ -157,7 +157,7 @@ void StorageDistributedDirectoryMonitor::run() std::chrono::milliseconds{Int64(default_sleep_time.count() * std::exp2(error_count))}, std::chrono::milliseconds{max_sleep_time}); tryLogCurrentException(getLoggerName().data()); - }; + } if (do_sleep) cond.wait_for(lock, sleep_time, quit_requested); @@ -174,7 +174,7 @@ void StorageDistributedDirectoryMonitor::run() ConnectionPoolPtr StorageDistributedDirectoryMonitor::createPool(const std::string & name, const StorageDistributed & storage) { - auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(storage.context.getSettingsRef()); + auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(storage.global_context.getSettingsRef()); const auto pool_factory = [&storage, &timeouts] (const std::string & host, const UInt16 port, const Protocol::Secure secure, const std::string & user, const std::string & password, @@ -519,9 +519,9 @@ bool StorageDistributedDirectoryMonitor::isFileBrokenErrorCode(int code) void StorageDistributedDirectoryMonitor::markAsBroken(const std::string & file_path) const { const auto last_path_separator_pos = file_path.rfind('/'); - const auto & path = file_path.substr(0, last_path_separator_pos + 1); + const auto & base_path = file_path.substr(0, last_path_separator_pos + 1); const auto & file_name = file_path.substr(last_path_separator_pos + 1); - const auto & broken_path = path + "broken/"; + const auto & broken_path = base_path + "broken/"; const auto & broken_file_path = broken_path + file_name; Poco::File{broken_path}.createDirectory(); diff --git a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp index 528c83167af..19ba8e48653 100644 --- a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -6,7 +6,7 @@ #include #include -#include +#include #include #include #include @@ -62,8 +62,9 @@ namespace ErrorCodes DistributedBlockOutputStream::DistributedBlockOutputStream( StorageDistributed & storage, const ASTPtr & query_ast, const ClusterPtr & cluster_, const Settings & settings_, bool insert_sync_, UInt64 insert_timeout_) - : storage(storage), query_ast(query_ast), cluster(cluster_), settings(settings_), insert_sync(insert_sync_), - insert_timeout(insert_timeout_), log(&Logger::get("DistributedBlockOutputStream")) + : storage(storage), query_ast(query_ast), query_string(queryToString(query_ast)), + cluster(cluster_), settings(settings_), insert_sync(insert_sync_), + insert_timeout(insert_timeout_), log(&Logger::get("DistributedBlockOutputStream")) { } @@ -283,7 +284,7 @@ ThreadPool::Job DistributedBlockOutputStream::runWritingJob(DistributedBlockOutp if (!job.stream) { /// Forward user settings - job.local_context = std::make_unique(storage.context); + job.local_context = std::make_unique(storage.global_context); job.local_context->setSettings(settings); InterpreterInsertQuery interp(query_ast, *job.local_context); @@ -313,7 +314,6 @@ void DistributedBlockOutputStream::writeSync(const Block & block) initWritingJobs(block); pool.emplace(remote_jobs_count + local_jobs_count); - query_string = queryToString(query_ast); if (!throttler && (settings.max_network_bandwidth || settings.max_network_bytes)) { @@ -505,7 +505,7 @@ void DistributedBlockOutputStream::writeAsyncImpl(const Block & block, const siz void DistributedBlockOutputStream::writeToLocal(const Block & block, const size_t repeats) { /// Async insert does not support settings forwarding yet whereas sync one supports - InterpreterInsertQuery interp(query_ast, storage.context); + InterpreterInsertQuery interp(query_ast, storage.global_context); auto block_io = interp.execute(); block_io.out->writePrefix(); @@ -525,7 +525,6 @@ void DistributedBlockOutputStream::writeToShard(const Block & block, const std:: std::string first_file_tmp_path{}; auto first = true; - const auto & query_string = queryToString(query_ast); /// write first file, hardlink the others for (const auto & dir_name : dir_names) diff --git a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.h b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.h index b4f830082ec..1c3dd8217e1 100644 --- a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.h +++ b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.h @@ -80,6 +80,7 @@ private: private: StorageDistributed & storage; ASTPtr query_ast; + String query_string; ClusterPtr cluster; const Settings & settings; size_t inserted_blocks = 0; @@ -93,7 +94,6 @@ private: Stopwatch watch_current_block; std::optional pool; ThrottlerPtr throttler; - String query_string; struct JobReplica { diff --git a/dbms/src/Storages/IStorage.h b/dbms/src/Storages/IStorage.h index 23f56b47a8f..07637acee87 100644 --- a/dbms/src/Storages/IStorage.h +++ b/dbms/src/Storages/IStorage.h @@ -214,7 +214,7 @@ public: /** Clear the table data and leave it empty. * Must be called under lockForAlter. */ - virtual void truncate(const ASTPtr & /*query*/) + virtual void truncate(const ASTPtr & /*query*/, const Context & /* context */) { throw Exception("Truncate is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED); } diff --git a/dbms/src/Storages/ITableDeclaration.cpp b/dbms/src/Storages/ITableDeclaration.cpp index c05f56c7939..d68693a82bc 100644 --- a/dbms/src/Storages/ITableDeclaration.cpp +++ b/dbms/src/Storages/ITableDeclaration.cpp @@ -229,7 +229,6 @@ void ITableDeclaration::check(const Block & block, bool need_all) const const NamesAndTypesList & available_columns = getColumns().getAllPhysical(); const auto columns_map = getColumnsMap(available_columns); - using NameSet = std::unordered_set; NameSet names_in_block; block.checkNumberOfRows(); diff --git a/dbms/src/Storages/Kafka/StorageKafka.cpp b/dbms/src/Storages/Kafka/StorageKafka.cpp index fd129b38519..e6ccf544ba1 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.cpp +++ b/dbms/src/Storages/Kafka/StorageKafka.cpp @@ -276,17 +276,17 @@ StorageKafka::StorageKafka( const String & format_name_, char row_delimiter_, const String & schema_name_, size_t num_consumers_, size_t max_block_size_) : IStorage{columns_}, - table_name(table_name_), database_name(database_name_), context(context_), - topics(context.getMacros()->expand(topics_)), - brokers(context.getMacros()->expand(brokers_)), - group(context.getMacros()->expand(group_)), - format_name(context.getMacros()->expand(format_name_)), + table_name(table_name_), database_name(database_name_), global_context(context_), + topics(global_context.getMacros()->expand(topics_)), + brokers(global_context.getMacros()->expand(brokers_)), + group(global_context.getMacros()->expand(group_)), + format_name(global_context.getMacros()->expand(format_name_)), row_delimiter(row_delimiter_), - schema_name(context.getMacros()->expand(schema_name_)), + schema_name(global_context.getMacros()->expand(schema_name_)), num_consumers(num_consumers_), max_block_size(max_block_size_), log(&Logger::get("StorageKafka (" + table_name_ + ")")), semaphore(0, num_consumers_), mutex(), consumers() { - task = context.getSchedulePool().createTask(log->name(), [this]{ streamThread(); }); + task = global_context.getSchedulePool().createTask(log->name(), [this]{ streamThread(); }); task->deactivate(); } @@ -296,7 +296,7 @@ BlockInputStreams StorageKafka::read( const SelectQueryInfo & /*query_info*/, const Context & context, QueryProcessingStage::Enum /*processed_stage*/, - size_t max_block_size, + size_t /*max_block_size*/, unsigned num_streams) { check(column_names); @@ -304,7 +304,7 @@ BlockInputStreams StorageKafka::read( if (num_created_consumers == 0) return BlockInputStreams(); - const size_t stream_count = std::min(num_streams, num_created_consumers); + const size_t stream_count = std::min(size_t(num_streams), num_created_consumers); BlockInputStreams streams; streams.reserve(stream_count); @@ -313,10 +313,11 @@ BlockInputStreams StorageKafka::read( for (size_t i = 0; i < stream_count; ++i) { // Use block size of 1, otherwise LIMIT won't work properly as it will buffer excess messages in the last block + // TODO That leads to awful performance. streams.emplace_back(std::make_shared(*this, context, schema_name, 1)); } - LOG_DEBUG(log, "Starting reading " << streams.size() << " streams, " << max_block_size << " block size"); + LOG_DEBUG(log, "Starting reading " << streams.size() << " streams"); return streams; } @@ -397,7 +398,7 @@ void StorageKafka::consumerConfiguration(struct rd_kafka_conf_s * conf) rd_kafka_conf_set(conf, "enable.auto.commit", "false", nullptr, 0); // Update consumer configuration from the configuration - const auto & config = context.getConfigRef(); + const auto & config = global_context.getConfigRef(); if (config.has(CONFIG_PREFIX)) loadFromConfig(conf, config, CONFIG_PREFIX); @@ -427,30 +428,30 @@ StorageKafka::ConsumerPtr StorageKafka::tryClaimConsumer(long wait_ms) semaphore.wait(); // Take the first available consumer from the list - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); auto consumer = consumers.back(); consumers.pop_back(); return consumer; } -void StorageKafka::pushConsumer(StorageKafka::ConsumerPtr c) +void StorageKafka::pushConsumer(StorageKafka::ConsumerPtr consumer) { - std::lock_guard lock(mutex); - consumers.push_back(c); + std::lock_guard lock(mutex); + consumers.push_back(consumer); semaphore.set(); } -bool StorageKafka::checkDependencies(const String & database_name, const String & table_name) +bool StorageKafka::checkDependencies(const String & current_database_name, const String & current_table_name) { // Check if all dependencies are attached - auto dependencies = context.getDependencies(database_name, table_name); + auto dependencies = global_context.getDependencies(current_database_name, current_table_name); if (dependencies.size() == 0) return true; // Check the dependencies are ready? for (const auto & db_tab : dependencies) { - auto table = context.tryGetTable(db_tab.first, db_tab.second); + auto table = global_context.tryGetTable(db_tab.first, db_tab.second); if (!table) return false; @@ -472,7 +473,7 @@ void StorageKafka::streamThread() try { // Check if at least one direct dependency is attached - auto dependencies = context.getDependencies(database_name, table_name); + auto dependencies = global_context.getDependencies(database_name, table_name); // Keep streaming as long as there are attached views and streaming is not cancelled while (!stream_cancelled && num_created_consumers > 0 && dependencies.size() > 0) @@ -500,7 +501,7 @@ void StorageKafka::streamThread() bool StorageKafka::streamToViews() { - auto table = context.getTable(database_name, table_name); + auto table = global_context.getTable(database_name, table_name); if (!table) throw Exception("Engine table " + database_name + "." + table_name + " doesn't exist.", ErrorCodes::LOGICAL_ERROR); @@ -511,7 +512,7 @@ bool StorageKafka::streamToViews() insert->no_destination = true; // Only insert into dependent views // Limit the number of batched messages to allow early cancellations - const Settings & settings = context.getSettingsRef(); + const Settings & settings = global_context.getSettingsRef(); size_t block_size = max_block_size; if (block_size == 0) block_size = settings.max_block_size.value; @@ -521,7 +522,7 @@ bool StorageKafka::streamToViews() streams.reserve(num_created_consumers); for (size_t i = 0; i < num_created_consumers; ++i) { - auto stream = std::make_shared(*this, context, schema_name, block_size); + auto stream = std::make_shared(*this, global_context, schema_name, block_size); streams.emplace_back(stream); // Limit read batch to maximum block size to allow DDL @@ -540,7 +541,7 @@ bool StorageKafka::streamToViews() in = streams[0]; // Execute the query - InterpreterInsertQuery interpreter{insert, context}; + InterpreterInsertQuery interpreter{insert, global_context}; auto block_io = interpreter.execute(); copyData(*in, *block_io.out, &stream_cancelled); @@ -576,27 +577,25 @@ StorageKafka::Consumer::~Consumer() } -void StorageKafka::Consumer::subscribe(const Names & topics) +void StorageKafka::Consumer::subscribe(const Names & topics_to_subscribe) { if (stream == nullptr) throw Exception("Cannot subscribe to topics when consumer is closed", ErrorCodes::UNKNOWN_EXCEPTION); // Create a list of partitions - auto * topicList = rd_kafka_topic_partition_list_new(topics.size()); - for (const auto & t : topics) - { - rd_kafka_topic_partition_list_add(topicList, t.c_str(), RD_KAFKA_PARTITION_UA); - } + auto * topic_list = rd_kafka_topic_partition_list_new(topics_to_subscribe.size()); + for (const auto & topic : topics_to_subscribe) + rd_kafka_topic_partition_list_add(topic_list, topic.c_str(), RD_KAFKA_PARTITION_UA); // Subscribe to requested topics - auto err = rd_kafka_subscribe(stream, topicList); + auto err = rd_kafka_subscribe(stream, topic_list); if (err) { - rd_kafka_topic_partition_list_destroy(topicList); + rd_kafka_topic_partition_list_destroy(topic_list); throw Exception("Failed to subscribe: " + String(rd_kafka_err2str(err)), ErrorCodes::UNKNOWN_EXCEPTION); } - rd_kafka_topic_partition_list_destroy(topicList); + rd_kafka_topic_partition_list_destroy(topic_list); } diff --git a/dbms/src/Storages/Kafka/StorageKafka.h b/dbms/src/Storages/Kafka/StorageKafka.h index 0a1162be2c0..561349ac474 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.h +++ b/dbms/src/Storages/Kafka/StorageKafka.h @@ -72,7 +72,7 @@ private: // Configuration and state String table_name; String database_name; - Context & context; + Context global_context; Names topics; const String brokers; const String group; diff --git a/dbms/src/Storages/MergeTree/DiskSpaceMonitor.h b/dbms/src/Storages/MergeTree/DiskSpaceMonitor.h index d06d25598c3..d518fea5490 100644 --- a/dbms/src/Storages/MergeTree/DiskSpaceMonitor.h +++ b/dbms/src/Storages/MergeTree/DiskSpaceMonitor.h @@ -41,7 +41,7 @@ public: { try { - std::lock_guard lock(DiskSpaceMonitor::mutex); + std::lock_guard lock(DiskSpaceMonitor::mutex); if (DiskSpaceMonitor::reserved_bytes < size) { DiskSpaceMonitor::reserved_bytes = 0; @@ -70,7 +70,7 @@ public: /// Change amount of reserved space. When new_size is greater than before, availability of free space is not checked. void update(UInt64 new_size) { - std::lock_guard lock(DiskSpaceMonitor::mutex); + std::lock_guard lock(DiskSpaceMonitor::mutex); DiskSpaceMonitor::reserved_bytes -= size; size = new_size; DiskSpaceMonitor::reserved_bytes += size; @@ -84,7 +84,7 @@ public: Reservation(UInt64 size_) : size(size_), metric_increment(CurrentMetrics::DiskSpaceReservedForMerge, size) { - std::lock_guard lock(DiskSpaceMonitor::mutex); + std::lock_guard lock(DiskSpaceMonitor::mutex); DiskSpaceMonitor::reserved_bytes += size; ++DiskSpaceMonitor::reservation_count; } @@ -108,7 +108,7 @@ public: /// Heuristic by Michael Kolupaev: reserve 30 MB more, because statvfs shows few megabytes more space than df. res -= std::min(res, static_cast(30 * (1ul << 20))); - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); if (reserved_bytes > res) res = 0; @@ -120,13 +120,13 @@ public: static UInt64 getReservedSpace() { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); return reserved_bytes; } static UInt64 getReservationCount() { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); return reservation_count; } diff --git a/dbms/src/Storages/MergeTree/KeyCondition.cpp b/dbms/src/Storages/MergeTree/KeyCondition.cpp index 31a4e08707f..d386b865231 100644 --- a/dbms/src/Storages/MergeTree/KeyCondition.cpp +++ b/dbms/src/Storages/MergeTree/KeyCondition.cpp @@ -918,9 +918,9 @@ bool KeyCondition::mayBeTrueInRange( std::cerr << "+inf)\n";*/ return forAnyParallelogram(used_key_size, left_key, right_key, true, right_bounded, key_ranges, 0, - [&] (const std::vector & key_ranges) + [&] (const std::vector & key_ranges_parallelogram) { - auto res = mayBeTrueInParallelogram(key_ranges, data_types); + auto res = mayBeTrueInParallelogram(key_ranges_parallelogram, data_types); /* std::cerr << "Parallelogram: "; for (size_t i = 0, size = key_ranges.size(); i != size; ++i) @@ -1125,9 +1125,9 @@ String KeyCondition::RPNElement::toString() const return "false"; case ALWAYS_TRUE: return "true"; - default: - throw Exception("Unknown function in RPNElement", ErrorCodes::LOGICAL_ERROR); } + + __builtin_unreachable(); } diff --git a/dbms/src/Storages/MergeTree/MergeList.h b/dbms/src/Storages/MergeTree/MergeList.h index 0972c76a121..1f41cc20fe8 100644 --- a/dbms/src/Storages/MergeTree/MergeList.h +++ b/dbms/src/Storages/MergeTree/MergeList.h @@ -124,13 +124,13 @@ public: template EntryPtr insert(Args &&... args) { - std::lock_guard lock{mutex}; + std::lock_guard lock{mutex}; return std::make_unique(*this, merges.emplace(merges.end(), std::forward(args)...)); } info_container_t get() const { - std::lock_guard lock{mutex}; + std::lock_guard lock{mutex}; info_container_t res; for (const auto & merge_element : merges) res.emplace_back(merge_element.getInfo()); @@ -141,7 +141,7 @@ public: inline MergeListEntry::~MergeListEntry() { - std::lock_guard lock{list.mutex}; + std::lock_guard lock{list.mutex}; list.merges.erase(it); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeBaseSelectBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeBaseSelectBlockInputStream.cpp index a7613a740d5..1c847eb0e11 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBaseSelectBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeBaseSelectBlockInputStream.cpp @@ -73,32 +73,32 @@ Block MergeTreeBaseSelectBlockInputStream::readFromPart() if (task->size_predictor) task->size_predictor->startBlock(); - const auto max_block_size_rows = this->max_block_size_rows; - const auto preferred_block_size_bytes = this->preferred_block_size_bytes; - const auto preferred_max_column_in_block_size_bytes = this->preferred_max_column_in_block_size_bytes; + const auto current_max_block_size_rows = max_block_size_rows; + const auto current_preferred_block_size_bytes = preferred_block_size_bytes; + const auto current_preferred_max_column_in_block_size_bytes = preferred_max_column_in_block_size_bytes; const auto index_granularity = storage.index_granularity; const double min_filtration_ratio = 0.00001; - auto estimateNumRows = [preferred_block_size_bytes, max_block_size_rows, - index_granularity, preferred_max_column_in_block_size_bytes, min_filtration_ratio]( - MergeTreeReadTask & task, MergeTreeRangeReader & reader) + auto estimateNumRows = [current_preferred_block_size_bytes, current_max_block_size_rows, + index_granularity, current_preferred_max_column_in_block_size_bytes, min_filtration_ratio]( + MergeTreeReadTask & current_task, MergeTreeRangeReader & current_reader) { - if (!task.size_predictor) - return max_block_size_rows; + if (!current_task.size_predictor) + return current_max_block_size_rows; /// Calculates number of rows will be read using preferred_block_size_bytes. /// Can't be less than index_granularity. - UInt64 rows_to_read = task.size_predictor->estimateNumRows(preferred_block_size_bytes); + UInt64 rows_to_read = current_task.size_predictor->estimateNumRows(current_preferred_block_size_bytes); if (!rows_to_read) return rows_to_read; rows_to_read = std::max(index_granularity, rows_to_read); - if (preferred_max_column_in_block_size_bytes) + if (current_preferred_max_column_in_block_size_bytes) { /// Calculates number of rows will be read using preferred_max_column_in_block_size_bytes. UInt64 rows_to_read_for_max_size_column - = task.size_predictor->estimateNumRowsForMaxSizeColumn(preferred_max_column_in_block_size_bytes); - double filtration_ratio = std::max(min_filtration_ratio, 1.0 - task.size_predictor->filtered_rows_ratio); + = current_task.size_predictor->estimateNumRowsForMaxSizeColumn(current_preferred_max_column_in_block_size_bytes); + double filtration_ratio = std::max(min_filtration_ratio, 1.0 - current_task.size_predictor->filtered_rows_ratio); auto rows_to_read_for_max_size_column_with_filtration = static_cast(rows_to_read_for_max_size_column / filtration_ratio); @@ -106,12 +106,12 @@ Block MergeTreeBaseSelectBlockInputStream::readFromPart() rows_to_read = std::min(rows_to_read, rows_to_read_for_max_size_column_with_filtration); } - UInt64 unread_rows_in_current_granule = reader.numPendingRowsInCurrentGranule(); + UInt64 unread_rows_in_current_granule = current_reader.numPendingRowsInCurrentGranule(); if (unread_rows_in_current_granule >= rows_to_read) return rows_to_read; - UInt64 granule_to_read = (rows_to_read + reader.numReadRowsInCurrentGranule() + index_granularity / 2) / index_granularity; - return index_granularity * granule_to_read - reader.numReadRowsInCurrentGranule(); + UInt64 granule_to_read = (rows_to_read + current_reader.numReadRowsInCurrentGranule() + index_granularity / 2) / index_granularity; + return index_granularity * granule_to_read - current_reader.numReadRowsInCurrentGranule(); }; if (!task->range_reader.isInitialized()) @@ -148,7 +148,7 @@ Block MergeTreeBaseSelectBlockInputStream::readFromPart() } UInt64 recommended_rows = estimateNumRows(*task, task->range_reader); - UInt64 rows_to_read = std::max(UInt64(1), std::min(max_block_size_rows, recommended_rows)); + UInt64 rows_to_read = std::max(UInt64(1), std::min(current_max_block_size_rows, recommended_rows)); auto read_result = task->range_reader.read(rows_to_read, task->mark_ranges); diff --git a/dbms/src/Storages/MergeTree/MergeTreeBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeBlockOutputStream.cpp index c9fc5b39fae..b3c2336bb2f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeBlockOutputStream.cpp @@ -24,7 +24,7 @@ void MergeTreeBlockOutputStream::write(const Block & block) MergeTreeData::MutableDataPartPtr part = storage.writer.writeTempPart(current_block); storage.data.renameTempPartAndAdd(part, &storage.increment); - PartLog::addNewPart(storage.context, part, watch.elapsed()); + PartLog::addNewPart(storage.global_context, part, watch.elapsed()); /// Initiate async merge - it will be done if it's good time for merge and if there are space in 'background_pool'. storage.background_task_handle->wake(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 729d4b321b0..75a54a5020f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include #include #include @@ -99,7 +99,7 @@ MergeTreeData::MergeTreeData( bool require_part_metadata_, bool attach, BrokenPartCallback broken_part_callback_) - : context(context_), + : global_context(context_), merging_params(merging_params_), index_granularity(settings_.index_granularity), settings(settings_), @@ -126,8 +126,8 @@ MergeTreeData::MergeTreeData( && !attach && !settings.compatibility_allow_sampling_expression_not_in_primary_key) /// This is for backward compatibility. throw Exception("Sampling expression must be present in the primary key", ErrorCodes::BAD_ARGUMENTS); - auto syntax = SyntaxAnalyzer(context, {}).analyze(sample_by_ast, getColumns().getAllPhysical()); - columns_required_for_sampling = ExpressionAnalyzer(sample_by_ast, syntax, context) + auto syntax = SyntaxAnalyzer(global_context).analyze(sample_by_ast, getColumns().getAllPhysical()); + columns_required_for_sampling = ExpressionAnalyzer(sample_by_ast, syntax, global_context) .getRequiredSourceColumns(); } @@ -282,8 +282,8 @@ void MergeTreeData::setPrimaryKeyAndColumns( if (!added_key_column_expr_list->children.empty()) { - auto syntax = SyntaxAnalyzer(context, {}).analyze(added_key_column_expr_list, all_columns); - Names used_columns = ExpressionAnalyzer(added_key_column_expr_list, syntax, context) + auto syntax = SyntaxAnalyzer(global_context).analyze(added_key_column_expr_list, all_columns); + Names used_columns = ExpressionAnalyzer(added_key_column_expr_list, syntax, global_context) .getRequiredSourceColumns(); NamesAndTypesList deleted_columns; @@ -305,17 +305,17 @@ void MergeTreeData::setPrimaryKeyAndColumns( } } - auto new_sorting_key_syntax = SyntaxAnalyzer(context, {}).analyze(new_sorting_key_expr_list, all_columns); - auto new_sorting_key_expr = ExpressionAnalyzer(new_sorting_key_expr_list, new_sorting_key_syntax, context) + auto new_sorting_key_syntax = SyntaxAnalyzer(global_context).analyze(new_sorting_key_expr_list, all_columns); + auto new_sorting_key_expr = ExpressionAnalyzer(new_sorting_key_expr_list, new_sorting_key_syntax, global_context) .getActions(false); auto new_sorting_key_sample = - ExpressionAnalyzer(new_sorting_key_expr_list, new_sorting_key_syntax, context) + ExpressionAnalyzer(new_sorting_key_expr_list, new_sorting_key_syntax, global_context) .getActions(true)->getSampleBlock(); checkKeyExpression(*new_sorting_key_expr, new_sorting_key_sample, "Sorting"); - auto new_primary_key_syntax = SyntaxAnalyzer(context, {}).analyze(new_primary_key_expr_list, all_columns); - auto new_primary_key_expr = ExpressionAnalyzer(new_primary_key_expr_list, new_primary_key_syntax, context) + auto new_primary_key_syntax = SyntaxAnalyzer(global_context).analyze(new_primary_key_expr_list, all_columns); + auto new_primary_key_expr = ExpressionAnalyzer(new_primary_key_expr_list, new_primary_key_syntax, global_context) .getActions(false); Block new_primary_key_sample; @@ -376,8 +376,8 @@ void MergeTreeData::initPartitionKey() return; { - auto syntax_result = SyntaxAnalyzer(context, {}).analyze(partition_key_expr_list, getColumns().getAllPhysical()); - partition_key_expr = ExpressionAnalyzer(partition_key_expr_list, syntax_result, context).getActions(false); + auto syntax_result = SyntaxAnalyzer(global_context).analyze(partition_key_expr_list, getColumns().getAllPhysical()); + partition_key_expr = ExpressionAnalyzer(partition_key_expr_list, syntax_result, global_context).getActions(false); } for (const ASTPtr & ast : partition_key_expr_list->children) @@ -390,7 +390,7 @@ void MergeTreeData::initPartitionKey() /// Add all columns used in the partition key to the min-max index. const NamesAndTypesList & minmax_idx_columns_with_types = partition_key_expr->getRequiredColumnsWithTypes(); - minmax_idx_expr = std::make_shared(minmax_idx_columns_with_types, context); + minmax_idx_expr = std::make_shared(minmax_idx_columns_with_types, global_context); for (const NameAndTypePair & column : minmax_idx_columns_with_types) { minmax_idx_columns.emplace_back(column.name); @@ -549,16 +549,15 @@ String MergeTreeData::MergingParams::getModeName() const case Replacing: return "Replacing"; case Graphite: return "Graphite"; case VersionedCollapsing: return "VersionedCollapsing"; - - default: - throw Exception("Unknown mode of operation for MergeTreeData: " + toString(mode), ErrorCodes::LOGICAL_ERROR); } + + __builtin_unreachable(); } Int64 MergeTreeData::getMaxBlockNumber() { - std::lock_guard lock_all(data_parts_mutex); + std::lock_guard lock_all(data_parts_mutex); Int64 max_block_num = 0; for (const DataPartPtr & part : data_parts_by_info) @@ -587,7 +586,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) DataPartsVector broken_parts_to_detach; size_t suspicious_broken_parts = 0; - std::lock_guard lock(data_parts_mutex); + std::lock_guard lock(data_parts_mutex); data_parts_indexes.clear(); for (const String & file_name : part_file_names) @@ -766,7 +765,7 @@ static bool isOldPartDirectory(Poco::File & directory, time_t threshold) void MergeTreeData::clearOldTemporaryDirectories(ssize_t custom_directories_lifetime_seconds) { /// If the method is already called from another thread, then we don't need to do anything. - std::unique_lock lock(clear_old_temporary_directories_mutex, std::defer_lock); + std::unique_lock lock(clear_old_temporary_directories_mutex, std::defer_lock); if (!lock.try_lock()) return; @@ -805,7 +804,7 @@ MergeTreeData::DataPartsVector MergeTreeData::grabOldParts() DataPartsVector res; /// If the method is already called from another thread, then we don't need to do anything. - std::unique_lock lock(grab_old_parts_mutex, std::defer_lock); + std::unique_lock lock(grab_old_parts_mutex, std::defer_lock); if (!lock.try_lock()) return res; @@ -813,7 +812,7 @@ MergeTreeData::DataPartsVector MergeTreeData::grabOldParts() std::vector parts_to_delete; { - std::lock_guard lock_parts(data_parts_mutex); + std::lock_guard lock_parts(data_parts_mutex); auto outdated_parts_range = getDataPartsStateRange(DataPartState::Outdated); for (auto it = outdated_parts_range.begin(); it != outdated_parts_range.end(); ++it) @@ -847,7 +846,7 @@ MergeTreeData::DataPartsVector MergeTreeData::grabOldParts() void MergeTreeData::rollbackDeletingParts(const MergeTreeData::DataPartsVector & parts) { - std::lock_guard lock(data_parts_mutex); + std::lock_guard lock(data_parts_mutex); for (auto & part : parts) { /// We should modify it under data_parts_mutex @@ -859,7 +858,7 @@ void MergeTreeData::rollbackDeletingParts(const MergeTreeData::DataPartsVector & void MergeTreeData::removePartsFinally(const MergeTreeData::DataPartsVector & parts) { { - std::lock_guard lock(data_parts_mutex); + std::lock_guard lock(data_parts_mutex); /// TODO: use data_parts iterators instead of pointers for (auto & part : parts) @@ -876,7 +875,7 @@ void MergeTreeData::removePartsFinally(const MergeTreeData::DataPartsVector & pa /// Data parts is still alive (since DataPartsVector holds shared_ptrs) and contain useful metainformation for logging /// NOTE: There is no need to log parts deletion somewhere else, all deleting parts pass through this function and pass away - if (auto part_log = context.getPartLog(database_name)) + if (auto part_log = global_context.getPartLog(database_name)) { PartLogElement part_log_elem; @@ -918,7 +917,7 @@ void MergeTreeData::setPath(const String & new_full_path) Poco::File(full_path).renameTo(new_full_path); - context.dropCaches(); + global_context.dropCaches(); full_path = new_full_path; } @@ -926,14 +925,14 @@ void MergeTreeData::dropAllData() { LOG_TRACE(log, "dropAllData: waiting for locks."); - std::lock_guard lock(data_parts_mutex); + std::lock_guard lock(data_parts_mutex); LOG_TRACE(log, "dropAllData: removing data from memory."); data_parts_indexes.clear(); column_sizes.clear(); - context.dropCaches(); + global_context.dropCaches(); LOG_TRACE(log, "dropAllData: removing data from filesystem."); @@ -1148,7 +1147,7 @@ void MergeTreeData::createConvertExpression(const DataPartPtr & part, const Name /// Need to modify column type. if (!out_expression) - out_expression = std::make_shared(NamesAndTypesList(), context); + out_expression = std::make_shared(NamesAndTypesList(), global_context); out_expression->addInput(ColumnWithTypeAndName(nullptr, column.type, column.name)); @@ -1159,7 +1158,7 @@ void MergeTreeData::createConvertExpression(const DataPartPtr & part, const Name out_expression->add(ExpressionAction::addColumn( { DataTypeString().createColumnConst(1, new_type_name), std::make_shared(), new_type_name_column })); - const auto & function = FunctionFactory::instance().get("CAST", context); + const auto & function = FunctionFactory::instance().get("CAST", global_context); out_expression->add(ExpressionAction::applyFunction( function, Names{column.name, new_type_name_column}), out_names); @@ -1303,7 +1302,7 @@ MergeTreeData::AlterDataPartTransactionPtr MergeTreeData::alterDataPart( BlockInputStreamPtr part_in = std::make_shared( *this, part, expression->getRequiredColumns(), false, /* take_column_types_from_storage = */ false); - auto compression_codec = this->context.chooseCompressionCodec( + auto compression_codec = global_context.chooseCompressionCodec( part->bytes_on_disk, static_cast(part->bytes_on_disk) / this->getTotalActiveSizeInBytes()); ExpressionBlockInputStream in(part_in, expression); @@ -1411,7 +1410,7 @@ void MergeTreeData::AlterDataPartTransaction::commit() mutable_part.bytes_on_disk = new_checksums.getTotalSizeOnDisk(); /// TODO: we can skip resetting caches when the column is added. - data_part->storage.context.dropCaches(); + data_part->storage.global_context.dropCaches(); clear(); } @@ -1630,7 +1629,7 @@ MergeTreeData::DataPartsVector MergeTreeData::renameTempPartAndReplace( DataPartsVector covered_parts; { - std::unique_lock lock(data_parts_mutex); + std::unique_lock lock(data_parts_mutex); renameTempPartAndReplace(part, increment, out_transaction, lock, &covered_parts); } return covered_parts; @@ -1844,7 +1843,7 @@ void MergeTreeData::tryRemovePartImmediately(DataPartPtr && part) { DataPartPtr part_to_delete; { - std::lock_guard lock_parts(data_parts_mutex); + std::lock_guard lock_parts(data_parts_mutex); LOG_TRACE(log, "Trying to immediately remove part " << part->getNameWithState()); @@ -1880,7 +1879,7 @@ size_t MergeTreeData::getTotalActiveSizeInBytes() const { size_t res = 0; { - std::lock_guard lock(data_parts_mutex); + std::lock_guard lock(data_parts_mutex); for (auto & part : getDataPartsStateRange(DataPartState::Committed)) res += part->bytes_on_disk; @@ -1892,7 +1891,7 @@ size_t MergeTreeData::getTotalActiveSizeInBytes() const size_t MergeTreeData::getMaxPartsCountForPartition() const { - std::lock_guard lock(data_parts_mutex); + std::lock_guard lock(data_parts_mutex); size_t res = 0; size_t cur_count = 0; @@ -2016,7 +2015,7 @@ MergeTreeData::DataPartsVector MergeTreeData::getDataPartsVectorInPartition(Merg { DataPartStateAndPartitionID state_with_partition{state, partition_id}; - std::lock_guard lock(data_parts_mutex); + std::lock_guard lock(data_parts_mutex); return DataPartsVector( data_parts_by_state_and_info.lower_bound(state_with_partition), data_parts_by_state_and_info.upper_bound(state_with_partition)); @@ -2025,7 +2024,7 @@ MergeTreeData::DataPartsVector MergeTreeData::getDataPartsVectorInPartition(Merg MergeTreeData::DataPartPtr MergeTreeData::getPartIfExists(const MergeTreePartInfo & part_info, const MergeTreeData::DataPartStates & valid_states) { - std::lock_guard lock(data_parts_mutex); + std::lock_guard lock(data_parts_mutex); auto it = data_parts_by_info.find(part_info); if (it == data_parts_by_info.end()) @@ -2266,11 +2265,11 @@ MergeTreeData::DataPartsVector MergeTreeData::getDataPartsVector(const DataPartS DataPartsVector res; DataPartsVector buf; { - std::lock_guard lock(data_parts_mutex); + std::lock_guard lock(data_parts_mutex); for (auto state : affordable_states) { - buf = std::move(res); + std::swap(buf, res); res.clear(); auto range = getDataPartsStateRange(state); @@ -2292,7 +2291,7 @@ MergeTreeData::DataPartsVector MergeTreeData::getAllDataPartsVector(MergeTreeDat { DataPartsVector res; { - std::lock_guard lock(data_parts_mutex); + std::lock_guard lock(data_parts_mutex); res.assign(data_parts_by_info.begin(), data_parts_by_info.end()); if (out_states != nullptr) @@ -2310,7 +2309,7 @@ MergeTreeData::DataParts MergeTreeData::getDataParts(const DataPartStates & affo { DataParts res; { - std::lock_guard lock(data_parts_mutex); + std::lock_guard lock(data_parts_mutex); for (auto state : affordable_states) { auto range = getDataPartsStateRange(state); diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index ed51a8cbd1a..d0a8c67b58a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -531,7 +531,7 @@ public: size_t getColumnCompressedSize(const std::string & name) const { - std::lock_guard lock{data_parts_mutex}; + std::lock_guard lock{data_parts_mutex}; const auto it = column_sizes.find(name); return it == std::end(column_sizes) ? 0 : it->second.data_compressed; @@ -540,14 +540,14 @@ public: using ColumnSizeByName = std::unordered_map; ColumnSizeByName getColumnSizes() const { - std::lock_guard lock{data_parts_mutex}; + std::lock_guard lock{data_parts_mutex}; return column_sizes; } /// Calculates column sizes in compressed form for the current state of data_parts. void recalculateColumnSizes() { - std::lock_guard lock{data_parts_mutex}; + std::lock_guard lock{data_parts_mutex}; calculateColumnSizesImpl(); } @@ -564,7 +564,7 @@ public: MergeTreeDataFormatVersion format_version; - Context & context; + Context global_context; /// Merging params - what additional actions to perform during merge. const MergingParams merging_params; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index ebdef7b7d37..895a068621f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -561,7 +561,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor size_t sum_input_rows_upper_bound = merge_entry->total_size_marks * data.index_granularity; - MergeAlgorithm merge_alg = chooseMergeAlgorithm(data, parts, sum_input_rows_upper_bound, gathering_columns, deduplicate); + MergeAlgorithm merge_alg = chooseMergeAlgorithm(parts, sum_input_rows_upper_bound, gathering_columns, deduplicate); LOG_DEBUG(log, "Selected MergeAlgorithm: " << ((merge_alg == MergeAlgorithm::Vertical) ? "Vertical" : "Horizontal")); @@ -570,7 +570,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor /// (which is locked in shared mode when input streams are created) and when inserting new data /// the order is reverse. This annoys TSan even though one lock is locked in shared mode and thus /// deadlock is impossible. - auto compression_codec = data.context.chooseCompressionCodec( + auto compression_codec = data.global_context.chooseCompressionCodec( merge_entry->total_size_bytes_compressed, static_cast (merge_entry->total_size_bytes_compressed) / data.getTotalActiveSizeInBytes()); @@ -687,9 +687,6 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor merged_stream = std::make_unique( src_streams, sort_description, data.merging_params.sign_column, DEFAULT_MERGE_BLOCK_SIZE, rows_sources_write_buf.get()); break; - - default: - throw Exception("Unknown mode of operation for MergeTreeData: " + toString(data.merging_params.mode), ErrorCodes::LOGICAL_ERROR); } if (deduplicate) @@ -974,8 +971,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor new_data_part->checksums.add(std::move(changed_checksums)); { /// Write file with checksums. - WriteBufferFromFile out(new_part_tmp_path + "checksums.txt", 4096); - new_data_part->checksums.write(out); + WriteBufferFromFile out_checksums(new_part_tmp_path + "checksums.txt", 4096); + new_data_part->checksums.write(out_checksums); } /// Write the columns list of the resulting part in the same order as all_columns. @@ -991,8 +988,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor } { /// Write a file with a description of columns. - WriteBufferFromFile out(new_part_tmp_path + "columns.txt", 4096); - new_data_part->columns.writeText(out); + WriteBufferFromFile out_columns(new_part_tmp_path + "columns.txt", 4096); + new_data_part->columns.writeText(out_columns); } new_data_part->rows_count = source_part->rows_count; @@ -1009,7 +1006,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor MergeTreeDataMergerMutator::MergeAlgorithm MergeTreeDataMergerMutator::chooseMergeAlgorithm( - const MergeTreeData & data, const MergeTreeData::DataPartsVector & parts, size_t sum_rows_upper_bound, + const MergeTreeData::DataPartsVector & parts, size_t sum_rows_upper_bound, const NamesAndTypesList & gathering_columns, bool deduplicate) const { if (deduplicate) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.h b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.h index 0d6cdd3f557..97f8466be20 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.h @@ -126,8 +126,8 @@ public: private: MergeAlgorithm chooseMergeAlgorithm( - const MergeTreeData & data, const MergeTreeData::DataPartsVector & parts, - size_t rows_upper_bound, const NamesAndTypesList & gathering_columns, bool deduplicate) const; + const MergeTreeData::DataPartsVector & parts, + size_t rows_upper_bound, const NamesAndTypesList & gathering_columns, bool deduplicate) const; private: MergeTreeData & data; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp index aca955fad7c..78ddd3f8f70 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp @@ -2,8 +2,8 @@ #include #include -#include -#include +#include +#include #include #include #include @@ -45,15 +45,15 @@ static ReadBufferFromFile openForReading(const String & path) return ReadBufferFromFile(path, std::min(static_cast(DBMS_DEFAULT_BUFFER_SIZE), Poco::File(path).getSize())); } -void MergeTreeDataPart::MinMaxIndex::load(const MergeTreeData & storage, const String & part_path) +void MergeTreeDataPart::MinMaxIndex::load(const MergeTreeData & data, const String & part_path) { - size_t minmax_idx_size = storage.minmax_idx_column_types.size(); + size_t minmax_idx_size = data.minmax_idx_column_types.size(); parallelogram.reserve(minmax_idx_size); for (size_t i = 0; i < minmax_idx_size; ++i) { - String file_name = part_path + "minmax_" + escapeForFileName(storage.minmax_idx_columns[i]) + ".idx"; + String file_name = part_path + "minmax_" + escapeForFileName(data.minmax_idx_columns[i]) + ".idx"; ReadBufferFromFile file = openForReading(file_name); - const DataTypePtr & type = storage.minmax_idx_column_types[i]; + const DataTypePtr & type = data.minmax_idx_column_types[i]; Field min_val; type->deserializeBinary(min_val, file); @@ -65,24 +65,24 @@ void MergeTreeDataPart::MinMaxIndex::load(const MergeTreeData & storage, const S initialized = true; } -void MergeTreeDataPart::MinMaxIndex::store(const MergeTreeData & storage, const String & part_path, Checksums & checksums) const +void MergeTreeDataPart::MinMaxIndex::store(const MergeTreeData & data, const String & part_path, Checksums & out_checksums) const { if (!initialized) throw Exception("Attempt to store uninitialized MinMax index for part " + part_path + ". This is a bug.", ErrorCodes::LOGICAL_ERROR); - for (size_t i = 0; i < storage.minmax_idx_columns.size(); ++i) + for (size_t i = 0; i < data.minmax_idx_columns.size(); ++i) { - String file_name = "minmax_" + escapeForFileName(storage.minmax_idx_columns[i]) + ".idx"; - const DataTypePtr & type = storage.minmax_idx_column_types[i]; + String file_name = "minmax_" + escapeForFileName(data.minmax_idx_columns[i]) + ".idx"; + const DataTypePtr & type = data.minmax_idx_column_types[i]; WriteBufferFromFile out(part_path + file_name); HashingWriteBuffer out_hashing(out); type->serializeBinary(parallelogram[i].left, out_hashing); type->serializeBinary(parallelogram[i].right, out_hashing); out_hashing.next(); - checksums.files[file_name].file_size = out_hashing.count(); - checksums.files[file_name].file_hash = out_hashing.getHash(); + out_checksums.files[file_name].file_size = out_hashing.count(); + out_checksums.files[file_name].file_hash = out_hashing.getHash(); } } @@ -138,7 +138,8 @@ MergeTreeDataPart::MergeTreeDataPart(MergeTreeData & storage_, const String & na /// Takes into account the fact that several columns can e.g. share their .size substreams. /// When calculating totals these should be counted only once. -MergeTreeDataPart::ColumnSize MergeTreeDataPart::getColumnSizeImpl(const String & name, const IDataType & type, std::unordered_set * processed_substreams) const +MergeTreeDataPart::ColumnSize MergeTreeDataPart::getColumnSizeImpl( + const String & column_name, const IDataType & type, std::unordered_set * processed_substreams) const { ColumnSize size; if (checksums.empty()) @@ -146,7 +147,7 @@ MergeTreeDataPart::ColumnSize MergeTreeDataPart::getColumnSizeImpl(const String type.enumerateStreams([&](const IDataType::SubstreamPath & substream_path) { - String file_name = IDataType::getFileNameForStream(name, substream_path); + String file_name = IDataType::getFileNameForStream(column_name, substream_path); if (processed_substreams && !processed_substreams->insert(file_name).second) return; @@ -166,9 +167,9 @@ MergeTreeDataPart::ColumnSize MergeTreeDataPart::getColumnSizeImpl(const String return size; } -MergeTreeDataPart::ColumnSize MergeTreeDataPart::getColumnSize(const String & name, const IDataType & type) const +MergeTreeDataPart::ColumnSize MergeTreeDataPart::getColumnSize(const String & column_name, const IDataType & type) const { - return getColumnSizeImpl(name, type, nullptr); + return getColumnSizeImpl(column_name, type, nullptr); } MergeTreeDataPart::ColumnSize MergeTreeDataPart::getTotalColumnsSize() const @@ -189,11 +190,11 @@ MergeTreeDataPart::ColumnSize MergeTreeDataPart::getTotalColumnsSize() const */ String MergeTreeDataPart::getColumnNameWithMinumumCompressedSize() const { - const auto & columns = storage.getColumns().getAllPhysical(); + const auto & storage_columns = storage.getColumns().getAllPhysical(); const std::string * minimum_size_column = nullptr; UInt64 minimum_size = std::numeric_limits::max(); - for (const auto & column : columns) + for (const auto & column : storage_columns) { if (!hasColumnFiles(column.name)) continue; @@ -793,9 +794,9 @@ String MergeTreeDataPart::stateToString(MergeTreeDataPart::State state) return "Outdated"; case State::Deleting: return "Deleting"; - default: - throw Exception("Unknown part state " + toString(static_cast(state)), ErrorCodes::LOGICAL_ERROR); } + + __builtin_unreachable(); } String MergeTreeDataPart::stateString() const @@ -808,8 +809,8 @@ void MergeTreeDataPart::assertState(const std::initializer_list #include #include -#include -#include +#include +#include #include @@ -359,7 +359,7 @@ bool MinimalisticDataPartChecksums::deserialize(ReadBuffer & in) return true; } -void MinimalisticDataPartChecksums::computeTotalChecksums(const MergeTreeDataPartChecksums & full_checksums) +void MinimalisticDataPartChecksums::computeTotalChecksums(const MergeTreeDataPartChecksums & full_checksums_) { num_compressed_files = 0; num_uncompressed_files = 0; @@ -368,7 +368,7 @@ void MinimalisticDataPartChecksums::computeTotalChecksums(const MergeTreeDataPar SipHash hash_of_uncompressed_files_; SipHash uncompressed_hash_of_compressed_files_; - for (const auto & elem : full_checksums.files) + for (const auto & elem : full_checksums_.files) { const String & name = elem.first; const auto & checksum = elem.second; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index dd5a35ad710..01fb3169013 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -488,7 +488,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::readFromParts( } ASTPtr query = filter_function; - auto syntax_result = SyntaxAnalyzer(context, {}).analyze(query, available_real_columns); + auto syntax_result = SyntaxAnalyzer(context).analyze(query, available_real_columns); filter_expression = ExpressionAnalyzer(filter_function, syntax_result, context).getActions(false); /// Add columns needed for `sample_by_ast` to `column_names_to_read`. @@ -848,7 +848,7 @@ void MergeTreeDataSelectExecutor::createPositiveSignCondition( arguments->children.push_back(one); ASTPtr query = function; - auto syntax_result = SyntaxAnalyzer(context, {}).analyze(query, data.getColumns().getAllPhysical()); + auto syntax_result = SyntaxAnalyzer(context).analyze(query, data.getColumns().getAllPhysical()); out_expression = ExpressionAnalyzer(query, syntax_result, context).getActions(false); out_column = function->getColumnName(); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 3399e162f0d..2b7ede696ad 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -209,7 +209,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa /// This effectively chooses minimal compression method: /// either default lz4 or compression method with zero thresholds on absolute and relative part size. - auto compression_codec = data.context.chooseCompressionCodec(0, 0); + auto compression_codec = data.global_context.chooseCompressionCodec(0, 0); NamesAndTypesList columns = data.getColumns().getAllPhysical().filter(block.getNames()); MergedBlockOutputStream out(data, new_data_part->getFullPath(), columns, compression_codec); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.h b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.h index 93d8ecb98ae..95371021939 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.h @@ -4,7 +4,7 @@ #include #include -#include +#include #include diff --git a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp index 21412b4f2c4..2be5b01119f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -5,7 +5,7 @@ #include #include -#if __SSE2__ +#ifdef __SSE2__ #include #endif @@ -254,12 +254,12 @@ void MergeTreeRangeReader::ReadResult::optimize() } } -size_t MergeTreeRangeReader::ReadResult::countZeroTails(const IColumn::Filter & filter, NumRows & zero_tails) const +size_t MergeTreeRangeReader::ReadResult::countZeroTails(const IColumn::Filter & filter_vec, NumRows & zero_tails) const { zero_tails.resize(0); zero_tails.reserve(rows_per_granule.size()); - auto filter_data = filter.data(); + auto filter_data = filter_vec.data(); size_t total_zero_rows_in_tails = 0; @@ -274,11 +274,11 @@ size_t MergeTreeRangeReader::ReadResult::countZeroTails(const IColumn::Filter & return total_zero_rows_in_tails; } -void MergeTreeRangeReader::ReadResult::collapseZeroTails(const IColumn::Filter & filter, IColumn::Filter & new_filter, +void MergeTreeRangeReader::ReadResult::collapseZeroTails(const IColumn::Filter & filter_vec, IColumn::Filter & new_filter_vec, const NumRows & zero_tails) { - auto filter_data = filter.data(); - auto new_filter_data = new_filter.data(); + auto filter_data = filter_vec.data(); + auto new_filter_data = new_filter_vec.data(); for (auto i : ext::range(0, rows_per_granule.size())) { @@ -294,14 +294,14 @@ void MergeTreeRangeReader::ReadResult::collapseZeroTails(const IColumn::Filter & filter_data += filtered_rows_num_at_granule_end; } - new_filter.resize(new_filter_data - new_filter.data()); + new_filter_vec.resize(new_filter_data - new_filter_vec.data()); } size_t MergeTreeRangeReader::ReadResult::numZerosInTail(const UInt8 * begin, const UInt8 * end) { size_t count = 0; -#if __SSE2__ && __POPCNT__ +#if defined(__SSE2__) && defined(__POPCNT__) const __m128i zero16 = _mm_setzero_si128(); while (end - begin >= 64) { diff --git a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp index 84f8511f5ae..85d33896967 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp @@ -34,14 +34,14 @@ MergeTreeReadPool::MergeTreeReadPool( std::reverse(std::begin(part_ranges.ranges), std::end(part_ranges.ranges)); /// parts don't contain duplicate MergeTreeDataPart's. - const auto per_part_sum_marks = fillPerPartInfo(parts, prewhere_info, check_columns); + const auto per_part_sum_marks = fillPerPartInfo(parts, check_columns); fillPerThreadInfo(threads, sum_marks, per_part_sum_marks, parts, min_marks_for_concurrent_read); } MergeTreeReadTaskPtr MergeTreeReadPool::getTask(const size_t min_marks_to_read, const size_t thread, const Names & ordered_names) { - const std::lock_guard lock{mutex}; + const std::lock_guard lock{mutex}; /// If number of threads was lowered due to backoff, then will assign work only for maximum 'backoff_state.current_threads' threads. if (thread >= backoff_state.current_threads) @@ -61,7 +61,7 @@ MergeTreeReadTaskPtr MergeTreeReadPool::getTask(const size_t min_marks_to_read, auto & thread_task = thread_tasks.parts_and_ranges.back(); const auto part_idx = thread_task.part_idx; - auto & part = parts[part_idx]; + auto & part = parts_with_idx[part_idx]; auto & marks_in_part = thread_tasks.sum_marks_in_parts.back(); /// Get whole part to read if it is small enough. @@ -164,7 +164,7 @@ void MergeTreeReadPool::profileFeedback(const ReadBufferFromFileBase::ProfileInf if (info.nanoseconds < backoff_settings.min_read_latency_ms * 1000000) return; - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); if (backoff_state.current_threads <= 1) return; @@ -198,7 +198,7 @@ void MergeTreeReadPool::profileFeedback(const ReadBufferFromFileBase::ProfileInf std::vector MergeTreeReadPool::fillPerPartInfo( - RangesInDataParts & parts, const PrewhereInfoPtr & prewhere_info, const bool check_columns) + RangesInDataParts & parts, const bool check_columns) { std::vector per_part_sum_marks; Block sample_block = data.getSampleBlock(); @@ -277,7 +277,7 @@ std::vector MergeTreeReadPool::fillPerPartInfo( per_part_should_reorder.push_back(should_reoder); - this->parts.push_back({ part.data_part, part.part_index_in_query }); + parts_with_idx.push_back({ part.data_part, part.part_index_in_query }); if (predict_block_size_bytes) { diff --git a/dbms/src/Storages/MergeTree/MergeTreeReadPool.h b/dbms/src/Storages/MergeTree/MergeTreeReadPool.h index 531702a8dd4..ab2280d6cef 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReadPool.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReadPool.h @@ -87,7 +87,7 @@ public: private: std::vector fillPerPartInfo( - RangesInDataParts & parts, const PrewhereInfoPtr & prewhere_info, const bool check_columns); + RangesInDataParts & parts, const bool check_columns); void fillPerThreadInfo( const size_t threads, const size_t sum_marks, std::vector per_part_sum_marks, @@ -96,7 +96,6 @@ private: std::vector> per_part_columns_lock; const MergeTreeData & data; Names column_names; - Names ordered_names; bool do_not_steal_tasks; bool predict_block_size_bytes; std::vector per_part_column_name_set; @@ -112,7 +111,7 @@ private: size_t part_index_in_query; }; - std::vector parts; + std::vector parts_with_idx; struct ThreadTask { diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp index 537a4d82722..de39ca1b7e7 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include #include #include @@ -49,7 +49,7 @@ MergeTreeReader::MergeTreeReader(const String & path, throw Exception("Part " + path + " is missing", ErrorCodes::NOT_FOUND_EXPECTED_DATA_PART); for (const NameAndTypePair & column : columns) - addStreams(column.name, *column.type, all_mark_ranges, profile_callback, clock_type); + addStreams(column.name, *column.type, profile_callback, clock_type); } catch (...) { @@ -261,34 +261,34 @@ const MarkInCompressedFile & MergeTreeReader::Stream::getMark(size_t index) void MergeTreeReader::Stream::loadMarks() { - std::string path = path_prefix + ".mrk"; + std::string mrk_path = path_prefix + ".mrk"; auto load = [&]() -> MarkCache::MappedPtr { /// Memory for marks must not be accounted as memory usage for query, because they are stored in shared cache. auto temporarily_disable_memory_tracker = getCurrentMemoryTrackerActionLock(); - size_t file_size = Poco::File(path).getSize(); + size_t file_size = Poco::File(mrk_path).getSize(); size_t expected_file_size = sizeof(MarkInCompressedFile) * marks_count; if (expected_file_size != file_size) throw Exception( - "bad size of marks file `" + path + "':" + std::to_string(file_size) + ", must be: " + std::to_string(expected_file_size), + "bad size of marks file `" + mrk_path + "':" + std::to_string(file_size) + ", must be: " + std::to_string(expected_file_size), ErrorCodes::CORRUPTED_DATA); auto res = std::make_shared(marks_count); /// Read directly to marks. - ReadBufferFromFile buffer(path, file_size, -1, reinterpret_cast(res->data())); + ReadBufferFromFile buffer(mrk_path, file_size, -1, reinterpret_cast(res->data())); if (buffer.eof() || buffer.buffer().size() != file_size) - throw Exception("Cannot read all marks from file " + path, ErrorCodes::CANNOT_READ_ALL_DATA); + throw Exception("Cannot read all marks from file " + mrk_path, ErrorCodes::CANNOT_READ_ALL_DATA); return res; }; if (mark_cache) { - auto key = mark_cache->hash(path); + auto key = mark_cache->hash(mrk_path); if (save_marks_in_cache) { marks = mark_cache->getOrSet(key, load); @@ -304,7 +304,7 @@ void MergeTreeReader::Stream::loadMarks() marks = load(); if (!marks) - throw Exception("Failed to load marks: " + path, ErrorCodes::LOGICAL_ERROR); + throw Exception("Failed to load marks: " + mrk_path, ErrorCodes::LOGICAL_ERROR); } @@ -353,7 +353,7 @@ void MergeTreeReader::Stream::seekToStart() } -void MergeTreeReader::addStreams(const String & name, const IDataType & type, const MarkRanges & all_mark_ranges, +void MergeTreeReader::addStreams(const String & name, const IDataType & type, const ReadBufferFromFileBase::ProfileCallback & profile_callback, clockid_t clock_type) { IDataType::StreamCallback callback = [&] (const IDataType::SubstreamPath & substream_path) @@ -377,8 +377,8 @@ void MergeTreeReader::addStreams(const String & name, const IDataType & type, co uncompressed_cache, aio_threshold, max_read_buffer_size, profile_callback, clock_type)); }; - IDataType::SubstreamPath path; - type.enumerateStreams(callback, path); + IDataType::SubstreamPath substream_path; + type.enumerateStreams(callback, substream_path); } @@ -389,13 +389,13 @@ void MergeTreeReader::readData( { auto get_stream_getter = [&](bool stream_for_prefix) -> IDataType::InputStreamGetter { - return [&, stream_for_prefix](const IDataType::SubstreamPath & path) -> ReadBuffer * + return [&, stream_for_prefix](const IDataType::SubstreamPath & substream_path) -> ReadBuffer * { /// If offsets for arrays have already been read. - if (!with_offsets && path.size() == 1 && path[0].type == IDataType::Substream::ArraySizes) + if (!with_offsets && substream_path.size() == 1 && substream_path[0].type == IDataType::Substream::ArraySizes) return nullptr; - String stream_name = IDataType::getFileNameForStream(name, path); + String stream_name = IDataType::getFileNameForStream(name, substream_path); auto it = streams.find(stream_name); if (it == streams.end()) @@ -513,8 +513,7 @@ void MergeTreeReader::fillMissingColumns(Block & res, bool & should_reorder, boo { ColumnPtr offsets_column = offset_columns[offsets_name]; DataTypePtr nested_type = typeid_cast(*column_to_add.type).getNestedType(); - size_t nested_rows = offsets_column->empty() ? 0 - : typeid_cast(*offsets_column).getData().back(); + size_t nested_rows = typeid_cast(*offsets_column).getData().back(); ColumnPtr nested_column = nested_type->createColumnConstWithDefaultValue(nested_rows)->convertToFullColumnIfConst(); @@ -566,7 +565,7 @@ void MergeTreeReader::evaluateMissingDefaults(Block & res) { try { - DB::evaluateMissingDefaults(res, columns, storage.getColumns().defaults, storage.context); + DB::evaluateMissingDefaults(res, columns, storage.getColumns().defaults, storage.global_context); } catch (Exception & e) { diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.h b/dbms/src/Storages/MergeTree/MergeTreeReader.h index 988ca570ddd..ac5d46fb664 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReader.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReader.h @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include @@ -120,7 +120,7 @@ private: size_t max_read_buffer_size; size_t index_granularity; - void addStreams(const String & name, const IDataType & type, const MarkRanges & all_mark_ranges, + void addStreams(const String & name, const IDataType & type, const ReadBufferFromFileBase::ProfileCallback & profile_callback, clockid_t clock_type); void readData( diff --git a/dbms/src/Storages/MergeTree/MergeTreeSelectBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeSelectBlockInputStream.cpp index 76d63743079..6ca9b18b70c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSelectBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSelectBlockInputStream.cpp @@ -22,18 +22,18 @@ MergeTreeSelectBlockInputStream::MergeTreeSelectBlockInputStream( Names column_names, const MarkRanges & mark_ranges_, bool use_uncompressed_cache_, - const PrewhereInfoPtr & prewhere_info, + const PrewhereInfoPtr & prewhere_info_, bool check_columns, size_t min_bytes_to_use_direct_io_, size_t max_read_buffer_size_, bool save_marks_in_cache_, - const Names & virt_column_names, + const Names & virt_column_names_, size_t part_index_in_query_, bool quiet) : - MergeTreeBaseSelectBlockInputStream{storage_, prewhere_info, max_block_size_rows_, + MergeTreeBaseSelectBlockInputStream{storage_, prewhere_info_, max_block_size_rows_, preferred_block_size_bytes_, preferred_max_column_in_block_size_bytes_, min_bytes_to_use_direct_io_, - max_read_buffer_size_, use_uncompressed_cache_, save_marks_in_cache_, virt_column_names}, + max_read_buffer_size_, use_uncompressed_cache_, save_marks_in_cache_, virt_column_names_}, required_columns{column_names}, data_part{owned_data_part_}, part_columns_lock(data_part->columns_lock), @@ -167,9 +167,9 @@ try if (!reader) { if (use_uncompressed_cache) - owned_uncompressed_cache = storage.context.getUncompressedCache(); + owned_uncompressed_cache = storage.global_context.getUncompressedCache(); - owned_mark_cache = storage.context.getMarkCache(); + owned_mark_cache = storage.global_context.getMarkCache(); reader = std::make_unique( path, data_part, columns, owned_uncompressed_cache.get(), diff --git a/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp index d984a75bf23..aecd822bca2 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp @@ -20,7 +20,7 @@ MergeTreeSequentialBlockInputStream::MergeTreeSequentialBlockInputStream( , part_columns_lock(data_part->columns_lock) , columns_to_read(columns_to_read_) , read_with_direct_io(read_with_direct_io_) - , mark_cache(storage.context.getMarkCache()) + , mark_cache(storage.global_context.getMarkCache()) { if (!quiet) LOG_TRACE(log, "Reading " << data_part->marks_count << " marks from part " << data_part->name diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp b/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp index 36b222add71..65b800460ed 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -68,7 +68,7 @@ void MergeTreeSettings::loadFromQuery(ASTStorage & storage_def) == changes.end()) \ changes.push_back(ASTSetQuery::Change{#NAME, NAME.value}); - APPLY_FOR_IMMUTABLE_MERGE_TREE_SETTINGS(ADD_IF_ABSENT); + APPLY_FOR_IMMUTABLE_MERGE_TREE_SETTINGS(ADD_IF_ABSENT) #undef ADD_IF_ABSENT } diff --git a/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.cpp index 432206270d9..964adc16edf 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.cpp @@ -11,18 +11,18 @@ MergeTreeThreadSelectBlockInputStream::MergeTreeThreadSelectBlockInputStream( const size_t thread, const MergeTreeReadPoolPtr & pool, const size_t min_marks_to_read_, - const size_t max_block_size_rows, - size_t preferred_block_size_bytes, - size_t preferred_max_column_in_block_size_bytes, - const MergeTreeData & storage, - const bool use_uncompressed_cache, - const PrewhereInfoPtr & prewhere_info, + const size_t max_block_size_rows_, + size_t preferred_block_size_bytes_, + size_t preferred_max_column_in_block_size_bytes_, + const MergeTreeData & storage_, + const bool use_uncompressed_cache_, + const PrewhereInfoPtr & prewhere_info_, const Settings & settings, - const Names & virt_column_names) + const Names & virt_column_names_) : - MergeTreeBaseSelectBlockInputStream{storage, prewhere_info, max_block_size_rows, - preferred_block_size_bytes, preferred_max_column_in_block_size_bytes, settings.min_bytes_to_use_direct_io, - settings.max_read_buffer_size, use_uncompressed_cache, true, virt_column_names}, + MergeTreeBaseSelectBlockInputStream{storage_, prewhere_info_, max_block_size_rows_, + preferred_block_size_bytes_, preferred_max_column_in_block_size_bytes_, settings.min_bytes_to_use_direct_io, + settings.max_read_buffer_size, use_uncompressed_cache_, true, virt_column_names_}, thread{thread}, pool{pool} { @@ -74,8 +74,8 @@ bool MergeTreeThreadSelectBlockInputStream::getNewTask() auto rest_mark_ranges = pool->getRestMarks(path, task->mark_ranges[0]); if (use_uncompressed_cache) - owned_uncompressed_cache = storage.context.getUncompressedCache(); - owned_mark_cache = storage.context.getMarkCache(); + owned_uncompressed_cache = storage.global_context.getUncompressedCache(); + owned_mark_cache = storage.global_context.getMarkCache(); reader = std::make_unique( path, task->data_part, task->columns, owned_uncompressed_cache.get(), owned_mark_cache.get(), save_marks_in_cache, diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 9d98e9f4555..ff47b97fd60 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -41,7 +41,7 @@ void IMergedBlockOutputStream::addStreams( const String & path, const String & name, const IDataType & type, - const CompressionCodecPtr & codec, + const CompressionCodecPtr & effective_codec, size_t estimated_size, bool skip_offsets) { @@ -60,7 +60,7 @@ void IMergedBlockOutputStream::addStreams( stream_name, path + stream_name, DATA_FILE_EXTENSION, path + stream_name, MARKS_FILE_EXTENSION, - codec, + effective_codec, max_compress_block_size, estimated_size, aio_threshold); @@ -99,7 +99,7 @@ void IMergedBlockOutputStream::writeData( bool skip_offsets, IDataType::SerializeBinaryBulkStatePtr & serialization_state) { - auto & settings = storage.context.getSettingsRef(); + auto & settings = storage.global_context.getSettingsRef(); IDataType::SerializeBinaryBulkSettings serialize_settings; serialize_settings.getter = createStreamGetter(name, offset_columns, skip_offsets); serialize_settings.low_cardinality_max_dictionary_size = settings.low_cardinality_max_dictionary_size; @@ -233,9 +233,9 @@ MergedBlockOutputStream::MergedBlockOutputStream( const NamesAndTypesList & columns_list_, CompressionCodecPtr default_codec_) : IMergedBlockOutputStream( - storage_, storage_.context.getSettings().min_compress_block_size, - storage_.context.getSettings().max_compress_block_size, default_codec_, - storage_.context.getSettings().min_bytes_to_use_direct_io), + storage_, storage_.global_context.getSettings().min_compress_block_size, + storage_.global_context.getSettings().max_compress_block_size, default_codec_, + storage_.global_context.getSettings().min_bytes_to_use_direct_io), columns_list(columns_list_), part_path(part_path_) { init(); @@ -254,8 +254,8 @@ MergedBlockOutputStream::MergedBlockOutputStream( const MergeTreeData::DataPart::ColumnToSize & merged_column_to_size_, size_t aio_threshold_) : IMergedBlockOutputStream( - storage_, storage_.context.getSettings().min_compress_block_size, - storage_.context.getSettings().max_compress_block_size, default_codec_, + storage_, storage_.global_context.getSettings().min_compress_block_size, + storage_.global_context.getSettings().max_compress_block_size, default_codec_, aio_threshold_), columns_list(columns_list_), part_path(part_path_) { @@ -312,7 +312,7 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( /// Finish columns serialization. if (!serialization_states.empty()) { - auto & settings = storage.context.getSettingsRef(); + auto & settings = storage.global_context.getSettingsRef(); IDataType::SerializeBinaryBulkSettings serialize_settings; serialize_settings.low_cardinality_max_dictionary_size = settings.low_cardinality_max_dictionary_size; serialize_settings.low_cardinality_use_single_dictionary_for_part = settings.low_cardinality_use_single_dictionary_for_part != 0; @@ -517,9 +517,9 @@ MergedColumnOnlyOutputStream::MergedColumnOnlyOutputStream( CompressionCodecPtr default_codec_, bool skip_offsets_, WrittenOffsetColumns & already_written_offset_columns) : IMergedBlockOutputStream( - storage_, storage_.context.getSettings().min_compress_block_size, - storage_.context.getSettings().max_compress_block_size, default_codec_, - storage_.context.getSettings().min_bytes_to_use_direct_io), + storage_, storage_.global_context.getSettings().min_compress_block_size, + storage_.global_context.getSettings().max_compress_block_size, default_codec_, + storage_.global_context.getSettings().min_bytes_to_use_direct_io), header(header_), part_path(part_path_), sync(sync_), skip_offsets(skip_offsets_), already_written_offset_columns(already_written_offset_columns) { @@ -570,7 +570,7 @@ void MergedColumnOnlyOutputStream::writeSuffix() MergeTreeData::DataPart::Checksums MergedColumnOnlyOutputStream::writeSuffixAndGetChecksums() { /// Finish columns serialization. - auto & settings = storage.context.getSettingsRef(); + auto & settings = storage.global_context.getSettingsRef(); IDataType::SerializeBinaryBulkSettings serialize_settings; serialize_settings.low_cardinality_max_dictionary_size = settings.low_cardinality_max_dictionary_size; serialize_settings.low_cardinality_use_single_dictionary_for_part = settings.low_cardinality_use_single_dictionary_for_part != 0; diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.h b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.h index b09abfc88a9..0ae8e83aafb 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.h +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include #include #include diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp index e7fe9dba256..438bca365ea 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp @@ -27,7 +27,7 @@ ReplicatedMergeTreeAlterThread::ReplicatedMergeTreeAlterThread(StorageReplicated , log_name(storage.database_name + "." + storage.table_name + " (ReplicatedMergeTreeAlterThread)") , log(&Logger::get(log_name)) { - task = storage_.context.getSchedulePool().createTask(log_name, [this]{ run(); }); + task = storage_.global_context.getSchedulePool().createTask(log_name, [this]{ run(); }); } void ReplicatedMergeTreeAlterThread::run() diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp index 831c2c36448..518731051a0 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp @@ -162,11 +162,11 @@ void ReplicatedMergeTreeBlockOutputStream::write(const Block & block) /// Set a special error code if the block is duplicate int error = (deduplicate && last_block_is_duplicate) ? ErrorCodes::INSERT_WAS_DEDUPLICATED : 0; - PartLog::addNewPart(storage.context, part, watch.elapsed(), ExecutionStatus(error)); + PartLog::addNewPart(storage.global_context, part, watch.elapsed(), ExecutionStatus(error)); } catch (...) { - PartLog::addNewPart(storage.context, part, watch.elapsed(), ExecutionStatus::fromCurrentException(__PRETTY_FUNCTION__)); + PartLog::addNewPart(storage.global_context, part, watch.elapsed(), ExecutionStatus::fromCurrentException(__PRETTY_FUNCTION__)); throw; } } @@ -190,11 +190,11 @@ void ReplicatedMergeTreeBlockOutputStream::writeExistingPart(MergeTreeData::Muta try { commitPart(zookeeper, part, ""); - PartLog::addNewPart(storage.context, part, watch.elapsed()); + PartLog::addNewPart(storage.global_context, part, watch.elapsed()); } catch (...) { - PartLog::addNewPart(storage.context, part, watch.elapsed(), ExecutionStatus::fromCurrentException(__PRETTY_FUNCTION__)); + PartLog::addNewPart(storage.global_context, part, watch.elapsed(), ExecutionStatus::fromCurrentException(__PRETTY_FUNCTION__)); throw; } } diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index 24356912aad..2f1ee2a2943 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -23,7 +23,7 @@ ReplicatedMergeTreeCleanupThread::ReplicatedMergeTreeCleanupThread(StorageReplic , log_name(storage.database_name + "." + storage.table_name + " (ReplicatedMergeTreeCleanupThread)") , log(&Logger::get(log_name)) { - task = storage.context.getSchedulePool().createTask(log_name, [this]{ run(); }); + task = storage.global_context.getSchedulePool().createTask(log_name, [this]{ run(); }); } void ReplicatedMergeTreeCleanupThread::run() @@ -86,7 +86,6 @@ void ReplicatedMergeTreeCleanupThread::clearOldLogs() /// We will keep logs after and including this threshold. UInt64 min_saved_log_pointer = std::numeric_limits::max(); - UInt64 min_log_pointer_lost_candidate = std::numeric_limits::max(); Strings entries = zookeeper->getChildren(storage.zookeeper_path + "/log"); @@ -118,7 +117,7 @@ void ReplicatedMergeTreeCleanupThread::clearOldLogs() zookeeper->get(storage.zookeeper_path + "/replicas/" + replica + "/host", &host_stat); String pointer = zookeeper->get(storage.zookeeper_path + "/replicas/" + replica + "/log_pointer"); - UInt32 log_pointer = 0; + UInt64 log_pointer = 0; if (!pointer.empty()) log_pointer = parse(pointer); @@ -190,7 +189,7 @@ void ReplicatedMergeTreeCleanupThread::clearOldLogs() for (const String & replica : recovering_replicas) { String pointer = zookeeper->get(storage.zookeeper_path + "/replicas/" + replica + "/log_pointer"); - UInt32 log_pointer = 0; + UInt64 log_pointer = 0; if (!pointer.empty()) log_pointer = parse(pointer); min_saved_log_pointer = std::min(min_saved_log_pointer, log_pointer); diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index c115264d393..0c529124698 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -27,7 +27,7 @@ ReplicatedMergeTreePartCheckThread::ReplicatedMergeTreePartCheckThread(StorageRe , log_name(storage.database_name + "." + storage.table_name + " (ReplicatedMergeTreePartCheckThread)") , log(&Logger::get(log_name)) { - task = storage.context.getSchedulePool().createTask(log_name, [this] { run(); }); + task = storage.global_context.getSchedulePool().createTask(log_name, [this] { run(); }); task->schedule(); } @@ -38,7 +38,7 @@ ReplicatedMergeTreePartCheckThread::~ReplicatedMergeTreePartCheckThread() void ReplicatedMergeTreePartCheckThread::start() { - std::lock_guard lock(start_stop_mutex); + std::lock_guard lock(start_stop_mutex); need_stop = false; task->activateAndSchedule(); } @@ -48,14 +48,14 @@ void ReplicatedMergeTreePartCheckThread::stop() //based on discussion on https://github.com/yandex/ClickHouse/pull/1489#issuecomment-344756259 //using the schedule pool there is no problem in case stop is called two time in row and the start multiple times - std::lock_guard lock(start_stop_mutex); + std::lock_guard lock(start_stop_mutex); need_stop = true; task->deactivate(); } void ReplicatedMergeTreePartCheckThread::enqueuePart(const String & name, time_t delay_to_check_seconds) { - std::lock_guard lock(parts_mutex); + std::lock_guard lock(parts_mutex); if (parts_set.count(name)) return; @@ -68,7 +68,7 @@ void ReplicatedMergeTreePartCheckThread::enqueuePart(const String & name, time_t size_t ReplicatedMergeTreePartCheckThread::size() const { - std::lock_guard lock(parts_mutex); + std::lock_guard lock(parts_mutex); return parts_set.size(); } @@ -295,7 +295,7 @@ void ReplicatedMergeTreePartCheckThread::run() time_t min_check_time = std::numeric_limits::max(); { - std::lock_guard lock(parts_mutex); + std::lock_guard lock(parts_mutex); if (parts_queue.empty()) { @@ -331,7 +331,7 @@ void ReplicatedMergeTreePartCheckThread::run() /// Remove the part from check queue. { - std::lock_guard lock(parts_mutex); + std::lock_guard lock(parts_mutex); if (parts_queue.empty()) { diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 2499b3239e7..fcdce191169 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -294,7 +294,7 @@ void ReplicatedMergeTreeQueue::removeProcessedEntry(zkutil::ZooKeeperPtr zookeep size_t queue_size = 0; { - std::unique_lock lock(state_mutex); + std::unique_lock lock(state_mutex); /// Remove the job from the queue in the RAM. /// You can not just refer to a pre-saved iterator, because someone else might be able to delete the task. @@ -335,7 +335,7 @@ bool ReplicatedMergeTreeQueue::remove(zkutil::ZooKeeperPtr zookeeper, const Stri std::optional max_processed_insert_time_changed; { - std::unique_lock lock(state_mutex); + std::unique_lock lock(state_mutex); virtual_parts.remove(part_name); @@ -370,7 +370,7 @@ bool ReplicatedMergeTreeQueue::remove(zkutil::ZooKeeperPtr zookeeper, const Stri bool ReplicatedMergeTreeQueue::removeFromVirtualParts(const MergeTreePartInfo & part_info) { - std::unique_lock lock(state_mutex); + std::unique_lock lock(state_mutex); return virtual_parts.remove(part_info); } @@ -420,10 +420,10 @@ void ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, C /// The average size of the node value in this case is less than 10 kilobytes. static constexpr auto MAX_MULTI_OPS = 100; - for (size_t i = 0, size = log_entries.size(); i < size; i += MAX_MULTI_OPS) + for (size_t entry_idx = 0, num_entries = log_entries.size(); entry_idx < num_entries; entry_idx += MAX_MULTI_OPS) { - auto begin = log_entries.begin() + i; - auto end = i + MAX_MULTI_OPS >= log_entries.size() + auto begin = log_entries.begin() + entry_idx; + auto end = entry_idx + MAX_MULTI_OPS >= log_entries.size() ? log_entries.end() : (begin + MAX_MULTI_OPS); auto last = end - 1; @@ -463,7 +463,7 @@ void ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, C const auto & entry = *copied_entries.back(); if (entry.type == LogEntry::GET_PART) { - std::lock_guard lock(state_mutex); + std::lock_guard state_lock(state_mutex); if (entry.create_time && (!min_unprocessed_insert_time || entry.create_time < min_unprocessed_insert_time)) { min_unprocessed_insert_time = entry.create_time; @@ -485,17 +485,17 @@ void ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, C try { - std::lock_guard lock(state_mutex); + std::lock_guard state_lock(state_mutex); log_pointer = last_entry_index + 1; - for (size_t i = 0, size = copied_entries.size(); i < size; ++i) + for (size_t copied_entry_idx = 0, num_copied_entries = copied_entries.size(); copied_entry_idx < num_copied_entries; ++copied_entry_idx) { - String path_created = dynamic_cast(*responses[i]).path_created; - copied_entries[i]->znode_name = path_created.substr(path_created.find_last_of('/') + 1); + String path_created = dynamic_cast(*responses[copied_entry_idx]).path_created; + copied_entries[copied_entry_idx]->znode_name = path_created.substr(path_created.find_last_of('/') + 1); std::optional unused = false; - insertUnlocked(copied_entries[i], unused, lock); + insertUnlocked(copied_entries[copied_entry_idx], unused, state_lock); } last_queue_update = time(nullptr); @@ -552,7 +552,7 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C /// Compare with the local state, delete obsolete entries and determine which new entries to load. Strings entries_to_load; { - std::lock_guard lock(state_mutex); + std::lock_guard state_lock(state_mutex); for (auto it = mutations_by_znode.begin(); it != mutations_by_znode.end();) { @@ -599,7 +599,7 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C bool some_mutations_are_probably_done = false; { - std::lock_guard lock(state_mutex); + std::lock_guard state_lock(state_mutex); for (const ReplicatedMergeTreeMutationEntryPtr & entry : new_mutations) { @@ -648,7 +648,7 @@ ReplicatedMergeTreeQueue::StringSet ReplicatedMergeTreeQueue::moveSiblingPartsFo /// Let's find the action to merge this part with others. Let's remember others. StringSet parts_for_merge; - Queue::iterator merge_entry; + Queue::iterator merge_entry = queue.end(); for (Queue::iterator it = queue.begin(); it != queue.end(); ++it) { if ((*it)->type == LogEntry::MERGE_PARTS || (*it)->type == LogEntry::MUTATE_PART) @@ -711,7 +711,7 @@ void ReplicatedMergeTreeQueue::removePartProducingOpsInRange(zkutil::ZooKeeperPt std::optional max_processed_insert_time_changed; /// Remove operations with parts, contained in the range to be deleted, from the queue. - std::unique_lock lock(state_mutex); + std::unique_lock lock(state_mutex); for (Queue::iterator it = queue.begin(); it != queue.end();) { auto type = (*it)->type; @@ -785,7 +785,7 @@ size_t ReplicatedMergeTreeQueue::getConflictsCountForRange( void ReplicatedMergeTreeQueue::checkThereAreNoConflictsInRange(const MergeTreePartInfo & range, const LogEntry & entry) { String conflicts_description; - std::lock_guard lock(state_mutex); + std::lock_guard lock(state_mutex); if (0 != getConflictsCountForRange(range, entry, &conflicts_description, lock)) throw Exception(conflicts_description, ErrorCodes::UNFINISHED); @@ -1013,7 +1013,7 @@ ReplicatedMergeTreeQueue::SelectedEntry ReplicatedMergeTreeQueue::selectEntryToP { LogEntryPtr entry; - std::lock_guard lock(state_mutex); + std::lock_guard lock(state_mutex); for (auto it = queue.begin(); it != queue.end(); ++it) { @@ -1635,8 +1635,8 @@ bool ReplicatedMergeTreeMergePredicate::isMutationFinished(const ReplicatedMerge ReplicatedMergeTreeQueue::SubscriberHandler ReplicatedMergeTreeQueue::addSubscriber(ReplicatedMergeTreeQueue::SubscriberCallBack && callback) { - std::lock_guard lock(state_mutex); - std::lock_guard lock_subscribers(subscribers_mutex); + std::lock_guard lock(state_mutex); + std::lock_guard lock_subscribers(subscribers_mutex); auto it = subscribers.emplace(subscribers.end(), std::move(callback)); @@ -1648,13 +1648,13 @@ ReplicatedMergeTreeQueue::addSubscriber(ReplicatedMergeTreeQueue::SubscriberCall ReplicatedMergeTreeQueue::SubscriberHandler::~SubscriberHandler() { - std::lock_guard lock(queue.subscribers_mutex); + std::lock_guard lock(queue.subscribers_mutex); queue.subscribers.erase(it); } void ReplicatedMergeTreeQueue::notifySubscribers(size_t new_queue_size) { - std::lock_guard lock_subscribers(subscribers_mutex); + std::lock_guard lock_subscribers(subscribers_mutex); for (auto & subscriber_callback : subscribers) subscriber_callback(new_queue_size); } diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index 1243094641e..091319b0596 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -51,7 +51,7 @@ ReplicatedMergeTreeRestartingThread::ReplicatedMergeTreeRestartingThread(Storage if (check_period_ms > static_cast(storage.data.settings.check_delay_period) * 1000) check_period_ms = storage.data.settings.check_delay_period * 1000; - task = storage.context.getSchedulePool().createTask(log_name, [this]{ run(); }); + task = storage.global_context.getSchedulePool().createTask(log_name, [this]{ run(); }); } void ReplicatedMergeTreeRestartingThread::run() @@ -84,7 +84,7 @@ void ReplicatedMergeTreeRestartingThread::run() { try { - storage.setZooKeeper(storage.context.getZooKeeper()); + storage.setZooKeeper(storage.global_context.getZooKeeper()); } catch (const Coordination::Exception &) { diff --git a/dbms/src/Storages/MergeTree/checkDataPart.cpp b/dbms/src/Storages/MergeTree/checkDataPart.cpp index eac9145692b..e88f027ad75 100644 --- a/dbms/src/Storages/MergeTree/checkDataPart.cpp +++ b/dbms/src/Storages/MergeTree/checkDataPart.cpp @@ -5,7 +5,7 @@ #include #include -#include +#include #include #include diff --git a/dbms/src/Storages/StorageBuffer.cpp b/dbms/src/Storages/StorageBuffer.cpp index 3749bb92d80..5d76279c95f 100644 --- a/dbms/src/Storages/StorageBuffer.cpp +++ b/dbms/src/Storages/StorageBuffer.cpp @@ -59,7 +59,7 @@ StorageBuffer::StorageBuffer(const std::string & name_, const ColumnsDescription size_t num_shards_, const Thresholds & min_thresholds_, const Thresholds & max_thresholds_, const String & destination_database_, const String & destination_table_, bool allow_materialized_) : IStorage{columns_}, - name(name_), context(context_), + name(name_), global_context(context_), num_shards(num_shards_), buffers(num_shards_), min_thresholds(min_thresholds_), max_thresholds(max_thresholds_), destination_database(destination_database_), destination_table(destination_table_), @@ -99,7 +99,7 @@ protected: return res; has_been_read = true; - std::lock_guard lock(buffer.mutex); + std::lock_guard lock(buffer.mutex); if (!buffer.data.rows()) return res; @@ -306,7 +306,7 @@ public: StoragePtr destination; if (!storage.no_destination) { - destination = storage.context.tryGetTable(storage.destination_database, storage.destination_table); + destination = storage.global_context.tryGetTable(storage.destination_database, storage.destination_table); if (destination.get() == &storage) throw Exception("Destination table is myself. Write will cause infinite loop.", ErrorCodes::INFINITE_LOOP); } @@ -336,7 +336,7 @@ public: for (size_t try_no = 0; try_no < storage.num_shards; ++try_no) { - std::unique_lock lock(storage.buffers[shard_num].mutex, std::try_to_lock); + std::unique_lock lock(storage.buffers[shard_num].mutex, std::try_to_lock); if (lock.owns_lock()) { @@ -356,7 +356,7 @@ public: if (!least_busy_buffer) { least_busy_buffer = &storage.buffers[start_shard_num]; - least_busy_lock = std::unique_lock(least_busy_buffer->mutex); + least_busy_lock = std::unique_lock(least_busy_buffer->mutex); } insertIntoBuffer(block, *least_busy_buffer); } @@ -403,7 +403,7 @@ bool StorageBuffer::mayBenefitFromIndexForIn(const ASTPtr & left_in_operand) con if (no_destination) return false; - auto destination = context.getTable(destination_database, destination_table); + auto destination = global_context.getTable(destination_database, destination_table); if (destination.get() == this) throw Exception("Destination table is myself. Read will cause infinite loop.", ErrorCodes::INFINITE_LOOP); @@ -414,7 +414,7 @@ bool StorageBuffer::mayBenefitFromIndexForIn(const ASTPtr & left_in_operand) con void StorageBuffer::startup() { - if (context.getSettingsRef().readonly) + if (global_context.getSettingsRef().readonly) { LOG_WARNING(log, "Storage " << getName() << " is run with readonly settings, it will not be able to insert data." << " Set apropriate system_profile to fix this."); @@ -433,7 +433,7 @@ void StorageBuffer::shutdown() try { - optimize(nullptr /*query*/, {} /*partition*/, false /*final*/, false /*deduplicate*/, context); + optimize(nullptr /*query*/, {} /*partition*/, false /*final*/, false /*deduplicate*/, global_context); } catch (...) { @@ -527,7 +527,7 @@ void StorageBuffer::flushBuffer(Buffer & buffer, bool check_thresholds, bool loc size_t bytes = 0; time_t time_passed = 0; - std::unique_lock lock(buffer.mutex, std::defer_lock); + std::unique_lock lock(buffer.mutex, std::defer_lock); if (!locked) lock.lock(); @@ -570,7 +570,7 @@ void StorageBuffer::flushBuffer(Buffer & buffer, bool check_thresholds, bool loc */ try { - writeBlockToDestination(block_to_write, context.tryGetTable(destination_database, destination_table)); + writeBlockToDestination(block_to_write, global_context.tryGetTable(destination_database, destination_table)); } catch (...) { @@ -625,7 +625,7 @@ void StorageBuffer::writeBlockToDestination(const Block & block, StoragePtr tabl << " have different type of column " << backQuoteIfNeed(column.name) << " (" << dst_col.type->getName() << " != " << column.type->getName() << "). Block of data is converted."); - column.column = castColumn(column, dst_col.type, context); + column.column = castColumn(column, dst_col.type, global_context); column.type = dst_col.type; } @@ -650,7 +650,7 @@ void StorageBuffer::writeBlockToDestination(const Block & block, StoragePtr tabl for (const auto & column : block_to_write) list_of_columns->children.push_back(std::make_shared(column.name)); - InterpreterInsertQuery interpreter{insert, context, allow_materialized}; + InterpreterInsertQuery interpreter{insert, global_context, allow_materialized}; auto block_io = interpreter.execute(); block_io.out->writePrefix(); diff --git a/dbms/src/Storages/StorageBuffer.h b/dbms/src/Storages/StorageBuffer.h index 3aa32b122e7..9992d1b49bd 100644 --- a/dbms/src/Storages/StorageBuffer.h +++ b/dbms/src/Storages/StorageBuffer.h @@ -85,7 +85,7 @@ public: private: String name; - Context & context; + Context global_context; struct Buffer { diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index f681565d40e..8f4f31d458c 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -101,27 +101,27 @@ ASTPtr createInsertToRemoteTableQuery(const std::string & database, const std::s /// Calculate maximum number in file names in directory and all subdirectories. /// To ensure global order of data blocks yet to be sent across server restarts. -UInt64 getMaximumFileNumber(const std::string & path) +UInt64 getMaximumFileNumber(const std::string & dir_path) { UInt64 res = 0; - boost::filesystem::recursive_directory_iterator begin(path); + boost::filesystem::recursive_directory_iterator begin(dir_path); boost::filesystem::recursive_directory_iterator end; for (auto it = begin; it != end; ++it) { - const auto & path = it->path(); + const auto & file_path = it->path(); - if (it->status().type() != boost::filesystem::regular_file || !endsWith(path.filename().string(), ".bin")) + if (it->status().type() != boost::filesystem::regular_file || !endsWith(file_path.filename().string(), ".bin")) continue; UInt64 num = 0; try { - num = parse(path.filename().stem().string()); + num = parse(file_path.filename().stem().string()); } catch (Exception & e) { - e.addMessage("Unexpected file name " + path.filename().string() + " found at " + path.parent_path().string() + ", should have numeric base name."); + e.addMessage("Unexpected file name " + file_path.filename().string() + " found at " + file_path.parent_path().string() + ", should have numeric base name."); throw; } @@ -170,13 +170,13 @@ StorageDistributed::~StorageDistributed() = default; static ExpressionActionsPtr buildShardingKeyExpression(const ASTPtr & sharding_key, const Context & context, NamesAndTypesList columns, bool project) { ASTPtr query = sharding_key; - auto syntax_result = SyntaxAnalyzer(context, {}).analyze(query, columns); + auto syntax_result = SyntaxAnalyzer(context).analyze(query, columns); return ExpressionAnalyzer(query, syntax_result, context).getActions(project); } StorageDistributed::StorageDistributed( const String & database_name, - const String & table_name_, + const String & table_name, const ColumnsDescription & columns_, const String & remote_database_, const String & remote_table_, @@ -185,18 +185,17 @@ StorageDistributed::StorageDistributed( const ASTPtr & sharding_key_, const String & data_path_, bool attach) - : IStorage{columns_}, - table_name(table_name_), + : IStorage{columns_}, table_name(table_name), remote_database(remote_database_), remote_table(remote_table_), - context(context_), cluster_name(context.getMacros()->expand(cluster_name_)), has_sharding_key(sharding_key_), - sharding_key_expr(sharding_key_ ? buildShardingKeyExpression(sharding_key_, context, getColumns().getAllPhysical(), false) : nullptr), + global_context(context_), cluster_name(global_context.getMacros()->expand(cluster_name_)), has_sharding_key(sharding_key_), + sharding_key_expr(sharding_key_ ? buildShardingKeyExpression(sharding_key_, global_context, getColumns().getAllPhysical(), false) : nullptr), sharding_key_column_name(sharding_key_ ? sharding_key_->getColumnName() : String{}), path(data_path_.empty() ? "" : (data_path_ + escapeForFileName(table_name) + '/')) { /// Sanity check. Skip check if the table is already created to allow the server to start. if (!attach && !cluster_name.empty()) { - size_t num_local_shards = context.getCluster(cluster_name)->getLocalShardCount(); + size_t num_local_shards = global_context.getCluster(cluster_name)->getLocalShardCount(); if (num_local_shards && remote_database == database_name && remote_table == table_name) throw Exception("Distributed table " + table_name + " looks at itself", ErrorCodes::INFINITE_LOOP); } @@ -336,13 +335,13 @@ BlockOutputStreamPtr StorageDistributed::write(const ASTPtr &, const Settings & } -void StorageDistributed::alter(const AlterCommands & params, const String & database_name, const String & table_name, const Context & context) +void StorageDistributed::alter(const AlterCommands & params, const String & database_name, const String & current_table_name, const Context & context) { auto lock = lockStructureForAlter(); ColumnsDescription new_columns = getColumns(); params.apply(new_columns); - context.getDatabase(database_name)->alterTable(context, table_name, new_columns, {}); + context.getDatabase(database_name)->alterTable(context, current_table_name, new_columns, {}); setColumns(std::move(new_columns)); } @@ -360,7 +359,7 @@ void StorageDistributed::shutdown() } -void StorageDistributed::truncate(const ASTPtr &) +void StorageDistributed::truncate(const ASTPtr &, const Context &) { std::lock_guard lock(cluster_nodes_mutex); @@ -440,7 +439,7 @@ size_t StorageDistributed::getShardCount() const ClusterPtr StorageDistributed::getCluster() const { - return owned_cluster ? owned_cluster : context.getCluster(cluster_name); + return owned_cluster ? owned_cluster : global_context.getCluster(cluster_name); } void StorageDistributed::ClusterNodeData::requireConnectionPool(const std::string & name, const StorageDistributed & storage) diff --git a/dbms/src/Storages/StorageDistributed.h b/dbms/src/Storages/StorageDistributed.h index e14d9f7081f..caa52209804 100644 --- a/dbms/src/Storages/StorageDistributed.h +++ b/dbms/src/Storages/StorageDistributed.h @@ -76,7 +76,7 @@ public: void drop() override {} /// Removes temporary data in local filesystem. - void truncate(const ASTPtr &) override; + void truncate(const ASTPtr &, const Context &) override; void rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & new_table_name) override { table_name = new_table_name; } /// in the sub-tables, you need to manually add and delete columns @@ -111,7 +111,7 @@ public: String remote_table; ASTPtr remote_table_function_ptr; - const Context & context; + Context global_context; Logger * log = &Logger::get("StorageDistributed"); /// Used to implement TableFunctionRemote. diff --git a/dbms/src/Storages/StorageHDFS.cpp b/dbms/src/Storages/StorageHDFS.cpp index 03c9626a582..2e94f8bd211 100644 --- a/dbms/src/Storages/StorageHDFS.cpp +++ b/dbms/src/Storages/StorageHDFS.cpp @@ -39,38 +39,36 @@ StorageHDFS::StorageHDFS(const String & uri_, namespace { - class StorageHDFSBlockInputStream : public IProfilingBlockInputStream + class HDFSBlockInputStream : public IProfilingBlockInputStream { public: - StorageHDFSBlockInputStream(const String & uri, + HDFSBlockInputStream(const String & uri, const String & format, - const String & name_, const Block & sample_block, const Context & context, size_t max_block_size) - : name(name_) { // Assume no query and fragment in uri, todo, add sanity check - String fuzzyFileNames; - String uriPrefix = uri.substr(0, uri.find_last_of('/')); - if (uriPrefix.length() == uri.length()) + String glob_file_names; + String url_prefix = uri.substr(0, uri.find_last_of('/')); + if (url_prefix.length() == uri.length()) { - fuzzyFileNames = uri; - uriPrefix.clear(); + glob_file_names = uri; + url_prefix.clear(); } else { - uriPrefix += "/"; - fuzzyFileNames = uri.substr(uriPrefix.length()); + url_prefix += "/"; + glob_file_names = uri.substr(url_prefix.length()); } - std::vector fuzzyNameList = parseRemoteDescription(fuzzyFileNames, 0, fuzzyFileNames.length(), ',' , 100/* hard coded max files */); + std::vector glob_names_list = parseRemoteDescription(glob_file_names, 0, glob_file_names.length(), ',' , 100/* hard coded max files */); BlockInputStreams inputs; - for (auto & name: fuzzyNameList) + for (const auto & name : glob_names_list) { - std::unique_ptr read_buf = std::make_unique(uriPrefix + name); + std::unique_ptr read_buf = std::make_unique(url_prefix + name); inputs.emplace_back( std::make_shared>( @@ -93,7 +91,7 @@ namespace String getName() const override { - return name; + return "HDFS"; } Block readImpl() override @@ -113,14 +111,13 @@ namespace void readSuffixImpl() override { - auto explicitReader = dynamic_cast(reader.get()); - if (explicitReader) explicitReader->cancel(false); // skip Union read suffix assertion + if (auto concrete_reader = dynamic_cast(reader.get())) + concrete_reader->cancel(false); // skip Union read suffix assertion reader->readSuffix(); } private: - String name; BlockInputStreamPtr reader; }; @@ -135,10 +132,9 @@ BlockInputStreams StorageHDFS::read( size_t max_block_size, unsigned /*num_streams*/) { - return {std::make_shared( + return {std::make_shared( uri, format_name, - getName(), getSampleBlock(), context, max_block_size)}; @@ -149,7 +145,6 @@ void StorageHDFS::rename(const String & /*new_path_to_db*/, const String & /*new BlockOutputStreamPtr StorageHDFS::write(const ASTPtr & /*query*/, const Settings & /*settings*/) { throw Exception("StorageHDFS write is not supported yet", ErrorCodes::NOT_IMPLEMENTED); - return {}; } void registerStorageHDFS(StorageFactory & factory) diff --git a/dbms/src/Storages/StorageJoin.cpp b/dbms/src/Storages/StorageJoin.cpp index e05146ad431..8d1d6d52fbf 100644 --- a/dbms/src/Storages/StorageJoin.cpp +++ b/dbms/src/Storages/StorageJoin.cpp @@ -51,7 +51,7 @@ StorageJoin::StorageJoin( } -void StorageJoin::truncate(const ASTPtr &) +void StorageJoin::truncate(const ASTPtr &, const Context &) { Poco::File(path).remove(true); Poco::File(path).createDirectories(); @@ -190,8 +190,8 @@ size_t rawSize(const StringRef & t) class JoinBlockInputStream : public IProfilingBlockInputStream { public: - JoinBlockInputStream(const Join & parent_, size_t max_block_size_, Block & sample_block_) - : parent(parent_), lock(parent.rwlock), max_block_size(max_block_size_), sample_block(sample_block_) + JoinBlockInputStream(const Join & parent_, size_t max_block_size_, Block && sample_block_) + : parent(parent_), lock(parent.rwlock), max_block_size(max_block_size_), sample_block(std::move(sample_block_)) { columns.resize(sample_block.columns()); column_indices.resize(sample_block.columns()); @@ -362,8 +362,7 @@ BlockInputStreams StorageJoin::read( unsigned /*num_streams*/) { check(column_names); - Block sample_block = getSampleBlockForColumns(column_names); - return {std::make_shared(*join, max_block_size, sample_block)}; + return {std::make_shared(*join, max_block_size, getSampleBlockForColumns(column_names))}; } } diff --git a/dbms/src/Storages/StorageJoin.h b/dbms/src/Storages/StorageJoin.h index 25c5128a349..ebe9dc68ce4 100644 --- a/dbms/src/Storages/StorageJoin.h +++ b/dbms/src/Storages/StorageJoin.h @@ -25,7 +25,7 @@ class StorageJoin : public ext::shared_ptr_helper, public StorageSe public: String getName() const override { return "Join"; } - void truncate(const ASTPtr &) override; + void truncate(const ASTPtr &, const Context &) override; /// Access the innards. JoinPtr & getJoin() { return join; } diff --git a/dbms/src/Storages/StorageLog.cpp b/dbms/src/Storages/StorageLog.cpp index 462f20ad582..e04d02e7de9 100644 --- a/dbms/src/Storages/StorageLog.cpp +++ b/dbms/src/Storages/StorageLog.cpp @@ -6,8 +6,8 @@ #include #include -#include -#include +#include +#include #include #include @@ -408,7 +408,7 @@ void LogBlockOutputStream::writeMarks(MarksForColumns && marks) writeIntBinary(mark.second.offset, marks_stream); size_t column_index = mark.first; - storage.files[storage.column_names[column_index]].marks.push_back(mark.second); + storage.files[storage.column_names_by_idx[column_index]].marks.push_back(mark.second); } } @@ -452,13 +452,13 @@ void StorageLog::addFiles(const String & column_name, const IDataType & type) column_data.data_file = Poco::File{ path + escapeForFileName(name) + '/' + stream_name + DBMS_STORAGE_LOG_DATA_FILE_EXTENSION}; - column_names.push_back(stream_name); + column_names_by_idx.push_back(stream_name); ++file_count; } }; - IDataType::SubstreamPath path; - type.enumerateStreams(stream_callback, path); + IDataType::SubstreamPath substream_path; + type.enumerateStreams(stream_callback, substream_path); } @@ -520,7 +520,7 @@ void StorageLog::rename(const String & new_path_to_db, const String & /*new_data marks_file = Poco::File(path + escapeForFileName(name) + '/' + DBMS_STORAGE_LOG_MARKS_FILE_NAME); } -void StorageLog::truncate(const ASTPtr &) +void StorageLog::truncate(const ASTPtr &, const Context &) { std::shared_lock lock(rwlock); @@ -554,12 +554,12 @@ const StorageLog::Marks & StorageLog::getMarksWithRealRowCount() const * If this is a data type with multiple stream, get the first stream, that we assume have real row count. * (Example: for Array data type, first stream is array sizes; and number of array sizes is the number of arrays). */ - IDataType::SubstreamPath path; + IDataType::SubstreamPath substream_root_path; column_type.enumerateStreams([&](const IDataType::SubstreamPath & substream_path) { if (filename.empty()) filename = IDataType::getFileNameForStream(column_name, substream_path); - }, path); + }, substream_root_path); Files_t::const_iterator it = files.find(filename); if (files.end() == it) diff --git a/dbms/src/Storages/StorageLog.h b/dbms/src/Storages/StorageLog.h index 91a03670e59..73a4d387dc5 100644 --- a/dbms/src/Storages/StorageLog.h +++ b/dbms/src/Storages/StorageLog.h @@ -40,7 +40,7 @@ public: bool checkData() const override; - void truncate(const ASTPtr &) override; + void truncate(const ASTPtr &, const Context &) override; std::string full_path() const { return path + escapeForFileName(name) + '/';} @@ -88,7 +88,7 @@ private: Files_t files; /// name -> data - Names column_names; /// column_index -> name + Names column_names_by_idx; /// column_index -> name Poco::File marks_file; diff --git a/dbms/src/Storages/StorageMaterializedView.cpp b/dbms/src/Storages/StorageMaterializedView.cpp index b73ee12b473..5df31fbab4d 100644 --- a/dbms/src/Storages/StorageMaterializedView.cpp +++ b/dbms/src/Storages/StorageMaterializedView.cpp @@ -229,7 +229,7 @@ void StorageMaterializedView::drop() executeDropQuery(ASTDropQuery::Kind::Drop, global_context, target_database_name, target_table_name); } -void StorageMaterializedView::truncate(const ASTPtr &) +void StorageMaterializedView::truncate(const ASTPtr &, const Context &) { if (has_inner_table) executeDropQuery(ASTDropQuery::Kind::Truncate, global_context, target_database_name, target_table_name); diff --git a/dbms/src/Storages/StorageMaterializedView.h b/dbms/src/Storages/StorageMaterializedView.h index 176a95c2d03..d308bd3550a 100644 --- a/dbms/src/Storages/StorageMaterializedView.h +++ b/dbms/src/Storages/StorageMaterializedView.h @@ -31,7 +31,7 @@ public: BlockOutputStreamPtr write(const ASTPtr & query, const Settings & settings) override; void drop() override; - void truncate(const ASTPtr &) override; + void truncate(const ASTPtr &, const Context &) override; bool optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & context) override; diff --git a/dbms/src/Storages/StorageMemory.cpp b/dbms/src/Storages/StorageMemory.cpp index ac8f6b79f74..29d084e7480 100644 --- a/dbms/src/Storages/StorageMemory.cpp +++ b/dbms/src/Storages/StorageMemory.cpp @@ -66,7 +66,7 @@ public: void write(const Block & block) override { storage.check(block, true); - std::lock_guard lock(storage.mutex); + std::lock_guard lock(storage.mutex); storage.data.push_back(block); } private: @@ -90,7 +90,7 @@ BlockInputStreams StorageMemory::read( { check(column_names); - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); size_t size = data.size(); @@ -123,13 +123,13 @@ BlockOutputStreamPtr StorageMemory::write( void StorageMemory::drop() { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); data.clear(); } -void StorageMemory::truncate(const ASTPtr &) +void StorageMemory::truncate(const ASTPtr &, const Context &) { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); data.clear(); } diff --git a/dbms/src/Storages/StorageMemory.h b/dbms/src/Storages/StorageMemory.h index 27c948bf87f..39bcce76894 100644 --- a/dbms/src/Storages/StorageMemory.h +++ b/dbms/src/Storages/StorageMemory.h @@ -40,7 +40,7 @@ public: void drop() override; - void truncate(const ASTPtr &) override; + void truncate(const ASTPtr &, const Context &) override; void rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & new_table_name) override { table_name = new_table_name; } diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp index 4a6ad3c7c8a..754d5e4fdfe 100644 --- a/dbms/src/Storages/StorageMerge.cpp +++ b/dbms/src/Storages/StorageMerge.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -51,7 +52,7 @@ StorageMerge::StorageMerge( const Context & context_) : IStorage{columns_}, name(name_), source_database(source_database_), - table_name_regexp(table_name_regexp_), context(context_) + table_name_regexp(table_name_regexp_), global_context(context_) { } @@ -95,8 +96,8 @@ bool StorageMerge::hasColumn(const String & column_name) const template StoragePtr StorageMerge::getFirstTable(F && predicate) const { - auto database = context.getDatabase(source_database); - auto iterator = database->getIterator(context); + auto database = global_context.getDatabase(source_database); + auto iterator = database->getIterator(global_context); while (iterator->isValid()) { @@ -185,14 +186,12 @@ BlockInputStreams StorageMerge::read( Names real_column_names; real_column_names.reserve(column_names.size()); - for (const auto & name : column_names) + for (const auto & column_name : column_names) { - if (name == "_table") - { + if (column_name == "_table") has_table_virtual_column = true; - } else - real_column_names.push_back(name); + real_column_names.push_back(column_name); } /** Just in case, turn off optimization "transfer to PREWHERE", @@ -221,7 +220,7 @@ BlockInputStreams StorageMerge::read( size_t current_need_streams = tables_count >= num_streams ? 1 : (num_streams / tables_count); size_t current_streams = std::min(current_need_streams, remaining_streams); remaining_streams -= current_streams; - current_streams = std::max(1, current_streams); + current_streams = std::max(size_t(1), current_streams); StoragePtr storage = it->first; TableStructureReadLockPtr struct_lock = it->second; @@ -336,8 +335,8 @@ BlockInputStreams StorageMerge::createSourceStreams(const SelectQueryInfo & quer StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables() const { StorageListWithLocks selected_tables; - auto database = context.getDatabase(source_database); - auto iterator = database->getIterator(context); + auto database = global_context.getDatabase(source_database); + auto iterator = database->getIterator(global_context); while (iterator->isValid()) { @@ -358,8 +357,8 @@ StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables() const StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables(const ASTPtr & query, bool has_virtual_column, bool get_lock) const { StorageListWithLocks selected_tables; - DatabasePtr database = context.getDatabase(source_database); - DatabaseIteratorPtr iterator = database->getIterator(context); + DatabasePtr database = global_context.getDatabase(source_database); + DatabaseIteratorPtr iterator = database->getIterator(global_context); auto virtual_column = ColumnString::create(); @@ -385,7 +384,7 @@ StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables(const ASTPtr if (has_virtual_column) { Block virtual_columns_block = Block{ColumnWithTypeAndName(std::move(virtual_column), std::make_shared(), "_table")}; - VirtualColumnUtils::filterBlockWithQuery(query, virtual_columns_block, context); + VirtualColumnUtils::filterBlockWithQuery(query, virtual_columns_block, global_context); auto values = VirtualColumnUtils::extractSingleValueFromBlock(virtual_columns_block, "_table"); /// Remove unused tables from the list @@ -454,11 +453,11 @@ void StorageMerge::convertingSourceStream(const Block & header, const Context & NamesAndTypesList source_columns = getSampleBlock().getNamesAndTypesList(); NameAndTypePair virtual_column = getColumn("_table"); source_columns.insert(source_columns.end(), virtual_column); - auto syntax_result = SyntaxAnalyzer(context, {}).analyze(where_expression, source_columns); + auto syntax_result = SyntaxAnalyzer(context).analyze(where_expression, source_columns); ExpressionActionsPtr actions = ExpressionAnalyzer{where_expression, syntax_result, context}.getActions(false, false); Names required_columns = actions->getRequiredColumns(); - for (const auto required_column : required_columns) + for (const auto & required_column : required_columns) { if (required_column == header_column.name) throw Exception("Block structure mismatch in Merge Storage: different types:\n" + before_block_header.dumpStructure() diff --git a/dbms/src/Storages/StorageMerge.h b/dbms/src/Storages/StorageMerge.h index 5cb1a46cbb5..247bb8ff930 100644 --- a/dbms/src/Storages/StorageMerge.h +++ b/dbms/src/Storages/StorageMerge.h @@ -52,7 +52,7 @@ private: String name; String source_database; OptimizedRegularExpression table_name_regexp; - const Context & context; + Context global_context; using StorageListWithLocks = std::list>; diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index ccdb8dd6c51..7a91ee41338 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -62,13 +62,13 @@ StorageMergeTree::StorageMergeTree( const MergeTreeSettings & settings_, bool has_force_restore_data_flag) : path(path_), database_name(database_name_), table_name(table_name_), full_path(path + escapeForFileName(table_name) + '/'), - context(context_), background_pool(context_.getBackgroundPool()), + global_context(context_), background_pool(context_.getBackgroundPool()), data(database_name, table_name, full_path, columns_, context_, date_column_name, partition_by_ast_, order_by_ast_, primary_key_ast_, sample_by_ast_, merging_params_, settings_, false, attach), - reader(data), writer(data), merger_mutator(data, context.getBackgroundPool()), + reader(data), writer(data), merger_mutator(data, global_context.getBackgroundPool()), log(&Logger::get(database_name_ + "." + table_name + " (StorageMergeTree)")) { if (path_.empty()) @@ -132,14 +132,14 @@ BlockOutputStreamPtr StorageMergeTree::write(const ASTPtr & /*query*/, const Set void StorageMergeTree::checkTableCanBeDropped() const { const_cast(getData()).recalculateColumnSizes(); - context.checkTableCanBeDropped(database_name, table_name, getData().getTotalActiveSizeInBytes()); + global_context.checkTableCanBeDropped(database_name, table_name, getData().getTotalActiveSizeInBytes()); } void StorageMergeTree::checkPartitionCanBeDropped(const ASTPtr & partition) { const_cast(getData()).recalculateColumnSizes(); - const String partition_id = data.getPartitionIDFromQuery(partition, context); + const String partition_id = data.getPartitionIDFromQuery(partition, global_context); auto parts_to_remove = data.getDataPartsVectorInPartition(MergeTreeDataPartState::Committed, partition_id); UInt64 partition_size = 0; @@ -148,7 +148,7 @@ void StorageMergeTree::checkPartitionCanBeDropped(const ASTPtr & partition) { partition_size += part->bytes_on_disk; } - context.checkPartitionCanBeDropped(database_name, table_name, partition_size); + global_context.checkPartitionCanBeDropped(database_name, table_name, partition_size); } void StorageMergeTree::drop() @@ -157,7 +157,7 @@ void StorageMergeTree::drop() data.dropAllData(); } -void StorageMergeTree::truncate(const ASTPtr &) +void StorageMergeTree::truncate(const ASTPtr &, const Context &) { { /// Asks to complete merges and does not allow them to start. @@ -191,8 +191,8 @@ void StorageMergeTree::rename(const String & new_path_to_db, const String & /*ne void StorageMergeTree::alter( const AlterCommands & params, - const String & database_name, - const String & table_name, + const String & current_database_name, + const String & current_table_name, const Context & context) { if (!params.is_mutable()) @@ -200,7 +200,7 @@ void StorageMergeTree::alter( auto table_soft_lock = lockStructureForAlter(); auto new_columns = getColumns(); params.apply(new_columns); - context.getDatabase(database_name)->alterTable(context, table_name, new_columns, {}); + context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, {}); setColumns(std::move(new_columns)); return; } @@ -239,7 +239,7 @@ void StorageMergeTree::alter( storage_ast.set(storage_ast.primary_key, new_primary_key_ast); }; - context.getDatabase(database_name)->alterTable(context, table_name, new_columns, storage_modifier); + context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, storage_modifier); /// Reinitialize primary key because primary key column types might have changed. data.setPrimaryKeyAndColumns(new_order_by_ast, new_primary_key_ast, new_columns); @@ -277,7 +277,7 @@ struct CurrentlyMergingPartsTagger ~CurrentlyMergingPartsTagger() { - std::lock_guard lock(storage->currently_merging_mutex); + std::lock_guard lock(storage->currently_merging_mutex); for (const auto & part : parts) { @@ -386,7 +386,7 @@ bool StorageMergeTree::merge( std::optional merging_tagger; { - std::lock_guard lock(currently_merging_mutex); + std::lock_guard lock(currently_merging_mutex); auto can_merge = [this, &lock] (const MergeTreeData::DataPartPtr & left, const MergeTreeData::DataPartPtr & right, String *) { @@ -414,7 +414,7 @@ bool StorageMergeTree::merge( merging_tagger.emplace(future_part.parts, MergeTreeDataMergerMutator::estimateNeededDiskSpace(future_part.parts), *this); } - MergeList::EntryPtr merge_entry = context.getMergeList().insert(database_name, table_name, future_part.name, future_part.parts); + MergeList::EntryPtr merge_entry = global_context.getMergeList().insert(database_name, table_name, future_part.name, future_part.parts); /// Logging Stopwatch stopwatch; @@ -424,7 +424,7 @@ bool StorageMergeTree::merge( { try { - auto part_log = context.getPartLog(database_name); + auto part_log = global_context.getPartLog(database_name); if (!part_log) return; @@ -492,7 +492,7 @@ bool StorageMergeTree::tryMutatePart() { auto disk_space = DiskSpaceMonitor::getUnreservedFreeSpace(full_path); - std::lock_guard lock(currently_merging_mutex); + std::lock_guard lock(currently_merging_mutex); if (current_mutations_by_version.empty()) return false; @@ -536,7 +536,7 @@ bool StorageMergeTree::tryMutatePart() { try { - auto part_log = context.getPartLog(database_name); + auto part_log = global_context.getPartLog(database_name); if (!part_log) return; @@ -574,7 +574,7 @@ bool StorageMergeTree::tryMutatePart() try { - new_part = merger_mutator.mutatePartToTemporaryPart(future_part, commands, context); + new_part = merger_mutator.mutatePartToTemporaryPart(future_part, commands, global_context); data.renameTempPartAndReplace(new_part); write_part_log({}); } @@ -728,10 +728,6 @@ void StorageMergeTree::clearColumnInPartition(const ASTPtr & partition, const Fi bool StorageMergeTree::optimize( const ASTPtr & /*query*/, const ASTPtr & partition, bool final, bool deduplicate, const Context & context) { - String partition_id; - if (partition) - partition_id = data.getPartitionIDFromQuery(partition, context); - String disable_reason; if (!partition && final) { @@ -753,6 +749,10 @@ bool StorageMergeTree::optimize( } else { + String partition_id; + if (partition) + partition_id = data.getPartitionIDFromQuery(partition, context); + if (!merge(true, partition_id, final, deduplicate, &disable_reason)) { if (context.getSettingsRef().optimize_throw_if_noop) @@ -844,13 +844,13 @@ void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, cons } -void StorageMergeTree::attachPartition(const ASTPtr & partition, bool part, const Context & context) +void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_part, const Context & context) { // TODO: should get some locks to prevent race with 'alter … modify column' String partition_id; - if (part) + if (attach_part) partition_id = typeid_cast(*partition).value.safeGet(); else partition_id = data.getPartitionIDFromQuery(partition, context); @@ -859,7 +859,7 @@ void StorageMergeTree::attachPartition(const ASTPtr & partition, bool part, cons /// Let's make a list of parts to add. Strings parts; - if (part) + if (attach_part) { parts.push_back(partition_id); } @@ -958,11 +958,11 @@ void StorageMergeTree::replacePartitionFrom(const StoragePtr & source_table, con data.removePartsInRangeFromWorkingSet(drop_range, true, false, data_parts_lock); } - PartLog::addNewParts(this->context, dst_parts, watch.elapsed()); + PartLog::addNewParts(global_context, dst_parts, watch.elapsed()); } catch (...) { - PartLog::addNewParts(this->context, dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentException()); + PartLog::addNewParts(global_context, dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentException()); throw; } } diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index c0a2b8b5463..aaf68c509f6 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -67,7 +67,7 @@ public: std::vector getMutationsStatus() const; void drop() override; - void truncate(const ASTPtr &) override; + void truncate(const ASTPtr &, const Context &) override; void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override; @@ -101,7 +101,7 @@ private: String table_name; String full_path; - Context & context; + Context global_context; BackgroundProcessingPool & background_pool; MergeTreeData data; diff --git a/dbms/src/Storages/StorageMySQL.cpp b/dbms/src/Storages/StorageMySQL.cpp index bad1fed4a4c..a009e3568c7 100644 --- a/dbms/src/Storages/StorageMySQL.cpp +++ b/dbms/src/Storages/StorageMySQL.cpp @@ -42,7 +42,7 @@ StorageMySQL::StorageMySQL(const std::string & name, , replace_query{replace_query} , on_duplicate_clause{on_duplicate_clause} , pool(std::move(pool)) - , context(context) + , global_context(context) { } @@ -60,9 +60,9 @@ BlockInputStreams StorageMySQL::read( *query_info.query, getColumns().ordinary, IdentifierQuotingStyle::Backticks, remote_database_name, remote_table_name, context); Block sample_block; - for (const String & name : column_names) + for (const String & column_name : column_names) { - auto column_data = getColumn(name); + auto column_data = getColumn(column_name); sample_block.insert({ column_data.type, column_data.name }); } @@ -114,7 +114,7 @@ public: sqlbuf << backQuoteIfNeed(remote_database_name) << "." << backQuoteIfNeed(remote_table_name); sqlbuf << " (" << dumpNamesWithBackQuote(block) << ") VALUES "; - auto writer = FormatFactory::instance().getOutput("Values", sqlbuf, storage.getSampleBlock(), storage.context); + auto writer = FormatFactory::instance().getOutput("Values", sqlbuf, storage.getSampleBlock(), storage.global_context); writer->write(block); if (!storage.on_duplicate_clause.empty()) diff --git a/dbms/src/Storages/StorageMySQL.h b/dbms/src/Storages/StorageMySQL.h index 574f019db4a..ee6f0ed3fe8 100644 --- a/dbms/src/Storages/StorageMySQL.h +++ b/dbms/src/Storages/StorageMySQL.h @@ -53,7 +53,7 @@ private: mysqlxx::Pool pool; - const Context & context; + Context global_context; }; } diff --git a/dbms/src/Storages/StorageNull.cpp b/dbms/src/Storages/StorageNull.cpp index 6aab8ae0ada..d23680b1c1a 100644 --- a/dbms/src/Storages/StorageNull.cpp +++ b/dbms/src/Storages/StorageNull.cpp @@ -30,13 +30,13 @@ void registerStorageNull(StorageFactory & factory) }); } -void StorageNull::alter(const AlterCommands & params, const String & database_name, const String & table_name, const Context & context) +void StorageNull::alter(const AlterCommands & params, const String & current_database_name, const String & current_table_name, const Context & context) { auto lock = lockStructureForAlter(); ColumnsDescription new_columns = getColumns(); params.apply(new_columns); - context.getDatabase(database_name)->alterTable(context, table_name, new_columns, {}); + context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, {}); setColumns(std::move(new_columns)); } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 6731513388d..5350b1db579 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -174,13 +174,13 @@ thread_local void StorageReplicatedMergeTree::setZooKeeper(zkutil::ZooKeeperPtr zookeeper) { - std::lock_guard lock(current_zookeeper_mutex); + std::lock_guard lock(current_zookeeper_mutex); current_zookeeper = zookeeper; } zkutil::ZooKeeperPtr StorageReplicatedMergeTree::tryGetZooKeeper() { - std::lock_guard lock(current_zookeeper_mutex); + std::lock_guard lock(current_zookeeper_mutex); return current_zookeeper; } @@ -208,18 +208,18 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( const MergeTreeData::MergingParams & merging_params_, const MergeTreeSettings & settings_, bool has_force_restore_data_flag) - : context(context_), + : global_context(context_), database_name(database_name_), table_name(name_), full_path(path_ + escapeForFileName(table_name) + '/'), - zookeeper_path(context.getMacros()->expand(zookeeper_path_, database_name, table_name)), - replica_name(context.getMacros()->expand(replica_name_, database_name, table_name)), + zookeeper_path(global_context.getMacros()->expand(zookeeper_path_, database_name, table_name)), + replica_name(global_context.getMacros()->expand(replica_name_, database_name, table_name)), data(database_name, table_name, full_path, columns_, context_, date_column_name, partition_by_ast_, order_by_ast_, primary_key_ast_, sample_by_ast_, merging_params_, settings_, true, attach, [this] (const std::string & name) { enqueuePartForCheck(name); }), - reader(data), writer(data), merger_mutator(data, context.getBackgroundPool()), queue(*this), + reader(data), writer(data), merger_mutator(data, global_context.getBackgroundPool()), queue(*this), fetcher(data), cleanup_thread(*this), alter_thread(*this), part_check_thread(*this), restarting_thread(*this), log(&Logger::get(database_name + "." + table_name + " (StorageReplicatedMergeTree)")) @@ -234,18 +234,18 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( zookeeper_path = "/" + zookeeper_path; replica_path = zookeeper_path + "/replicas/" + replica_name; - queue_updating_task = context.getSchedulePool().createTask(database_name + "." + table_name + " (StorageReplicatedMergeTree::queueUpdatingTask)", [this]{ queueUpdatingTask(); }); + queue_updating_task = global_context.getSchedulePool().createTask(database_name + "." + table_name + " (StorageReplicatedMergeTree::queueUpdatingTask)", [this]{ queueUpdatingTask(); }); - mutations_updating_task = context.getSchedulePool().createTask(database_name + "." + table_name + " (StorageReplicatedMergeTree::mutationsUpdatingTask)", [this]{ mutationsUpdatingTask(); }); + mutations_updating_task = global_context.getSchedulePool().createTask(database_name + "." + table_name + " (StorageReplicatedMergeTree::mutationsUpdatingTask)", [this]{ mutationsUpdatingTask(); }); - merge_selecting_task = context.getSchedulePool().createTask(database_name + "." + table_name + " (StorageReplicatedMergeTree::mergeSelectingTask)", [this] { mergeSelectingTask(); }); + merge_selecting_task = global_context.getSchedulePool().createTask(database_name + "." + table_name + " (StorageReplicatedMergeTree::mergeSelectingTask)", [this] { mergeSelectingTask(); }); /// Will be activated if we win leader election. merge_selecting_task->deactivate(); - mutations_finalizing_task = context.getSchedulePool().createTask(database_name + "." + table_name + " (StorageReplicatedMergeTree::mutationsFinalizingTask)", [this] { mutationsFinalizingTask(); }); + mutations_finalizing_task = global_context.getSchedulePool().createTask(database_name + "." + table_name + " (StorageReplicatedMergeTree::mutationsFinalizingTask)", [this] { mutationsFinalizingTask(); }); - if (context.hasZooKeeper()) - current_zookeeper = context.getZooKeeper(); + if (global_context.hasZooKeeper()) + current_zookeeper = global_context.getZooKeeper(); bool skip_sanity_checks = false; @@ -454,7 +454,7 @@ void StorageReplicatedMergeTree::setTableStructure(ColumnsDescription new_column }; } - context.getDatabase(database_name)->alterTable(context, table_name, new_columns, storage_modifier); + global_context.getDatabase(database_name)->alterTable(global_context, table_name, new_columns, storage_modifier); /// Even if the primary/sorting keys didn't change we must reinitialize it /// because primary key column types might have changed. @@ -922,11 +922,11 @@ void StorageReplicatedMergeTree::writePartLog( const String & new_part_name, const MergeTreeData::DataPartPtr & result_part, const MergeTreeData::DataPartsVector & source_parts, - const MergeListEntry * merge_entry) const + const MergeListEntry * merge_entry) { try { - auto part_log = context.getPartLog(database_name); + auto part_log = global_context.getPartLog(database_name); if (!part_log) return; @@ -1041,7 +1041,7 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) auto table_lock = lockStructure(false); - MergeList::EntryPtr merge_entry = context.getMergeList().insert(database_name, table_name, entry.new_part_name, parts); + MergeList::EntryPtr merge_entry = global_context.getMergeList().insert(database_name, table_name, entry.new_part_name, parts); MergeTreeDataMergerMutator::FuturePart future_merged_part(parts); if (future_merged_part.name != entry.new_part_name) @@ -1190,7 +1190,7 @@ bool StorageReplicatedMergeTree::tryExecutePartMutation(const StorageReplicatedM try { - new_part = merger_mutator.mutatePartToTemporaryPart(future_mutated_part, commands, context); + new_part = merger_mutator.mutatePartToTemporaryPart(future_mutated_part, commands, global_context); data.renameTempPartAndReplace(new_part, nullptr, &transaction); try @@ -1621,7 +1621,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) auto clone_data_parts_from_source_table = [&] () -> size_t { - source_table = context.tryGetTable(entry_replace.from_database, entry_replace.from_table); + source_table = global_context.tryGetTable(entry_replace.from_database, entry_replace.from_table); if (!source_table) { LOG_DEBUG(log, "Can't use " << source_table_name << " as source table for REPLACE PARTITION command. It does not exist."); @@ -1779,17 +1779,17 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) } else if (!part_desc->replica.empty()) { - String replica_path = zookeeper_path + "/replicas/" + part_desc->replica; - ReplicatedMergeTreeAddress address(getZooKeeper()->get(replica_path + "/host")); - auto timeouts = ConnectionTimeouts::getHTTPTimeouts(context.getSettingsRef()); - auto [user, password] = context.getInterserverCredentials(); - String interserver_scheme = context.getInterserverScheme(); + String source_replica_path = zookeeper_path + "/replicas/" + part_desc->replica; + ReplicatedMergeTreeAddress address(getZooKeeper()->get(source_replica_path + "/host")); + auto timeouts = ConnectionTimeouts::getHTTPTimeouts(global_context.getSettingsRef()); + auto [user, password] = global_context.getInterserverCredentials(); + String interserver_scheme = global_context.getInterserverScheme(); if (interserver_scheme != address.scheme) throw Exception("Interserver schemes are different '" + interserver_scheme + "' != '" + address.scheme + "', can't fetch part from " + address.host, ErrorCodes::LOGICAL_ERROR); - part_desc->res_part = fetcher.fetchPart(part_desc->found_new_part_name, replica_path, - address.host, address.replication_port, timeouts, user, password, interserver_scheme, false, TMP_PREFIX + "fetch_"); + part_desc->res_part = fetcher.fetchPart(part_desc->found_new_part_name, source_replica_path, + address.host, address.replication_port, timeouts, user, password, interserver_scheme, false, TMP_PREFIX + "fetch_"); /// TODO: check columns_version of fetched part @@ -1838,11 +1838,11 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) parts_to_remove = data.removePartsInRangeFromWorkingSet(drop_range, true, false, data_parts_lock); } - PartLog::addNewParts(this->context, res_parts, watch.elapsed()); + PartLog::addNewParts(global_context, res_parts, watch.elapsed()); } catch (...) { - PartLog::addNewParts(this->context, res_parts, watch.elapsed(), ExecutionStatus::fromCurrentException()); + PartLog::addNewParts(global_context, res_parts, watch.elapsed(), ExecutionStatus::fromCurrentException()); throw; } @@ -1969,9 +1969,9 @@ void StorageReplicatedMergeTree::cloneReplicaIfNeeded(zkutil::ZooKeeperPtr zooke Coordination::Stat source_is_lost_stat; source_is_lost_stat.version = -1; - for (const String & replica_name : zookeeper->getChildren(zookeeper_path + "/replicas")) + for (const String & source_replica_name : zookeeper->getChildren(zookeeper_path + "/replicas")) { - String source_replica_path = zookeeper_path + "/replicas/" + replica_name; + String source_replica_path = zookeeper_path + "/replicas/" + source_replica_name; /// Do not clone from myself. if (source_replica_path != replica_path) @@ -1981,7 +1981,7 @@ void StorageReplicatedMergeTree::cloneReplicaIfNeeded(zkutil::ZooKeeperPtr zooke if (!zookeeper->tryGet(source_replica_path + "/is_lost", source_replica_is_lost_value, &source_is_lost_stat) || source_replica_is_lost_value == "0") { - source_replica = replica_name; + source_replica = source_replica_name; break; } } @@ -2083,11 +2083,11 @@ BackgroundProcessingPoolTaskResult StorageReplicatedMergeTree::queueTask() time_t prev_attempt_time = entry->last_attempt_time; - bool res = queue.processEntry([this]{ return getZooKeeper(); }, entry, [&](LogEntryPtr & entry) + bool res = queue.processEntry([this]{ return getZooKeeper(); }, entry, [&](LogEntryPtr & entry_to_process) { try { - return executeLogEntry(*entry); + return executeLogEntry(*entry_to_process); } catch (const Exception & e) { @@ -2144,7 +2144,7 @@ void StorageReplicatedMergeTree::mergeSelectingTask() { /// We must select parts for merge under merge_selecting_mutex because other threads /// (OPTIMIZE queries) can assign new merges. - std::lock_guard merge_selecting_lock(merge_selecting_mutex); + std::lock_guard merge_selecting_lock(merge_selecting_mutex); auto zookeeper = getZooKeeper(); @@ -2373,7 +2373,7 @@ void StorageReplicatedMergeTree::enterLeaderElection() try { leader_election = std::make_shared( - context.getSchedulePool(), + global_context.getSchedulePool(), zookeeper_path + "/leader_election", *current_zookeeper, /// current_zookeeper lives for the lifetime of leader_election, /// since before changing `current_zookeeper`, `leader_election` object is destroyed in `partialShutdown` method. @@ -2631,7 +2631,7 @@ void StorageReplicatedMergeTree::updateQuorum(const String & part_name) } -bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const String & replica_path, bool to_detached, size_t quorum) +bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const String & source_replica_path, bool to_detached, size_t quorum) { const auto part_info = MergeTreePartInfo::fromPartName(part_name, data.format_version); @@ -2644,7 +2644,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Strin } { - std::lock_guard lock(currently_fetching_parts_mutex); + std::lock_guard lock(currently_fetching_parts_mutex); if (!currently_fetching_parts.insert(part_name).second) { LOG_DEBUG(log, "Part " << part_name << " is already fetching right now"); @@ -2654,11 +2654,11 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Strin SCOPE_EXIT ({ - std::lock_guard lock(currently_fetching_parts_mutex); + std::lock_guard lock(currently_fetching_parts_mutex); currently_fetching_parts.erase(part_name); }); - LOG_DEBUG(log, "Fetching part " << part_name << " from " << replica_path); + LOG_DEBUG(log, "Fetching part " << part_name << " from " << source_replica_path); TableStructureReadLockPtr table_lock; if (!to_detached) @@ -2691,7 +2691,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Strin MinimalisticDataPartChecksums source_part_checksums; source_part_checksums.computeTotalChecksums(source_part->checksums); - String desired_checksums_str = getZooKeeper()->get(replica_path + "/parts/" + part_name + "/checksums"); + String desired_checksums_str = getZooKeeper()->get(source_replica_path + "/parts/" + part_name + "/checksums"); auto desired_checksums = MinimalisticDataPartChecksums::deserializeFrom(desired_checksums_str); if (source_part_checksums == desired_checksums) { @@ -2712,10 +2712,10 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Strin } else { - ReplicatedMergeTreeAddress address(getZooKeeper()->get(replica_path + "/host")); - auto timeouts = ConnectionTimeouts::getHTTPTimeouts(context.getSettingsRef()); - auto user_password = context.getInterserverCredentials(); - String interserver_scheme = context.getInterserverScheme(); + ReplicatedMergeTreeAddress address(getZooKeeper()->get(source_replica_path + "/host")); + auto timeouts = ConnectionTimeouts::getHTTPTimeouts(global_context.getSettingsRef()); + auto user_password = global_context.getInterserverCredentials(); + String interserver_scheme = global_context.getInterserverScheme(); get_part = [&, address, timeouts, user_password, interserver_scheme]() { @@ -2725,7 +2725,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Strin ErrorCodes::LOGICAL_ERROR); return fetcher.fetchPart( - part_name, replica_path, + part_name, source_replica_path, address.host, address.replication_port, timeouts, user_password.first, user_password.second, interserver_scheme, to_detached); }; @@ -2781,7 +2781,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Strin if (part_to_clone) LOG_DEBUG(log, "Cloned part " << part_name << " from " << part_to_clone->name << (to_detached ? " (to 'detached' directory)" : "")); else - LOG_DEBUG(log, "Fetched part " << part_name << " from " << replica_path << (to_detached ? " (to 'detached' directory)" : "")); + LOG_DEBUG(log, "Fetched part " << part_name << " from " << source_replica_path << (to_detached ? " (to 'detached' directory)" : "")); return true; } @@ -2803,9 +2803,9 @@ void StorageReplicatedMergeTree::startup() StoragePtr ptr = shared_from_this(); InterserverIOEndpointPtr data_parts_exchange_endpoint = std::make_shared(data, ptr); data_parts_exchange_endpoint_holder = std::make_shared( - data_parts_exchange_endpoint->getId(replica_path), data_parts_exchange_endpoint, context.getInterserverIOHandler()); + data_parts_exchange_endpoint->getId(replica_path), data_parts_exchange_endpoint, global_context.getInterserverIOHandler()); - queue_task_handle = context.getBackgroundPool().addTask([this] { return queueTask(); }); + queue_task_handle = global_context.getBackgroundPool().addTask([this] { return queueTask(); }); /// In this thread replica will be activated. restarting_thread.start(); @@ -2824,7 +2824,7 @@ void StorageReplicatedMergeTree::shutdown() restarting_thread.shutdown(); if (queue_task_handle) - context.getBackgroundPool().removeTask(queue_task_handle); + global_context.getBackgroundPool().removeTask(queue_task_handle); queue_task_handle.reset(); if (data_parts_exchange_endpoint_holder) @@ -2934,13 +2934,13 @@ BlockOutputStreamPtr StorageReplicatedMergeTree::write(const ASTPtr & /*query*/, } -bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & context) +bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & query_context) { assertNotReadonly(); if (!is_leader) { - sendRequestToLeaderReplica(query, context.getSettingsRef()); + sendRequestToLeaderReplica(query, query_context); return true; } @@ -2948,14 +2948,14 @@ bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const ASTPtr & p { /// We must select parts for merge under merge_selecting_mutex because other threads /// (merge_selecting_thread or OPTIMIZE queries) could assign new merges. - std::lock_guard merge_selecting_lock(merge_selecting_mutex); + std::lock_guard merge_selecting_lock(merge_selecting_mutex); auto zookeeper = getZooKeeper(); ReplicatedMergeTreeMergePredicate can_merge = queue.getMergePredicate(zookeeper); auto handle_noop = [&] (const String & message) { - if (context.getSettingsRef().optimize_throw_if_noop) + if (query_context.getSettingsRef().optimize_throw_if_noop) throw Exception(message, ErrorCodes::CANNOT_ASSIGN_OPTIMIZE); return false; }; @@ -2993,7 +2993,7 @@ bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const ASTPtr & p else { UInt64 disk_space = DiskSpaceMonitor::getUnreservedFreeSpace(full_path); - String partition_id = data.getPartitionIDFromQuery(partition, context); + String partition_id = data.getPartitionIDFromQuery(partition, query_context); selected = merger_mutator.selectAllPartsToMergeWithinPartition( future_merged_part, disk_space, can_merge, partition_id, final, &disable_reason); } @@ -3010,7 +3010,7 @@ bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const ASTPtr & p } /// TODO: Bad setting name for such purpose - if (context.getSettingsRef().replication_alter_partitions_sync != 0) + if (query_context.getSettingsRef().replication_alter_partitions_sync != 0) waitForAllReplicasToProcessLogEntry(merge_entry); return true; @@ -3018,7 +3018,7 @@ bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const ASTPtr & p void StorageReplicatedMergeTree::alter(const AlterCommands & params, - const String & /*database_name*/, const String & /*table_name*/, const Context & context) + const String & /*database_name*/, const String & /*table_name*/, const Context & query_context) { assertNotReadonly(); @@ -3113,7 +3113,7 @@ void StorageReplicatedMergeTree::alter(const AlterCommands & params, std::set inactive_replicas; std::set timed_out_replicas; - time_t replication_alter_columns_timeout = context.getSettingsRef().replication_alter_columns_timeout; + time_t replication_alter_columns_timeout = query_context.getSettingsRef().replication_alter_columns_timeout; for (const String & replica : replicas) { @@ -3277,7 +3277,7 @@ void StorageReplicatedMergeTree::alter(const AlterCommands & params, LOG_DEBUG(log, "ALTER finished"); } -void StorageReplicatedMergeTree::alterPartition(const ASTPtr & query, const PartitionCommands & commands, const Context & context) +void StorageReplicatedMergeTree::alterPartition(const ASTPtr & query, const PartitionCommands & commands, const Context & query_context) { for (const PartitionCommand & command : commands) { @@ -3285,46 +3285,43 @@ void StorageReplicatedMergeTree::alterPartition(const ASTPtr & query, const Part { case PartitionCommand::DROP_PARTITION: checkPartitionCanBeDropped(command.partition); - dropPartition(query, command.partition, command.detach, context); + dropPartition(query, command.partition, command.detach, query_context); break; case PartitionCommand::ATTACH_PARTITION: - attachPartition(command.partition, command.part, context); + attachPartition(command.partition, command.part, query_context); break; case PartitionCommand::REPLACE_PARTITION: { checkPartitionCanBeDropped(command.partition); - String from_database = command.from_database.empty() ? context.getCurrentDatabase() : command.from_database; - auto from_storage = context.getTable(from_database, command.from_table); - replacePartitionFrom(from_storage, command.partition, command.replace, context); + String from_database = command.from_database.empty() ? query_context.getCurrentDatabase() : command.from_database; + auto from_storage = query_context.getTable(from_database, command.from_table); + replacePartitionFrom(from_storage, command.partition, command.replace, query_context); } break; case PartitionCommand::FETCH_PARTITION: - fetchPartition(command.partition, command.from_zookeeper_path, context); + fetchPartition(command.partition, command.from_zookeeper_path, query_context); break; case PartitionCommand::FREEZE_PARTITION: { auto lock = lockStructure(false); - data.freezePartition(command.partition, command.with_name, context); + data.freezePartition(command.partition, command.with_name, query_context); } break; case PartitionCommand::CLEAR_COLUMN: - clearColumnInPartition(command.partition, command.column_name, context); + clearColumnInPartition(command.partition, command.column_name, query_context); break; case PartitionCommand::FREEZE_ALL_PARTITIONS: { auto lock = lockStructure(false); - data.freezeAll(command.with_name, context); + data.freezeAll(command.with_name, query_context); } break; - - default: - IStorage::alterPartition(query, commands, context); // should throw an exception. } } } @@ -3382,13 +3379,13 @@ bool StorageReplicatedMergeTree::getFakePartCoveringAllPartsInPartition(const St void StorageReplicatedMergeTree::clearColumnInPartition( - const ASTPtr & partition, const Field & column_name, const Context & context) + const ASTPtr & partition, const Field & column_name, const Context & query_context) { assertNotReadonly(); /// We don't block merges, so anyone can manage this task (not only leader) - String partition_id = data.getPartitionIDFromQuery(partition, context); + String partition_id = data.getPartitionIDFromQuery(partition, query_context); MergeTreePartInfo drop_range_info; if (!getFakePartCoveringAllPartsInPartition(partition_id, drop_range_info)) @@ -3409,9 +3406,9 @@ void StorageReplicatedMergeTree::clearColumnInPartition( entry.znode_name = log_znode_path.substr(log_znode_path.find_last_of('/') + 1); /// If necessary, wait until the operation is performed on itself or on all replicas. - if (context.getSettingsRef().replication_alter_partitions_sync != 0) + if (query_context.getSettingsRef().replication_alter_partitions_sync != 0) { - if (context.getSettingsRef().replication_alter_partitions_sync == 1) + if (query_context.getSettingsRef().replication_alter_partitions_sync == 1) waitForReplicaToProcessLogEntry(replica_name, entry); else waitForAllReplicasToProcessLogEntry(entry); @@ -3419,7 +3416,7 @@ void StorageReplicatedMergeTree::clearColumnInPartition( } -void StorageReplicatedMergeTree::dropPartition(const ASTPtr & query, const ASTPtr & partition, bool detach, const Context & context) +void StorageReplicatedMergeTree::dropPartition(const ASTPtr & query, const ASTPtr & partition, bool detach, const Context & query_context) { assertNotReadonly(); @@ -3429,19 +3426,19 @@ void StorageReplicatedMergeTree::dropPartition(const ASTPtr & query, const ASTPt { // TODO: we can manually reconstruct the query from outside the |dropPartition()| and remove the |query| argument from interface. // It's the only place where we need this argument. - sendRequestToLeaderReplica(query, context.getSettingsRef()); + sendRequestToLeaderReplica(query, query_context); return; } - String partition_id = data.getPartitionIDFromQuery(partition, context); + String partition_id = data.getPartitionIDFromQuery(partition, query_context); LogEntry entry; if (dropPartsInPartition(*zookeeper, partition_id, entry, detach)) { /// If necessary, wait until the operation is performed on itself or on all replicas. - if (context.getSettingsRef().replication_alter_partitions_sync != 0) + if (query_context.getSettingsRef().replication_alter_partitions_sync != 0) { - if (context.getSettingsRef().replication_alter_partitions_sync == 1) + if (query_context.getSettingsRef().replication_alter_partitions_sync == 1) waitForReplicaToProcessLogEntry(replica_name, entry); else waitForAllReplicasToProcessLogEntry(entry); @@ -3450,7 +3447,7 @@ void StorageReplicatedMergeTree::dropPartition(const ASTPtr & query, const ASTPt } -void StorageReplicatedMergeTree::truncate(const ASTPtr & query) +void StorageReplicatedMergeTree::truncate(const ASTPtr & query, const Context & query_context) { assertNotReadonly(); @@ -3458,7 +3455,7 @@ void StorageReplicatedMergeTree::truncate(const ASTPtr & query) if (!is_leader) { - sendRequestToLeaderReplica(query, context.getSettingsRef()); + sendRequestToLeaderReplica(query, query_context); return; } @@ -3474,7 +3471,7 @@ void StorageReplicatedMergeTree::truncate(const ASTPtr & query) } -void StorageReplicatedMergeTree::attachPartition(const ASTPtr & partition, bool attach_part, const Context & context) +void StorageReplicatedMergeTree::attachPartition(const ASTPtr & partition, bool attach_part, const Context & query_context) { // TODO: should get some locks to prevent race with 'alter … modify column' @@ -3485,7 +3482,7 @@ void StorageReplicatedMergeTree::attachPartition(const ASTPtr & partition, bool if (attach_part) partition_id = typeid_cast(*partition).value.safeGet(); else - partition_id = data.getPartitionIDFromQuery(partition, context); + partition_id = data.getPartitionIDFromQuery(partition, query_context); String source_dir = "detached/"; @@ -3548,7 +3545,7 @@ void StorageReplicatedMergeTree::checkTableCanBeDropped() const { /// Consider only synchronized data const_cast(getData()).recalculateColumnSizes(); - context.checkTableCanBeDropped(database_name, table_name, getData().getTotalActiveSizeInBytes()); + global_context.checkTableCanBeDropped(database_name, table_name, getData().getTotalActiveSizeInBytes()); } @@ -3556,16 +3553,15 @@ void StorageReplicatedMergeTree::checkPartitionCanBeDropped(const ASTPtr & parti { const_cast(getData()).recalculateColumnSizes(); - const String partition_id = data.getPartitionIDFromQuery(partition, context); + const String partition_id = data.getPartitionIDFromQuery(partition, global_context); auto parts_to_remove = data.getDataPartsVectorInPartition(MergeTreeDataPartState::Committed, partition_id); UInt64 partition_size = 0; for (const auto & part : parts_to_remove) - { partition_size += part->bytes_on_disk; - } - context.checkPartitionCanBeDropped(database_name, table_name, partition_size); + + global_context.checkPartitionCanBeDropped(database_name, table_name, partition_size); } @@ -3620,7 +3616,7 @@ void StorageReplicatedMergeTree::rename(const String & new_path_to_db, const Str bool StorageReplicatedMergeTree::existsNodeCached(const std::string & path) { { - std::lock_guard lock(existing_nodes_cache_mutex); + std::lock_guard lock(existing_nodes_cache_mutex); if (existing_nodes_cache.count(path)) return true; } @@ -3629,7 +3625,7 @@ bool StorageReplicatedMergeTree::existsNodeCached(const std::string & path) if (res) { - std::lock_guard lock(existing_nodes_cache_mutex); + std::lock_guard lock(existing_nodes_cache_mutex); existing_nodes_cache.insert(path); } @@ -3783,8 +3779,8 @@ void StorageReplicatedMergeTree::waitForReplicaToProcessLogEntry(const String & { zkutil::EventPtr event = std::make_shared(); - String log_pointer = getZooKeeper()->get(zookeeper_path + "/replicas/" + replica + "/log_pointer", nullptr, event); - if (!log_pointer.empty() && parse(log_pointer) > log_index) + String log_pointer_new = getZooKeeper()->get(zookeeper_path + "/replicas/" + replica + "/log_pointer", nullptr, event); + if (!log_pointer_new.empty() && parse(log_pointer_new) > log_index) break; event->wait(); @@ -3886,7 +3882,7 @@ void StorageReplicatedMergeTree::getStatus(Status & res, bool with_zk_fields) /// TODO: Probably it is better to have queue in ZK with tasks for leader (like DDL) -void StorageReplicatedMergeTree::sendRequestToLeaderReplica(const ASTPtr & query, const Settings & settings) +void StorageReplicatedMergeTree::sendRequestToLeaderReplica(const ASTPtr & query, const Context & query_context) { auto live_replicas = getZooKeeper()->getChildren(zookeeper_path + "/leader_election"); if (live_replicas.empty()) @@ -3920,18 +3916,27 @@ void StorageReplicatedMergeTree::sendRequestToLeaderReplica(const ASTPtr & query else throw Exception("Can't proxy this query. Unsupported query type", ErrorCodes::NOT_IMPLEMENTED); - /// Query send with current user credentials + auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithoutFailover(global_context.getSettingsRef()); + + const auto & query_settings = query_context.getSettingsRef(); + const auto & query_client_info = query_context.getClientInfo(); + String user = query_client_info.current_user; + String password = query_client_info.current_password; + if (auto address = findClusterAddress(leader_address); address) + { + user = address->user; + password = address->password; + } - auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithoutFailover(context.getSettingsRef()); Connection connection( leader_address.host, leader_address.queries_port, leader_address.database, - context.getClientInfo().current_user, context.getClientInfo().current_password, timeouts, "ClickHouse replica"); + user, password, timeouts, "ClickHouse replica"); std::stringstream new_query_ss; formatAST(*new_query, new_query_ss, false, true); - RemoteBlockInputStream stream(connection, new_query_ss.str(), {}, context, &settings); + RemoteBlockInputStream stream(connection, new_query_ss.str(), {}, global_context, &query_settings); NullBlockOutputStream output({}); copyData(stream, output); @@ -3939,6 +3944,26 @@ void StorageReplicatedMergeTree::sendRequestToLeaderReplica(const ASTPtr & query } +std::optional StorageReplicatedMergeTree::findClusterAddress(const ReplicatedMergeTreeAddress & leader_address) const +{ + for (auto & iter : global_context.getClusters().getContainer()) + { + const auto & shards = iter.second->getShardsAddresses(); + + for (size_t shard_num = 0; shard_num < shards.size(); ++shard_num) + { + for (size_t replica_num = 0; replica_num < shards[shard_num].size(); ++replica_num) + { + const Cluster::Address & address = shards[shard_num][replica_num]; + /// user is actually specified, not default + if (address.host_name == leader_address.host && address.port == leader_address.queries_port && address.user_specified) + return address; + } + } + } + return {}; +} + void StorageReplicatedMergeTree::getQueue(LogEntriesData & res, String & replica_name_) { replica_name_ = replica_name; @@ -4052,9 +4077,9 @@ void StorageReplicatedMergeTree::getReplicaDelays(time_t & out_absolute_delay, t } -void StorageReplicatedMergeTree::fetchPartition(const ASTPtr & partition, const String & from_, const Context & context) +void StorageReplicatedMergeTree::fetchPartition(const ASTPtr & partition, const String & from_, const Context & query_context) { - String partition_id = data.getPartitionIDFromQuery(partition, context); + String partition_id = data.getPartitionIDFromQuery(partition, query_context); String from = from_; if (from.back() == '/') @@ -4146,7 +4171,7 @@ void StorageReplicatedMergeTree::fetchPartition(const ASTPtr & partition, const if (try_no) LOG_INFO(log, "Some of parts (" << missing_parts.size() << ") are missing. Will try to fetch covering parts."); - if (try_no >= context.getSettings().max_fetch_partition_retries_count) + if (try_no >= query_context.getSettings().max_fetch_partition_retries_count) throw Exception("Too many retries to fetch parts from " + best_replica_path, ErrorCodes::TOO_MANY_RETRIES_TO_FETCH_PARTS); Strings parts = getZooKeeper()->getChildren(best_replica_path + "/parts"); @@ -4637,7 +4662,7 @@ void StorageReplicatedMergeTree::replacePartitionFrom(const StoragePtr & source_ /// It does not provides strong guarantees, but is suitable for intended use case (assume merges are quite rare). { - std::lock_guard merge_selecting_lock(merge_selecting_mutex); + std::lock_guard merge_selecting_lock(merge_selecting_mutex); queue.disableMergesInRange(drop_range_fake_part_name); } } @@ -4736,11 +4761,11 @@ void StorageReplicatedMergeTree::replacePartitionFrom(const StoragePtr & source_ parts_to_remove = data.removePartsInRangeFromWorkingSet(drop_range, true, false, data_parts_lock); } - PartLog::addNewParts(this->context, dst_parts, watch.elapsed()); + PartLog::addNewParts(global_context, dst_parts, watch.elapsed()); } catch (...) { - PartLog::addNewParts(this->context, dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentException()); + PartLog::addNewParts(global_context, dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentException()); throw; } @@ -4800,15 +4825,15 @@ void StorageReplicatedMergeTree::getCommitPartOps( ReplicatedMergeTreeAddress StorageReplicatedMergeTree::getReplicatedMergeTreeAddress() const { - auto host_port = context.getInterserverIOAddress(); + auto host_port = global_context.getInterserverIOAddress(); ReplicatedMergeTreeAddress res; res.host = host_port.first; res.replication_port = host_port.second; - res.queries_port = context.getTCPPort(); + res.queries_port = global_context.getTCPPort(); res.database = database_name; res.table = table_name; - res.scheme = context.getInterserverScheme(); + res.scheme = global_context.getInterserverScheme(); return res; } @@ -4884,7 +4909,7 @@ bool StorageReplicatedMergeTree::dropPartsInPartition( */ String drop_range_fake_part_name = getPartNamePossiblyFake(data.format_version, drop_range_info); { - std::lock_guard merge_selecting_lock(merge_selecting_mutex); + std::lock_guard merge_selecting_lock(merge_selecting_mutex); queue.disableMergesInRange(drop_range_fake_part_name); } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index 37566ddd1cf..c71ddeb45b0 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -113,11 +114,11 @@ public: BlockOutputStreamPtr write(const ASTPtr & query, const Settings & settings) override; - bool optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & context) override; + bool optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & query_context) override; - void alter(const AlterCommands & params, const String & database_name, const String & table_name, const Context & context) override; + void alter(const AlterCommands & params, const String & database_name, const String & table_name, const Context & query_context) override; - void alterPartition(const ASTPtr & query, const PartitionCommands & commands, const Context & context) override; + void alterPartition(const ASTPtr & query, const PartitionCommands & commands, const Context & query_context) override; void mutate(const MutationCommands & commands, const Context & context) override; @@ -127,7 +128,7 @@ public: */ void drop() override; - void truncate(const ASTPtr &) override; + void truncate(const ASTPtr &, const Context &) override; void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override; @@ -217,7 +218,7 @@ private: using LogEntry = ReplicatedMergeTreeLogEntry; using LogEntryPtr = LogEntry::Ptr; - Context & context; + Context global_context; zkutil::ZooKeeperPtr current_zookeeper; /// Use only the methods below. std::mutex current_zookeeper_mutex; /// To recreate the session in the background thread. @@ -396,7 +397,7 @@ private: const String & new_part_name, const MergeTreeData::DataPartPtr & result_part, const MergeTreeData::DataPartsVector & source_parts, - const MergeListEntry * merge_entry) const; + const MergeListEntry * merge_entry); void executeDropRange(const LogEntry & entry); @@ -502,7 +503,7 @@ private: void waitForReplicaToProcessLogEntry(const String & replica_name, const ReplicatedMergeTreeLogEntryData & entry); /// Choose leader replica, send requst to it and wait. - void sendRequestToLeaderReplica(const ASTPtr & query, const Settings & settings); + void sendRequestToLeaderReplica(const ASTPtr & query, const Context & query_context); /// Throw an exception if the table is readonly. void assertNotReadonly() const; @@ -526,12 +527,15 @@ private: bool dropPartsInPartition(zkutil::ZooKeeper & zookeeper, String & partition_id, StorageReplicatedMergeTree::LogEntry & entry, bool detach); + /// Find cluster address for host + std::optional findClusterAddress(const ReplicatedMergeTreeAddress & leader_address) const; + // Partition helpers - void clearColumnInPartition(const ASTPtr & partition, const Field & column_name, const Context & context); - void dropPartition(const ASTPtr & query, const ASTPtr & partition, bool detach, const Context & context); - void attachPartition(const ASTPtr & partition, bool part, const Context & context); - void replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, const Context & context); - void fetchPartition(const ASTPtr & partition, const String & from, const Context & context); + void clearColumnInPartition(const ASTPtr & partition, const Field & column_name, const Context & query_context); + void dropPartition(const ASTPtr & query, const ASTPtr & partition, bool detach, const Context & query_context); + void attachPartition(const ASTPtr & partition, bool part, const Context & query_context); + void replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, const Context & query_context); + void fetchPartition(const ASTPtr & partition, const String & from, const Context & query_context); protected: /** If not 'attach', either creates a new table in ZK, or adds a replica to an existing table. diff --git a/dbms/src/Storages/StorageSet.cpp b/dbms/src/Storages/StorageSet.cpp index 2348520674d..3a33270b1cd 100644 --- a/dbms/src/Storages/StorageSet.cpp +++ b/dbms/src/Storages/StorageSet.cpp @@ -1,9 +1,9 @@ #include #include #include -#include +#include #include -#include +#include #include #include #include @@ -119,7 +119,7 @@ void StorageSet::insertBlock(const Block & block) { set->insertFromBlock(block); size_t StorageSet::getSize() const { return set->getTotalRowCount(); } -void StorageSet::truncate(const ASTPtr &) +void StorageSet::truncate(const ASTPtr &, const Context &) { Poco::File(path).remove(true); Poco::File(path).createDirectories(); diff --git a/dbms/src/Storages/StorageSet.h b/dbms/src/Storages/StorageSet.h index 400e9670349..c5c7560e0f2 100644 --- a/dbms/src/Storages/StorageSet.h +++ b/dbms/src/Storages/StorageSet.h @@ -65,7 +65,7 @@ public: /// Access the insides. SetPtr & getSet() { return set; } - void truncate(const ASTPtr &) override; + void truncate(const ASTPtr &, const Context &) override; private: SetPtr set; diff --git a/dbms/src/Storages/StorageStripeLog.cpp b/dbms/src/Storages/StorageStripeLog.cpp index 9cf512880fe..0512682f9de 100644 --- a/dbms/src/Storages/StorageStripeLog.cpp +++ b/dbms/src/Storages/StorageStripeLog.cpp @@ -11,8 +11,8 @@ #include #include -#include -#include +#include +#include #include #include @@ -288,7 +288,7 @@ bool StorageStripeLog::checkData() const return file_checker.check(); } -void StorageStripeLog::truncate(const ASTPtr &) +void StorageStripeLog::truncate(const ASTPtr &, const Context &) { if (name.empty()) throw Exception("Logical error: table name is empty", ErrorCodes::LOGICAL_ERROR); diff --git a/dbms/src/Storages/StorageStripeLog.h b/dbms/src/Storages/StorageStripeLog.h index 22010b39b63..0282235de51 100644 --- a/dbms/src/Storages/StorageStripeLog.h +++ b/dbms/src/Storages/StorageStripeLog.h @@ -53,7 +53,7 @@ public: String getDataPath() const override { return full_path(); } - void truncate(const ASTPtr &) override; + void truncate(const ASTPtr &, const Context &) override; private: String path; diff --git a/dbms/src/Storages/StorageTinyLog.cpp b/dbms/src/Storages/StorageTinyLog.cpp index 5f66f0f5049..d3259ff859c 100644 --- a/dbms/src/Storages/StorageTinyLog.cpp +++ b/dbms/src/Storages/StorageTinyLog.cpp @@ -13,8 +13,8 @@ #include #include -#include -#include +#include +#include #include #include @@ -362,8 +362,8 @@ void StorageTinyLog::addFiles(const String & column_name, const IDataType & type } }; - IDataType::SubstreamPath path; - type.enumerateStreams(stream_callback, path); + IDataType::SubstreamPath substream_path; + type.enumerateStreams(stream_callback, substream_path); } @@ -407,7 +407,7 @@ bool StorageTinyLog::checkData() const return file_checker.check(); } -void StorageTinyLog::truncate(const ASTPtr &) +void StorageTinyLog::truncate(const ASTPtr &, const Context &) { if (name.empty()) throw Exception("Logical error: table name is empty", ErrorCodes::LOGICAL_ERROR); diff --git a/dbms/src/Storages/StorageTinyLog.h b/dbms/src/Storages/StorageTinyLog.h index c33c2c87d9a..b96570b4428 100644 --- a/dbms/src/Storages/StorageTinyLog.h +++ b/dbms/src/Storages/StorageTinyLog.h @@ -52,7 +52,7 @@ public: String getDataPath() const override { return full_path(); } - void truncate(const ASTPtr &) override; + void truncate(const ASTPtr &, const Context &) override; private: String path; diff --git a/dbms/src/Storages/System/StorageSystemPartsBase.cpp b/dbms/src/Storages/System/StorageSystemPartsBase.cpp index 8b7a016dd20..435a246018d 100644 --- a/dbms/src/Storages/System/StorageSystemPartsBase.cpp +++ b/dbms/src/Storages/System/StorageSystemPartsBase.cpp @@ -65,14 +65,14 @@ public: database_column_mut->insert(database.first); } block_to_filter.insert(ColumnWithTypeAndName( - std::move(database_column_mut), std::make_shared(), "database")); + std::move(database_column_mut), std::make_shared(), "database")); /// Filter block_to_filter with column 'database'. VirtualColumnUtils::filterBlockWithQuery(query_info.query, block_to_filter, context); rows = block_to_filter.rows(); /// Block contains new columns, update database_column. - ColumnPtr database_column = block_to_filter.getByName("database").column; + ColumnPtr database_column_ = block_to_filter.getByName("database").column; if (rows) { @@ -82,7 +82,7 @@ public: for (size_t i = 0; i < rows; ++i) { - String database_name = (*database_column)[i].get(); + String database_name = (*database_column_)[i].get(); const DatabasePtr database = databases.at(database_name); offsets[i] = i ? offsets[i - 1] : 0; @@ -147,10 +147,10 @@ public: info.database = (*database_column)[next_row].get(); info.table = (*table_column)[next_row].get(); - auto isSameTable = [& info, this] (size_t next_row) -> bool + auto isSameTable = [&info, this] (size_t row) -> bool { - return (*database_column)[next_row].get() == info.database && - (*table_column)[next_row].get() == info.table; + return (*database_column)[row].get() == info.database && + (*table_column)[row].get() == info.table; }; /// What 'active' value we need. diff --git a/dbms/src/Storages/VirtualColumnUtils.cpp b/dbms/src/Storages/VirtualColumnUtils.cpp index d78a7a36727..6ce3e58cc75 100644 --- a/dbms/src/Storages/VirtualColumnUtils.cpp +++ b/dbms/src/Storages/VirtualColumnUtils.cpp @@ -157,7 +157,7 @@ void filterBlockWithQuery(const ASTPtr & query, Block & block, const Context & c return; /// Let's analyze and calculate the expression. - auto syntax_result = SyntaxAnalyzer(context, {}).analyze(expression_ast, block.getNamesAndTypesList()); + auto syntax_result = SyntaxAnalyzer(context).analyze(expression_ast, block.getNamesAndTypesList()); ExpressionAnalyzer analyzer(expression_ast, syntax_result, context); ExpressionActionsPtr actions = analyzer.getActions(false); diff --git a/dbms/src/Storages/getStructureOfRemoteTable.cpp b/dbms/src/Storages/getStructureOfRemoteTable.cpp index 174ec49a4f1..1e5b37d62d0 100644 --- a/dbms/src/Storages/getStructureOfRemoteTable.cpp +++ b/dbms/src/Storages/getStructureOfRemoteTable.cpp @@ -1,6 +1,7 @@ #include "getStructureOfRemoteTable.h" #include #include +#include #include #include #include @@ -54,7 +55,10 @@ ColumnsDescription getStructureOfRemoteTable( ColumnsDescription res; - auto input = std::make_shared(shard_info.pool, query, InterpreterDescribeQuery::getSampleBlock(), context); + + auto new_context = ClusterProxy::removeUserRestrictionsFromSettings(context, context.getSettingsRef()); + /// Execute remote query without restrictions (because it's not real user query, but part of implementation) + auto input = std::make_shared(shard_info.pool, query, InterpreterDescribeQuery::getSampleBlock(), new_context); input->setPoolMode(PoolMode::GET_ONE); if (!table_func_ptr) input->setMainTable(QualifiedTableName{database, table}); diff --git a/dbms/src/Storages/tests/CMakeLists.txt b/dbms/src/Storages/tests/CMakeLists.txt index 2942ecfe6bc..007219a3b96 100644 --- a/dbms/src/Storages/tests/CMakeLists.txt +++ b/dbms/src/Storages/tests/CMakeLists.txt @@ -4,9 +4,6 @@ target_link_libraries (system_numbers PRIVATE dbms clickhouse_storages_system cl add_executable (storage_log storage_log.cpp) target_link_libraries (storage_log PRIVATE dbms) -add_executable (seek_speed_test seek_speed_test.cpp) -target_link_libraries (seek_speed_test PRIVATE dbms) - add_executable (part_checker part_checker.cpp) target_link_libraries (part_checker PRIVATE dbms) diff --git a/dbms/src/Storages/tests/gtest_row_source_bits_test.cpp b/dbms/src/Storages/tests/gtest_row_source_bits_test.cpp index 5e293a5c899..7b2f25061b6 100644 --- a/dbms/src/Storages/tests/gtest_row_source_bits_test.cpp +++ b/dbms/src/Storages/tests/gtest_row_source_bits_test.cpp @@ -1,6 +1,7 @@ #pragma GCC diagnostic ignored "-Wsign-compare" #ifdef __clang__ #pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" + #pragma clang diagnostic ignored "-Wundef" #endif #include diff --git a/dbms/src/Storages/tests/seek_speed_test.cpp b/dbms/src/Storages/tests/seek_speed_test.cpp deleted file mode 100644 index 70910d65b5d..00000000000 --- a/dbms/src/Storages/tests/seek_speed_test.cpp +++ /dev/null @@ -1,68 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -/** We test the hypothesis that skipping unnecessary parts of seek-forward never degrades overall read speed. - * Before the measurements, it is desirable to discard disk cache: `echo 3 > /proc/sys/vm/drop_caches`. - * - * Result: yes, even frequent relatively short seek forward does not worsen anything on all tested parameters - * - 1MiB of data, 16 0 0 16 vs 16 16 32 16 - * - 1GiB of data, 1048576 0 0 vs 1048576 512 1024 vs 1048576 1048576 1048576 - * - 1GiB of data, 1024 0 0 vs 1024 512 1024 - */ - -int main(int argc, const char ** argv) -{ - if (argc < 5 || argc > 6) - { - std::cerr << "Usage:\n" - << argv[0] << " file bytes_in_block min_skip_bytes max_skip_bytes [buffer_size]" << std::endl; - return 0; - } - - int block = atoi(argv[2]); - int min_skip = atoi(argv[3]); - int max_skip = atoi(argv[4]); - size_t buf_size = argc <= 5 ? DBMS_DEFAULT_BUFFER_SIZE : static_cast(atoi(argv[5])); - - UInt64 size = Poco::File(argv[1]).getSize(); - UInt64 pos = 0; - DB::ReadBufferFromFile in(argv[1], buf_size); - auto buf = std::make_unique(block); - int checksum = 0; - UInt64 bytes_read = 0; - - Stopwatch watch; - - while (!in.eof()) - { - UInt64 len = static_cast(rand() % (max_skip - min_skip + 1) + min_skip); - len = std::min(len, size - pos); - off_t seek_res = in.seek(len, SEEK_CUR); - pos += len; - if (seek_res != static_cast(pos)) - { - std::cerr << "Unexpected seek return value: " << seek_res << "; expeted " << pos << ", seeking by " << len << std::endl; - return 1; - } - len = std::min(static_cast(block), size - pos); - in.read(buf.get(), len); - checksum += buf[0] + buf[block - 1]; - pos += len; - bytes_read += len; - } - watch.stop(); - - std::cout << checksum << std::endl; /// don't optimize - - std::cout << "Read " << bytes_read << " out of " << size << " bytes in " - << std::setprecision(4) << watch.elapsedSeconds() << " seconds (" - << bytes_read / watch.elapsedSeconds() / 1000000 << " MB/sec.)" << std::endl; - - return 0; -} diff --git a/dbms/src/Storages/transformQueryForExternalDatabase.cpp b/dbms/src/Storages/transformQueryForExternalDatabase.cpp index f37e51b714e..0131d9f2162 100644 --- a/dbms/src/Storages/transformQueryForExternalDatabase.cpp +++ b/dbms/src/Storages/transformQueryForExternalDatabase.cpp @@ -28,7 +28,7 @@ static void replaceConstFunction(IAST & node, const Context & context, const Nam { NamesAndTypesList source_columns = all_columns; ASTPtr query = function->ptr(); - auto syntax_result = SyntaxAnalyzer(context, {}).analyze(query, source_columns); + auto syntax_result = SyntaxAnalyzer(context).analyze(query, source_columns); auto result_block = KeyCondition::getBlockWithConstants(query, syntax_result, context); if (!result_block.has(child->getColumnName())) return; @@ -92,7 +92,7 @@ String transformQueryForExternalDatabase( const Context & context) { auto clone_query = query.clone(); - auto syntax_result = SyntaxAnalyzer(context, {}).analyze(clone_query, available_columns); + auto syntax_result = SyntaxAnalyzer(context).analyze(clone_query, available_columns); ExpressionAnalyzer analyzer(clone_query, syntax_result, context); const Names & used_columns = analyzer.getRequiredSourceColumns(); diff --git a/dbms/src/TableFunctions/TableFunctionNumbers.cpp b/dbms/src/TableFunctions/TableFunctionNumbers.cpp index 1970a757b2d..8226542d9ee 100644 --- a/dbms/src/TableFunctions/TableFunctionNumbers.cpp +++ b/dbms/src/TableFunctions/TableFunctionNumbers.cpp @@ -34,7 +34,7 @@ StoragePtr TableFunctionNumbers::executeImpl(const ASTPtr & ast_function, const res->startup(); return res; } - throw new Exception("Table function 'numbers' requires 'limit' or 'offset, limit'.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + throw Exception("Table function 'numbers' requires 'limit' or 'offset, limit'.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); } void registerTableFunctionNumbers(TableFunctionFactory & factory) diff --git a/dbms/src/TableFunctions/parseRemoteDescription.cpp b/dbms/src/TableFunctions/parseRemoteDescription.cpp index d903fe72f03..1dce4c6f6e1 100644 --- a/dbms/src/TableFunctions/parseRemoteDescription.cpp +++ b/dbms/src/TableFunctions/parseRemoteDescription.cpp @@ -123,18 +123,19 @@ std::vector parseRemoteDescription(const String & description, size_t l, ErrorCodes::BAD_ARGUMENTS); bool add_leading_zeroes = false; size_t len = last_dot - 1 - (i + 1); - /// If the left and right borders have equal numbers, then you must add leading zeros. + /// If the left and right borders have equal numbers, then you must add leading zeros. + /// TODO The code is somewhat awful. if (last_dot - 1 - (i + 1) == m - (last_dot + 1)) add_leading_zeroes = true; for (size_t id = left; id <= right; ++id) { - String cur = toString(id); + String id_str = toString(id); if (add_leading_zeroes) { - while (cur.size() < len) - cur = "0" + cur; + while (id_str.size() < len) + id_str = "0" + id_str; } - buffer.push_back(cur); + buffer.push_back(id_str); } } else if (have_splitter) /// If there is a current delimiter inside, then generate a set of resulting rows diff --git a/dbms/tests/external_models/catboost/helpers/server_with_models.py b/dbms/tests/external_models/catboost/helpers/server_with_models.py index e0ed81980e1..ad9feea99fe 100644 --- a/dbms/tests/external_models/catboost/helpers/server_with_models.py +++ b/dbms/tests/external_models/catboost/helpers/server_with_models.py @@ -20,6 +20,13 @@ CLICKHOUSE_CONFIG = \ users.xml {tcp_port} {catboost_dynamic_library_path} + + trace + {path}/clickhouse-server.log + {path}/clickhouse-server.err.log + never + 50 + ''' diff --git a/dbms/tests/external_models/catboost/helpers/table.py b/dbms/tests/external_models/catboost/helpers/table.py index 2e9c454ab10..e6b05ac7b7b 100644 --- a/dbms/tests/external_models/catboost/helpers/table.py +++ b/dbms/tests/external_models/catboost/helpers/table.py @@ -56,7 +56,12 @@ class ClickHouseTable: columns = ', '.join(list(float_columns) + list(cat_columns)) query = "select modelEvaluate('{}', {}) from test.{} format TSV" result = self.client.query(query.format(model_name, columns, self.table_name)) - return tuple(map(float, filter(len, map(str.strip, result.split())))) + + def parse_row(row): + values = tuple(map(float, filter(len, map(str.strip, row.replace('(', '').replace(')', '').split(','))))) + return values if len(values) != 1 else values[0] + + return tuple(map(parse_row, filter(len, map(str.strip, result.split('\n'))))) def _drop_table(self): self.client.query('drop table test.{}'.format(self.table_name)) diff --git a/dbms/tests/external_models/catboost/test_apply_catboost_model/test.py b/dbms/tests/external_models/catboost/test_apply_catboost_model/test.py index 792ba9a13c8..00b9fe0dce1 100644 --- a/dbms/tests/external_models/catboost/test_apply_catboost_model/test.py +++ b/dbms/tests/external_models/catboost/test_apply_catboost_model/test.py @@ -234,3 +234,61 @@ def test_apply_float_features_with_mixed_cat_features(): print 'clickhouse predictions', pred_ch check_predictions(name, test_target, pred_python, pred_ch, 0.9) + + +def test_apply_multiclass(): + + name = 'test_apply_float_features_with_mixed_cat_features' + + train_size = 10000 + test_size = 10000 + + def gen_data(size, seed): + data = { + 'a': generate_uniform_float_column(size, 0., 1., seed + 1), + 'b': generate_uniform_float_column(size, 0., 1., seed + 2), + 'c': generate_uniform_string_column(size, ['a', 'b', 'c'], seed + 3), + 'd': generate_uniform_int_column(size, 1, 4, seed + 4) + } + return DataFrame.from_dict(data) + + def get_target(df): + def target_filter(row): + if row['a'] > .3 and row['b'] > .3 and row['c'] != 'a': + return 0 + elif row['a'] * row['b'] > 0.1 and row['c'] != 'b' and row['d'] != 2: + return 1 + else: + return 2 + + return df.apply(target_filter, axis=1).as_matrix() + + train_df = gen_data(train_size, 42) + test_df = gen_data(test_size, 43) + + train_target = get_target(train_df) + test_target = get_target(test_df) + + print + print 'train target', train_target + print 'test target', test_target + + params = { + 'iterations': 10, + 'depth': 4, + 'learning_rate': 1, + 'loss_function': 'MultiClass' + } + + model = train_catboost_model(train_df, train_target, ['c', 'd'], params) + pred_python = model.predict(test_df)[:,0].astype(int) + + server = ClickHouseServerWithCatboostModels(name, CLICKHOUSE_TESTS_SERVER_BIN_PATH, PORT) + server.add_model(name, model) + with server: + pred_ch = np.argmax(np.array(server.apply_model(name, test_df, [])), axis=1) + + print 'python predictions', pred_python + print 'clickhouse predictions', pred_ch + + check_predictions(name, test_target, pred_python, pred_ch, 0.9) diff --git a/dbms/tests/instructions/ninja_trace.txt b/dbms/tests/instructions/ninja_trace.txt new file mode 100644 index 00000000000..d36573e4a9c --- /dev/null +++ b/dbms/tests/instructions/ninja_trace.txt @@ -0,0 +1,5 @@ +To obtain the build trace in Chrome-trace format you need to do the following: +- ninja -C ${BUILD_DIR} +- ninjatrace ${BUILD_DIR}/.ninja_log > ninja_trace.json + +You can get `ninjatrace` from https://github.com/nico/ninjatracing.git - also there is a more detailed instruction. diff --git a/dbms/tests/instructions/sanitizers.md b/dbms/tests/instructions/sanitizers.md index e934ece7f10..f71d469182d 100644 --- a/dbms/tests/instructions/sanitizers.md +++ b/dbms/tests/instructions/sanitizers.md @@ -37,38 +37,26 @@ CC=clang CXX=clang++ cmake -D SANITIZE=thread .. ninja ``` -## Copy binary to your server - -``` -scp ./dbms/programs/clickhouse yourserver:~/clickhouse-tsan -``` - ## Start ClickHouse and run tests ``` -sudo -u clickhouse TSAN_OPTIONS='halt_on_error=1' ./clickhouse-tsan server --config /etc/clickhouse-server/config.xml +sudo -u clickhouse TSAN_OPTIONS='halt_on_error=1,suppressions=../dbms/tests/tsan_suppressions.txt' ./clickhouse-tsan server --config /etc/clickhouse-server/config.xml ``` # How to use Undefined Behaviour Sanitizer ``` -CC=clang CXX=clang++ mkdir build_ubsan && cd build_ubsan +mkdir build_ubsan && cd build_ubsan ``` Note: clang is mandatory, because gcc (in version 8) has false positives due to devirtualization and it has less amount of checks. ``` -cmake -D SANITIZE=undefined .. +CC=clang CXX=clang++ cmake -D SANITIZE=undefined .. ninja ``` -## Copy binary to your server - -``` -scp ./dbms/programs/clickhouse yourserver:~/clickhouse-ubsan -``` - ## Start ClickHouse and run tests ``` diff --git a/dbms/tests/integration/helpers/test_tools.py b/dbms/tests/integration/helpers/test_tools.py index c4fbf075eb6..e90580de44c 100644 --- a/dbms/tests/integration/helpers/test_tools.py +++ b/dbms/tests/integration/helpers/test_tools.py @@ -28,13 +28,13 @@ def assert_eq_with_retry(instance, query, expectation, retry_count=20, sleep_tim expectation_tsv = TSV(expectation) for i in xrange(retry_count): try: - if TSV(instance.query(query)) == expectation_tsv: + if TSV(instance.query(query, user=user, stdin=stdin, timeout=timeout, settings=settings, ignore_error=ignore_error)) == expectation_tsv: break time.sleep(sleep_time) except Exception as ex: print "assert_eq_with_retry retry {} exception {}".format(i + 1, ex) time.sleep(sleep_time) else: - val = TSV(instance.query(query)) + val = TSV(instance.query(query, user=user, stdin=stdin, timeout=timeout, settings=settings, ignore_error=ignore_error)) if expectation_tsv != val: raise AssertionError("'{}' != '{}'\n{}".format(expectation_tsv, val, '\n'.join(expectation_tsv.diff(val, n1="expectation", n2="query")))) diff --git a/dbms/tests/integration/test_concurrent_queries_for_user_restriction/__init__.py b/dbms/tests/integration/test_concurrent_queries_for_user_restriction/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_concurrent_queries_for_user_restriction/configs/user_restrictions.xml b/dbms/tests/integration/test_concurrent_queries_for_user_restriction/configs/user_restrictions.xml new file mode 100644 index 00000000000..bd91f1d495c --- /dev/null +++ b/dbms/tests/integration/test_concurrent_queries_for_user_restriction/configs/user_restrictions.xml @@ -0,0 +1,38 @@ + + + + 10000000000 + 0 + random + + + 10000000000 + 0 + random + 2 + + + + + + + ::/0 + + default + default + + + + + ::/0 + + good + default + + + + + + + + diff --git a/dbms/tests/integration/test_concurrent_queries_for_user_restriction/test.py b/dbms/tests/integration/test_concurrent_queries_for_user_restriction/test.py new file mode 100644 index 00000000000..4b7cc87c15a --- /dev/null +++ b/dbms/tests/integration/test_concurrent_queries_for_user_restriction/test.py @@ -0,0 +1,38 @@ +import time + +import pytest + +from multiprocessing.dummy import Pool +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +node1 = cluster.add_instance('node1', user_configs=['configs/user_restrictions.xml']) +node2 = cluster.add_instance('node2', user_configs=['configs/user_restrictions.xml']) + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + node1.query("create table nums (number UInt64) ENGINE = MergeTree() order by tuple()") + node1.query("insert into nums values(0),(1)") + + yield cluster + finally: + cluster.shutdown() + +def test_exception_message(started_cluster): + assert node1.query("select number from nums order by number") == "0\n1\n" + + def node_busy(_): + for i in xrange(10): + node1.query("select sleep(2)", user='default') + + busy_pool = Pool(3) + busy_pool.map_async(node_busy, xrange(3)) + time.sleep(1) # wait a little until polling starts + try: + assert node2.query("select number from remote('node1', 'default', 'nums')", user='good') == "0\n1\n" + except Exception as ex: + print ex.message + assert False, "Exception thrown while max_concurrent_queries_for_user is not exceeded" diff --git a/dbms/tests/integration/test_send_request_to_leader_replica/__init__.py b/dbms/tests/integration/test_send_request_to_leader_replica/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_send_request_to_leader_replica/configs/remote_servers.xml b/dbms/tests/integration/test_send_request_to_leader_replica/configs/remote_servers.xml new file mode 100644 index 00000000000..149569e4c28 --- /dev/null +++ b/dbms/tests/integration/test_send_request_to_leader_replica/configs/remote_servers.xml @@ -0,0 +1,30 @@ + + + + + true + + node1 + 9000 + awesome + + + node2 + 9000 + awesome + + + + true + + node3 + 9000 + + + node4 + 9000 + + + + + diff --git a/dbms/tests/integration/test_send_request_to_leader_replica/configs/user_good_allowed.xml b/dbms/tests/integration/test_send_request_to_leader_replica/configs/user_good_allowed.xml new file mode 100644 index 00000000000..e72fb61d2af --- /dev/null +++ b/dbms/tests/integration/test_send_request_to_leader_replica/configs/user_good_allowed.xml @@ -0,0 +1,38 @@ + + + + 10000000000 + 0 + random + + + 10000000000 + 0 + random + + + + + + + 127.0.0.1 + ::1 + + default + default + + + + + ::/0 + + good + default + + + + + + + + diff --git a/dbms/tests/integration/test_send_request_to_leader_replica/configs/user_good_restricted.xml b/dbms/tests/integration/test_send_request_to_leader_replica/configs/user_good_restricted.xml new file mode 100644 index 00000000000..ea0714aec11 --- /dev/null +++ b/dbms/tests/integration/test_send_request_to_leader_replica/configs/user_good_restricted.xml @@ -0,0 +1,52 @@ + + + + 10000000000 + 0 + random + + + 10000000000 + 0 + random + + + + 10000000000 + 0 + random + + + + + + + 127.0.0.1 + ::1 + + default + default + + + + + ::/0 + + good + default + + + + + ::/0 + + awesome + default + + + + + + + + diff --git a/dbms/tests/integration/test_send_request_to_leader_replica/test.py b/dbms/tests/integration/test_send_request_to_leader_replica/test.py new file mode 100644 index 00000000000..913a5e6ff7a --- /dev/null +++ b/dbms/tests/integration/test_send_request_to_leader_replica/test.py @@ -0,0 +1,74 @@ +import time + +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import assert_eq_with_retry + +cluster = ClickHouseCluster(__file__) + +node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], user_configs=['configs/user_good_restricted.xml'], with_zookeeper=True) +node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], user_configs=['configs/user_good_restricted.xml'], with_zookeeper=True) +node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml'], user_configs=['configs/user_good_allowed.xml'], with_zookeeper=True) +node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml'], user_configs=['configs/user_good_allowed.xml'], with_zookeeper=True) + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + + for node in [node1, node2]: + node.query(''' + CREATE TABLE sometable(date Date, id UInt32, value Int32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/sometable', '{replica}', date, id, 8192); + '''.format(replica=node.name), user='awesome') + + for node in [node3, node4]: + node.query(''' + CREATE TABLE someothertable(date Date, id UInt32, value Int32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/someothertable', '{replica}', date, id, 8192); + '''.format(replica=node.name), user='good') + + + yield cluster + + finally: + cluster.shutdown() + +@pytest.mark.parametrize("table,query,expected,n1,n2", [ + ("sometable","ALTER TABLE sometable DROP PARTITION 201706", '1', node1, node2), + ("sometable","TRUNCATE TABLE sometable", '0', node1, node2), + ("sometable", "OPTIMIZE TABLE sometable", '4', node1, node2), + ("someothertable","ALTER TABLE someothertable DROP PARTITION 201706", '1', node3, node4), + ("someothertable","TRUNCATE TABLE someothertable", '0', node3, node4), + ("someothertable", "OPTIMIZE TABLE someothertable", '4', node3, node4), +]) +def test_alter_table_drop_partition(started_cluster, table, query, expected, n1, n2): + to_insert = '''\ +2017-06-16 111 0 +2017-06-16 222 1 +2017-06-16 333 2 +2017-07-16 444 3 +''' + n1.query("INSERT INTO {} FORMAT TSV".format(table), stdin=to_insert, user='good') + + assert_eq_with_retry(n1, "SELECT COUNT(*) from {}".format(table), '4', user='good') + assert_eq_with_retry(n2, "SELECT COUNT(*) from {}".format(table), '4', user='good') + + ### It maybe leader and everything will be ok + n1.query(query, user='good') + + assert_eq_with_retry(n1, "SELECT COUNT(*) from {}".format(table), expected, user='good') + assert_eq_with_retry(n2, "SELECT COUNT(*) from {}".format(table), expected, user='good') + + n1.query("INSERT INTO {} FORMAT TSV".format(table), stdin=to_insert, user='good') + + assert_eq_with_retry(n1, "SELECT COUNT(*) from {}".format(table), '4', user='good') + assert_eq_with_retry(n2, "SELECT COUNT(*) from {}".format(table), '4', user='good') + + ### If node1 is leader than node2 will be slave + n2.query(query, user='good') + + assert_eq_with_retry(n1, "SELECT COUNT(*) from {}".format(table), expected, user='good') + assert_eq_with_retry(n2, "SELECT COUNT(*) from {}".format(table), expected, user='good') diff --git a/dbms/tests/performance/int_parsing/int_parsing.xml b/dbms/tests/performance/int_parsing/int_parsing.xml new file mode 100644 index 00000000000..7dbdfd7aa17 --- /dev/null +++ b/dbms/tests/performance/int_parsing/int_parsing.xml @@ -0,0 +1,107 @@ + + int_parsing + loop + + + + 3 + 10000 + + + 5 + 60000 + + + + + + + + + test.hits + + + SELECT count() FROM test.hits WHERE NOT ignore(toString(WatchID)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(JavaEnable)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(GoodEvent)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(CounterID)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(ClientIP)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(RegionID)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(UserID)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(CounterClass)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(OS)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(UserAgent)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(Refresh)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(IsRobot)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(ResolutionWidth)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(ResolutionHeight)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(ResolutionDepth)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(FlashMajor)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(FlashMinor)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(NetMajor)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(NetMinor)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(UserAgentMajor)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(CookieEnable)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(JavascriptEnable)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(IsMobile)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(MobilePhone)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(IPNetworkID)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(TraficSourceID)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(SearchEngineID)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(SearchPhrase)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(AdvEngineID)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(IsArtifical)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(WindowClientWidth)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(WindowClientHeight)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(ClientTimeZone)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(SilverlightVersion1)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(SilverlightVersion2)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(SilverlightVersion3)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(SilverlightVersion4)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(CodeVersion)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(IsLink)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(IsDownload)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(IsNotBounce)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(FUniqID)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(HID)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(IsOldCounter)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(IsEvent)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(IsParameter)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(DontCountHits)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(WithHash)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(Age)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(Sex)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(Income)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(Interests)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(Robotness)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(RemoteIP)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(WindowName)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(OpenerName)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(HistoryLength)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(HTTPError)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(SendTiming)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(DNSTiming)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(ConnectTiming)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(ResponseStartTiming)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(ResponseEndTiming)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(FetchTiming)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(RedirectTiming)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(DOMInteractiveTiming)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(DOMContentLoadedTiming)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(DOMCompleteTiming)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(LoadEventStartTiming)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(LoadEventEndTiming)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(NSToDOMContentLoadedTiming)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(FirstPaintTiming)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(RedirectCount)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(SocialSourceNetworkID)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(ParamPrice)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(ParamCurrencyID)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(HasGCLID)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(RefererHash)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(URLHash)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(CLID)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(YCLID)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(RequestNum)) SETTINGS max_threads = 1 + SELECT count() FROM test.hits WHERE NOT ignore(toString(RequestTry)) SETTINGS max_threads = 1 + diff --git a/dbms/tests/queries/0_stateless/00678_murmurhash.reference b/dbms/tests/queries/0_stateless/00678_murmurhash.reference index 548c5c1cae6..988c022f1bf 100644 --- a/dbms/tests/queries/0_stateless/00678_murmurhash.reference +++ b/dbms/tests/queries/0_stateless/00678_murmurhash.reference @@ -22,5 +22,8 @@ 11303473983767132390 956517343494314387 956517343494314387 +9631199822919835226 +4334672815104069193 +4334672815104069193 6145F501578671E2877DBA2BE487AF7E 16FE7483905CCE7A85670E43E4678877 diff --git a/dbms/tests/queries/0_stateless/00678_murmurhash.sql b/dbms/tests/queries/0_stateless/00678_murmurhash.sql index 9d20b56aa93..91b4deef9b3 100644 --- a/dbms/tests/queries/0_stateless/00678_murmurhash.sql +++ b/dbms/tests/queries/0_stateless/00678_murmurhash.sql @@ -28,5 +28,10 @@ SELECT murmurHash3_64('foo'); SELECT murmurHash3_64('\x01'); SELECT murmurHash3_64(1); +SELECT gccMurmurHash('foo'); +SELECT gccMurmurHash('\x01'); +SELECT gccMurmurHash(1); + SELECT hex(murmurHash3_128('foo')); SELECT hex(murmurHash3_128('\x01')); + diff --git a/dbms/tests/queries/0_stateless/00746_hashing_tuples.reference b/dbms/tests/queries/0_stateless/00746_hashing_tuples.reference index 4c84566e975..ebb03034add 100644 --- a/dbms/tests/queries/0_stateless/00746_hashing_tuples.reference +++ b/dbms/tests/queries/0_stateless/00746_hashing_tuples.reference @@ -13,3 +13,6 @@ 8163029322371165472 8788309436660676487 236561483980029756 +12384823029245979431 +4507350192761038840 +1188926775431157506 diff --git a/dbms/tests/queries/0_stateless/00746_hashing_tuples.sql b/dbms/tests/queries/0_stateless/00746_hashing_tuples.sql index 1582d74e030..d612d207ba3 100644 --- a/dbms/tests/queries/0_stateless/00746_hashing_tuples.sql +++ b/dbms/tests/queries/0_stateless/00746_hashing_tuples.sql @@ -17,3 +17,7 @@ SELECT murmurHash2_64(('a', [1, 2, 3], 4, (4, ['foo', 'bar'], 1, (1, 2)))); SELECT murmurHash3_64(1, 2, 3); SELECT murmurHash3_64(1, 3, 2); SELECT murmurHash3_64(('a', [1, 2, 3], 4, (4, ['foo', 'bar'], 1, (1, 2)))); + +SELECT gccMurmurHash(1, 2, 3); +SELECT gccMurmurHash(1, 3, 2); +SELECT gccMurmurHash(('a', [1, 2, 3], 4, (4, ['foo', 'bar'], 1, (1, 2)))); \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/00800_low_cardinality_distinct_numeric.reference b/dbms/tests/queries/0_stateless/00800_low_cardinality_distinct_numeric.reference new file mode 100644 index 00000000000..a39df1e16c0 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00800_low_cardinality_distinct_numeric.reference @@ -0,0 +1,123 @@ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +97 +98 +99 +100 +101 +102 +103 +104 +105 +106 +107 +108 +109 +110 +111 +112 +113 +114 +115 +116 +117 +118 +119 +120 +121 +122 diff --git a/dbms/tests/queries/0_stateless/00800_low_cardinality_distinct_numeric.sql b/dbms/tests/queries/0_stateless/00800_low_cardinality_distinct_numeric.sql new file mode 100644 index 00000000000..d3ba9138a8f --- /dev/null +++ b/dbms/tests/queries/0_stateless/00800_low_cardinality_distinct_numeric.sql @@ -0,0 +1,7 @@ +set allow_experimental_low_cardinality_type = 1; +drop table if exists test.lc; +create table test.lc (val LowCardinality(UInt64)) engine = MergeTree order by val; +insert into test.lc select number % 123 from system.numbers limit 100000; +select distinct(val) from test.lc order by val; +drop table if exists test.lc; + diff --git a/dbms/tests/queries/0_stateless/00800_low_cardinality_empty_array.reference b/dbms/tests/queries/0_stateless/00800_low_cardinality_empty_array.reference new file mode 100644 index 00000000000..c71bf50e82f --- /dev/null +++ b/dbms/tests/queries/0_stateless/00800_low_cardinality_empty_array.reference @@ -0,0 +1,2 @@ +[] +[] diff --git a/dbms/tests/queries/0_stateless/00800_low_cardinality_empty_array.sql b/dbms/tests/queries/0_stateless/00800_low_cardinality_empty_array.sql new file mode 100644 index 00000000000..0f02f6aa2d5 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00800_low_cardinality_empty_array.sql @@ -0,0 +1,7 @@ +drop table if exists test.lc; +create table test.lc (names Array(LowCardinality(String))) engine=MergeTree order by tuple(); +insert into test.lc values ([]); +insert into test.lc select emptyArrayString(); +select * from test.lc; +drop table if exists test.lc; + diff --git a/dbms/tests/queries/0_stateless/00815_left_join_on_stepanel.reference b/dbms/tests/queries/0_stateless/00815_left_join_on_stepanel.reference new file mode 100644 index 00000000000..6ed281c757a --- /dev/null +++ b/dbms/tests/queries/0_stateless/00815_left_join_on_stepanel.reference @@ -0,0 +1,2 @@ +1 +1 diff --git a/dbms/tests/queries/0_stateless/00815_left_join_on_stepanel.sql b/dbms/tests/queries/0_stateless/00815_left_join_on_stepanel.sql new file mode 100644 index 00000000000..695760c8633 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00815_left_join_on_stepanel.sql @@ -0,0 +1,19 @@ +USE test; + +DROP TABLE IF EXISTS fact_cpc_clicks; +DROP TABLE IF EXISTS dim_model; + +CREATE TABLE fact_cpc_clicks (model_id UInt8) ENGINE = Memory; +CREATE TABLE dim_model (model_id UInt8) ENGINE = Memory; + +INSERT INTO fact_cpc_clicks VALUES (1); +INSERT INTO dim_model VALUES (1); + +select f.model_id from fact_cpc_clicks as f left join dim_model as d on f.model_id=d.model_id limit 10; + +USE default; + +select f.model_id from test.fact_cpc_clicks as f left join test.dim_model as d on f.model_id=d.model_id limit 10; + +DROP TABLE test.fact_cpc_clicks; +DROP TABLE test.dim_model; diff --git a/dbms/tests/queries/0_stateless/00816_join_column_names_sarg.reference b/dbms/tests/queries/0_stateless/00816_join_column_names_sarg.reference new file mode 100644 index 00000000000..4257477d2cf --- /dev/null +++ b/dbms/tests/queries/0_stateless/00816_join_column_names_sarg.reference @@ -0,0 +1,3 @@ +1 1 +1 1 456 +1 456 diff --git a/dbms/tests/queries/0_stateless/00816_join_column_names_sarg.sql b/dbms/tests/queries/0_stateless/00816_join_column_names_sarg.sql new file mode 100644 index 00000000000..a9d30e7152e --- /dev/null +++ b/dbms/tests/queries/0_stateless/00816_join_column_names_sarg.sql @@ -0,0 +1,23 @@ +USE test; + +drop table if exists t1; +drop table if exists t2; +create table t1 (a Int8, val Float32) engine=Memory(); +create table t2 (a Int8, val Float32) engine=Memory(); + +INSERT INTO t1 VALUES (1, 123); +INSERT INTO t2 VALUES (1, 456); + + +select t1.a, t2.a from t1 all inner join t2 on t1.a=t2.a; +-- Received exception from server (version 18.14.1): +-- Code: 47. DB::Exception: Received from localhost:9000, 127.0.0.1. DB::Exception: Unknown identifier: test.t2.a. + +-- this query works fine +select t1.a, t2.* from t1 all inner join t2 on t1.a=t2.a; + +-- and this +select t1.a, t2.val from t1 all inner join t2 on t1.a=t2.a; + +DROP TABLE t1; +DROP TABLE t2; diff --git a/dbms/tests/queries/0_stateless/00817_with_simple.reference b/dbms/tests/queries/0_stateless/00817_with_simple.reference new file mode 100644 index 00000000000..8b1acc12b63 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00817_with_simple.reference @@ -0,0 +1,10 @@ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 diff --git a/dbms/tests/queries/0_stateless/00817_with_simple.sql b/dbms/tests/queries/0_stateless/00817_with_simple.sql new file mode 100644 index 00000000000..8f1b8e6fe6c --- /dev/null +++ b/dbms/tests/queries/0_stateless/00817_with_simple.sql @@ -0,0 +1 @@ +WITH number AS k SELECT k FROM system.numbers LIMIT 10; diff --git a/dbms/tests/queries/0_stateless/00820_multiple_joins.sql b/dbms/tests/queries/0_stateless/00820_multiple_joins.sql index 3885c7d3535..66b594f917f 100644 --- a/dbms/tests/queries/0_stateless/00820_multiple_joins.sql +++ b/dbms/tests/queries/0_stateless/00820_multiple_joins.sql @@ -20,11 +20,7 @@ SET allow_experimental_multiple_joins_emulation = 1; -- FIXME: wrong names qualification select a, b, c from table1 as t1 join table2 as t2 on t1.a = t2.a join table3 as t3 on b = t3.b; select a, b, c from table1 as t1 join table2 as t2 on t1.a = t2.a join table5 as t5 on a = t5.a AND b = t5.b; ---select a, b, c, d from table1 as t1 join table2 as t2 on t1.a = t2.a join table3 as t3 on b = t3.b join table5 as t5 on c = t5.c; - ---select t1.x, t2.y, t3.z from table1 as t1 join table2 as t2 on t1.x = t2.x join table3 as t3 on t2.y = t3.y; ---select t1.x, t2.y, t3.z from table1 as t1 join (select * from table2 as t2 join table3 as t3 on t2.y = t3.y) on t1.x = t2.x; ---select t1.x, j1.y, j1.z from table1 as t1 join (select * from table2 as t2 join table3 as t3 on t2.y = t3.y) as j1 on t1.x = j1.x; +--select a, b, c from table1 as t1 join table2 as t2 on t1.a = t2.a join table3 as t3 on b = t3.b join table5 as t5 on c = t5.c; DROP TABLE table1; DROP TABLE table2; diff --git a/dbms/tests/queries/0_stateless/00821_distributed_storage_with_join_on.reference b/dbms/tests/queries/0_stateless/00821_distributed_storage_with_join_on.reference new file mode 100644 index 00000000000..e8183f05f5d --- /dev/null +++ b/dbms/tests/queries/0_stateless/00821_distributed_storage_with_join_on.reference @@ -0,0 +1,3 @@ +1 +1 +1 diff --git a/dbms/tests/queries/0_stateless/00821_distributed_storage_with_join_on.sql b/dbms/tests/queries/0_stateless/00821_distributed_storage_with_join_on.sql new file mode 100644 index 00000000000..cbd3eb6e467 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00821_distributed_storage_with_join_on.sql @@ -0,0 +1,12 @@ +DROP TABLE IF EXISTS test.table1; +DROP TABLE IF EXISTS test.table2; + +CREATE TABLE test.table1 AS system.columns ENGINE = Distributed('test_shard_localhost', system, columns); +CREATE TABLE test.table2 AS system.tables ENGINE = Distributed('test_shard_localhost', system, tables); + +SELECT 1 FROM test.table1 T1 ALL INNER JOIN test.table2 T2 ON T1.table = T2.name LIMIT 1; +SELECT 1 FROM cluster('test_shard_localhost', system.columns) T1 ALL INNER JOIN cluster('test_shard_localhost', system.tables) T2 ON T1.table = T2.name LIMIT 1; +SELECT 1 FROM (SELECT * FROM test.table1) T1 ALL INNER JOIN (SELECT * FROM test.table2) T2 ON T1.table = T2.name LIMIT 1; + +DROP TABLE IF EXISTS test.table1; +DROP TABLE IF EXISTS test.table2; diff --git a/dbms/tests/queries/0_stateless/00822_array_insert_default.reference b/dbms/tests/queries/0_stateless/00822_array_insert_default.reference new file mode 100644 index 00000000000..573541ac970 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00822_array_insert_default.reference @@ -0,0 +1 @@ +0 diff --git a/dbms/tests/queries/0_stateless/00822_array_insert_default.sql b/dbms/tests/queries/0_stateless/00822_array_insert_default.sql new file mode 100644 index 00000000000..762d423dd4b --- /dev/null +++ b/dbms/tests/queries/0_stateless/00822_array_insert_default.sql @@ -0,0 +1 @@ +SELECT sum(ignore(*)) FROM (SELECT arrayFirst(x -> empty(x), [[number]]) FROM numbers(10000000)); diff --git a/dbms/tests/queries/bugs/00816_concurrent_alter_column.sh b/dbms/tests/queries/bugs/00816_concurrent_alter_column.sh new file mode 100755 index 00000000000..63e8e48e0bb --- /dev/null +++ b/dbms/tests/queries/bugs/00816_concurrent_alter_column.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -e + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +echo "DROP TABLE IF EXISTS test.test" | ${CLICKHOUSE_CLIENT} +echo "CREATE TABLE test.test (ts DATETIME) ENGINE = MergeTree PARTITION BY toStartOfDay(ts) ORDER BY tuple()" | ${CLICKHOUSE_CLIENT} + +for i in {1..500}; do echo "ALTER TABLE test.test ADD COLUMN c$i DOUBLE;"; done | ${CLICKHOUSE_CLIENT} -n + +for i in {1..500}; do echo "ALTER TABLE test.test ADD COLUMN d DOUBLE" | ${CLICKHOUSE_CLIENT}; echo "ALTER TABLE test.test DROP COLUMN d" | ${CLICKHOUSE_CLIENT} -n; done & +for i in {1..500}; do echo "ALTER TABLE test.test ADD COLUMN e DOUBLE" | ${CLICKHOUSE_CLIENT}; echo "ALTER TABLE test.test DROP COLUMN e" | ${CLICKHOUSE_CLIENT} -n; done & + +wait + +echo "DROP TABLE test.test" | ${CLICKHOUSE_CLIENT} diff --git a/dbms/tests/queries/bugs/remote_scalar_subquery.sql b/dbms/tests/queries/bugs/remote_scalar_subquery.sql new file mode 100644 index 00000000000..eada5ed4b59 --- /dev/null +++ b/dbms/tests/queries/bugs/remote_scalar_subquery.sql @@ -0,0 +1 @@ +SELECT (SELECT 1) FROM remote('127.0.0.{1,2}', system.one); diff --git a/dbms/tests/queries/bugs/view_bad_types.sql b/dbms/tests/queries/bugs/view_bad_types.sql new file mode 100644 index 00000000000..38daabfd6b8 --- /dev/null +++ b/dbms/tests/queries/bugs/view_bad_types.sql @@ -0,0 +1,11 @@ +DROP TABLE IF EXISTS test.table; +CREATE TABLE test.table (x UInt16) ENGINE = TinyLog; +INSERT INTO test.table SELECT * FROM system.numbers LIMIT 10; + +DROP TABLE IF EXISTS test.view; +CREATE VIEW test.view (x UInt64) AS SELECT * FROM test.table; + +SELECT x, any(x) FROM test.view GROUP BY x; + +DROP TABLE test.view; +DROP TABLE test.table; diff --git a/dbms/tests/tsan_suppressions.txt b/dbms/tests/tsan_suppressions.txt new file mode 100644 index 00000000000..476e135de14 --- /dev/null +++ b/dbms/tests/tsan_suppressions.txt @@ -0,0 +1,2 @@ +# libc++ +race:locale diff --git a/debian/changelog b/debian/changelog index a6a05e3479e..837c798c45b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,5 @@ -clickhouse (18.16.0) unstable; urgency=low +clickhouse (19.1.0) unstable; urgency=low * Modified source code - -- Fri, 14 Dec 2018 20:26:45 +0300 + -- Tue, 01 Jan 2019 07:16:20 +0300 diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile index 883b00d4b99..d58f3197c33 100644 --- a/docker/client/Dockerfile +++ b/docker/client/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=18.16.0 +ARG version=19.1.0 RUN apt-get update \ && apt-get install --yes --no-install-recommends \ diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index 49b757e1b00..8927e79b01f 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -1,5 +1,7 @@ FROM ubuntu:18.04 +RUN echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-7 main" >> /etc/apt/sources.list + RUN apt-get update -y \ && env DEBIAN_FRONTEND=noninteractive \ apt-get install --yes --no-install-recommends \ @@ -11,11 +13,17 @@ RUN apt-get update -y \ gcc-8 \ g++-8 \ clang-6.0 \ - clang++-6.0 \ lld-6.0 \ libclang-6.0-dev \ - libicu-dev \ liblld-6.0-dev \ + llvm-6.0 \ + libllvm6.0 \ + llvm-6.0-dev \ + clang-7 \ + lld-7 \ + libclang-7-dev \ + liblld-7-dev \ + libicu-dev \ libreadline-dev \ ninja-build \ git diff --git a/docker/packager/deb/Dockerfile b/docker/packager/deb/Dockerfile index f7dfcc2513b..3d7ad768ec5 100644 --- a/docker/packager/deb/Dockerfile +++ b/docker/packager/deb/Dockerfile @@ -1,8 +1,10 @@ FROM ubuntu:18.04 -RUN apt-get update -y \ +RUN echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-7 main" >> /etc/apt/sources.list + +RUN apt-get --allow-unauthenticated update -y \ && env DEBIAN_FRONTEND=noninteractive \ - apt-get install --yes --no-install-recommends \ + apt-get --allow-unauthenticated install --yes --no-install-recommends \ bash \ fakeroot \ cmake \ @@ -18,6 +20,10 @@ RUN apt-get update -y \ llvm-6.0 \ libllvm6.0 \ llvm-6.0-dev \ + clang-7 \ + lld-7 \ + libclang-7-dev \ + liblld-7-dev \ libicu-dev \ libreadline-dev \ ninja-build \ @@ -26,8 +32,8 @@ RUN apt-get update -y \ devscripts \ debhelper \ git \ - libc++abi-dev \ libc++-dev \ + libc++abi-dev \ libboost-program-options-dev \ libboost-system-dev \ libboost-filesystem-dev \ diff --git a/docker/packager/packager b/docker/packager/packager index 38c4dc107c9..46e39dd6b26 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -86,7 +86,7 @@ if __name__ == "__main__": parser.add_argument("--clickhouse-repo-path", default="../../") parser.add_argument("--output-dir", required=True) parser.add_argument("--build-type", choices=("debug", ""), default="") - parser.add_argument("--compiler", choices=("clang-6.0", "gcc-7", "gcc-8"), default="gcc-7") + parser.add_argument("--compiler", choices=("clang-6.0", "clang-7", "gcc-7", "gcc-8"), default="gcc-7") parser.add_argument("--sanitizer", choices=("address", "thread", "memory", "undefined", ""), default="") parser.add_argument("--unbundled", action="store_true") parser.add_argument("--cache", choices=("", "ccache", "distcc"), default="") diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index de51fb31077..8e96407d2c6 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=18.16.0 +ARG version=19.1.0 ARG gosu_ver=1.10 RUN apt-get update \ diff --git a/docker/server/entrypoint.sh b/docker/server/entrypoint.sh index c44ec3e5a9f..f22942d6dfd 100644 --- a/docker/server/entrypoint.sh +++ b/docker/server/entrypoint.sh @@ -27,14 +27,16 @@ mkdir -p \ "$USER_PATH" \ "$FORMAT_SCHEMA_PATH" -# ensure proper directories permissions -chown -R $USER:$GROUP \ - "$DATA_DIR" \ - "$ERROR_LOG_DIR" \ - "$LOG_DIR" \ - "$TMP_DIR" \ - "$USER_PATH" \ - "$FORMAT_SCHEMA_PATH" +if [ "$CLICKHOUSE_DO_NOT_CHOWN" != "1" ]; then + # ensure proper directories permissions + chown -R $USER:$GROUP \ + "$DATA_DIR" \ + "$ERROR_LOG_DIR" \ + "$LOG_DIR" \ + "$TMP_DIR" \ + "$USER_PATH" \ + "$FORMAT_SCHEMA_PATH" +fi if [ -n "$(ls /docker-entrypoint-initdb.d/)" ]; then gosu clickhouse /usr/bin/clickhouse-server --config-file=$CLICKHOUSE_CONFIG & diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile index 352194b2be9..f0a09d82afa 100644 --- a/docker/test/Dockerfile +++ b/docker/test/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=18.16.0 +ARG version=19.1.0 RUN apt-get update && \ apt-get install -y apt-transport-https dirmngr && \ diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile index 4bdad6aa02c..90b4068fb93 100644 --- a/docker/test/stateless/Dockerfile +++ b/docker/test/stateless/Dockerfile @@ -25,6 +25,8 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone COPY zookeeper.xml /etc/clickhouse-server/config.d/zookeeper.xml COPY listen.xml /etc/clickhouse-server/config.d/listen.xml +COPY part_log.xml /etc/clickhouse-server/config.d/part_log.xml +COPY log_queries.xml /etc/clickhouse-server/users.d/log_queries.xml CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \ dpkg -i package_folder/clickhouse-server_*.deb; \ diff --git a/docker/test/stateless/log_queries.xml b/docker/test/stateless/log_queries.xml new file mode 100644 index 00000000000..25261072ade --- /dev/null +++ b/docker/test/stateless/log_queries.xml @@ -0,0 +1,7 @@ + + + + 1 + + + diff --git a/docker/test/stateless/part_log.xml b/docker/test/stateless/part_log.xml new file mode 100644 index 00000000000..6c6fc9c6982 --- /dev/null +++ b/docker/test/stateless/part_log.xml @@ -0,0 +1,8 @@ + + + system + part_log
+ + 7500 + + diff --git a/docker/test/stress/Dockerfile b/docker/test/stress/Dockerfile index 7987e042273..8f2524930c1 100644 --- a/docker/test/stress/Dockerfile +++ b/docker/test/stress/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:18.10 RUN apt-get update -y \ && env DEBIAN_FRONTEND=noninteractive \ @@ -20,6 +20,8 @@ RUN apt-get update -y \ telnet COPY ./stress /stress +COPY log_queries.xml /etc/clickhouse-server/users.d/log_queries.xml +COPY part_log.xml /etc/clickhouse-server/config.d/part_log.xml CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \ dpkg -i package_folder/clickhouse-server_*.deb; \ diff --git a/docker/test/stress/log_queries.xml b/docker/test/stress/log_queries.xml new file mode 100644 index 00000000000..25261072ade --- /dev/null +++ b/docker/test/stress/log_queries.xml @@ -0,0 +1,7 @@ + + + + 1 + + + diff --git a/docker/test/stress/part_log.xml b/docker/test/stress/part_log.xml new file mode 100644 index 00000000000..6c6fc9c6982 --- /dev/null +++ b/docker/test/stress/part_log.xml @@ -0,0 +1,8 @@ + + + system + part_log
+ + 7500 + + diff --git a/docs/en/interfaces/cli.md b/docs/en/interfaces/cli.md index d665fbd3e17..632049b5901 100644 --- a/docs/en/interfaces/cli.md +++ b/docs/en/interfaces/cli.md @@ -92,6 +92,7 @@ You can pass parameters to `clickhouse-client` (all parameters have a default va - `--time, -t` – If specified, print the query execution time to 'stderr' in non-interactive mode. - `--stacktrace` – If specified, also print the stack trace if an exception occurs. - `--config-file` – The name of the configuration file. +- `--secure` – If specified, will connect to server over secure connection. ### Configuration Files @@ -108,6 +109,7 @@ Example of a config file: username password + False ``` diff --git a/docs/en/operations/server_settings/settings.md b/docs/en/operations/server_settings/settings.md index 76fe4a62bb1..6cb2de77ba9 100644 --- a/docs/en/operations/server_settings/settings.md +++ b/docs/en/operations/server_settings/settings.md @@ -625,11 +625,11 @@ Path to temporary data for processing large queries. ``` -## uncompressed_cache_size +## uncompressed_cache_size {#server-settings-uncompressed_cache_size} Cache size (in bytes) for uncompressed data used by table engines from the [MergeTree](../../operations/table_engines/mergetree.md). -There is one shared cache for the server. Memory is allocated on demand. The cache is used if the option [use_uncompressed_cache](../settings/settings.md) is enabled. +There is one shared cache for the server. Memory is allocated on demand. The cache is used if the option [use_uncompressed_cache](../settings/settings.md#setting-use_uncompressed_cache) is enabled. The uncompressed cache is advantageous for very short queries in individual cases. diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index add73212c5d..36a4d6bc135 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -110,6 +110,56 @@ Used for the same purpose as `max_block_size`, but it sets the recommended block However, the block size cannot be more than `max_block_size` rows. Disabled by default (set to 0). It only works when reading from MergeTree engines. +## merge_tree_uniform_read_distribution {#setting-merge_tree_uniform_read_distribution} + +When reading from [MergeTree*](../table_engines/mergetree.md) tables, ClickHouse uses several threads. This setting turns on/off the uniform distribution of reading tasks over the working threads. The algorithm of the uniform distribution aims to make execution time for all the threads approximately equal in a `SELECT` query. + +**Possible values** + +- 0 — Uniform read distribution turned off. +- 1 — Uniform read distribution turned on. + +**Default value** — 1. + +## merge_tree_min_rows_for_concurrent_read {#setting-merge_tree_min_rows_for_concurrent_read} + +If a number of rows to be read from a file of [MergeTree*](../table_engines/mergetree.md) table exceeds `merge_tree_min_rows_for_concurrent_read` then ClickHouse tries to perform a concurrent reading from this file by several threads. + +**Possible values** + +Any positive integer. + +**Default value** — 163840. + +## merge_tree_min_rows_for_seek {#setting-merge_tree_min_rows_for_seek} + +If the distance between two data blocks to be read in one file less than `merge_tree_min_rows_for_seek` rows, then ClickHouse does not seek through the file, it reads the data sequentially. + +**Possible values** + +Any positive integer. + +**Default value** — 0. + +## merge_tree_coarse_index_granularity {#setting-merge_tree_coarse_index_granularity} + +When searching data, ClickHouse checks the data marks in the index file. If ClickHouse finds that required keys are in some range, it divides this range for `merge_tree_coarse_index_granularity` subranges and searches the required keys there recursively. + +**Possible values** + +Any positive even integer. + +**Default value** — 8. + +## merge_tree_max_rows_to_use_cache {#setting-merge_tree_max_rows_to_use_cache} + +If ClickHouse should read more than `merge_tree_max_rows_to_use_cache` rows in one query, it does not use the cash of uncompressed blocks. The [uncompressed_cache_size](../server_settings/settings.md#server-settings-uncompressed_cache_size) server setting defines the size of the cache of uncompressed blocks. + +**Possible values** + +Any positive integer. + +**Default value** — 1048576. ## log_queries @@ -242,10 +292,10 @@ Whether to count extreme values (the minimums and maximums in columns of a query For more information, see the section "Extreme values". -## use_uncompressed_cache +## use_uncompressed_cache {#setting-use_uncompressed_cache} Whether to use a cache of uncompressed blocks. Accepts 0 or 1. By default, 0 (disabled). -The uncompressed cache (only for tables in the MergeTree family) allows significantly reducing latency and increasing throughput when working with a large number of short queries. Enable this setting for users who send frequent short requests. Also pay attention to the 'uncompressed_cache_size' configuration parameter (only set in the config file) – the size of uncompressed cache blocks. By default, it is 8 GiB. The uncompressed cache is filled in as needed; the least-used data is automatically deleted. +The uncompressed cache (only for tables in the MergeTree family) allows significantly reducing latency and increasing throughput when working with a large number of short queries. Enable this setting for users who send frequent short requests. Also pay attention to the [uncompressed_cache_size](../server_settings/settings.md#server-settings-uncompressed_cache_size) configuration parameter (only set in the config file) – the size of uncompressed cache blocks. By default, it is 8 GiB. The uncompressed cache is filled in as needed; the least-used data is automatically deleted. For queries that read at least a somewhat large volume of data (one million rows or more), the uncompressed cache is disabled automatically in order to save space for truly small queries. So you can keep the 'use_uncompressed_cache' setting always set to 1. diff --git a/docs/en/roadmap.md b/docs/en/roadmap.md index 46931d5983b..b59b58ee8dd 100644 --- a/docs/en/roadmap.md +++ b/docs/en/roadmap.md @@ -1,13 +1,10 @@ # Roadmap -## Q4 2018 +## Q1 2019 - JOIN syntax compatible with SQL standard: - Mutliple `JOIN`s in single `SELECT` - Protobuf and Parquet input and output formats - -## Q1 2019 - - Import/export from HDFS and S3 - Lower metadata size in ZooKeeper - Adaptive index granularity for MergeTree engine family diff --git a/docs/ru/interfaces/cli.md b/docs/ru/interfaces/cli.md index eaf2cf132af..4cb4742078e 100644 --- a/docs/ru/interfaces/cli.md +++ b/docs/ru/interfaces/cli.md @@ -95,6 +95,7 @@ cat file.csv | clickhouse-client --database=test --query="INSERT INTO test FORMA - `--time, -t` - если указано, в неинтерактивном режиме вывести время выполнения запроса в stderr. - `--stacktrace` - если указано, в случае исключения, выводить также его стек трейс. - `--config-file` - имя конфигурационного файла. +- `--secure` - если указано, будет использован безопасный канал. ### Конфигурационные файлы @@ -111,6 +112,7 @@ cat file.csv | clickhouse-client --database=test --query="INSERT INTO test FORMA username password + False ``` [Оригинальная статья](https://clickhouse.yandex/docs/ru/interfaces/cli/) diff --git a/docs/toc_zh.yml b/docs/toc_zh.yml index 01eda540d27..764195a3f04 100644 --- a/docs/toc_zh.yml +++ b/docs/toc_zh.yml @@ -178,11 +178,11 @@ nav: - '开发者指南': - 'hidden': 'development/index.md' - - 'Overview of ClickHouse architecture': 'development/architecture.md' - - 'How to build ClickHouse on Linux': 'development/build.md' - - 'How to build ClickHouse on Mac OS X': 'development/build_osx.md' - - 'How to write C++ code': 'development/style.md' - - 'How to run ClickHouse tests': 'development/tests.md' + - 'ClickHouse架构概述': 'development/architecture.md' + - '如何在Linux中编译ClickHouse': 'development/build.md' + - '如何在Mac OS X中编译ClickHouse': 'development/build_osx.md' + - '如何编写C++代码': 'development/style.md' + - '如何运行ClickHouse测试': 'development/tests.md' - '新功能特性': - '路线图': 'roadmap.md' diff --git a/docs/tools/mkdocs-material-theme/base.html b/docs/tools/mkdocs-material-theme/base.html index a8950c53c0a..05708186299 100644 --- a/docs/tools/mkdocs-material-theme/base.html +++ b/docs/tools/mkdocs-material-theme/base.html @@ -40,8 +40,10 @@ {% block htmltitle %} {% if page and page.meta and page.meta.title %} {{ page.meta.title }} - {% elif page and page.title and not page.is_homepage %} + {% elif page and page.title and not page.is_homepage and page.title != 'hidden' %} {{ page.title }} - {{ config.site_name }} + {% elif page and page.title and not page.is_homepage and page.title == 'hidden' and page.ancestors %} + {{ (page.ancestors | first).title }} - {{ config.site_name }} {% else %} {{ config.site_name }} {% endif %} diff --git a/docs/tools/requirements.txt b/docs/tools/requirements.txt index e8680473939..85cd355dbdc 100644 --- a/docs/tools/requirements.txt +++ b/docs/tools/requirements.txt @@ -18,7 +18,7 @@ mkdocs==1.0.4 Pygments==2.2.0 python-slugify==1.2.6 pytz==2017.3 -PyYAML==3.12 +PyYAML==4.2b1 recommonmark==0.4.0 requests==2.21.0 singledispatch==3.4.0.3 diff --git a/docs/zh/development/tests.md b/docs/zh/development/tests.md deleted file mode 120000 index c03d36c3916..00000000000 --- a/docs/zh/development/tests.md +++ /dev/null @@ -1 +0,0 @@ -../../en/development/tests.md \ No newline at end of file diff --git a/docs/zh/development/tests.md b/docs/zh/development/tests.md new file mode 100644 index 00000000000..2b5fb7ca0e6 --- /dev/null +++ b/docs/zh/development/tests.md @@ -0,0 +1,257 @@ +# ClickHouse 测试 + + +## 功能性测试 + +功能性测试是最简便使用的。绝大部分 ClickHouse 的功能可以通过功能性测试来测试，任何代码的更改都必须通过该测试。 + +每个功能测试会向正在运行的 ClickHouse服 务器发送一个或多个查询，并将结果与预期结果进行比较。 + +测试用例在 `dbms/src/tests/queries` 目录中。这里有两个子目录：`stateless` 和 `stateful`目录。 无状态的测试无需预加载测试数据集 - 通常是在测试运行期间动态创建小量的数据集。有状态测试需要来自 Yandex.Metrica 的预加载测试数据，而不向一般公众提供。 我们倾向于仅使用“无状态”测试并避免添加新的“有状态”测试。 + +每个测试用例可以是两种类型之一：`.sql` 和 `.sh`。`.sql` 测试文件是用于管理`clickhouse-client --multiquery --testmode`的简单SQL脚本。`.sh` 测试文件是一个可以自己运行的脚本。 + +要运行所有测试，请使用 `dbms/tests/clickhouse-test` 工具，用 `--help` 可以获取所有的选项列表。您可以简单地运行所有测试或运行测试名称中的子字符串过滤的测试子集：`./clickhouse-test substring`。 + +调用功能测试最简单的方法是将 `clickhouse-client` 复制到`/usr/bin/`，运行`clickhouse-server`，然后从自己的目录运行`./ clickhouse-test`。 + +要添加新测试，请在 `dbms/src/tests/queries/0_stateless` 目录内添加新的 `.sql` 或 `.sh` 文件，手动检查，然后按以下方式生成 `.reference` 文件： `clickhouse-client -n --testmode < 00000_test.sql > 00000_test.reference` or `./00000_test.sh > ./00000_test.reference`。 + +测试应该只使用（创建，删除等）`test` 数据库中的表，这些表假定是事先创建的; 测试也可以使用临时表。 + +如果要在功能测试中使用分布式查询，可以利用 `remote` 表函数和 `127.0.0.{1..2}` 地址为服务器查询自身; 或者您可以在服务器配置文件中使用预定义的测试集群，例如`test_shard_localhost`。 + +有些测试在名称中标有 `zookeeper`，`shard` 或 `long`。`zookeeper` 用于使用ZooKeeper的测试; `shard` 用于需要服务器监听`127.0.0.*`的测试。`long` 适用于运行时间稍长一秒的测试。 + + +## 已知的bug + +如果我们知道一些可以通过功能测试轻松复制的错误，我们将准备好的功能测试放在 `dbms/src/tests/queries/bugs` 目录中。当修复错误时，这些测试将被移动到 `dbms/src/tests/queries/0_stateless` 目录中。 + + +## 集成测试 + +集成测试允许在集群配置中测试 ClickHouse，并与其他服务器（如MySQL，Postgres，MongoDB）进行 ClickHouse 交互。它们可用于模拟网络拆分，数据包丢弃等。这些测试在Docker 下运行，并使用各种软件创建多个容器。 + +参考 `dbms/tests/integration/README.md` 文档关于如何使用集成测试。 + +请注意，ClickHouse 与第三方驱动程序的集成未经过测试。此外，我们目前还没有与 JDBC 和ODBC 驱动程序进行集成测试。 + + +## 单元测试 + +当您想要测试整个 ClickHouse，而不是单个独立的库或类时，单元测试非常有用。您可以使用`ENABLE_TESTS` CMake 选项启用或禁用测试构建。单元测试（和其他测试程序）位于代码中的`tests` 子目录中。要运行单元测试，请键入 `ninja test`。有些测试使用 `gtest`，但有些只是在测试失败时返回非零状态码。 + +如果代码已经被功能测试覆盖（并且功能测试通常使用起来要简单得多），则不一定要进行单元测试。 + + +## 性能测试 + +性能测试允许测量和比较综合查询中 ClickHouse 的某些独立部分的性能。测试位于`dbms/tests/performance` 目录中。每个测试都由 `.xml` 文件表示，并附有测试用例的描述。使用 `clickhouse performance-test` 工具（嵌入在 `clickhouse` 二进制文件中）运行测试。请参阅 `--help` 以进行调用。 + +每个测试在循环中运行一个或多个查询（可能带有参数组合），并具有一些停止条件（如“最大执行速度不会在三秒内更改”）并测量一些有关查询性能的指标（如“最大执行速度”））。某些测试可以包含预加载的测试数据集的前提条件。 + +如果要在某些情况下提高 ClickHouse 的性能，并且如果可以在简单查询上观察到改进，则强烈建议编写性能测试。在测试过程中使用 `perf top` 或其他 perf 工具总是有意义的。 + + +性能测试不是基于每个提交运行的。不收集性能测试结果，我们手动比较它们。 + + +## 测试工具和脚本 + +`tests`目录中的一些程序不是准备测试，而是测试工具。例如，对于`Lexer`，有一个工具`dbms/src/Parsers/tests/lexer` 标准输出。您可以使用这些工具作为代码示例以及探索和手动测试。 + +您还可以将一对文件 `.sh` 和 `.reference` 与工具放在一些预定义的输入上运行它 - 然后可以将脚本结果与 `.reference` 文件进行比较。这些测试不是自动化的。 + + +## 杂项测试 + +有一些外部字典的测试位于 `dbms/tests/external_dictionaries`，机器学习模型在`dbms/tests/external_models`目录。这些测试未更新，必须转移到集成测试。 + +对于分布式数据的插入，有单独的测试。此测试在单独的服务器上运行 ClickHouse 集群并模拟各种故障情况：网络拆分，数据包丢弃（ClickHouse 节点之间，ClickHouse 和 ZooKeeper之间，ClickHouse 服务器和客户端之间等），进行 `kill -9`，`kill -STOP` 和`kill -CONT` 等操作，类似[Jepsen](https://aphyr.com/tags/Jepsen)。然后，测试检查是否已写入所有已确认的插入，并且所有已拒绝的插入都未写入。 + + +在 ClickHouse 开源之前，分布式测试是由单独的团队编写的，但该团队不再使用 ClickHouse，测试是在 Java 中意外编写的。由于这些原因，必须重写分布式测试并将其移至集成测试。 + + +## 手动测试 + +当您开发了新的功能，做手动测试也是合理的。可以按照以下步骤来进行： + +编译 ClickHouse。在命令行中运行 ClickHouse：进入 `dbms/src/programs/clickhouse-server` 目录并运行 `./clickhouse-server`。它会默认使用当前目录的配置文件 (`config.xml`， `users.xml` 以及在 `config.d` 和 `users.d` 目录的文件)。可以使用 `dbms/src/programs/clickhouse-client/clickhouse-client` 来连接数据库。 + +或者，您可以安装 ClickHouse 软件包：从 Yandex 存储库中获得稳定版本，或者您可以在ClickHouse源根目录中使用 `./release` 构建自己的软件包。然后使用 `sudo service clickhouse-server start` 启动服务器（或停止服务器）。在 `/etc/clickhouse-server/clickhouse-server.log` 中查找日志。 + +当您的系统上已经安装了 ClickHouse 时，您可以构建一个新的 `clickhouse` 二进制文件并替换现有的二进制文件： + +``` +sudo service clickhouse-server stop +sudo cp ./clickhouse /usr/bin/ +sudo service clickhouse-server start +``` + +您也可以停止 clickhouse-server 并使用相同的配置运行您自己的服务器，日志打印到终端： +``` +sudo service clickhouse-server stop +sudo -u clickhouse /usr/bin/clickhouse server --config-file /etc/clickhouse-server/config.xml +``` + +使用 gdb 的一个示例: +``` +sudo -u clickhouse gdb --args /usr/bin/clickhouse server --config-file /etc/clickhouse-server/config.xml +``` + +如果 clickhouse-server 已经运行并且您不想停止它，您可以更改 `config.xml` 中的端口号（或在 `config.d` 目录中的文件中覆盖它们），配置适当的数据路径，然后运行它。 + +`clickhouse` 二进制文件几乎没有依赖关系，适用于各种 Linux 发行版。要快速地测试服务器上的更改，您可以简单地将新建的 `clickhouse` 二进制文件 `scp` 到其他服务器，然后按照上面的示例运行它。 + + +## 测试环境 + +在将版本发布为稳定之前，我们将其部署在测试环境中 测试环境是一个处理[Yandex.Metrica]（https://metrica.yandex.com/）总数据的1/39部分大小的集群。 我们与 Yandex.Metrica 团队公用我们的测试环境。ClickHouse 在现有数据的基础上无需停机即可升级。 我们首先看到数据处理成功而不会实时滞后，复制继续工作，并且 Yandex.Metrica 团队无法看到问题。 首先的检查可以通过以下方式完成： + +``` +SELECT hostName() AS h, any(version()), any(uptime()), max(UTCEventTime), count() FROM remote('example01-01-{1..3}t', merge, hits) WHERE EventDate >= today() - 2 GROUP BY h ORDER BY h; +``` + +在某些情况下，我们还部署到 Yandex 的合作团队的测试环境：市场，云等。此外，我们还有一些用于开发目的的硬件服务器。 + +## 负载测试 + +部署到测试环境后，我们使用生产群集中的查询运行负载测试。 这是手动完成的。 + +确保在生产集群中开启了 `query_log` 选项。 + +收集一天或更多的查询日志： +``` +clickhouse-client --query="SELECT DISTINCT query FROM system.query_log WHERE event_date = today() AND query LIKE '%ym:%' AND query NOT LIKE '%system.query_log%' AND type = 2 AND is_initial_query" > queries.tsv +``` + +这是一个复杂的例子。`type = 2` 将过滤成功执行的查询。`query LIKE'％ym：％'` 用于从 Yandex.Metrica 中选择相关查询。`is_initial_query` 是仅选择由客户端发起的查询，而不是由 ClickHouse 本身（作为分布式查询处理的一部分）。 + +`scp` 这份日志到测试机器，并运行以下操作： + +``` +clickhouse benchmark --concurrency 16 < queries.tsv +``` +(可能你需要指定运行的用户 `--user`) + +然后离开它一晚或周末休息一下。 + +你要检查下 `clickhouse-server` 是否崩溃，内存占用是否合理，性能也不会随着时间的推移而降低。 + +由于查询和环境的高度可变性，不会记录精确的查询执行时序并且不进行比较。 + + +## 编译测试 + +构建测试允许检查构建在各种替代配置和某些外部系统上是否被破坏。测试位于`ci`目录。 它们从 Docker，Vagrant 中的源代码运行构建，有时在 Docker 中运行 `qemu-user-static`。这些测试正在开发中，测试运行不是自动化的。 + +动机： + +通常我们会在 ClickHouse 构建的单个版本上发布并运行所有测试。 但是有一些未经过彻底测试的替代构建版本。 例子： + +- 在 FreeBSD 中的构建； +- 在 Debian 中使用系统包中的库进行构建； +- 使用库的共享链接构建； +- 在 AArch64 平台进行构建。 + +例如，使用系统包构建是不好的做法，因为我们无法保证系统具有的确切版本的软件包。但 Debian 维护者确实需要这样做。出于这个原因，我们至少必须支持这种构建。另一个例子：共享链接是一个常见的麻烦来源，但是对于一些爱好者来说需要它。 + +虽然我们无法对所有构建版本运行所有测试，但我们想要检查至少不会破坏各种构建变体。为此，我们使用构建测试。 + + +## 测试协议兼容性 + +当我们扩展 ClickHouse 网络协议时，我们手动测试旧的 clickhouse-client 与新的 clickhouse-server 和新的clickhouse-client 一起使用旧的 clickhouse-server (只需从相应的包中运行二进制文件) + + +## 来自编译器的帮助 + +ClickHouse 主要的代码 (位于`dbms`目录中) 使用 `-Wall -Wextra -Werror` 构建，并带有一些其他已启用的警告。 虽然没有为第三方库启用这些选项。 + +Clang 有更多有用的警告 - 您可以使用 `-Weverything` 查找它们并选择默认构建的东西。 + +对于生产构建，使用 gcc（它仍然生成比 clang 稍高效的代码）。对于开发来说，clang 通常更方便使用。您可以使用调试模式在自己的机器上构建（以节省笔记本电脑的电量），但请注意，由于更好的控制流程和过程分析，编译器使用 `-O3` 会生成更多警告。 当使用 clang 构建时，使用 `libc++` 而不是 `libstdc++`，并且在使用调试模式构建时，使用调试版本的 `libc++`，它允许在运行时捕获更多错误。 + +## Sanitizers + +**Address sanitizer**. +我们在每个提交的基础上在 ASan 下运行功能和集成测试。 + +**Valgrind (Memcheck)**. +我们在 Valgrind 过夜进行功能测试。 这需要几个小时。 目前在 `re2` 库中有一个已知的误报，请参阅 [文章](https://research.swtch.com/sparse)。 + +**Thread sanitizer**. +我们在 TSan 下进行功能测试。ClickHouse 必须通过所有测试。在 TSan 下运行不是自动化的，只是偶尔执行。 + +**Memory sanitizer**. +目前我们不使用 MSan。 + +**Undefined behaviour sanitizer.** +我们仍然不会在每次提交的基础上使用 UBSan。 有一些地方需要解决。 + +**Debug allocator.** +您可以使用 `DEBUG_TCMALLOC` CMake 选项启用 `tcmalloc` 的调试版本。我们在每次提交的基础上使用调试分配器运行测试。 + +更多请参阅 `dbms/tests/instructions/sanitizers.txt`。 + + +## 模糊测试 + +我们使用简单的模糊测试来生成随机SQL查询并检查服务器是否正常，使用 Address sanitizer 执行模糊测试。你可以在`00746_sql_fuzzy.pl` 找到它。 测试应连续进行（过夜和更长时间）。 + +截至2018年12月，我们仍然不使用库代码的孤立模糊测试。 + +## 安全审计 + +Yandex Cloud 部门的人员从安全角度对 ClickHouse 功能进行了一些基本概述。 + + +## 静态分析 + +我们偶尔使用静态分析。我们已经评估过 `clang-tidy`， `Coverity`， `cppcheck`， `PVS-Studio`， `tscancode`。您将在 `dbms/tests/instructions/` 目录中找到使用说明。你也可以阅读[俄文文章](https://habr.com/company/yandex/blog/342018/). + +如果您使用 `CLion` 作为 IDE，您可以开箱即用一些 `clang-tidy` 检查。 + +## 其他强化 + +默认情况下使用 `FORTIFY_SOURCE`。它几乎没用，但在极少数情况下仍然有意义，我们不会禁用它。 + + +## 代码风格 + +代码风格在[这里](https://clickhouse.yandex/docs/en/development/style/) 有说明。 + +要检查一些常见的样式冲突，您可以使用 `utils/check-style` 脚本。 + +为了强制你的代码的正确风格，你可以使用 `clang-format` 文件。`.clang-format` 位于源代码根目录， 它主要与我们的实际代码风格对应。但不建议将 `clang-format` 应用于现有文件，因为它会使格式变得更糟。您可以使用 `clang-format-diff` 工具，您可以在 clang 源代码库中找到 + +或者，您可以尝试`uncrustify` 工具来格式化您的代码。配置文件在源代码的根目录中的`uncrustify.cfg`。它比 `clang-format` 经过更少的测试。 + +`CLion` 有自己的代码格式化程序，必须调整为我们的代码风格。 + + +## Metrica B2B 测试 + +每个 ClickHouse 版本都经过 Yandex Metrica 和 AppMetrica 引擎的测试。测试和稳定版本的 ClickHouse 部署在虚拟机上，并使用处理输入数据固定样本的度量引擎的小副本运行。 将度量引擎的两个实例的结果一起进行比较 + +这些测试是由单独的团队自动完成的。由于移动部件的数量很多，大部分时间的测试都是完全无关的，很难弄清楚。很可能这些测试对我们来说是负值。然而，这些测试被证明是有用的大约一个或两个倍的数百。 + + +## 测试覆盖率 + +截至2018年7月，我们不会跟踪测试复盖率。 + + +## 自动化测试 + +我们使用 Yandex 内部 CI 和名为"沙箱"的作业自动化系统运行测试。 我们还继续使用 Jenkins（可在Yandex内部使用）。 + +构建作业和测试在沙箱中按每次提交的基础上运行。结果包和测试结果发布在 GitHub 上，可以通过直接链接下载，结果会被永久存储。当您在 GitHub 上发送拉取请求时，我们将其标记为"可以测试"，我们的 CI 系统将为您构建 ClickHouse 包（发布，调试，地址消除等）。 + +由于时间和计算能力的限制，我们不使用 Travis CI。 + +在 Jenkins，我们运行字典测试，指标B2B测试。 我们使用 Jenkins 来准备和发布版本。 Jenkins是一种传统的技术，所有的工作将被转移到沙箱中。 + +[来源文章](https://clickhouse.yandex/docs/zh/development/tests/) diff --git a/libs/libcommon/include/common/JSON.h b/libs/libcommon/include/common/JSON.h index 84fa089bcda..9bb109a840f 100644 --- a/libs/libcommon/include/common/JSON.h +++ b/libs/libcommon/include/common/JSON.h @@ -111,11 +111,11 @@ public: template T getWithDefault(const std::string & key, const T & default_ = T()) const; - double getDouble() const; - Int64 getInt() const; /// Отбросить дробную часть. - UInt64 getUInt() const; /// Отбросить дробную часть. Если число отрицательное - исключение. + double getDouble() const; + Int64 getInt() const; /// Отбросить дробную часть. + UInt64 getUInt() const; /// Отбросить дробную часть. Если число отрицательное - исключение. std::string getString() const; - bool getBool() const; + bool getBool() const; std::string getName() const; /// Получить имя name-value пары. JSON getValue() const; /// Получить значение name-value пары. @@ -123,9 +123,9 @@ public: StringRef getRawName() const; /// Получить значение элемента; если элемент - строка, то распарсить значение из строки; если не строка или число - то исключение. - double toDouble() const; - Int64 toInt() const; - UInt64 toUInt() const; + double toDouble() const; + Int64 toInt() const; + UInt64 toUInt() const; /** Преобразовать любой элемент в строку. * Для строки возвращается её значение, для всех остальных элементов - сериализованное представление. diff --git a/libs/libcommon/include/common/ThreadPool.h b/libs/libcommon/include/common/ThreadPool.h index 980fdcba355..dd82c0c0399 100644 --- a/libs/libcommon/include/common/ThreadPool.h +++ b/libs/libcommon/include/common/ThreadPool.h @@ -56,6 +56,8 @@ private: void worker(); + + void finalize(); }; diff --git a/libs/libcommon/include/common/Types.h b/libs/libcommon/include/common/Types.h index a6bfcc6ae31..70c9c3d2f3c 100644 --- a/libs/libcommon/include/common/Types.h +++ b/libs/libcommon/include/common/Types.h @@ -1,6 +1,5 @@ #pragma once #include -#include using Int8 = int8_t; using Int16 = int16_t; @@ -11,30 +10,3 @@ using UInt8 = uint8_t; using UInt16 = uint16_t; using UInt32 = uint32_t; using UInt64 = uint64_t; - - -/** This is not the best way to overcome an issue of different definitions - * of uint64_t and size_t on Linux and Mac OS X (both 64 bit). - * - * Note that on both platforms, long and long long are 64 bit types. - * But they are always different types (with the same physical representation). - */ -namespace std -{ - inline UInt64 max(unsigned long x, unsigned long long y) { return x > y ? x : y; } - inline UInt64 max(unsigned long long x, unsigned long y) { return x > y ? x : y; } - inline UInt64 min(unsigned long x, unsigned long long y) { return x < y ? x : y; } - inline UInt64 min(unsigned long long x, unsigned long y) { return x < y ? x : y; } - - inline Int64 max(long x, long long y) { return x > y ? x : y; } - inline Int64 max(long long x, long y) { return x > y ? x : y; } - inline Int64 min(long x, long long y) { return x < y ? x : y; } - inline Int64 min(long long x, long y) { return x < y ? x : y; } -} - - -/// Workaround for the issue, that KDevelop doesn't see time_t and size_t types (for syntax highlight). -#ifdef IN_KDEVELOP_PARSER - using time_t = Int64; - using size_t = UInt64; -#endif diff --git a/libs/libcommon/include/common/itoa.h b/libs/libcommon/include/common/itoa.h new file mode 100644 index 00000000000..18764fc5e98 --- /dev/null +++ b/libs/libcommon/include/common/itoa.h @@ -0,0 +1,350 @@ +#pragma once + +// Based on https://github.com/amdn/itoa and combined with our optimizations +// +//=== itoa.h - Fast integer to ascii conversion --*- C++ -*-// +// +// The MIT License (MIT) +// Copyright (c) 2016 Arturo Martin-de-Nicolas +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +//===----------------------------------------------------------------------===// + +#include +#include +#include "likely.h" +#include "unaligned.h" + +using uint128_t = unsigned __int128; + +namespace impl +{ +// Using a lookup table to convert binary numbers from 0 to 99 +// into ascii characters as described by Andrei Alexandrescu in +// https://www.facebook.com/notes/facebook-engineering/three-optimization-tips-for-c/10151361643253920/ + +static const char digits[201] = "00010203040506070809" + "10111213141516171819" + "20212223242526272829" + "30313233343536373839" + "40414243444546474849" + "50515253545556575859" + "60616263646566676869" + "70717273747576777879" + "80818283848586878889" + "90919293949596979899"; + +static inline uint16_t const & dd(uint8_t u) +{ + return reinterpret_cast(digits)[u]; +} + +template +static constexpr T pow10(size_t x) +{ + return x ? 10 * pow10(x - 1) : 1; +} + +// Division by a power of 10 is implemented using a multiplicative inverse. +// This strength reduction is also done by optimizing compilers, but +// presently the fastest results are produced by using the values +// for the multiplication and the shift as given by the algorithm +// described by Agner Fog in "Optimizing Subroutines in Assembly Language" +// +// http://www.agner.org/optimize/optimizing_assembly.pdf +// +// "Integer division by a constant (all processors) +// A floating point number can be divided by a constant by multiplying +// with the reciprocal. If we want to do the same with integers, we have +// to scale the reciprocal by 2n and then shift the product to the right +// by n. There are various algorithms for finding a suitable value of n +// and compensating for rounding errors. The algorithm described below +// was invented by Terje Mathisen, Norway, and not published elsewhere." + +template +struct MulInv +{ + using type = UInt; + static constexpr bool a{A}; + static constexpr UInt m{M}; + static constexpr unsigned s{S}; +}; +template +struct UT; +template +struct UT +{ + using U = T; +}; +template +struct UT +{ + using U = typename UT::U; +}; +template +using MI = typename UT< + N, + 1, + MulInv, + MulInv, + MulInv, + MulInv, + MulInv>::U; +template +using U = typename MI::type; + +// struct QR holds the result of dividing an unsigned N-byte variable +// by 10^N resulting in +template +struct QR +{ + U q; // quotient with fewer than 2*N decimal digits + U r; // remainder with at most N decimal digits +}; +template +QR static inline split(U u) +{ + constexpr MI mi{}; + U q = (mi.m * (U<2 * N>(u) + mi.a)) >> mi.s; + return {q, U(u - q * pow10>(N))}; +} + +template +static inline char * out(char * p, T && obj) +{ + memcpy(p, reinterpret_cast(&obj), sizeof(T)); + p += sizeof(T); + return p; +} + +struct convert +{ + //===----------------------------------------------------------===// + // head: find most significant digit, skip leading zeros + //===----------------------------------------------------------===// + + // "x" contains quotient and remainder after division by 10^N + // quotient is less than 10^N + template + static inline char * head(char * p, QR x) + { + return tail(head(p, U(x.q)), x.r); + } + + // "u" is less than 10^2*N + template + static inline char * head(char * p, UInt u) + { + return (u < pow10>(N) ? (head(p, U(u))) : (head(p, split(u)))); + } + + // recursion base case, selected when "u" is one byte + static inline char * head(char * p, U<1> u) { return (u < 10 ? (out(p, '0' + u)) : (out(p, dd(u)))); } + + //===----------------------------------------------------------===// + // tail: produce all digits including leading zeros + //===----------------------------------------------------------===// + + // recursive step, "u" is less than 10^2*N + template + static inline char * tail(char * p, UInt u) + { + QR x = split(u); + return tail(tail(p, U(x.q)), x.r); + } + + // recursion base case, selected when "u" is one byte + static inline char * tail(char * p, U<1> u) { return out(p, dd(u)); } + + //===----------------------------------------------------------===// + // large values are >= 10^2*N + // where x contains quotient and remainder after division by 10^N + //===----------------------------------------------------------===// + + template + static inline char * large(char * p, QR x) + { + QR y = split(x.q); + return tail(tail(head(p, U(y.q)), y.r), x.r); + } + + //===----------------------------------------------------------===// + // handle values of "u" that might be >= 10^2*N + // where N is the size of "u" in bytes + //===----------------------------------------------------------===// + + template + static inline char * itoa(char * p, UInt u) + { + if (u < pow10>(N)) + return head(p, U(u)); + QR x = split(u); + return (u < pow10>(2 * N) ? (head(p, x)) : (large(p, x))); + } + + // selected when "u" is one byte + static inline char * itoa(char * p, U<1> u) + { + if (u < 10) + return out(p, '0' + u); + if (u < 100) + return out(p, dd(u)); + return out(out(p, '0' + u / 100), dd(u % 100)); + } + + //===----------------------------------------------------------===// + // handle unsigned and signed integral operands + //===----------------------------------------------------------===// + + // itoa: handle unsigned integral operands (selected by SFINAE) + template ::value && std::is_integral::value> * = nullptr> + static inline char * itoa(U u, char * p) + { + return convert::itoa(p, u); + } + + // itoa: handle signed integral operands (selected by SFINAE) + template ::value && std::is_integral::value> * = nullptr> + static inline char * itoa(I i, char * p) + { + // Need "mask" to be filled with a copy of the sign bit. + // If "i" is a negative value, then the result of "operator >>" + // is implementation-defined, though usually it is an arithmetic + // right shift that replicates the sign bit. + // Use a conditional expression to be portable, + // a good optimizing compiler generates an arithmetic right shift + // and avoids the conditional branch. + U mask = i < 0 ? ~U(0) : 0; + // Now get the absolute value of "i" and cast to unsigned type U. + // Cannot use std::abs() because the result is undefined + // in 2's complement systems for the most-negative value. + // Want to avoid conditional branch for performance reasons since + // CPU branch prediction will be ineffective when negative values + // occur randomly. + // Let "u" be "i" cast to unsigned type U. + // Subtract "u" from 2*u if "i" is positive or 0 if "i" is negative. + // This yields the absolute value with the desired type without + // using a conditional branch and without invoking undefined or + // implementation defined behavior: + U u = ((2 * U(i)) & ~mask) - U(i); + // Unconditionally store a minus sign when producing digits + // in a forward direction and increment the pointer only if + // the value is in fact negative. + // This avoids a conditional branch and is safe because we will + // always produce at least one digit and it will overwrite the + // minus sign when the value is not negative. + *p = '-'; + p += (mask & 1); + p = convert::itoa(p, u); + return p; + } +}; + +static inline int digits10(uint128_t x) +{ + if (x < 10ULL) + return 1; + if (x < 100ULL) + return 2; + if (x < 1000ULL) + return 3; + + if (x < 1000000000000ULL) + { + if (x < 100000000ULL) + { + if (x < 1000000ULL) + { + if (x < 10000ULL) + return 4; + else + return 5 + (x >= 100000ULL); + } + + return 7 + (x >= 10000000ULL); + } + + if (x < 10000000000ULL) + return 9 + (x >= 1000000000ULL); + + return 11 + (x >= 100000000000ULL); + } + + return 12 + digits10(x / 1000000000000ULL); +} + +static inline char * writeUIntText(uint128_t x, char * p) +{ + int len = digits10(x); + auto pp = p + len; + while (x >= 100) + { + const auto i = x % 100; + x /= 100; + pp -= 2; + unalignedStore(pp, dd(i)); + } + if (x < 10) + *p = '0' + x; + else + unalignedStore(p, dd(x)); + return p + len; +} + +static inline char * writeLeadingMinus(char * pos) +{ + *pos = '-'; + return pos + 1; +} + +static inline char * writeSIntText(__int128 x, char * pos) +{ + static const __int128 min_int128 = __int128(0x8000000000000000ll) << 64; + + if (unlikely(x == min_int128)) + { + memcpy(pos, "-170141183460469231731687303715884105728", 40); + return pos + 40; + } + + if (x < 0) + { + x = -x; + pos = writeLeadingMinus(pos); + } + return writeUIntText(static_cast(x), pos); +} + +} + +template +char * itoa(I i, char * p) +{ + return impl::convert::itoa(i, p); +} + +static inline char * itoa(uint128_t i, char * p) +{ + return impl::writeUIntText(i, p); +} + +static inline char * itoa(__int128 i, char * p) +{ + return impl::writeSIntText(i, p); +} diff --git a/libs/libcommon/include/common/mremap.h b/libs/libcommon/include/common/mremap.h index edb62070247..f569ff05d4e 100644 --- a/libs/libcommon/include/common/mremap.h +++ b/libs/libcommon/include/common/mremap.h @@ -2,7 +2,7 @@ #include #include -#if !_MSC_VER +#if !defined(_MSC_VER) #include #endif diff --git a/libs/libcommon/src/ThreadPool.cpp b/libs/libcommon/src/ThreadPool.cpp index 4da7c9689b8..e45e64853dc 100644 --- a/libs/libcommon/src/ThreadPool.cpp +++ b/libs/libcommon/src/ThreadPool.cpp @@ -6,8 +6,17 @@ ThreadPool::ThreadPool(size_t m_size) : m_size(m_size) { threads.reserve(m_size); - for (size_t i = 0; i < m_size; ++i) - threads.emplace_back([this] { worker(); }); + + try + { + for (size_t i = 0; i < m_size; ++i) + threads.emplace_back([this] { worker(); }); + } + catch (...) + { + finalize(); + throw; + } } void ThreadPool::schedule(Job job) @@ -40,6 +49,11 @@ void ThreadPool::wait() } ThreadPool::~ThreadPool() +{ + finalize(); +} + +void ThreadPool::finalize() { { std::unique_lock lock(mutex); @@ -50,6 +64,8 @@ ThreadPool::~ThreadPool() for (auto & thread : threads) thread.join(); + + threads.clear(); } size_t ThreadPool::active() const diff --git a/libs/libglibc-compatibility/CMakeLists.txt b/libs/libglibc-compatibility/CMakeLists.txt index cfb2f221c97..15aa9fe7e3d 100644 --- a/libs/libglibc-compatibility/CMakeLists.txt +++ b/libs/libglibc-compatibility/CMakeLists.txt @@ -19,7 +19,8 @@ musl/sched_cpucount.c musl/glob.c musl/exp2f.c musl/pwritev.c -musl/getrandom.c) +musl/getrandom.c +musl/fcntl.c) if (MAKE_STATIC_LIBRARIES) set (GLIBC_COMPATIBILITY_SOURCES ${GLIBC_COMPATIBILITY_SOURCES} diff --git a/libs/libglibc-compatibility/musl/fcntl.c b/libs/libglibc-compatibility/musl/fcntl.c new file mode 100644 index 00000000000..f9bf82ccb84 --- /dev/null +++ b/libs/libglibc-compatibility/musl/fcntl.c @@ -0,0 +1,61 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include "syscall.h" + +#include +#if defined(__GLIBC__) && __GLIBC__ >= 2 +# define GLIBC_MINOR __GLIBC_MINOR__ +#elif defined (__GNU_LIBRARY__) && __GNU_LIBRARY__ >= 2 +# define GLIBC_MINOR __GNU_LIBRARY_MINOR__ +#endif + +#if defined(GLIBC_MINOR) && GLIBC_MINOR >= 28 + +int fcntl64(int fd, int cmd, ...) +{ + unsigned long arg; + va_list ap; + va_start(ap, cmd); + arg = va_arg(ap, unsigned long); + va_end(ap); + if (cmd == F_SETFL) arg |= O_LARGEFILE; + if (cmd == F_SETLKW) return syscall(SYS_fcntl, fd, cmd, (void *)arg); + if (cmd == F_GETOWN) { + struct f_owner_ex ex; + int ret = __syscall(SYS_fcntl, fd, F_GETOWN_EX, &ex); + if (ret == -EINVAL) return __syscall(SYS_fcntl, fd, cmd, (void *)arg); + if (ret) return __syscall_ret(ret); + return ex.type == F_OWNER_PGRP ? -ex.pid : ex.pid; + } + if (cmd == F_DUPFD_CLOEXEC) { + int ret = __syscall(SYS_fcntl, fd, F_DUPFD_CLOEXEC, arg); + if (ret != -EINVAL) { + if (ret >= 0) + __syscall(SYS_fcntl, ret, F_SETFD, FD_CLOEXEC); + return __syscall_ret(ret); + } + ret = __syscall(SYS_fcntl, fd, F_DUPFD_CLOEXEC, 0); + if (ret != -EINVAL) { + if (ret >= 0) __syscall(SYS_close, ret); + return __syscall_ret(-EINVAL); + } + ret = __syscall(SYS_fcntl, fd, F_DUPFD, arg); + if (ret >= 0) __syscall(SYS_fcntl, ret, F_SETFD, FD_CLOEXEC); + return __syscall_ret(ret); + } + switch (cmd) { + case F_SETLK: + case F_GETLK: + case F_GETOWN_EX: + case F_SETOWN_EX: + return syscall(SYS_fcntl, fd, cmd, (void *)arg); + default: + return syscall(SYS_fcntl, fd, cmd, arg); + } +} + +#endif diff --git a/libs/libmysqlxx/include/mysqlxx/Value.h b/libs/libmysqlxx/include/mysqlxx/Value.h index 889a97ea05b..c3460f917a1 100644 --- a/libs/libmysqlxx/include/mysqlxx/Value.h +++ b/libs/libmysqlxx/include/mysqlxx/Value.h @@ -69,7 +69,7 @@ public: if (unlikely(isNull())) throwException("Value is NULL"); - return readUIntText(m_data, m_length);; + return readUIntText(m_data, m_length); } /// Получить целое со знаком или дату или дату-время (в unix timestamp согласно текущей тайм-зоне). @@ -190,7 +190,7 @@ private: Int64 getIntImpl() const { - return readIntText(m_data, m_length);; + return readIntText(m_data, m_length); } diff --git a/libs/libwidechar_width/widechar_width.h b/libs/libwidechar_width/widechar_width.h index f49180e7c54..39cf0ded05b 100644 --- a/libs/libwidechar_width/widechar_width.h +++ b/libs/libwidechar_width/widechar_width.h @@ -492,10 +492,10 @@ static const struct widechar_range widechar_widened_table[] = { {0x1F910, 0x1F918}, {0x1F980, 0x1F984}, {0x1F9C0, 0x1F9C0} }; -template +template bool widechar_in_table(const Collection &arr, int32_t c) { auto where = std::lower_bound(std::begin(arr), std::end(arr), c, - [](widechar_range p, int32_t c) { return p.hi < c; }); + [](widechar_range p, int32_t chr) { return p.hi < chr; }); return where != std::end(arr) && where->lo <= c; } diff --git a/release b/release index 23bfd6f2dd6..e3c8842a820 100755 --- a/release +++ b/release @@ -32,7 +32,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) cd $CURDIR -source "./release_lib.sh" +source "./utils/release/release_lib.sh" PBUILDER_AUTOUPDATE=${PBUILDER_AUTOUPDATE=4320} diff --git a/utils/check-marks/CMakeLists.txt b/utils/check-marks/CMakeLists.txt index 9c534364691..86cff8fb233 100644 --- a/utils/check-marks/CMakeLists.txt +++ b/utils/check-marks/CMakeLists.txt @@ -1,2 +1,2 @@ add_executable (check-marks main.cpp) -target_link_libraries(check-marks PRIVATE clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY}) +target_link_libraries(check-marks PRIVATE clickhouse_compression clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY}) diff --git a/utils/check-marks/main.cpp b/utils/check-marks/main.cpp index 63e785b8e9d..52020dfff25 100644 --- a/utils/check-marks/main.cpp +++ b/utils/check-marks/main.cpp @@ -2,14 +2,14 @@ #include -#include -#include +#include +#include #include #include #include #include #include -#include +#include /** This program checks correctness of .mrk (marks) file for corresponding compressed .bin file. diff --git a/utils/compressor/CMakeLists.txt b/utils/compressor/CMakeLists.txt index cd140051203..2dec2117943 100644 --- a/utils/compressor/CMakeLists.txt +++ b/utils/compressor/CMakeLists.txt @@ -13,7 +13,7 @@ add_executable (mutator mutator.cpp) target_link_libraries(mutator PRIVATE clickhouse_common_io) add_executable (decompress_perf decompress_perf.cpp) -target_link_libraries(decompress_perf PRIVATE clickhouse_common_io ${LZ4_LIBRARY}) +target_link_libraries(decompress_perf PRIVATE clickhouse_common_io clickhouse_compression ${LZ4_LIBRARY}) if (NOT USE_INTERNAL_ZSTD_LIBRARY) target_include_directories (zstd_test BEFORE PRIVATE ${ZSTD_INCLUDE_DIR}) diff --git a/utils/compressor/decompress_perf.cpp b/utils/compressor/decompress_perf.cpp index 91c2128cef1..3e598b10876 100644 --- a/utils/compressor/decompress_perf.cpp +++ b/utils/compressor/decompress_perf.cpp @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/utils/compressor/mutator.cpp b/utils/compressor/mutator.cpp index c8cca3e6ecf..65125d073d0 100644 --- a/utils/compressor/mutator.cpp +++ b/utils/compressor/mutator.cpp @@ -106,7 +106,7 @@ static void mutate(pcg64 & generator, void * src, size_t length) && isAlphaASCII(pos[2])) { auto res = rand(generator, 0, 3); - if (res == 2) + if (res == 2) { std::swap(pos[0], pos[1]); } @@ -118,7 +118,7 @@ static void mutate(pcg64 & generator, void * src, size_t length) else if (pos + 5 <= end && pos[0] >= 0xC0 && pos[0] <= 0xDF && pos[1] >= 0x80 && pos[1] <= 0xBF && pos[2] >= 0x20 && pos[2] < 0x80 && !isAlphaASCII(pos[2]) - && pos[3] >= 0xC0 && pos[0] <= 0xDF && pos[4] >= 0x80 && pos[4] <= 0xBF) + && pos[3] >= 0xC0 && pos[3] <= 0xDF && pos[4] >= 0x80 && pos[4] <= 0xBF) { auto res = rand(generator, 0, 3); if (res == 2) diff --git a/release_lib.sh b/utils/release/release_lib.sh similarity index 99% rename from release_lib.sh rename to utils/release/release_lib.sh index ecdc10deefe..45a01e3f745 100644 --- a/release_lib.sh +++ b/utils/release/release_lib.sh @@ -9,7 +9,7 @@ function gen_version_string { } function get_version { - BASEDIR=$(dirname "${BASH_SOURCE[0]}") + BASEDIR=$(dirname "${BASH_SOURCE[0]}")/../../ VERSION_REVISION=`grep "set(VERSION_REVISION" ${BASEDIR}/dbms/cmake/version.cmake | sed 's/^.*VERSION_REVISION \(.*\)$/\1/' | sed 's/[) ].*//'` VERSION_MAJOR=`grep "set(VERSION_MAJOR" ${BASEDIR}/dbms/cmake/version.cmake | sed 's/^.*VERSION_MAJOR \(.*\)/\1/' | sed 's/[) ].*//'` VERSION_MINOR=`grep "set(VERSION_MINOR" ${BASEDIR}/dbms/cmake/version.cmake | sed 's/^.*VERSION_MINOR \(.*\)/\1/' | sed 's/[) ].*//'` diff --git a/website/index.html b/website/index.html index dab2cbf30bb..c4fd39267ec 100644 --- a/website/index.html +++ b/website/index.html @@ -459,7 +459,7 @@ clickhouse-client ClickHouse source code is published under Apache 2.0 License. Software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

- + diff --git a/website/tutorial.html b/website/tutorial.html index 558d9a0d0fe..2d8a2584f06 100644 --- a/website/tutorial.html +++ b/website/tutorial.html @@ -596,7 +596,7 @@ ENGINE = ReplicatedMergeTree( ClickHouse source code is published under Apache 2.0 License. Software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

- +