thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-26 09:32:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into update_format_settings_docs

Browse Source
This commit is contained in:
Kruglov Pavel 2022-06-02 13:12:58 +02:00 committed by GitHub
commit 23290c6047
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
250 changed files with 3378 additions and 2082 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
.github/workflows/master.yml vendored
View File

@ -215,8 +215,8 @@ jobs:
fetch-depth: 0 # For a proper version and performance artifacts
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -259,8 +259,8 @@ jobs:
fetch-depth: 0 # For a proper version and performance artifacts
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -305,8 +305,8 @@ jobs:
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -350,8 +350,8 @@ jobs:
# uses: actions/checkout@v2
# - name: Build
# run: |
# git -C "$GITHUB_WORKSPACE" submodule sync
# git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
# git -C "$GITHUB_WORKSPACE" submodule sync --recursive
# git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
# sudo rm -fr "$TEMP_PATH"
# mkdir -p "$TEMP_PATH"
# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -395,8 +395,8 @@ jobs:
uses: actions/checkout@v2
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -440,8 +440,8 @@ jobs:
uses: actions/checkout@v2
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -485,8 +485,8 @@ jobs:
uses: actions/checkout@v2
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -530,8 +530,8 @@ jobs:
uses: actions/checkout@v2
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -575,8 +575,8 @@ jobs:
uses: actions/checkout@v2
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -623,8 +623,8 @@ jobs:
uses: actions/checkout@v2
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -668,8 +668,8 @@ jobs:
uses: actions/checkout@v2
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -715,8 +715,8 @@ jobs:
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -762,8 +762,8 @@ jobs:
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -809,8 +809,8 @@ jobs:
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -856,8 +856,8 @@ jobs:
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -903,8 +903,8 @@ jobs:
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"

7
.github/workflows/nightly.yml vendored
View File

@ -81,7 +81,6 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF'
BUILD_NAME=coverity
CACHES_PATH=${{runner.temp}}/../ccaches
CHECK_NAME=ClickHouse build check (actions)
IMAGES_PATH=${{runner.temp}}/images_path
REPO_COPY=${{runner.temp}}/build_check/ClickHouse
TEMP_PATH=${{runner.temp}}/build_check
@ -99,13 +98,15 @@ jobs:
id: coverity-checkout
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci" && python3 build_check.py "$CHECK_NAME" "$BUILD_NAME"
cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
- name: Upload Coverity Analysis
if: ${{ success() || failure() }}
run: |

64
.github/workflows/pull_request.yml vendored
View File

@ -277,8 +277,8 @@ jobs:
fetch-depth: 0 # for performance artifact
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -322,8 +322,8 @@ jobs:
uses: actions/checkout@v2
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -367,8 +367,8 @@ jobs:
# uses: actions/checkout@v2
# - name: Build
# run: |
# git -C "$GITHUB_WORKSPACE" submodule sync
# git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
# git -C "$GITHUB_WORKSPACE" submodule sync --recursive
# git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
# sudo rm -fr "$TEMP_PATH"
# mkdir -p "$TEMP_PATH"
# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -414,8 +414,8 @@ jobs:
fetch-depth: 0 # for performance artifact
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -459,8 +459,8 @@ jobs:
uses: actions/checkout@v2
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -504,8 +504,8 @@ jobs:
uses: actions/checkout@v2
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -549,8 +549,8 @@ jobs:
uses: actions/checkout@v2
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -594,8 +594,8 @@ jobs:
uses: actions/checkout@v2
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -639,8 +639,8 @@ jobs:
uses: actions/checkout@v2
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -687,8 +687,8 @@ jobs:
uses: actions/checkout@v2
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -732,8 +732,8 @@ jobs:
uses: actions/checkout@v2
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -777,8 +777,8 @@ jobs:
uses: actions/checkout@v2
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -822,8 +822,8 @@ jobs:
uses: actions/checkout@v2
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -867,8 +867,8 @@ jobs:
uses: actions/checkout@v2
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -912,8 +912,8 @@ jobs:
uses: actions/checkout@v2
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -957,8 +957,8 @@ jobs:
uses: actions/checkout@v2
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"

6
.gitmodules vendored
View File

@ -79,10 +79,10 @@
url = https://github.com/ClickHouse/snappy.git
[submodule "contrib/cppkafka"]
path = contrib/cppkafka
url = https://github.com/ClickHouse/cppkafka.git
url = https://github.com/mfontanini/cppkafka.git
[submodule "contrib/brotli"]
path = contrib/brotli
url = https://github.com/ClickHouse/brotli.git
url = https://github.com/google/brotli.git
[submodule "contrib/h3"]
path = contrib/h3
url = https://github.com/ClickHouse/h3
@ -144,7 +144,7 @@
ignore = untracked
[submodule "contrib/msgpack-c"]
path = contrib/msgpack-c
url = https://github.com/ClickHouse/msgpack-c
url = https://github.com/msgpack/msgpack-c
[submodule "contrib/libcpuid"]
path = contrib/libcpuid
url = https://github.com/ClickHouse/libcpuid.git

2
CMakeLists.txt
View File

@ -36,7 +36,7 @@ set_property(GLOBAL PROPERTY USE_FOLDERS ON)
# Check that submodules are present
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/sysroot/README.md")
message (FATAL_ERROR "Submodules are not initialized. Run\n\tgit submodule update --init --recursive")
message (FATAL_ERROR "Submodules are not initialized. Run\n\tgit submodule update --init")
endif ()
# Take care to add prlimit in command line before ccache, or else ccache thinks that

5
base/base/CMakeLists.txt
View File

@ -17,15 +17,12 @@ set (SRCS
sleep.cpp
terminalColors.cpp
errnoToString.cpp
ReplxxLineReader.cpp
StringRef.cpp
safeExit.cpp
throwError.cpp
)
if (ENABLE_REPLXX)
list (APPEND SRCS ReplxxLineReader.cpp)
endif ()
if (USE_DEBUG_HELPERS)
get_target_property(MAGIC_ENUM_INCLUDE_DIR ch_contrib::magic_enum INTERFACE_INCLUDE_DIRECTORIES)
# CMake generator expression will do insane quoting when it encounters special character like quotes, spaces, etc.

3
contrib/abseil-cpp-cmake/CMakeLists.txt
View File

@ -1,7 +1,4 @@
set(ABSL_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/abseil-cpp")
if(NOT EXISTS "${ABSL_ROOT_DIR}/CMakeLists.txt")
message(FATAL_ERROR " submodule third_party/abseil-cpp is missing. To fix try run: \n git submodule update --init --recursive")
endif()
set(BUILD_TESTING OFF)
set(ABSL_PROPAGATE_CXX_STD ON)
add_subdirectory("${ABSL_ROOT_DIR}" "${ClickHouse_BINARY_DIR}/contrib/abseil-cpp")

16
contrib/amqpcpp-cmake/CMakeLists.txt
View File

@ -5,6 +5,7 @@ if (NOT ENABLE_AMQPCPP)
return()
endif()
# can be removed once libuv build on MacOS with GCC is possible
if (NOT TARGET ch_contrib::uv)
message(STATUS "Not using AMQP-CPP because libuv is disabled")
return()
@ -37,21 +38,6 @@ set (SRCS
add_library(_amqp-cpp ${SRCS})
target_compile_options (_amqp-cpp
PRIVATE
-Wno-old-style-cast
-Wno-inconsistent-missing-destructor-override
-Wno-deprecated
-Wno-unused-parameter
-Wno-shadow
-Wno-tautological-type-limit-compare
-Wno-extra-semi
# NOTE: disable all warnings at last because the warning:
# "conversion function converting 'XXX' to itself will never be used"
# doesn't have it's own diagnostic flag yet.
-w
)
target_include_directories (_amqp-cpp SYSTEM BEFORE PUBLIC "${LIBRARY_DIR}/include" "${LIBRARY_DIR}")
target_link_libraries (_amqp-cpp PUBLIC OpenSSL::Crypto OpenSSL::SSL ch_contrib::uv)
add_library (ch_contrib::amqp_cpp ALIAS _amqp-cpp)

2
contrib/arrow vendored

@ -1 +1 @@
Subproject commit 6f274b737c66a6c39bab0d3bdf6cf7d139ef06f5
Subproject commit efdcd015cfdee1b6aa349c9ca227ca12c3d697f5

2
contrib/arrow-cmake/CMakeLists.txt
View File

@ -20,7 +20,7 @@ endif()
option (ENABLE_PARQUET "Enable parquet" ${ENABLE_PARQUET_DEFAULT})
if (NOT ENABLE_PARQUET)
message(STATUS "Building without Parquet support")
message(STATUS "Not using parquet")
return()
endif()

8
contrib/avro-cmake/CMakeLists.txt
View File

@ -60,14 +60,6 @@ target_compile_definitions (_avrocpp PUBLIC SNAPPY_CODEC_AVAILABLE)
target_include_directories (_avrocpp PRIVATE ${SNAPPY_INCLUDE_DIR})
target_link_libraries (_avrocpp PRIVATE ch_contrib::snappy)
if (COMPILER_GCC)
set (SUPPRESS_WARNINGS -Wno-non-virtual-dtor)
elseif (COMPILER_CLANG)
set (SUPPRESS_WARNINGS -Wno-non-virtual-dtor)
endif ()
target_compile_options(_avrocpp PRIVATE ${SUPPRESS_WARNINGS})
# create a symlink to include headers with <avro/...>
set(AVRO_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/include")
ADD_CUSTOM_TARGET(avro_symlink_headers ALL

14
contrib/azure-cmake/CMakeLists.txt
View File

@ -52,20 +52,6 @@ include("${AZURE_DIR}/cmake-modules/AzureTransportAdapters.cmake")
add_library(_azure_sdk ${AZURE_SDK_UNIFIED_SRC})
if (COMPILER_CLANG)
target_compile_options(_azure_sdk PRIVATE
-Wno-deprecated-copy-dtor
-Wno-extra-semi
-Wno-suggest-destructor-override
-Wno-inconsistent-missing-destructor-override
-Wno-error=unknown-warning-option
)
if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 13)
target_compile_options(_azure_sdk PRIVATE -Wno-reserved-identifier)
endif()
endif()
# Originally, on Windows azure-core is built with bcrypt and crypt32 by default
if (TARGET OpenSSL::SSL)
target_link_libraries(_azure_sdk PRIVATE OpenSSL::Crypto OpenSSL::SSL)

7
contrib/boringssl-cmake/CMakeLists.txt
View File

@ -1,7 +1,12 @@
# Needed for:
# - securely connecting to an external server, e.g. clickhouse-client --host ... --secure
# - lots of thirdparty libraries
option(ENABLE_SSL "Enable ssl" ${ENABLE_LIBRARIES})
# Actually, so many 3rd party libraries + unit tests need SSL that we cannot disable it
# without breaking the build ...
option(ENABLE_SSL "Enable ssl" ON) # breaks if OFF
# TODO: Making SSL dependent on ENABLE_LIBRARIES is desirable but needs fixing dependent libs + tests.
# option(ENABLE_SSL "Enable ssl" ${ENABLE_LIBRARIES})
if(NOT ENABLE_SSL)
message(STATUS "Not using openssl")

2
contrib/brotli vendored

@ -1 +1 @@
Subproject commit 5bd78768449751a78d4b4c646b0612917986f5b1
Subproject commit 63be8a99401992075c23e99f7c84de1c653e39e2

3
contrib/brotli-cmake/CMakeLists.txt
View File

@ -45,7 +45,4 @@ add_library(ch_contrib::brotli ALIAS _brotli)
target_include_directories(_brotli SYSTEM BEFORE PUBLIC "${BROTLI_SOURCE_DIR}/include")
if(M_LIBRARY)
target_link_libraries(_brotli PRIVATE ${M_LIBRARY})
endif()
target_compile_definitions(_brotli PRIVATE BROTLI_BUILD_PORTABLE=1)

6
contrib/bzip2-cmake/CMakeLists.txt
View File

@ -1,6 +1,6 @@
option(ENABLE_BZIP2 "Enable bzip2 compression support" ${ENABLE_LIBRARIES})
if (NOT ENABLE_BZIP2)
message (STATUS "bzip2 compression disabled")
message (STATUS "Not using bzip2")
return()
endif()
@ -26,8 +26,4 @@ configure_file (
add_library(_bzip2 ${SRCS})
add_library(ch_contrib::bzip2 ALIAS _bzip2)
# To avoid -Wreserved-id-macro we use SYSTEM:
#
# clickhouse/contrib/bzip2/bzlib.h:23:9: error: macro name is a reserved identifier [-Werror,-Wreserved-id-macro]
# #define _BZLIB_H
target_include_directories(_bzip2 SYSTEM BEFORE PUBLIC "${BZIP2_SOURCE_DIR}" "${BZIP2_BINARY_DIR}")

12
contrib/capnproto-cmake/CMakeLists.txt
View File

@ -81,16 +81,12 @@ set (CAPNPC_SRCS
add_library(_capnpc ${CAPNPC_SRCS})
target_link_libraries(_capnpc PUBLIC _capnp)
# The library has substandard code
if (COMPILER_GCC)
set (SUPPRESS_WARNINGS -w)
elseif (COMPILER_CLANG)
set (SUPPRESS_WARNINGS -w)
if (COMPILER_CLANG)
set (CAPNP_PRIVATE_CXX_FLAGS -fno-char8_t)
endif ()
target_compile_options(_kj PRIVATE ${SUPPRESS_WARNINGS} ${CAPNP_PRIVATE_CXX_FLAGS})
target_compile_options(_capnp PRIVATE ${SUPPRESS_WARNINGS} ${CAPNP_PRIVATE_CXX_FLAGS})
target_compile_options(_capnpc PRIVATE ${SUPPRESS_WARNINGS} ${CAPNP_PRIVATE_CXX_FLAGS})
target_compile_options(_kj PRIVATE ${CAPNP_PRIVATE_CXX_FLAGS})
target_compile_options(_capnp PRIVATE ${CAPNP_PRIVATE_CXX_FLAGS})
target_compile_options(_capnpc PRIVATE ${CAPNP_PRIVATE_CXX_FLAGS})
add_library(ch_contrib::capnp ALIAS _capnpc)

1
contrib/cassandra-cmake/CMakeLists.txt
View File

@ -5,6 +5,7 @@ if (NOT ENABLE_CASSANDRA)
return()
endif()
# can be removed once libuv build on MacOS with GCC is possible
if (NOT TARGET ch_contrib::uv)
message(STATUS "Not using cassandra because libuv is disabled")
return()

2
contrib/cppkafka vendored

@ -1 +1 @@
Subproject commit 64bd67db12b9c705e9127439a5b05b351d9df7da
Subproject commit 5a119f689f8a4d90d10a9635e7ee2bee5c127de1

2
contrib/cppkafka-cmake/CMakeLists.txt
View File

@ -1,5 +1,5 @@
if (NOT ENABLE_KAFKA)
message(STATUS "Not using librdkafka (skip cppkafka)")
message(STATUS "Not using kafka")
return()
endif()

2
contrib/fastops-cmake/CMakeLists.txt
View File

@ -5,7 +5,7 @@ elseif(ENABLE_FASTOPS)
endif()
if(NOT ENABLE_FASTOPS)
message(STATUS "Not using fast vectorized mathematical functions library by Mikhail Parakhin")
message(STATUS "Not using fastops")
return()
endif()

32
contrib/fmtlib-cmake/CMakeLists.txt
View File

@ -1,22 +1,24 @@
set(FMT_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/fmtlib")
set (SRCS
# NOTE: do not build module for now:
# ../fmtlib/src/fmt.cc
../fmtlib/src/format.cc
../fmtlib/src/os.cc
${FMT_SOURCE_DIR}/src/format.cc
${FMT_SOURCE_DIR}/src/os.cc
../fmtlib/include/fmt/args.h
../fmtlib/include/fmt/chrono.h
../fmtlib/include/fmt/color.h
../fmtlib/include/fmt/compile.h
../fmtlib/include/fmt/core.h
../fmtlib/include/fmt/format.h
../fmtlib/include/fmt/format-inl.h
../fmtlib/include/fmt/locale.h
../fmtlib/include/fmt/os.h
../fmtlib/include/fmt/ostream.h
../fmtlib/include/fmt/printf.h
../fmtlib/include/fmt/ranges.h
../fmtlib/include/fmt/xchar.h
${FMT_SOURCE_DIR}/include/fmt/args.h
${FMT_SOURCE_DIR}/include/fmt/chrono.h
${FMT_SOURCE_DIR}/include/fmt/color.h
${FMT_SOURCE_DIR}/include/fmt/compile.h
${FMT_SOURCE_DIR}/include/fmt/core.h
${FMT_SOURCE_DIR}/include/fmt/format.h
${FMT_SOURCE_DIR}/include/fmt/format-inl.h
${FMT_SOURCE_DIR}/include/fmt/locale.h
${FMT_SOURCE_DIR}/include/fmt/os.h
${FMT_SOURCE_DIR}/include/fmt/ostream.h
${FMT_SOURCE_DIR}/include/fmt/printf.h
${FMT_SOURCE_DIR}/include/fmt/ranges.h
${FMT_SOURCE_DIR}/include/fmt/xchar.h
)
add_library(_fmt ${SRCS})

37
contrib/h3-cmake/CMakeLists.txt
View File

@ -9,23 +9,23 @@ set(H3_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/h3/src/h3lib")
set(H3_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/h3/src/h3lib")
set(SRCS
"${H3_SOURCE_DIR}/lib/algos.c"
"${H3_SOURCE_DIR}/lib/coordijk.c"
"${H3_SOURCE_DIR}/lib/bbox.c"
"${H3_SOURCE_DIR}/lib/polygon.c"
"${H3_SOURCE_DIR}/lib/h3Index.c"
"${H3_SOURCE_DIR}/lib/vec2d.c"
"${H3_SOURCE_DIR}/lib/vec3d.c"
"${H3_SOURCE_DIR}/lib/vertex.c"
"${H3_SOURCE_DIR}/lib/linkedGeo.c"
"${H3_SOURCE_DIR}/lib/localij.c"
"${H3_SOURCE_DIR}/lib/latLng.c"
"${H3_SOURCE_DIR}/lib/directedEdge.c"
"${H3_SOURCE_DIR}/lib/mathExtensions.c"
"${H3_SOURCE_DIR}/lib/iterators.c"
"${H3_SOURCE_DIR}/lib/vertexGraph.c"
"${H3_SOURCE_DIR}/lib/faceijk.c"
"${H3_SOURCE_DIR}/lib/baseCells.c"
"${H3_SOURCE_DIR}/lib/algos.c"
"${H3_SOURCE_DIR}/lib/coordijk.c"
"${H3_SOURCE_DIR}/lib/bbox.c"
"${H3_SOURCE_DIR}/lib/polygon.c"
"${H3_SOURCE_DIR}/lib/h3Index.c"
"${H3_SOURCE_DIR}/lib/vec2d.c"
"${H3_SOURCE_DIR}/lib/vec3d.c"
"${H3_SOURCE_DIR}/lib/vertex.c"
"${H3_SOURCE_DIR}/lib/linkedGeo.c"
"${H3_SOURCE_DIR}/lib/localij.c"
"${H3_SOURCE_DIR}/lib/latLng.c"
"${H3_SOURCE_DIR}/lib/directedEdge.c"
"${H3_SOURCE_DIR}/lib/mathExtensions.c"
"${H3_SOURCE_DIR}/lib/iterators.c"
"${H3_SOURCE_DIR}/lib/vertexGraph.c"
"${H3_SOURCE_DIR}/lib/faceijk.c"
"${H3_SOURCE_DIR}/lib/baseCells.c"
)
configure_file("${H3_SOURCE_DIR}/include/h3api.h.in" "${H3_BINARY_DIR}/include/h3api.h")
@ -34,8 +34,5 @@ add_library(_h3 ${SRCS})
target_include_directories(_h3 SYSTEM PUBLIC "${H3_SOURCE_DIR}/include")
target_include_directories(_h3 SYSTEM PUBLIC "${H3_BINARY_DIR}/include")
target_compile_definitions(_h3 PRIVATE H3_HAVE_VLA)
if(M_LIBRARY)
target_link_libraries(_h3 PRIVATE ${M_LIBRARY})
endif()
add_library(ch_contrib::h3 ALIAS _h3)

2
contrib/hive-metastore-cmake/CMakeLists.txt
View File

@ -5,7 +5,7 @@ elseif(ENABLE_HIVE)
endif()
if (NOT ENABLE_HIVE)
message("Hive disabled")
message(STATUS "Not using hive")
return()
endif()

4
contrib/icu-cmake/CMakeLists.txt
View File

@ -481,10 +481,6 @@ target_include_directories(_icui18n SYSTEM PUBLIC "${ICU_SOURCE_DIR}/i18n/")
target_compile_definitions(_icuuc PRIVATE -DU_COMMON_IMPLEMENTATION)
target_compile_definitions(_icui18n PRIVATE -DU_I18N_IMPLEMENTATION)
if (COMPILER_CLANG)
target_compile_options(_icudata PRIVATE -Wno-unused-command-line-argument)
endif ()
add_library(_icu INTERFACE)
target_link_libraries(_icu INTERFACE _icui18n _icuuc _icudata)
add_library(ch_contrib::icu ALIAS _icu)

1
contrib/jemalloc-cmake/CMakeLists.txt
View File

@ -180,7 +180,6 @@ if (USE_UNWIND)
target_link_libraries (_jemalloc PRIVATE unwind)
endif ()
target_compile_options(_jemalloc PRIVATE -Wno-redundant-decls)
# for RTLD_NEXT
target_compile_options(_jemalloc PRIVATE -D_GNU_SOURCE)

5
contrib/libcpuid-cmake/CMakeLists.txt
View File

@ -6,7 +6,7 @@ elseif(ENABLE_CPUID)
endif()
if (NOT ENABLE_CPUID)
message("Not using cpuid")
message(STATUS "Not using cpuid")
return()
endif()
@ -27,8 +27,5 @@ add_library (_cpuid ${SRCS})
target_include_directories (_cpuid SYSTEM PUBLIC "${LIBRARY_DIR}")
target_compile_definitions (_cpuid PRIVATE VERSION="v0.4.1")
if (COMPILER_CLANG)
target_compile_options (_cpuid PRIVATE -Wno-reserved-id-macro)
endif ()
add_library(ch_contrib::cpuid ALIAS _cpuid)

2
contrib/libgsasl-cmake/CMakeLists.txt
View File

@ -1,7 +1,7 @@
option(ENABLE_GSASL_LIBRARY "Enable gsasl library" ${ENABLE_LIBRARIES})
if (NOT ENABLE_GSASL_LIBRARY)
message(STATUS "Not using gsasl library")
message(STATUS "Not using gsasl")
return()
endif()

1
contrib/libuv-cmake/CMakeLists.txt
View File

@ -1,3 +1,4 @@
# once fixed, please remove similar places in CMakeLists of libuv users (search "ch_contrib::uv")
if (OS_DARWIN AND COMPILER_GCC)
message (WARNING "libuv cannot be built with GCC in macOS due to a bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=93082")
return()

3
contrib/libxml2-cmake/CMakeLists.txt
View File

@ -53,9 +53,6 @@ set(SRCS
add_library(_libxml2 ${SRCS})
target_link_libraries(_libxml2 PRIVATE ch_contrib::zlib)
if(M_LIBRARY)
target_link_libraries(_libxml2 PRIVATE ${M_LIBRARY})
endif()
target_include_directories(_libxml2 BEFORE PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64/include")
target_include_directories(_libxml2 BEFORE PUBLIC "${LIBXML2_SOURCE_DIR}/include")

2
contrib/minizip-ng-cmake/CMakeLists.txt
View File

@ -1,6 +1,6 @@
option(ENABLE_MINIZIP "Enable minizip-ng the zip manipulation library" ${ENABLE_LIBRARIES})
if (NOT ENABLE_MINIZIP)
message (STATUS "minizip-ng disabled")
message (STATUS "Not using minizip-ng")
return()
endif()

2
contrib/msgpack-c vendored

@ -1 +1 @@
Subproject commit 790b3fe58ebded7a8bd130782ef28bec5784c248
Subproject commit 46684265d50b5d1b062d4c5c428ba08462844b1d

4
contrib/nanodbc-cmake/CMakeLists.txt
View File

@ -2,12 +2,12 @@ if (NOT ENABLE_ODBC)
return ()
endif ()
set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/nanodbc")
if (NOT TARGET ch_contrib::unixodbc)
message(FATAL_ERROR "Configuration error: unixodbc is not a target")
endif()
set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/nanodbc")
set (SRCS
"${LIBRARY_DIR}/nanodbc/nanodbc.cpp"
)

2
contrib/rapidjson vendored

@ -1 +1 @@
Subproject commit b571bd5c1a3b1fc931d77ae36932537a3c9018c3
Subproject commit c4ef90ccdbc21d5d5a628d08316bfd301e32d6fa

12
contrib/replxx-cmake/CMakeLists.txt
View File

@ -1,10 +1,3 @@
option (ENABLE_REPLXX "Enable replxx support" ${ENABLE_LIBRARIES})
if (NOT ENABLE_REPLXX)
message (STATUS "Not using replxx")
return()
endif()
set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/replxx")
set(SRCS
@ -22,9 +15,4 @@ set(SRCS
add_library (_replxx ${SRCS})
target_include_directories(_replxx SYSTEM PUBLIC "${LIBRARY_DIR}/include")
if (COMPILER_CLANG)
target_compile_options(_replxx PRIVATE -Wno-documentation)
endif ()
add_library(ch_contrib::replxx ALIAS _replxx)

4
contrib/s2geometry-cmake/CMakeLists.txt
View File

@ -149,7 +149,3 @@ target_link_libraries(_s2 PRIVATE
target_include_directories(_s2 SYSTEM BEFORE PUBLIC "${S2_SOURCE_DIR}/")
target_include_directories(_s2 SYSTEM PUBLIC "${ABSL_SOURCE_DIR}")
if(M_LIBRARY)
target_link_libraries(_s2 PRIVATE ${M_LIBRARY})
endif()

2
contrib/snappy vendored

@ -1 +1 @@
Subproject commit 3786173af204d21da97180977ad6ab4321138b3d
Subproject commit fb057edfed820212076239fd32cb2ff23e9016bf

2
contrib/thrift-cmake/CMakeLists.txt
View File

@ -1,7 +1,7 @@
option(ENABLE_THRIFT "Enable Thrift" ${ENABLE_LIBRARIES})
if (NOT ENABLE_THRIFT)
message (STATUS "thrift disabled")
message (STATUS "Not using thrift")
return()
endif()

10
contrib/unixodbc-cmake/CMakeLists.txt
View File

@ -294,14 +294,6 @@ target_include_directories (_unixodbc
"${LIBRARY_DIR}/include"
)
target_compile_definitions (_unixodbc PRIVATE -DHAVE_CONFIG_H)
target_compile_options (_unixodbc
PRIVATE
-Wno-dangling-else
-Wno-parentheses
-Wno-misleading-indentation
-Wno-unknown-warning-option
-Wno-reserved-id-macro
-O2
)
target_compile_options (_unixodbc PRIVATE -O2) # intended?
add_library (ch_contrib::unixodbc ALIAS _unixodbc)

2
docker/test/stateless/run.sh
View File

@ -92,8 +92,6 @@ function run_tests()
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
ADDITIONAL_OPTIONS+=('--replicated-database')
# Cannot be used with replicated database, due to distributed_ddl_output_mode=none
ADDITIONAL_OPTIONS+=('--no-left-queries-check')
ADDITIONAL_OPTIONS+=('--jobs')
ADDITIONAL_OPTIONS+=('2')
else

2
docs/en/development/tests.md
View File

@ -81,7 +81,7 @@ $ ./src/unit_tests_dbms --gtest_filter=LocalAddress*
## Performance Tests {#performance-tests}
Performance tests allow to measure and compare performance of some isolated part of ClickHouse on synthetic queries. Tests are located at `tests/performance`. Each test is represented by `.xml` file with description of test case. Tests are run with `docker/tests/performance-comparison` tool . See the readme file for invocation.
Performance tests allow to measure and compare performance of some isolated part of ClickHouse on synthetic queries. Tests are located at `tests/performance`. Each test is represented by `.xml` file with description of test case. Tests are run with `docker/test/performance-comparison` tool . See the readme file for invocation.
Each test run one or multiple queries (possibly with combinations of parameters) in a loop.

2
docs/en/operations/caches.md
View File

@ -5,7 +5,7 @@ sidebar_label: Caches
# Cache Types {#cache-types}
When performing queries, ClichHouse uses different caches.
When performing queries, ClickHouse uses different caches.
Main cache types:

10
docs/en/operations/server-configuration-parameters/settings.md
View File

@ -1745,13 +1745,3 @@ Possible values:
- Positive integer.
Default value: `10000`.
## global_memory_usage_overcommit_max_wait_microseconds {#global_memory_usage_overcommit_max_wait_microseconds}
Sets maximum waiting time for global overcommit tracker.
Possible values:
- Positive integer.
Default value: `200`.

2
docs/en/operations/settings/settings.md
View File

@ -3318,7 +3318,7 @@ Maximum time thread will wait for memory to be freed in the case of memory overc
If the timeout is reached and memory is not freed, an exception is thrown.
Read more about [memory overcommit](memory-overcommit.md).
Default value: `200`.
Default value: `5000000`.
## memory_overcommit_ratio_denominator_for_user

6
docs/en/operations/system-tables/part_log.md
View File

@ -14,6 +14,11 @@ The `system.part_log` table contains the following columns:
- `REMOVE_PART` — Removing or detaching a data part using [DETACH PARTITION](../../sql-reference/statements/alter/partition.md#alter_detach-partition).
- `MUTATE_PART` — Mutating of a data part.
- `MOVE_PART` — Moving the data part from the one disk to another one.
- `merge_reason` ([Enum8](../../sql-reference/data-types/enum.md)) — The reason for the event with type `MERGE_PARTS`. Can have one of the following values:
- `NOT_A_MERGE` — The current event has the type other than `MERGE_PARTS`.
- `REGULAR_MERGE` — Some regular merge.
- `TTL_DELETE_MERGE` — Cleaning up expired data.
- `TTL_RECOMPRESS_MERGE` — Recompressing data part with the.
- `event_date` ([Date](../../sql-reference/data-types/date.md)) — Event date.
- `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Event time.
- `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — Event time with microseconds precision.
@ -46,6 +51,7 @@ Row 1:
──────
query_id: 983ad9c7-28d5-4ae1-844e-603116b7de31
event_type: NewPart
merge_reason: NotAMerge
event_date: 2021-02-02
event_time: 2021-02-02 11:14:28
event_time_microseconds: 2021-02-02 11:14:28.861919

68
docs/zh/sql-reference/statements/insert-into.md
View File

@ -71,7 +71,7 @@ INSERT INTO [db.]table [(c1, c2, c3)] FORMAT format_name data_set
INSERT INTO [db.]table [(c1, c2, c3)] FORMAT Values (v11, v12, v13), (v21, v22, v23), ...
```
ClickHouse会清除数据前所有的空白字符与一行摘要信息（如果需要的话）。所以在进行查询时，我们建议您将数据放入到输入输出格式名称后的新的一行中去（如果数据是以空白字符开始的，这将非常重要）。
ClickHouse会清除数据前所有的空白字符与一个换行符（如果有换行符的话）。所以在进行查询时，我们建议您将数据放入到输入输出格式名称后的新的一行中去（如果数据是以空白字符开始的，这将非常重要）。
示例:
@ -83,6 +83,10 @@ INSERT INTO t FORMAT TabSeparated
在使用命令行客户端或HTTP客户端时，你可以将具体的查询语句与数据分开发送。更多具体信息，请参考«[客户端](../../interfaces/index.md#interfaces)»部分。
### 限制 {#constraints}
如果表中有一些[限制](../../sql-reference/statements/create/table.md#constraints),，数据插入时会逐行进行数据校验，如果这里面包含了不符合限制条件的数据，服务将会抛出包含限制信息的异常，这个语句也会被停止执行。
### 使用`SELECT`的结果写入 {#insert_query_insert-select}
``` sql
@ -96,6 +100,66 @@ INSERT INTO [db.]table [(c1, c2, c3)] SELECT ...
系统不支持的其他用于修改数据的查询：`UPDATE`, `DELETE`, `REPLACE`, `MERGE`, `UPSERT`, `INSERT UPDATE`。
但是，您可以使用 `ALTER TABLE ... DROP PARTITION`查询来删除一些旧的数据。
如果 `SELECT` 查询中包含了 [input()](../../sql-reference/table-functions/input.md) 函数，那么 `FORMAT` 必须出现在查询语句的最后。
如果某一列限制了值不能是NULL，那么插入NULL的时候就会插入这个列类型的默认数据，可以通过设置 [insert_null_as_default](../../operations/settings/settings.md#insert_null_as_default) 插入NULL。
### 从文件向表中插入数据 {#inserting-data-from-a-file}
``` sql
INSERT INTO [db.]table [(c1, c2, c3)] FROM INFILE file_name [COMPRESSION type] FORMAT format_name
```
使用上面的语句可以从客户端的文件上读取数据并插入表中，`file_name` 和 `type` 都是 `String` 类型，输入文件的[格式](../../interfaces/formats.md) 一定要在 `FORMAT` 语句中设置。
支持读取压缩文件。默认会去读文件的拓展名作为文件的压缩方式，或者也可以在 `COMPRESSION` 语句中指明，支持的文件压缩格式如下：`'none'`， `'gzip'`， `'deflate'`， `'br'`， `'xz'`， `'zstd'`， `'lz4'`， `'bz2'`。
这个功能在 [command-line client](../../interfaces/cli.md) 和 [clickhouse-local](../../operations/utilities/clickhouse-local.md) 是可用的。
**样例**
```bash
echo 1,A > input.csv ; echo 2,B >> input.csv
clickhouse-client --query="CREATE TABLE table_from_file (id UInt32, text String) ENGINE=MergeTree() ORDER BY id;"
clickhouse-client --query="INSERT INTO table_from_file FROM INFILE 'input.csv' FORMAT CSV;"
clickhouse-client --query="SELECT * FROM table_from_file FORMAT PrettyCompact;"
```
结果:
```text
┌─id─┬─text─┐
│ 1 │ A │
│ 2 │ B │
└────┴──────┘
```
### 插入表函数 {#inserting-into-table-function}
数据可以通过 [table functions](../../sql-reference/table-functions/index.md) 方法插入。
``` sql
INSERT INTO [TABLE] FUNCTION table_func ...
```
**例如**
可以这样使用[remote](../../sql-reference/table-functions/index.md#remote) 表函数:
``` sql
CREATE TABLE simple_table (id UInt32, text String) ENGINE=MergeTree() ORDER BY id;
INSERT INTO TABLE FUNCTION remote('localhost', default.simple_table)
VALUES (100, 'inserted via remote()');
SELECT * FROM simple_table;
```
结果:
``` text
┌──id─┬─text──────────────────┐
│ 100 │ inserted via remote() │
└─────┴───────────────────────┘
```
### 性能的注意事项 {#xing-neng-de-zhu-yi-shi-xiang}
在进行`INSERT`时将会对写入的数据进行一些处理，按照主键排序，按照月份对数据进行分区等。所以如果在您的写入数据中包含多个月份的混合数据时，将会显著的降低`INSERT`的性能。为了避免这种情况：
@ -108,4 +172,6 @@ INSERT INTO [db.]table [(c1, c2, c3)] SELECT ...
- 数据总是被实时的写入。
- 写入的数据已经按照时间排序。
也可以异步的、小规模的插入数据，这些数据会被合并成多个批次，然后安全地写入到表中，通过设置[async_insert](../../operations/settings/settings.md#async-insert)，可以使用异步插入的方式，请注意，异步插入的方式只支持HTTP协议，并且不支持数据去重。
[来源文章](https://clickhouse.com/docs/en/query_language/insert_into/) <!--hide-->

29
programs/server/Server.cpp
View File

@ -546,8 +546,9 @@ static void sanityChecks(Server & server)
#if defined(OS_LINUX)
try
{
if (readString("/sys/devices/system/clocksource/clocksource0/current_clocksource").find("tsc") == std::string::npos)
server.context()->addWarningMessage("Linux is not using a fast TSC clock source. Performance can be degraded.");
const char * filename = "/sys/devices/system/clocksource/clocksource0/current_clocksource";
if (readString(filename).find("tsc") == std::string::npos)
server.context()->addWarningMessage("Linux is not using a fast TSC clock source. Performance can be degraded. Check " + String(filename));
}
catch (...)
{
@ -555,8 +556,9 @@ static void sanityChecks(Server & server)
try
{
if (readNumber("/proc/sys/vm/overcommit_memory") == 2)
server.context()->addWarningMessage("Linux memory overcommit is disabled.");
const char * filename = "/proc/sys/vm/overcommit_memory";
if (readNumber(filename) == 2)
server.context()->addWarningMessage("Linux memory overcommit is disabled. Check " + String(filename));
}
catch (...)
{
@ -564,8 +566,9 @@ static void sanityChecks(Server & server)
try
{
if (readString("/sys/kernel/mm/transparent_hugepage/enabled").find("[always]") != std::string::npos)
server.context()->addWarningMessage("Linux transparent hugepages are set to \"always\".");
const char * filename = "/sys/kernel/mm/transparent_hugepage/enabled";
if (readString(filename).find("[always]") != std::string::npos)
server.context()->addWarningMessage("Linux transparent hugepages are set to \"always\". Check " + String(filename));
}
catch (...)
{
@ -573,8 +576,9 @@ static void sanityChecks(Server & server)
try
{
if (readNumber("/proc/sys/kernel/pid_max") < 30000)
server.context()->addWarningMessage("Linux max PID is too low.");
const char * filename = "/proc/sys/kernel/pid_max";
if (readNumber(filename) < 30000)
server.context()->addWarningMessage("Linux max PID is too low. Check " + String(filename));
}
catch (...)
{
@ -582,8 +586,9 @@ static void sanityChecks(Server & server)
try
{
if (readNumber("/proc/sys/kernel/threads-max") < 30000)
server.context()->addWarningMessage("Linux threads max count is too low.");
const char * filename = "/proc/sys/kernel/threads-max";
if (readNumber(filename) < 30000)
server.context()->addWarningMessage("Linux threads max count is too low. Check " + String(filename));
}
catch (...)
{
@ -591,7 +596,7 @@ static void sanityChecks(Server & server)
std::string dev_id = getBlockDeviceId(data_path);
if (getBlockDeviceType(dev_id) == BlockDeviceType::ROT && getBlockDeviceReadAheadBytes(dev_id) == 0)
server.context()->addWarningMessage("Rotational disk with disabled readahead is in use. Performance can be degraded.");
server.context()->addWarningMessage("Rotational disk with disabled readahead is in use. Performance can be degraded. Used for data: " + String(data_path));
#endif
try
@ -1095,8 +1100,6 @@ int Server::main(const std::vector<std::string> & /*args*/)
total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking);
auto * global_overcommit_tracker = global_context->getGlobalOvercommitTracker();
UInt64 max_overcommit_wait_time = config->getUInt64("global_memory_usage_overcommit_max_wait_microseconds", 200);
global_overcommit_tracker->setMaxWaitTime(max_overcommit_wait_time);
total_memory_tracker.setOvercommitTracker(global_overcommit_tracker);
// FIXME logging-related things need synchronization -- see the 'Logger * log' saved

3
src/Access/ContextAccess.cpp
View File

@ -445,10 +445,11 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
const AccessFlags dictionary_ddl = AccessType::CREATE_DICTIONARY | AccessType::DROP_DICTIONARY;
const AccessFlags function_ddl = AccessType::CREATE_FUNCTION | AccessType::DROP_FUNCTION;
const AccessFlags table_and_dictionary_ddl = table_ddl | dictionary_ddl;
const AccessFlags table_and_dictionary_and_function_ddl = table_ddl | dictionary_ddl | function_ddl;
const AccessFlags write_table_access = AccessType::INSERT | AccessType::OPTIMIZE;
const AccessFlags write_dcl_access = AccessType::ACCESS_MANAGEMENT - AccessType::SHOW_ACCESS;
const AccessFlags not_readonly_flags = write_table_access | table_and_dictionary_ddl | write_dcl_access | AccessType::SYSTEM | AccessType::KILL_QUERY;
const AccessFlags not_readonly_flags = write_table_access | table_and_dictionary_and_function_ddl | write_dcl_access | AccessType::SYSTEM | AccessType::KILL_QUERY;
const AccessFlags not_readonly_1_flags = AccessType::CREATE_TEMPORARY_TABLE;
const AccessFlags ddl_flags = table_ddl | dictionary_ddl | function_ddl;

9
src/Common/ColumnsHashing.h
View File

@ -6,9 +6,11 @@
#include <Common/Arena.h>
#include <Common/LRUCache.h>
#include <Common/assert_cast.h>
#include "Columns/IColumn.h"
#include <base/unaligned.h>
#include <Columns/ColumnString.h>
#include <Columns/ColumnConst.h>
#include <Columns/ColumnFixedString.h>
#include <Columns/ColumnLowCardinality.h>
@ -83,8 +85,11 @@ struct HashMethodString
HashMethodString(const ColumnRawPtrs & key_columns, const Sizes & /*key_sizes*/, const HashMethodContextPtr &)
{
const IColumn & column = *key_columns[0];
const ColumnString & column_string = assert_cast<const ColumnString &>(column);
const IColumn * column = key_columns[0];
if (isColumnConst(*column))
column = &assert_cast<const ColumnConst &>(*column).getDataColumn();
const ColumnString & column_string = assert_cast<const ColumnString &>(*column);
offsets = column_string.getOffsets().data();
chars = column_string.getChars().data();
}

2
src/Common/CpuId.h
View File

@ -221,7 +221,7 @@ bool haveAVX512F() noexcept
&& (our_xgetbv(0) & 6u) == 6u // XMM state and YMM state are enabled by OS
&& ((our_xgetbv(0) >> 5) & 7u) == 7u // ZMM state is enabled by OS
&& CpuInfo(0x0).registers.eax >= 0x7 // leaf 7 is present
&& ((CpuInfo(0x7).registers.ebx >> 16) & 1u); // AVX512F bit
&& ((CpuInfo(0x7, 0).registers.ebx >> 16) & 1u); // AVX512F bit
#else
return false;
#endif

44
src/Common/MemoryTracker.cpp
View File

@ -16,6 +16,7 @@
#include <cmath>
#include <random>
#include <cstdlib>
#include <string>
#ifdef MEMORY_TRACKER_DEBUG_CHECKS
@ -52,11 +53,37 @@ namespace DB
}
}
namespace
{
inline std::string_view toDescription(OvercommitResult result)
{
switch (result)
{
case OvercommitResult::NONE:
return "Memory overcommit isn't used. OvercommitTracker isn't set.";
case OvercommitResult::DISABLED:
return "Memory overcommit isn't used. Waiting time or orvercommit denominator are set to zero.";
case OvercommitResult::MEMORY_FREED:
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "OvercommitResult::MEMORY_FREED shouldn't be asked for description");
case OvercommitResult::SELECTED:
return "Query was selected to stop by OvercommitTracker.";
case OvercommitResult::TIMEOUTED:
return "Waiting timeout for memory to be freed is reached.";
case OvercommitResult::NOT_ENOUGH_FREED:
return "Memory overcommit has freed not enough memory.";
}
}
}
namespace ProfileEvents
{
extern const Event QueryMemoryLimitExceeded;
}
using namespace std::chrono_literals;
static constexpr size_t log_peak_memory_usage_every = 1ULL << 30;
MemoryTracker total_memory_tracker(nullptr, VariableContext::Global);
@ -189,11 +216,11 @@ void MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded, MemoryT
if (unlikely(current_hard_limit && will_be > current_hard_limit) && memoryTrackerCanThrow(level, false) && throw_if_memory_exceeded)
{
bool need_to_throw = true;
OvercommitResult overcommit_result = OvercommitResult::NONE;
if (auto * overcommit_tracker_ptr = overcommit_tracker.load(std::memory_order_relaxed); overcommit_tracker_ptr != nullptr && query_tracker != nullptr)
need_to_throw = overcommit_tracker_ptr->needToStopQuery(query_tracker, size);
overcommit_result = overcommit_tracker_ptr->needToStopQuery(query_tracker, size);
if (need_to_throw)
if (overcommit_result != OvercommitResult::MEMORY_FREED)
{
/// Prevent recursion. Exception::ctor -> std::string -> new[] -> MemoryTracker::alloc
MemoryTrackerBlockerInThread untrack_lock(VariableContext::Global);
@ -201,12 +228,13 @@ void MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded, MemoryT
const auto * description = description_ptr.load(std::memory_order_relaxed);
throw DB::Exception(
DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED,
"Memory limit{}{} exceeded: would use {} (attempt to allocate chunk of {} bytes), maximum: {}",
"Memory limit{}{} exceeded: would use {} (attempt to allocate chunk of {} bytes), maximum: {}. OvercommitTracker decision: {}.",
description ? " " : "",
description ? description : "",
formatReadableSizeWithBinarySuffix(will_be),
size,
formatReadableSizeWithBinarySuffix(current_hard_limit));
formatReadableSizeWithBinarySuffix(current_hard_limit),
toDescription(overcommit_result));
}
else
{
@ -337,6 +365,12 @@ OvercommitRatio MemoryTracker::getOvercommitRatio(Int64 limit)
}
void MemoryTracker::setOvercommitWaitingTime(UInt64 wait_time)
{
max_wait_time.store(wait_time * 1us, std::memory_order_relaxed);
}
void MemoryTracker::resetCounters()
{
amount.store(0, std::memory_order_relaxed);

10
src/Common/MemoryTracker.h
View File

@ -1,6 +1,7 @@
#pragma once
#include <atomic>
#include <chrono>
#include <base/types.h>
#include <Common/CurrentMetrics.h>
#include <Common/VariableContext.h>
@ -73,6 +74,8 @@ private:
/// This description will be used as prefix into log messages (if isn't nullptr)
std::atomic<const char *> description_ptr = nullptr;
std::atomic<std::chrono::microseconds> max_wait_time;
std::atomic<OvercommitTracker *> overcommit_tracker = nullptr;
bool updatePeak(Int64 will_be, bool log_memory_usage);
@ -186,6 +189,13 @@ public:
OvercommitRatio getOvercommitRatio();
OvercommitRatio getOvercommitRatio(Int64 limit);
std::chrono::microseconds getOvercommitWaitingTime()
{
return max_wait_time.load(std::memory_order_relaxed);
}
void setOvercommitWaitingTime(UInt64 wait_time);
void setOvercommitTracker(OvercommitTracker * tracker) noexcept
{
overcommit_tracker.store(tracker, std::memory_order_relaxed);

35
src/Common/OvercommitTracker.cpp
View File

@ -2,15 +2,20 @@
#include <chrono>
#include <mutex>
#include <Common/ProfileEvents.h>
#include <Interpreters/ProcessList.h>
namespace ProfileEvents
{
extern const Event MemoryOvercommitWaitTimeMicroseconds;
}
using namespace std::chrono_literals;
constexpr std::chrono::microseconds ZERO_MICROSEC = 0us;
OvercommitTracker::OvercommitTracker(std::mutex & global_mutex_)
: max_wait_time(ZERO_MICROSEC)
, picked_tracker(nullptr)
: picked_tracker(nullptr)
, cancellation_state(QueryCancellationState::NONE)
, global_mutex(global_mutex_)
, freed_memory(0)
@ -18,13 +23,7 @@ OvercommitTracker::OvercommitTracker(std::mutex & global_mutex_)
, allow_release(true)
{}
void OvercommitTracker::setMaxWaitTime(UInt64 wait_time)
{
std::lock_guard guard(overcommit_m);
max_wait_time = wait_time * 1us;
}
bool OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int64 amount)
OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int64 amount)
{
// NOTE: Do not change the order of locks
//
@ -35,8 +34,10 @@ bool OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int64 amount)
std::unique_lock<std::mutex> global_lock(global_mutex);
std::unique_lock<std::mutex> lk(overcommit_m);
auto max_wait_time = tracker->getOvercommitWaitingTime();
if (max_wait_time == ZERO_MICROSEC)
return true;
return OvercommitResult::DISABLED;
pickQueryToExclude();
assert(cancellation_state != QueryCancellationState::NONE);
@ -50,7 +51,7 @@ bool OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int64 amount)
// picked_tracker to be not null pointer.
assert(cancellation_state == QueryCancellationState::SELECTED);
cancellation_state = QueryCancellationState::NONE;
return true;
return OvercommitResult::DISABLED;
}
if (picked_tracker == tracker)
{
@ -58,17 +59,20 @@ bool OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int64 amount)
// It may happen even when current state is RUNNING, because
// ThreadStatus::~ThreadStatus may call MemoryTracker::alloc.
cancellation_state = QueryCancellationState::RUNNING;
return true;
return OvercommitResult::SELECTED;
}
allow_release = true;
required_memory += amount;
required_per_thread[tracker] = amount;
auto wait_start_time = std::chrono::system_clock::now();
bool timeout = !cv.wait_for(lk, max_wait_time, [this, tracker]()
{
return required_per_thread[tracker] == 0 || cancellation_state == QueryCancellationState::NONE;
});
auto wait_end_time = std::chrono::system_clock::now();
ProfileEvents::increment(ProfileEvents::MemoryOvercommitWaitTimeMicroseconds, (wait_end_time - wait_start_time) / 1us);
LOG_DEBUG(getLogger(), "Memory was{} freed within timeout", (timeout ? " not" : ""));
required_memory -= amount;
@ -84,7 +88,12 @@ bool OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int64 amount)
// As we don't need to free memory, we can continue execution of the selected query.
if (required_memory == 0 && cancellation_state == QueryCancellationState::SELECTED)
reset();
return timeout || still_need != 0;
if (timeout)
return OvercommitResult::TIMEOUTED;
if (still_need != 0)
return OvercommitResult::NOT_ENOUGH_FREED;
else
return OvercommitResult::MEMORY_FREED;
}
void OvercommitTracker::tryContinueQueryExecutionAfterFree(Int64 amount)

16
src/Common/OvercommitTracker.h
View File

@ -36,6 +36,16 @@ struct OvercommitRatio
class MemoryTracker;
enum class OvercommitResult
{
NONE,
DISABLED,
MEMORY_FREED,
SELECTED,
TIMEOUTED,
NOT_ENOUGH_FREED,
};
enum class QueryCancellationState
{
NONE = 0, // Hard limit is not reached, there is no selected query to kill.
@ -52,9 +62,7 @@ enum class QueryCancellationState
// is killed to free memory.
struct OvercommitTracker : boost::noncopyable
{
void setMaxWaitTime(UInt64 wait_time);
bool needToStopQuery(MemoryTracker * tracker, Int64 amount);
OvercommitResult needToStopQuery(MemoryTracker * tracker, Int64 amount);
void tryContinueQueryExecutionAfterFree(Int64 amount);
@ -72,8 +80,6 @@ protected:
std::mutex overcommit_m;
std::condition_variable cv;
std::chrono::microseconds max_wait_time;
// Specifies memory tracker of the chosen to stop query.
// If soft limit is not set, all the queries which reach hard limit must stop.
// This case is represented as picked tracker pointer is set to nullptr and

1
src/Common/ProfileEvents.cpp
View File

@ -199,6 +199,7 @@
M(RealTimeMicroseconds, "Total (wall clock) time spent in processing (queries and other tasks) threads (not that this is a sum).") \
M(UserTimeMicroseconds, "Total time spent in processing (queries and other tasks) threads executing CPU instructions in user space. This include time CPU pipeline was stalled due to cache misses, branch mispredictions, hyper-threading, etc.") \
M(SystemTimeMicroseconds, "Total time spent in processing (queries and other tasks) threads executing CPU instructions in OS kernel space. This include time CPU pipeline was stalled due to cache misses, branch mispredictions, hyper-threading, etc.") \
M(MemoryOvercommitWaitTimeMicroseconds, "Total time spent in waiting for memory to be freed in OvercommitTracker.") \
M(SoftPageFaults, "") \
M(HardPageFaults, "") \
\

3
src/Common/TargetSpecific.cpp
View File

@ -16,6 +16,8 @@ UInt32 getSupportedArchs()
result |= static_cast<UInt32>(TargetArch::AVX2);
if (Cpu::CpuFlagsCache::have_AVX512F)
result |= static_cast<UInt32>(TargetArch::AVX512F);
if (Cpu::CpuFlagsCache::have_AVX512BW)
result |= static_cast<UInt32>(TargetArch::AVX512BW);
return result;
}
@ -34,6 +36,7 @@ String toString(TargetArch arch)
case TargetArch::AVX: return "avx";
case TargetArch::AVX2: return "avx2";
case TargetArch::AVX512F: return "avx512f";
case TargetArch::AVX512BW: return "avx512bw";
}
__builtin_unreachable();

1
src/Common/TargetSpecific.h
View File

@ -80,6 +80,7 @@ enum class TargetArch : UInt32
AVX = (1 << 1),
AVX2 = (1 << 2),
AVX512F = (1 << 3),
AVX512BW = (1 << 4),
};
/// Runtime detection.

58
src/Common/tests/gtest_overcommit_tracker.cpp
View File

@ -40,15 +40,17 @@ static constexpr UInt64 WAIT_TIME = 4'000'000;
template <typename T>
void free_not_continue_test(T & overcommit_tracker)
{
overcommit_tracker.setMaxWaitTime(WAIT_TIME);
static constexpr size_t THREADS = 5;
std::vector<MemoryTracker> trackers(THREADS);
for (auto & tracker : trackers)
tracker.setOvercommitWaitingTime(WAIT_TIME);
std::atomic<int> need_to_stop = 0;
std::vector<std::thread> threads;
threads.reserve(THREADS);
MemoryTracker picked;
picked.setOvercommitWaitingTime(WAIT_TIME);
overcommit_tracker.setCandidate(&picked);
for (size_t i = 0; i < THREADS; ++i)
@ -56,7 +58,7 @@ void free_not_continue_test(T & overcommit_tracker)
threads.push_back(std::thread(
[&, i]()
{
if (overcommit_tracker.needToStopQuery(&trackers[i], 100))
if (overcommit_tracker.needToStopQuery(&trackers[i], 100) != OvercommitResult::MEMORY_FREED)
++need_to_stop;
}
));
@ -96,15 +98,16 @@ TEST(OvercommitTracker, GlobalFreeNotContinue)
template <typename T>
void free_continue_test(T & overcommit_tracker)
{
overcommit_tracker.setMaxWaitTime(WAIT_TIME);
static constexpr size_t THREADS = 5;
std::vector<MemoryTracker> trackers(THREADS);
for (auto & tracker : trackers)
tracker.setOvercommitWaitingTime(WAIT_TIME);
std::atomic<int> need_to_stop = 0;
std::vector<std::thread> threads;
threads.reserve(THREADS);
MemoryTracker picked;
picked.setOvercommitWaitingTime(WAIT_TIME);
overcommit_tracker.setCandidate(&picked);
for (size_t i = 0; i < THREADS; ++i)
@ -112,7 +115,7 @@ void free_continue_test(T & overcommit_tracker)
threads.push_back(std::thread(
[&, i]()
{
if (overcommit_tracker.needToStopQuery(&trackers[i], 100))
if (overcommit_tracker.needToStopQuery(&trackers[i], 100) != OvercommitResult::MEMORY_FREED)
++need_to_stop;
}
));
@ -152,15 +155,16 @@ TEST(OvercommitTracker, GlobalFreeContinue)
template <typename T>
void free_continue_and_alloc_test(T & overcommit_tracker)
{
overcommit_tracker.setMaxWaitTime(WAIT_TIME);
static constexpr size_t THREADS = 5;
std::vector<MemoryTracker> trackers(THREADS);
for (auto & tracker : trackers)
tracker.setOvercommitWaitingTime(WAIT_TIME);
std::atomic<int> need_to_stop = 0;
std::vector<std::thread> threads;
threads.reserve(THREADS);
MemoryTracker picked;
picked.setOvercommitWaitingTime(WAIT_TIME);
overcommit_tracker.setCandidate(&picked);
for (size_t i = 0; i < THREADS; ++i)
@ -168,7 +172,7 @@ void free_continue_and_alloc_test(T & overcommit_tracker)
threads.push_back(std::thread(
[&, i]()
{
if (overcommit_tracker.needToStopQuery(&trackers[i], 100))
if (overcommit_tracker.needToStopQuery(&trackers[i], 100) != OvercommitResult::MEMORY_FREED)
++need_to_stop;
}
));
@ -179,9 +183,10 @@ void free_continue_and_alloc_test(T & overcommit_tracker)
[&]()
{
MemoryTracker failed;
failed.setOvercommitWaitingTime(WAIT_TIME);
std::this_thread::sleep_for(1000ms);
overcommit_tracker.tryContinueQueryExecutionAfterFree(5000);
stopped_next = overcommit_tracker.needToStopQuery(&failed, 100);
stopped_next = overcommit_tracker.needToStopQuery(&failed, 100) != OvercommitResult::MEMORY_FREED;
}
).join();
@ -212,15 +217,16 @@ TEST(OvercommitTracker, GlobalFreeContinueAndAlloc)
template <typename T>
void free_continue_and_alloc_2_test(T & overcommit_tracker)
{
overcommit_tracker.setMaxWaitTime(WAIT_TIME);
static constexpr size_t THREADS = 5;
std::vector<MemoryTracker> trackers(THREADS);
for (auto & tracker : trackers)
tracker.setOvercommitWaitingTime(WAIT_TIME);
std::atomic<int> need_to_stop = 0;
std::vector<std::thread> threads;
threads.reserve(THREADS);
MemoryTracker picked;
picked.setOvercommitWaitingTime(WAIT_TIME);
overcommit_tracker.setCandidate(&picked);
for (size_t i = 0; i < THREADS; ++i)
@ -228,7 +234,7 @@ void free_continue_and_alloc_2_test(T & overcommit_tracker)
threads.push_back(std::thread(
[&, i]()
{
if (overcommit_tracker.needToStopQuery(&trackers[i], 100))
if (overcommit_tracker.needToStopQuery(&trackers[i], 100) != OvercommitResult::MEMORY_FREED)
++need_to_stop;
}
));
@ -239,9 +245,10 @@ void free_continue_and_alloc_2_test(T & overcommit_tracker)
[&]()
{
MemoryTracker failed;
failed.setOvercommitWaitingTime(WAIT_TIME);
std::this_thread::sleep_for(1000ms);
overcommit_tracker.tryContinueQueryExecutionAfterFree(5000);
stopped_next = overcommit_tracker.needToStopQuery(&failed, 100);
stopped_next = overcommit_tracker.needToStopQuery(&failed, 100) != OvercommitResult::MEMORY_FREED;
}
));
@ -280,15 +287,16 @@ TEST(OvercommitTracker, GlobalFreeContinueAndAlloc2)
template <typename T>
void free_continue_and_alloc_3_test(T & overcommit_tracker)
{
overcommit_tracker.setMaxWaitTime(WAIT_TIME);
static constexpr size_t THREADS = 5;
std::vector<MemoryTracker> trackers(THREADS);
for (auto & tracker : trackers)
tracker.setOvercommitWaitingTime(WAIT_TIME);
std::atomic<int> need_to_stop = 0;
std::vector<std::thread> threads;
threads.reserve(THREADS);
MemoryTracker picked;
picked.setOvercommitWaitingTime(WAIT_TIME);
overcommit_tracker.setCandidate(&picked);
for (size_t i = 0; i < THREADS; ++i)
@ -296,7 +304,7 @@ void free_continue_and_alloc_3_test(T & overcommit_tracker)
threads.push_back(std::thread(
[&, i]()
{
if (overcommit_tracker.needToStopQuery(&trackers[i], 100))
if (overcommit_tracker.needToStopQuery(&trackers[i], 100) != OvercommitResult::MEMORY_FREED)
++need_to_stop;
}
));
@ -307,9 +315,10 @@ void free_continue_and_alloc_3_test(T & overcommit_tracker)
[&]()
{
MemoryTracker failed;
failed.setOvercommitWaitingTime(WAIT_TIME);
std::this_thread::sleep_for(1000ms);
overcommit_tracker.tryContinueQueryExecutionAfterFree(5000);
stopped_next = overcommit_tracker.needToStopQuery(&failed, 100);
stopped_next = overcommit_tracker.needToStopQuery(&failed, 100) != OvercommitResult::MEMORY_FREED;
}
));
@ -348,15 +357,16 @@ TEST(OvercommitTracker, GlobalFreeContinueAndAlloc3)
template <typename T>
void free_continue_2_test(T & overcommit_tracker)
{
overcommit_tracker.setMaxWaitTime(WAIT_TIME);
static constexpr size_t THREADS = 5;
std::vector<MemoryTracker> trackers(THREADS);
for (auto & tracker : trackers)
tracker.setOvercommitWaitingTime(WAIT_TIME);
std::atomic<int> need_to_stop = 0;
std::vector<std::thread> threads;
threads.reserve(THREADS);
MemoryTracker picked;
picked.setOvercommitWaitingTime(WAIT_TIME);
overcommit_tracker.setCandidate(&picked);
for (size_t i = 0; i < THREADS; ++i)
@ -364,7 +374,7 @@ void free_continue_2_test(T & overcommit_tracker)
threads.push_back(std::thread(
[&, i]()
{
if (overcommit_tracker.needToStopQuery(&trackers[i], 100))
if (overcommit_tracker.needToStopQuery(&trackers[i], 100) != OvercommitResult::MEMORY_FREED)
++need_to_stop;
}
));
@ -404,18 +414,18 @@ TEST(OvercommitTracker, GlobalFreeContinue2)
template <typename T>
void query_stop_not_continue_test(T & overcommit_tracker)
{
overcommit_tracker.setMaxWaitTime(WAIT_TIME);
std::atomic<int> need_to_stop = 0;
MemoryTracker picked;
picked.setOvercommitWaitingTime(WAIT_TIME);
overcommit_tracker.setCandidate(&picked);
MemoryTracker another;
another.setOvercommitWaitingTime(WAIT_TIME);
auto thread = std::thread(
[&]()
{
if (overcommit_tracker.needToStopQuery(&another, 100))
if (overcommit_tracker.needToStopQuery(&another, 100) != OvercommitResult::MEMORY_FREED)
++need_to_stop;
}
);

4
src/Core/Settings.h
View File

@ -372,7 +372,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
M(UInt64, memory_profiler_step, (4 * 1024 * 1024), "Whenever query memory usage becomes larger than every next step in number of bytes the memory profiler will collect the allocating stack trace. Zero means disabled memory profiler. Values lower than a few megabytes will slow down query processing.", 0) \
M(Float, memory_profiler_sample_probability, 0., "Collect random allocations and deallocations and write them into system.trace_log with 'MemorySample' trace_type. The probability is for every alloc/free regardless to the size of the allocation. Note that sampling happens only when the amount of untracked memory exceeds 'max_untracked_memory'. You may want to set 'max_untracked_memory' to 0 for extra fine grained sampling.", 0) \
\
M(UInt64, memory_usage_overcommit_max_wait_microseconds, 200, "Maximum time thread will wait for memory to be freed in the case of memory overcommit on user level. If timeout is reached and memory is not freed, exception is thrown.", 0) \
M(UInt64, memory_usage_overcommit_max_wait_microseconds, 5'000'000, "Maximum time thread will wait for memory to be freed in the case of memory overcommit. If timeout is reached and memory is not freed, exception is thrown.", 0) \
\
M(UInt64, max_network_bandwidth, 0, "The maximum speed of data exchange over the network in bytes per second for a query. Zero means unlimited.", 0) \
M(UInt64, max_network_bytes, 0, "The maximum number of bytes (compressed) to receive or transmit over the network for execution of the query.", 0) \
@ -695,7 +695,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
M(Bool, output_format_json_quote_denormals, false, "Enables '+nan', '-nan', '+inf', '-inf' outputs in JSON output format.", 0) \
\
M(Bool, output_format_json_escape_forward_slashes, true, "Controls escaping forward slashes for string outputs in JSON output format. This is intended for compatibility with JavaScript. Don't confuse with backslashes that are always escaped.", 0) \
M(Bool, output_format_json_named_tuples_as_objects, false, "Serialize named tuple columns as JSON objects.", 0) \
M(Bool, output_format_json_named_tuples_as_objects, true, "Serialize named tuple columns as JSON objects.", 0) \
M(Bool, output_format_json_array_of_rows, false, "Output a JSON array of all rows in JSONEachRow(Compact) format.", 0) \
\
M(UInt64, output_format_pretty_max_rows, 10000, "Rows limit for Pretty formats.", 0) \

13
src/Databases/MySQL/DatabaseMySQL.cpp
View File

@ -26,6 +26,7 @@
# include <Common/setThreadName.h>
# include <filesystem>
# include <Common/filesystemHelpers.h>
# include <Parsers/ASTIdentifier.h>
namespace fs = std::filesystem;
@ -148,8 +149,16 @@ ASTPtr DatabaseMySQL::getCreateTableQueryImpl(const String & table_name, Context
auto storage_engine_arguments = ast_storage->engine->arguments;
/// Add table_name to engine arguments
auto mysql_table_name = std::make_shared<ASTLiteral>(table_name);
storage_engine_arguments->children.insert(storage_engine_arguments->children.begin() + 2, mysql_table_name);
if (typeid_cast<ASTIdentifier *>(storage_engine_arguments->children[0].get()))
{
storage_engine_arguments->children.push_back(
makeASTFunction("equals", std::make_shared<ASTIdentifier>("table"), std::make_shared<ASTLiteral>(table_name)));
}
else
{
auto mysql_table_name = std::make_shared<ASTLiteral>(table_name);
storage_engine_arguments->children.insert(storage_engine_arguments->children.begin() + 2, mysql_table_name);
}
/// Unset settings
std::erase_if(storage_children, [&](const ASTPtr & element) { return element.get() == ast_storage->settings; });

86
src/Dictionaries/HTTPDictionarySource.cpp
View File

@ -8,6 +8,7 @@
#include <IO/WriteHelpers.h>
#include <Processors/Formats/IInputFormat.h>
#include <Interpreters/Context.h>
#include <Storages/ExternalDataSourceConfiguration.h>
#include <Poco/Net/HTTPRequest.h>
#include <Common/logger_useful.h>
#include "DictionarySourceFactory.h"
@ -228,45 +229,80 @@ void registerDictionarySourceHTTP(DictionarySourceFactory & factory)
if (dict_struct.has_expressions)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Dictionary source of type `http` does not support attribute expressions");
auto context = copyContextAndApplySettingsFromDictionaryConfig(global_context, config, config_prefix);
const auto & settings_config_prefix = config_prefix + ".http";
const auto & credentials_prefix = settings_config_prefix + ".credentials";
auto settings_config_prefix = config_prefix + ".http";
Poco::Net::HTTPBasicCredentials credentials;
if (config.has(credentials_prefix))
{
credentials.setUsername(config.getString(credentials_prefix + ".user", ""));
credentials.setPassword(config.getString(credentials_prefix + ".password", ""));
}
const auto & headers_prefix = settings_config_prefix + ".headers";
ReadWriteBufferFromHTTP::HTTPHeaderEntries header_entries;
String url;
String endpoint;
String format;
if (config.has(headers_prefix))
auto named_collection = created_from_ddl
? getURLBasedDataSourceConfiguration(config, settings_config_prefix, global_context)
: std::nullopt;
if (named_collection)
{
Poco::Util::AbstractConfiguration::Keys config_keys;
config.keys(headers_prefix, config_keys);
url = named_collection->configuration.url;
endpoint = named_collection->configuration.endpoint;
format = named_collection->configuration.format;
header_entries.reserve(config_keys.size());
for (const auto & key : config_keys)
{
const auto header_key = config.getString(headers_prefix + "." + key + ".name", "");
const auto header_value = config.getString(headers_prefix + "." + key + ".value", "");
header_entries.emplace_back(std::make_tuple(header_key, header_value));
}
credentials.setUsername(named_collection->configuration.user);
credentials.setPassword(named_collection->configuration.password);
header_entries.reserve(named_collection->configuration.headers.size());
for (const auto & header : named_collection->configuration.headers)
header_entries.emplace_back(std::make_tuple(header.first, header.second.get<String>()));
}
else
{
const auto & credentials_prefix = settings_config_prefix + ".credentials";
if (config.has(credentials_prefix))
{
credentials.setUsername(config.getString(credentials_prefix + ".user", ""));
credentials.setPassword(config.getString(credentials_prefix + ".password", ""));
}
const auto & headers_prefix = settings_config_prefix + ".headers";
if (config.has(headers_prefix))
{
Poco::Util::AbstractConfiguration::Keys config_keys;
config.keys(headers_prefix, config_keys);
header_entries.reserve(config_keys.size());
for (const auto & key : config_keys)
{
const auto header_key = config.getString(headers_prefix + "." + key + ".name", "");
const auto header_value = config.getString(headers_prefix + "." + key + ".value", "");
header_entries.emplace_back(std::make_tuple(header_key, header_value));
}
}
url = config.getString(settings_config_prefix + ".url", "");
endpoint = config.getString(settings_config_prefix + ".endpoint", "");
format =config.getString(settings_config_prefix + ".format", "");
}
if (url.ends_with('/'))
{
if (endpoint.starts_with('/'))
url.pop_back();
}
else if (!endpoint.empty() && !endpoint.starts_with('/'))
url.push_back('/');
auto configuration = HTTPDictionarySource::Configuration
{
.url = config.getString(settings_config_prefix + ".url", ""),
.format =config.getString(settings_config_prefix + ".format", ""),
.url = url + endpoint,
.format = format,
.update_field = config.getString(settings_config_prefix + ".update_field", ""),
.update_lag = config.getUInt64(settings_config_prefix + ".update_lag", 1),
.header_entries = std::move(header_entries) //-V1030
};
auto context = copyContextAndApplySettingsFromDictionaryConfig(global_context, config, config_prefix);
return std::make_unique<HTTPDictionarySource>(dict_struct, configuration, credentials, sample_block, context, created_from_ddl);
};
factory.registerSource("http", create_table_source);

20
src/Functions/FunctionHashID.h
View File

@ -51,9 +51,11 @@ public:
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; }
bool useDefaultImplementationForConstants() const override { return true; }
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
{
if (arguments.size() < 1)
if (arguments.empty())
throw Exception(ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION, "Function {} expects at least one argument", getName());
const auto & id_col = arguments[0];
@ -114,18 +116,16 @@ public:
const auto & numcolumn = arguments[0].column;
if (checkAndGetColumn<ColumnUInt8>(numcolumn.get()) || checkAndGetColumn<ColumnUInt16>(numcolumn.get())
|| checkAndGetColumn<ColumnUInt32>(numcolumn.get()) || checkAndGetColumn<ColumnUInt64>(numcolumn.get())
|| checkAndGetColumnConst<ColumnUInt8>(numcolumn.get()) || checkAndGetColumnConst<ColumnUInt16>(numcolumn.get())
|| checkAndGetColumnConst<ColumnUInt32>(numcolumn.get()) || checkAndGetColumnConst<ColumnUInt64>(numcolumn.get()))
|| checkAndGetColumn<ColumnUInt32>(numcolumn.get()) || checkAndGetColumn<ColumnUInt64>(numcolumn.get()))
{
std::string salt;
UInt8 minLength = 0;
UInt8 min_length = 0;
std::string alphabet;
if (arguments.size() >= 4)
{
const auto & alphabetcolumn = arguments[3].column;
if (auto alpha_col = checkAndGetColumnConst<ColumnString>(alphabetcolumn.get()))
if (const auto * alpha_col = checkAndGetColumnConst<ColumnString>(alphabetcolumn.get()))
{
alphabet = alpha_col->getValue<String>();
if (alphabet.find('\0') != std::string::npos)
@ -138,18 +138,18 @@ public:
if (arguments.size() >= 3)
{
const auto & minlengthcolumn = arguments[2].column;
if (auto min_length_col = checkAndGetColumnConst<ColumnUInt8>(minlengthcolumn.get()))
minLength = min_length_col->getValue<UInt8>();
if (const auto * min_length_col = checkAndGetColumnConst<ColumnUInt8>(minlengthcolumn.get()))
min_length = min_length_col->getValue<UInt8>();
}
if (arguments.size() >= 2)
{
const auto & saltcolumn = arguments[1].column;
if (auto salt_col = checkAndGetColumnConst<ColumnString>(saltcolumn.get()))
if (const auto * salt_col = checkAndGetColumnConst<ColumnString>(saltcolumn.get()))
salt = salt_col->getValue<String>();
}
hashidsxx::Hashids hash(salt, minLength, alphabet);
hashidsxx::Hashids hash(salt, min_length, alphabet);
auto col_res = ColumnString::create();

8
src/Functions/FunctionsStringArray.h
View File

@ -448,7 +448,7 @@ public:
class SplitByRegexpImpl
{
private:
Regexps::Pool::Pointer re;
Regexps::RegexpPtr re;
OptimizedRegularExpression::MatchVec matches;
Pos pos;
@ -477,7 +477,7 @@ public:
ErrorCodes::ILLEGAL_COLUMN);
if (!col->getValue<String>().empty())
re = Regexps::get<false, false, false>(col->getValue<String>());
re = std::make_shared<Regexps::Regexp>(Regexps::createRegexp<false, false, false>(col->getValue<String>()));
}
@ -532,7 +532,7 @@ public:
class ExtractAllImpl
{
private:
Regexps::Pool::Pointer re;
Regexps::RegexpPtr re;
OptimizedRegularExpression::MatchVec matches;
size_t capture;
@ -560,7 +560,7 @@ public:
+ " of first argument of function " + getName() + ". Must be constant string.",
ErrorCodes::ILLEGAL_COLUMN);
re = Regexps::get<false, false, false>(col->getValue<String>());
re = std::make_shared<Regexps::Regexp>(Regexps::createRegexp<false, false, false>(col->getValue<String>()));
capture = re->getNumberOfSubpatterns() > 0 ? 1 : 0;
matches.resize(capture + 1);

83
src/Functions/MatchImpl.h
View File

@ -18,8 +18,6 @@ namespace DB
namespace ErrorCodes
{
extern const int ILLEGAL_COLUMN;
extern const int LOGICAL_ERROR;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
}
namespace impl
@ -112,16 +110,14 @@ struct MatchImpl
const ColumnString::Chars & haystack_data,
const ColumnString::Offsets & haystack_offsets,
const String & needle,
const ColumnPtr & start_pos_,
[[maybe_unused]] const ColumnPtr & start_pos_,
PaddedPODArray<UInt8> & res)
{
const size_t haystack_size = haystack_offsets.size();
if (haystack_size != res.size())
throw Exception(ErrorCodes::LOGICAL_ERROR, "Function '{}' unexpectedly received a different number of haystacks and results", name);
assert(haystack_size == res.size());
if (start_pos_ != nullptr)
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function '{}' doesn't support start_pos argument", name);
assert(start_pos_ == nullptr);
if (haystack_offsets.empty())
return;
@ -166,17 +162,17 @@ struct MatchImpl
}
else
{
auto regexp = Regexps::get<is_like, true, case_insensitive>(needle);
const auto & regexp = Regexps::Regexp(Regexps::createRegexp<is_like, /*no_capture*/ true, case_insensitive>(needle));
String required_substring;
bool is_trivial;
bool required_substring_is_prefix; /// for `anchored` execution of the regexp.
regexp->getAnalyzeResult(required_substring, is_trivial, required_substring_is_prefix);
regexp.getAnalyzeResult(required_substring, is_trivial, required_substring_is_prefix);
if (required_substring.empty())
{
if (!regexp->getRE2()) /// An empty regexp. Always matches.
if (!regexp.getRE2()) /// An empty regexp. Always matches.
{
if (haystack_size)
memset(res.data(), !negate, haystack_size * sizeof(res[0]));
@ -186,7 +182,7 @@ struct MatchImpl
size_t prev_offset = 0;
for (size_t i = 0; i < haystack_size; ++i)
{
const bool match = regexp->getRE2()->Match(
const bool match = regexp.getRE2()->Match(
{reinterpret_cast<const char *>(&haystack_data[prev_offset]), haystack_offsets[i] - prev_offset - 1},
0,
haystack_offsets[i] - prev_offset - 1,
@ -241,7 +237,7 @@ struct MatchImpl
const size_t start_pos = (required_substring_is_prefix) ? (reinterpret_cast<const char *>(pos) - str_data) : 0;
const size_t end_pos = str_size;
const bool match = regexp->getRE2()->Match(
const bool match = regexp.getRE2()->Match(
{str_data, str_size},
start_pos,
end_pos,
@ -274,8 +270,7 @@ struct MatchImpl
{
const size_t haystack_size = haystack.size() / N;
if (haystack_size != res.size())
throw Exception(ErrorCodes::LOGICAL_ERROR, "Function '{}' unexpectedly received a different number of haystacks and results", name);
assert(haystack_size == res.size());
if (haystack.empty())
return;
@ -325,17 +320,17 @@ struct MatchImpl
}
else
{
auto regexp = Regexps::get<is_like, true, case_insensitive>(needle);
const auto & regexp = Regexps::Regexp(Regexps::createRegexp<is_like, /*no_capture*/ true, case_insensitive>(needle));
String required_substring;
bool is_trivial;
bool required_substring_is_prefix; /// for `anchored` execution of the regexp.
regexp->getAnalyzeResult(required_substring, is_trivial, required_substring_is_prefix);
regexp.getAnalyzeResult(required_substring, is_trivial, required_substring_is_prefix);
if (required_substring.empty())
{
if (!regexp->getRE2()) /// An empty regexp. Always matches.
if (!regexp.getRE2()) /// An empty regexp. Always matches.
{
if (haystack_size)
memset(res.data(), !negate, haystack_size * sizeof(res[0]));
@ -345,7 +340,7 @@ struct MatchImpl
size_t offset = 0;
for (size_t i = 0; i < haystack_size; ++i)
{
const bool match = regexp->getRE2()->Match(
const bool match = regexp.getRE2()->Match(
{reinterpret_cast<const char *>(&haystack[offset]), N},
0,
N,
@ -403,7 +398,7 @@ struct MatchImpl
const size_t start_pos = (required_substring_is_prefix) ? (reinterpret_cast<const char *>(pos) - str_data) : 0;
const size_t end_pos = N;
const bool match = regexp->getRE2()->Match(
const bool match = regexp.getRE2()->Match(
{str_data, N},
start_pos,
end_pos,
@ -433,16 +428,15 @@ struct MatchImpl
const ColumnString::Offsets & haystack_offsets,
const ColumnString::Chars & needle_data,
const ColumnString::Offsets & needle_offset,
const ColumnPtr & start_pos_,
[[maybe_unused]] const ColumnPtr & start_pos_,
PaddedPODArray<UInt8> & res)
{
const size_t haystack_size = haystack_offsets.size();
if (haystack_size != needle_offset.size() || haystack_size != res.size())
throw Exception(ErrorCodes::LOGICAL_ERROR, "Function '{}' unexpectedly received a different number of haystacks, needles and results", name);
assert(haystack_size == needle_offset.size());
assert(haystack_size == res.size());
if (start_pos_ != nullptr)
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function '{}' doesn't support start_pos argument", name);
assert(start_pos_ == nullptr);
if (haystack_offsets.empty())
return;
@ -454,6 +448,9 @@ struct MatchImpl
size_t prev_haystack_offset = 0;
size_t prev_needle_offset = 0;
Regexps::LocalCacheTable cache;
Regexps::RegexpPtr regexp;
for (size_t i = 0; i < haystack_size; ++i)
{
const auto * const cur_haystack_data = &haystack_data[prev_haystack_offset];
@ -479,22 +476,19 @@ struct MatchImpl
}
else
{
// each row is expected to contain a different like/re2 pattern
// --> bypass the regexp cache, instead construct the pattern on-the-fly
const int flags = Regexps::buildRe2Flags</*no_capture*/ true, case_insensitive>();
const auto & regexp = Regexps::Regexp(Regexps::createRegexp<is_like>(needle, flags));
cache.getOrSet<is_like, /*no_capture*/ true, case_insensitive>(needle, regexp);
regexp.getAnalyzeResult(required_substr, is_trivial, required_substring_is_prefix);
regexp->getAnalyzeResult(required_substr, is_trivial, required_substring_is_prefix);
if (required_substr.empty())
{
if (!regexp.getRE2()) /// An empty regexp. Always matches.
if (!regexp->getRE2()) /// An empty regexp. Always matches.
{
res[i] = !negate;
}
else
{
const bool match = regexp.getRE2()->Match(
const bool match = regexp->getRE2()->Match(
{reinterpret_cast<const char *>(cur_haystack_data), cur_haystack_length},
0,
cur_haystack_length,
@ -524,7 +518,7 @@ struct MatchImpl
const size_t start_pos = (required_substring_is_prefix) ? (match - cur_haystack_data) : 0;
const size_t end_pos = cur_haystack_length;
const bool match2 = regexp.getRE2()->Match(
const bool match2 = regexp->getRE2()->Match(
{reinterpret_cast<const char *>(cur_haystack_data), cur_haystack_length},
start_pos,
end_pos,
@ -547,16 +541,15 @@ struct MatchImpl
size_t N,
const ColumnString::Chars & needle_data,
const ColumnString::Offsets & needle_offset,
const ColumnPtr & start_pos_,
[[maybe_unused]] const ColumnPtr & start_pos_,
PaddedPODArray<UInt8> & res)
{
const size_t haystack_size = haystack.size()/N;
if (haystack_size != needle_offset.size() || haystack_size != res.size())
throw Exception(ErrorCodes::LOGICAL_ERROR, "Function '{}' unexpectedly received a different number of haystacks, needles and results", name);
assert(haystack_size == needle_offset.size());
assert(haystack_size == res.size());
if (start_pos_ != nullptr)
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function '{}' doesn't support start_pos argument", name);
assert(start_pos_ == nullptr);
if (haystack.empty())
return;
@ -568,6 +561,9 @@ struct MatchImpl
size_t prev_haystack_offset = 0;
size_t prev_needle_offset = 0;
Regexps::LocalCacheTable cache;
Regexps::RegexpPtr regexp;
for (size_t i = 0; i < haystack_size; ++i)
{
const auto * const cur_haystack_data = &haystack[prev_haystack_offset];
@ -593,22 +589,19 @@ struct MatchImpl
}
else
{
// each row is expected to contain a different like/re2 pattern
// --> bypass the regexp cache, instead construct the pattern on-the-fly
const int flags = Regexps::buildRe2Flags</*no_capture*/ true, case_insensitive>();
const auto & regexp = Regexps::Regexp(Regexps::createRegexp<is_like>(needle, flags));
cache.getOrSet<is_like, /*no_capture*/ true, case_insensitive>(needle, regexp);
regexp.getAnalyzeResult(required_substr, is_trivial, required_substring_is_prefix);
regexp->getAnalyzeResult(required_substr, is_trivial, required_substring_is_prefix);
if (required_substr.empty())
{
if (!regexp.getRE2()) /// An empty regexp. Always matches.
if (!regexp->getRE2()) /// An empty regexp. Always matches.
{
res[i] = !negate;
}
else
{
const bool match = regexp.getRE2()->Match(
const bool match = regexp->getRE2()->Match(
{reinterpret_cast<const char *>(cur_haystack_data), cur_haystack_length},
0,
cur_haystack_length,
@ -638,7 +631,7 @@ struct MatchImpl
const size_t start_pos = (required_substring_is_prefix) ? (match - cur_haystack_data) : 0;
const size_t end_pos = cur_haystack_length;
const bool match2 = regexp.getRE2()->Match(
const bool match2 = regexp->getRE2()->Match(
{reinterpret_cast<const char *>(cur_haystack_data), cur_haystack_length},
start_pos,
end_pos,

462
src/Functions/Regexps.h
View File

@ -9,7 +9,6 @@
#include <vector>
#include <Functions/likePatternToRegexp.h>
#include <Common/Exception.h>
#include <Common/ObjectPool.h>
#include <Common/OptimizedRegularExpression.h>
#include <Common/ProfileEvents.h>
#include <Common/config.h>
@ -38,254 +37,283 @@ namespace ErrorCodes
namespace Regexps
{
using Regexp = OptimizedRegularExpressionSingleThreaded;
using Pool = ObjectPoolMap<Regexp, String>;
using Regexp = OptimizedRegularExpressionSingleThreaded;
using RegexpPtr = std::shared_ptr<Regexp>;
template <bool like>
inline Regexp createRegexp(const std::string & pattern, int flags)
template <bool like, bool no_capture, bool case_insensitive>
inline Regexp createRegexp(const std::string & pattern)
{
int flags = OptimizedRegularExpression::RE_DOT_NL;
if constexpr (no_capture)
flags |= OptimizedRegularExpression::RE_NO_CAPTURE;
if constexpr (case_insensitive)
flags |= OptimizedRegularExpression::RE_CASELESS;
if constexpr (like)
return {likePatternToRegexp(pattern), flags};
else
return {pattern, flags};
}
/// Caches compiled re2 objects for given string patterns. Intended to support the common situation of a small set of patterns which are
/// evaluated over and over within the same query. In these situations, usage of the cache will save unnecessary pattern re-compilation.
/// However, we must be careful that caching does not add too much static overhead to overall pattern evaluation. Therefore, the cache is
/// intentionally very lightweight: a) no thread-safety/mutexes, b) small & fixed capacity, c) no collision list, d) but also no open
/// addressing, instead collisions simply replace the existing element.
class LocalCacheTable
{
public:
using RegexpPtr = std::shared_ptr<Regexp>;
LocalCacheTable()
: known_regexps(max_regexp_cache_size, {"", nullptr})
{
if constexpr (like)
return {likePatternToRegexp(pattern), flags};
else
return {pattern, flags};
}
template<bool no_capture, bool case_insensitive>
inline int buildRe2Flags()
{
int flags = OptimizedRegularExpression::RE_DOT_NL;
if constexpr (no_capture)
flags |= OptimizedRegularExpression::RE_NO_CAPTURE;
if constexpr (case_insensitive)
flags |= OptimizedRegularExpression::RE_CASELESS;
return flags;
}
/** Returns holder of an object from Pool.
* You must hold the ownership while using the object.
* In destructor, it returns the object back to the Pool for further reuse.
*/
template <bool like, bool no_capture, bool case_insensitive>
inline Pool::Pointer get(const std::string & pattern)
void getOrSet(const String & pattern, RegexpPtr & regexp)
{
/// the Singleton is thread-safe in C++11
static Pool known_regexps; /// Different variables for different pattern parameters.
StringAndRegexp & bucket = known_regexps[hasher(pattern) % max_regexp_cache_size];
return known_regexps.get(pattern, [&pattern]
if (likely(bucket.regexp != nullptr))
{
const int flags = buildRe2Flags<no_capture, case_insensitive>();
ProfileEvents::increment(ProfileEvents::RegexpCreated);
return new Regexp{createRegexp<like>(pattern, flags)};
});
if (pattern == bucket.pattern)
regexp = bucket.regexp;
else
{
regexp = std::make_shared<Regexp>(createRegexp<like, no_capture, case_insensitive>(pattern));
bucket = {pattern, regexp};
}
}
else
{
regexp = std::make_shared<Regexp>(createRegexp<like, no_capture, case_insensitive>(pattern));
bucket = {pattern, regexp};
}
}
private:
std::hash<std::string> hasher;
struct StringAndRegexp
{
std::string pattern;
RegexpPtr regexp;
};
using CacheTable = std::vector<StringAndRegexp>;
CacheTable known_regexps;
constexpr static size_t max_regexp_cache_size = 100; // collision probability
};
}
#if USE_HYPERSCAN
namespace MultiRegexps
{
template <typename Deleter, Deleter deleter>
struct HyperscanDeleter
template <typename Deleter, Deleter deleter>
struct HyperscanDeleter
{
template <typename T>
void operator()(T * ptr) const
{
template <typename T>
void operator()(T * ptr) const
{
deleter(ptr);
}
};
deleter(ptr);
}
};
/// Helper unique pointers to correctly delete the allocated space when hyperscan cannot compile something and we throw an exception.
using CompilerError = std::unique_ptr<hs_compile_error_t, HyperscanDeleter<decltype(&hs_free_compile_error), &hs_free_compile_error>>;
using ScratchPtr = std::unique_ptr<hs_scratch_t, HyperscanDeleter<decltype(&hs_free_scratch), &hs_free_scratch>>;
using DataBasePtr = std::unique_ptr<hs_database_t, HyperscanDeleter<decltype(&hs_free_database), &hs_free_database>>;
/// Helper unique pointers to correctly delete the allocated space when hyperscan cannot compile something and we throw an exception.
using CompilerError = std::unique_ptr<hs_compile_error_t, HyperscanDeleter<decltype(&hs_free_compile_error), &hs_free_compile_error>>;
using ScratchPtr = std::unique_ptr<hs_scratch_t, HyperscanDeleter<decltype(&hs_free_scratch), &hs_free_scratch>>;
using DataBasePtr = std::unique_ptr<hs_database_t, HyperscanDeleter<decltype(&hs_free_database), &hs_free_database>>;
/// Database is thread safe across multiple threads and Scratch is not but we can copy it whenever we use it in the searcher.
class Regexps
/// Database is thread safe across multiple threads and Scratch is not but we can copy it whenever we use it in the searcher.
class Regexps
{
public:
Regexps(hs_database_t * db_, hs_scratch_t * scratch_) : db{db_}, scratch{scratch_} { }
hs_database_t * getDB() const { return db.get(); }
hs_scratch_t * getScratch() const { return scratch.get(); }
private:
DataBasePtr db;
ScratchPtr scratch;
};
class RegexpsConstructor
{
public:
RegexpsConstructor() = default;
void setConstructor(std::function<Regexps()> constructor_) { constructor = std::move(constructor_); }
Regexps * operator()()
{
public:
Regexps(hs_database_t * db_, hs_scratch_t * scratch_) : db{db_}, scratch{scratch_} { }
hs_database_t * getDB() const { return db.get(); }
hs_scratch_t * getScratch() const { return scratch.get(); }
private:
DataBasePtr db;
ScratchPtr scratch;
};
class RegexpsConstructor
{
public:
RegexpsConstructor() = default;
void setConstructor(std::function<Regexps()> constructor_) { constructor = std::move(constructor_); }
Regexps * operator()()
{
std::unique_lock lock(mutex);
if (regexp)
return &*regexp;
regexp = constructor();
std::unique_lock lock(mutex);
if (regexp)
return &*regexp;
}
regexp = constructor();
return &*regexp;
}
private:
std::function<Regexps()> constructor;
std::optional<Regexps> regexp;
std::mutex mutex;
};
private:
std::function<Regexps()> constructor;
std::optional<Regexps> regexp;
std::mutex mutex;
};
struct Pool
struct Pool
{
/// Mutex for finding in map.
std::mutex mutex;
/// Patterns + possible edit_distance to database and scratch.
std::map<std::pair<std::vector<String>, std::optional<UInt32>>, RegexpsConstructor> storage;
};
template <bool save_indices, bool CompileForEditDistance>
inline Regexps constructRegexps(const std::vector<String> & str_patterns, std::optional<UInt32> edit_distance)
{
(void)edit_distance;
/// Common pointers
std::vector<const char *> patterns;
std::vector<unsigned int> flags;
/// Pointer for external edit distance compilation
std::vector<hs_expr_ext> ext_exprs;
std::vector<const hs_expr_ext *> ext_exprs_ptrs;
patterns.reserve(str_patterns.size());
flags.reserve(str_patterns.size());
if constexpr (CompileForEditDistance)
{
/// Mutex for finding in map.
std::mutex mutex;
/// Patterns + possible edit_distance to database and scratch.
std::map<std::pair<std::vector<String>, std::optional<UInt32>>, RegexpsConstructor> storage;
};
ext_exprs.reserve(str_patterns.size());
ext_exprs_ptrs.reserve(str_patterns.size());
}
template <bool save_indices, bool CompileForEditDistance>
inline Regexps constructRegexps(const std::vector<String> & str_patterns, std::optional<UInt32> edit_distance)
for (const StringRef ref : str_patterns)
{
(void)edit_distance;
/// Common pointers
std::vector<const char *> patterns;
std::vector<unsigned int> flags;
/// Pointer for external edit distance compilation
std::vector<hs_expr_ext> ext_exprs;
std::vector<const hs_expr_ext *> ext_exprs_ptrs;
patterns.reserve(str_patterns.size());
flags.reserve(str_patterns.size());
patterns.push_back(ref.data);
/* Flags below are the pattern matching flags.
* HS_FLAG_DOTALL is a compile flag where matching a . will not exclude newlines. This is a good
* performance practice according to Hyperscan API. https://intel.github.io/hyperscan/dev-reference/performance.html#dot-all-mode
* HS_FLAG_ALLOWEMPTY is a compile flag where empty strings are allowed to match.
* HS_FLAG_UTF8 is a flag where UTF8 literals are matched.
* HS_FLAG_SINGLEMATCH is a compile flag where each pattern match will be returned only once. it is a good performance practice
* as it is said in the Hyperscan documentation. https://intel.github.io/hyperscan/dev-reference/performance.html#single-match-flag
*/
flags.push_back(HS_FLAG_DOTALL | HS_FLAG_SINGLEMATCH | HS_FLAG_ALLOWEMPTY | HS_FLAG_UTF8);
if constexpr (CompileForEditDistance)
{
ext_exprs.reserve(str_patterns.size());
ext_exprs_ptrs.reserve(str_patterns.size());
/// Hyperscan currently does not support UTF8 matching with edit distance.
flags.back() &= ~HS_FLAG_UTF8;
ext_exprs.emplace_back();
/// HS_EXT_FLAG_EDIT_DISTANCE is a compile flag responsible for Levenstein distance.
ext_exprs.back().flags = HS_EXT_FLAG_EDIT_DISTANCE;
ext_exprs.back().edit_distance = edit_distance.value();
ext_exprs_ptrs.push_back(&ext_exprs.back());
}
for (const StringRef ref : str_patterns)
{
patterns.push_back(ref.data);
/* Flags below are the pattern matching flags.
* HS_FLAG_DOTALL is a compile flag where matching a . will not exclude newlines. This is a good
* performance practice according to Hyperscan API. https://intel.github.io/hyperscan/dev-reference/performance.html#dot-all-mode
* HS_FLAG_ALLOWEMPTY is a compile flag where empty strings are allowed to match.
* HS_FLAG_UTF8 is a flag where UTF8 literals are matched.
* HS_FLAG_SINGLEMATCH is a compile flag where each pattern match will be returned only once. it is a good performance practice
* as it is said in the Hyperscan documentation. https://intel.github.io/hyperscan/dev-reference/performance.html#single-match-flag
*/
flags.push_back(HS_FLAG_DOTALL | HS_FLAG_SINGLEMATCH | HS_FLAG_ALLOWEMPTY | HS_FLAG_UTF8);
if constexpr (CompileForEditDistance)
{
/// Hyperscan currently does not support UTF8 matching with edit distance.
flags.back() &= ~HS_FLAG_UTF8;
ext_exprs.emplace_back();
/// HS_EXT_FLAG_EDIT_DISTANCE is a compile flag responsible for Levenstein distance.
ext_exprs.back().flags = HS_EXT_FLAG_EDIT_DISTANCE;
ext_exprs.back().edit_distance = edit_distance.value();
ext_exprs_ptrs.push_back(&ext_exprs.back());
}
}
hs_database_t * db = nullptr;
hs_compile_error_t * compile_error;
std::unique_ptr<unsigned int[]> ids;
/// We mark the patterns to provide the callback results.
if constexpr (save_indices)
{
ids.reset(new unsigned int[patterns.size()]);
for (size_t i = 0; i < patterns.size(); ++i)
ids[i] = i + 1;
}
hs_error_t err;
if constexpr (!CompileForEditDistance)
err = hs_compile_multi(
patterns.data(),
flags.data(),
ids.get(),
patterns.size(),
HS_MODE_BLOCK,
nullptr,
&db,
&compile_error);
else
err = hs_compile_ext_multi(
patterns.data(),
flags.data(),
ids.get(),
ext_exprs_ptrs.data(),
patterns.size(),
HS_MODE_BLOCK,
nullptr,
&db,
&compile_error);
if (err != HS_SUCCESS)
{
/// CompilerError is a unique_ptr, so correct memory free after the exception is thrown.
CompilerError error(compile_error);
if (error->expression < 0)
throw Exception(String(error->message), ErrorCodes::LOGICAL_ERROR);
else
throw Exception(
"Pattern '" + str_patterns[error->expression] + "' failed with error '" + String(error->message),
ErrorCodes::BAD_ARGUMENTS);
}
ProfileEvents::increment(ProfileEvents::RegexpCreated);
/// We allocate the scratch space only once, then copy it across multiple threads with hs_clone_scratch
/// function which is faster than allocating scratch space each time in each thread.
hs_scratch_t * scratch = nullptr;
err = hs_alloc_scratch(db, &scratch);
/// If not HS_SUCCESS, it is guaranteed that the memory would not be allocated for scratch.
if (err != HS_SUCCESS)
throw Exception("Could not allocate scratch space for hyperscan", ErrorCodes::CANNOT_ALLOCATE_MEMORY);
return Regexps{db, scratch};
}
hs_database_t * db = nullptr;
hs_compile_error_t * compile_error;
/// If CompileForEditDistance is False, edit_distance must be nullopt
/// Also, we use templates here because each instantiation of function
/// template has its own copy of local static variables which must not be the same
/// for different hyperscan compilations.
template <bool save_indices, bool CompileForEditDistance>
inline Regexps * get(const std::vector<StringRef> & patterns, std::optional<UInt32> edit_distance)
std::unique_ptr<unsigned int[]> ids;
/// We mark the patterns to provide the callback results.
if constexpr (save_indices)
{
/// C++11 has thread-safe function-local static on most modern compilers.
static Pool known_regexps; /// Different variables for different pattern parameters.
std::vector<String> str_patterns;
str_patterns.reserve(patterns.size());
for (const StringRef & ref : patterns)
str_patterns.push_back(ref.toString());
/// Get the lock for finding database.
std::unique_lock lock(known_regexps.mutex);
auto it = known_regexps.storage.find({str_patterns, edit_distance});
/// If not found, compile and let other threads wait.
if (known_regexps.storage.end() == it)
{
it = known_regexps.storage
.emplace(std::piecewise_construct, std::make_tuple(std::move(str_patterns), edit_distance), std::make_tuple())
.first;
it->second.setConstructor([&str_patterns = it->first.first, edit_distance]()
{
return constructRegexps<save_indices, CompileForEditDistance>(str_patterns, edit_distance);
});
}
/// Unlock before possible construction.
lock.unlock();
return it->second();
ids.reset(new unsigned int[patterns.size()]);
for (size_t i = 0; i < patterns.size(); ++i)
ids[i] = i + 1;
}
hs_error_t err;
if constexpr (!CompileForEditDistance)
err = hs_compile_multi(
patterns.data(),
flags.data(),
ids.get(),
patterns.size(),
HS_MODE_BLOCK,
nullptr,
&db,
&compile_error);
else
err = hs_compile_ext_multi(
patterns.data(),
flags.data(),
ids.get(),
ext_exprs_ptrs.data(),
patterns.size(),
HS_MODE_BLOCK,
nullptr,
&db,
&compile_error);
if (err != HS_SUCCESS)
{
/// CompilerError is a unique_ptr, so correct memory free after the exception is thrown.
CompilerError error(compile_error);
if (error->expression < 0)
throw Exception(String(error->message), ErrorCodes::LOGICAL_ERROR);
else
throw Exception(
"Pattern '" + str_patterns[error->expression] + "' failed with error '" + String(error->message),
ErrorCodes::BAD_ARGUMENTS);
}
ProfileEvents::increment(ProfileEvents::RegexpCreated);
/// We allocate the scratch space only once, then copy it across multiple threads with hs_clone_scratch
/// function which is faster than allocating scratch space each time in each thread.
hs_scratch_t * scratch = nullptr;
err = hs_alloc_scratch(db, &scratch);
/// If not HS_SUCCESS, it is guaranteed that the memory would not be allocated for scratch.
if (err != HS_SUCCESS)
throw Exception("Could not allocate scratch space for hyperscan", ErrorCodes::CANNOT_ALLOCATE_MEMORY);
return Regexps{db, scratch};
}
/// If CompileForEditDistance is False, edit_distance must be nullopt
/// Also, we use templates here because each instantiation of function
/// template has its own copy of local static variables which must not be the same
/// for different hyperscan compilations.
template <bool save_indices, bool CompileForEditDistance>
inline Regexps * get(const std::vector<StringRef> & patterns, std::optional<UInt32> edit_distance)
{
/// C++11 has thread-safe function-local static on most modern compilers.
static Pool known_regexps; /// Different variables for different pattern parameters.
std::vector<String> str_patterns;
str_patterns.reserve(patterns.size());
for (const StringRef & ref : patterns)
str_patterns.push_back(ref.toString());
/// Get the lock for finding database.
std::unique_lock lock(known_regexps.mutex);
auto it = known_regexps.storage.find({str_patterns, edit_distance});
/// If not found, compile and let other threads wait.
if (known_regexps.storage.end() == it)
{
it = known_regexps.storage
.emplace(std::piecewise_construct, std::make_tuple(std::move(str_patterns), edit_distance), std::make_tuple())
.first;
it->second.setConstructor([&str_patterns = it->first.first, edit_distance]()
{
return constructRegexps<save_indices, CompileForEditDistance>(str_patterns, edit_distance);
});
}
/// Unlock before possible construction.
lock.unlock();
return it->second();
}
}
#endif // USE_HYPERSCAN

12
src/Functions/coalesce.cpp
View File

@ -112,7 +112,6 @@ public:
auto is_not_null = FunctionFactory::instance().get("isNotNull", context);
auto assume_not_null = FunctionFactory::instance().get("assumeNotNull", context);
auto multi_if = FunctionFactory::instance().get("multiIf", context);
ColumnsWithTypeAndName multi_if_args;
ColumnsWithTypeAndName tmp_args(1);
@ -144,7 +143,16 @@ public:
if (multi_if_args.size() == 1)
return multi_if_args.front().column;
ColumnPtr res = multi_if->build(multi_if_args)->execute(multi_if_args, result_type, input_rows_count);
/// If there was only two arguments (3 arguments passed to multiIf)
/// use function "if" instead, because it's implemented more efficient.
/// TODO: make "multiIf" the same efficient.
FunctionOverloadResolverPtr if_function;
if (multi_if_args.size() == 3)
if_function = FunctionFactory::instance().get("if", context);
else
if_function = FunctionFactory::instance().get("multiIf", context);
ColumnPtr res = if_function->build(multi_if_args)->execute(multi_if_args, result_type, input_rows_count);
/// if last argument is not nullable, result should be also not nullable
if (!multi_if_args.back().column->isNullable() && res->isNullable())

6
src/Functions/countMatches.h
View File

@ -55,7 +55,7 @@ public:
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
{
const ColumnConst * column_pattern = checkAndGetColumnConstStringOrFixedString(arguments[1].column.get());
Regexps::Pool::Pointer re = Regexps::get<false /* like */, true /* is_no_capture */, CountMatchesBase::case_insensitive>(column_pattern->getValue<String>());
const Regexps::Regexp re = Regexps::createRegexp</*is_like*/ false, /*no_capture*/ true, CountMatchesBase::case_insensitive>(column_pattern->getValue<String>());
OptimizedRegularExpression::MatchVec matches;
const IColumn * column_haystack = arguments[0].column.get();
@ -95,7 +95,7 @@ public:
throw Exception(ErrorCodes::LOGICAL_ERROR, "Error in FunctionCountMatches::getReturnTypeImpl()");
}
static uint64_t countMatches(StringRef src, Regexps::Pool::Pointer & re, OptimizedRegularExpression::MatchVec & matches)
static uint64_t countMatches(StringRef src, const Regexps::Regexp & re, OptimizedRegularExpression::MatchVec & matches)
{
/// Only one match is required, no need to copy more.
static const unsigned matches_limit = 1;
@ -108,7 +108,7 @@ public:
{
if (pos >= end)
break;
if (!re->match(pos, end - pos, matches, matches_limit))
if (!re.match(pos, end - pos, matches, matches_limit))
break;
/// Progress should be made, but with empty match the progress will not be done.
/// Also note that simply check is pattern empty is not enough,

6
src/Functions/extract.cpp
View File

@ -21,9 +21,9 @@ struct ExtractImpl
res_data.reserve(data.size() / 5);
res_offsets.resize(offsets.size());
const auto & regexp = Regexps::get<false, false, false>(pattern);
const Regexps::Regexp regexp = Regexps::createRegexp<false, false, false>(pattern);
unsigned capture = regexp->getNumberOfSubpatterns() > 0 ? 1 : 0;
unsigned capture = regexp.getNumberOfSubpatterns() > 0 ? 1 : 0;
OptimizedRegularExpression::MatchVec matches;
matches.reserve(capture + 1);
size_t prev_offset = 0;
@ -34,7 +34,7 @@ struct ExtractImpl
size_t cur_offset = offsets[i];
unsigned count
= regexp->match(reinterpret_cast<const char *>(&data[prev_offset]), cur_offset - prev_offset - 1, matches, capture + 1);
= regexp.match(reinterpret_cast<const char *>(&data[prev_offset]), cur_offset - prev_offset - 1, matches, capture + 1);
if (count > capture && matches[capture].offset != std::string::npos)
{
const auto & match = matches[capture];

4
src/Functions/extractAllGroups.h
View File

@ -95,8 +95,8 @@ public:
throw Exception("Length of 'needle' argument must be greater than 0.", ErrorCodes::BAD_ARGUMENTS);
using StringPiece = typename Regexps::Regexp::StringPieceType;
auto holder = Regexps::get<false, false, false>(needle);
const auto & regexp = holder->getRE2();
const Regexps::Regexp holder = Regexps::createRegexp<false, false, false>(needle);
const auto & regexp = holder.getRE2();
if (!regexp)
throw Exception("There are no groups in regexp: " + needle, ErrorCodes::BAD_ARGUMENTS);

4
src/Functions/extractGroups.cpp
View File

@ -63,8 +63,8 @@ public:
if (needle.empty())
throw Exception(getName() + " length of 'needle' argument must be greater than 0.", ErrorCodes::BAD_ARGUMENTS);
auto regexp = Regexps::get<false, false, false>(needle);
const auto & re2 = regexp->getRE2();
const Regexps::Regexp regexp = Regexps::createRegexp<false, false, false>(needle);
const auto & re2 = regexp.getRE2();
if (!re2)
throw Exception("There are no groups in regexp: " + needle, ErrorCodes::BAD_ARGUMENTS);

10
src/IO/ParallelReadBuffer.cpp
View File

@ -49,7 +49,15 @@ ParallelReadBuffer::ParallelReadBuffer(
, schedule(std::move(schedule_))
, reader_factory(std::move(reader_factory_))
{
addReaders();
try
{
addReaders();
}
catch (const Exception &)
{
finishAndWait();
throw;
}
}
bool ParallelReadBuffer::addReaderToPool()

4
src/IO/S3Common.cpp
View File

@ -765,9 +765,9 @@ namespace S3
const String & force_region,
const RemoteHostFilter & remote_host_filter,
unsigned int s3_max_redirects,
bool enable_s3_requestrs_logging)
bool enable_s3_requests_logging)
{
return PocoHTTPClientConfiguration(force_region, remote_host_filter, s3_max_redirects, enable_s3_requestrs_logging);
return PocoHTTPClientConfiguration(force_region, remote_host_filter, s3_max_redirects, enable_s3_requests_logging);
}
URI::URI(const Poco::URI & uri_)

2
src/IO/S3Common.h
View File

@ -45,7 +45,7 @@ public:
const String & force_region,
const RemoteHostFilter & remote_host_filter,
unsigned int s3_max_redirects,
bool enable_s3_requestrs_logging);
bool enable_s3_requests_logging);
private:
ClientFactory();

2
src/Interpreters/ActionsDAG.cpp
View File

@ -39,7 +39,7 @@ void ActionsDAG::Node::toTree(JSONBuilder::JSONMap & map) const
map.add("Result Type", result_type->getName());
if (!result_name.empty())
map.add("Result Type", magic_enum::enum_name(type));
map.add("Result Name", result_name);
if (column)
map.add("Column", column->getName());

3
src/Interpreters/ActionsVisitor.cpp
View File

@ -839,6 +839,9 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data &
if (node.name == "grouping")
{
if (data.only_consts)
return; // Can not perform constant folding, because this function can be executed only after GROUP BY
size_t arguments_size = node.arguments->children.size();
if (arguments_size == 0)
throw Exception(ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION, "Function GROUPING expects at least one argument");

10
src/Interpreters/Cluster.cpp
View File

@ -418,6 +418,8 @@ Cluster::Cluster(const Poco::Util::AbstractConfiguration & config,
if (address.is_local)
info.local_addresses.push_back(address);
info.all_addresses.push_back(address);
auto pool = ConnectionPoolFactory::instance().get(
settings.distributed_connections_pool_size,
address.host_name, address.port,
@ -485,6 +487,7 @@ Cluster::Cluster(const Poco::Util::AbstractConfiguration & config,
}
Addresses shard_local_addresses;
Addresses shard_all_addresses;
ConnectionPoolPtrs all_replicas_pools;
all_replicas_pools.reserve(replica_addresses.size());
@ -502,6 +505,7 @@ Cluster::Cluster(const Poco::Util::AbstractConfiguration & config,
all_replicas_pools.emplace_back(replica_pool);
if (replica.is_local)
shard_local_addresses.push_back(replica);
shard_all_addresses.push_back(replica);
}
ConnectionPoolWithFailoverPtr shard_pool = std::make_shared<ConnectionPoolWithFailover>(
@ -516,6 +520,7 @@ Cluster::Cluster(const Poco::Util::AbstractConfiguration & config,
current_shard_num,
weight,
std::move(shard_local_addresses),
std::move(shard_all_addresses),
std::move(shard_pool),
std::move(all_replicas_pools),
internal_replication
@ -571,6 +576,7 @@ Cluster::Cluster(
addresses_with_failover.emplace_back(current);
Addresses shard_local_addresses;
Addresses all_addresses;
ConnectionPoolPtrs all_replicas;
all_replicas.reserve(current.size());
@ -585,6 +591,7 @@ Cluster::Cluster(
all_replicas.emplace_back(replica_pool);
if (replica.is_local && !treat_local_as_remote)
shard_local_addresses.push_back(replica);
all_addresses.push_back(replica);
}
ConnectionPoolWithFailoverPtr shard_pool = std::make_shared<ConnectionPoolWithFailover>(
@ -597,6 +604,7 @@ Cluster::Cluster(
current_shard_num,
default_weight,
std::move(shard_local_addresses),
std::move(all_addresses),
std::move(shard_pool),
std::move(all_replicas),
false // has_internal_replication
@ -680,6 +688,8 @@ Cluster::Cluster(Cluster::ReplicasAsShardsTag, const Cluster & from, const Setti
if (address.is_local)
info.local_addresses.push_back(address);
info.all_addresses.push_back(address);
auto pool = ConnectionPoolFactory::instance().get(
settings.distributed_connections_pool_size,
address.host_name,

1
src/Interpreters/Cluster.h
View File

@ -202,6 +202,7 @@ public:
UInt32 shard_num = 0;
UInt32 weight = 1;
Addresses local_addresses;
Addresses all_addresses;
/// nullptr if there are no remote addresses
ConnectionPoolWithFailoverPtr pool;
/// Connection pool for each replica, contains nullptr for local replicas

65
src/Interpreters/ClusterProxy/IStreamFactory.h
View File

@ -1,65 +0,0 @@
#pragma once
#include <Client/ConnectionPool.h>
#include <Interpreters/Cluster.h>
#include <Parsers/IAST.h>
namespace DB
{
struct Settings;
class Cluster;
class Throttler;
struct SelectQueryInfo;
class Pipe;
using Pipes = std::vector<Pipe>;
class QueryPlan;
using QueryPlanPtr = std::unique_ptr<QueryPlan>;
struct StorageID;
namespace ClusterProxy
{
/// Base class for the implementation of the details of distributed query
/// execution that are specific to the query type.
class IStreamFactory
{
public:
virtual ~IStreamFactory() = default;
struct Shard
{
/// Query and header may be changed depending on shard.
ASTPtr query;
Block header;
size_t shard_num = 0;
size_t num_replicas = 0;
ConnectionPoolWithFailoverPtr pool;
ConnectionPoolPtrs per_replica_pools;
/// If we connect to replicas lazily.
/// (When there is a local replica with big delay).
bool lazy = false;
UInt32 local_delay = 0;
};
using Shards = std::vector<Shard>;
virtual void createForShard(
const Cluster::ShardInfo & shard_info,
const ASTPtr & query_ast,
const StorageID & main_table,
const ASTPtr & table_func_ptr,
ContextPtr context,
std::vector<QueryPlanPtr> & local_plans,
Shards & remote_shards,
UInt32 shard_count) = 0;
};
}
}

104
src/Interpreters/ClusterProxy/SelectStreamFactory.cpp
View File

@ -1,4 +1,5 @@
#include <Interpreters/ClusterProxy/SelectStreamFactory.h>
#include <Interpreters/Cluster.h>
#include <Interpreters/InterpreterSelectQuery.h>
#include <Storages/StorageReplicatedMergeTree.h>
#include <Storages/VirtualColumnUtils.h>
@ -10,14 +11,15 @@
#include <Interpreters/RequiredSourceColumnsVisitor.h>
#include <DataTypes/ObjectUtils.h>
#include <Client/IConnections.h>
#include <Common/logger_useful.h>
#include <Processors/QueryPlan/QueryPlan.h>
#include <Processors/QueryPlan/ReadFromRemote.h>
#include <Processors/QueryPlan/ExpressionStep.h>
#include <Processors/QueryPlan/BuildQueryPipelineSettings.h>
#include <Processors/QueryPlan/DistributedCreateLocalPlan.h>
#include <Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h>
namespace ProfileEvents
{
extern const Event DistributedConnectionMissingTable;
@ -63,7 +65,8 @@ void SelectStreamFactory::createForShard(
auto emplace_local_stream = [&]()
{
local_plans.emplace_back(createLocalPlan(query_ast, header, context, processed_stage, shard_info.shard_num, shard_count, /*coordinator=*/nullptr));
local_plans.emplace_back(createLocalPlan(
query_ast, header, context, processed_stage, shard_info.shard_num, shard_count, /*replica_num=*/0, /*replica_count=*/0, /*coordinator=*/nullptr));
};
auto emplace_remote_stream = [&](bool lazy = false, UInt32 local_delay = 0)
@ -71,10 +74,7 @@ void SelectStreamFactory::createForShard(
remote_shards.emplace_back(Shard{
.query = query_ast,
.header = header,
.shard_num = shard_info.shard_num,
.num_replicas = shard_info.getAllNodeCount(),
.pool = shard_info.pool,
.per_replica_pools = shard_info.per_replica_pools,
.shard_info = shard_info,
.lazy = lazy,
.local_delay = local_delay,
});
@ -173,5 +173,97 @@ void SelectStreamFactory::createForShard(
emplace_remote_stream();
}
SelectStreamFactory::ShardPlans SelectStreamFactory::createForShardWithParallelReplicas(
const Cluster::ShardInfo & shard_info,
const ASTPtr & query_ast,
const StorageID & main_table,
const ASTPtr & table_function_ptr,
const ThrottlerPtr & throttler,
ContextPtr context,
UInt32 shard_count)
{
SelectStreamFactory::ShardPlans result;
if (auto it = objects_by_shard.find(shard_info.shard_num); it != objects_by_shard.end())
replaceMissedSubcolumnsByConstants(storage_snapshot->object_columns, it->second, query_ast);
const auto & settings = context->getSettingsRef();
auto is_local_replica_obsolete = [&]()
{
auto resolved_id = context->resolveStorageID(main_table);
auto main_table_storage = DatabaseCatalog::instance().tryGetTable(resolved_id, context);
const auto * replicated_storage = dynamic_cast<const StorageReplicatedMergeTree *>(main_table_storage.get());
if (!replicated_storage)
return false;
UInt64 max_allowed_delay = settings.max_replica_delay_for_distributed_queries;
if (!max_allowed_delay)
return false;
UInt32 local_delay = replicated_storage->getAbsoluteDelay();
return local_delay >= max_allowed_delay;
};
size_t next_replica_number = 0;
size_t all_replicas_count = shard_info.getRemoteNodeCount();
auto coordinator = std::make_shared<ParallelReplicasReadingCoordinator>();
auto remote_plan = std::make_unique<QueryPlan>();
if (settings.prefer_localhost_replica && shard_info.isLocal())
{
/// We don't need more than one local replica in parallel reading
if (!is_local_replica_obsolete())
{
++all_replicas_count;
result.local_plan = createLocalPlan(
query_ast, header, context, processed_stage, shard_info.shard_num, shard_count, next_replica_number, all_replicas_count, coordinator);
++next_replica_number;
}
}
Scalars scalars = context->hasQueryContext() ? context->getQueryContext()->getScalars() : Scalars{};
scalars.emplace(
"_shard_count", Block{{DataTypeUInt32().createColumnConst(1, shard_count), std::make_shared<DataTypeUInt32>(), "_shard_count"}});
auto external_tables = context->getExternalTables();
auto shard = Shard{
.query = query_ast,
.header = header,
.shard_info = shard_info,
.lazy = false,
.local_delay = 0,
};
if (shard_info.hasRemoteConnections())
{
auto read_from_remote = std::make_unique<ReadFromParallelRemoteReplicasStep>(
coordinator,
shard,
header,
processed_stage,
main_table,
table_function_ptr,
context,
throttler,
std::move(scalars),
std::move(external_tables),
&Poco::Logger::get("ReadFromParallelRemoteReplicasStep"),
shard_count);
remote_plan->addStep(std::move(read_from_remote));
result.remote_plan = std::move(remote_plan);
}
return result;
}
}
}

59
src/Interpreters/ClusterProxy/SelectStreamFactory.h
View File

@ -1,22 +1,56 @@
#pragma once
#include <Core/QueryProcessingStage.h>
#include <Interpreters/ClusterProxy/IStreamFactory.h>
#include <Interpreters/StorageID.h>
#include <Storages/IStorage_fwd.h>
#include <Storages/StorageSnapshot.h>
#include <Client/ConnectionPool.h>
#include <Interpreters/Cluster.h>
#include <Parsers/IAST.h>
namespace DB
{
struct Settings;
class Cluster;
class Throttler;
struct SelectQueryInfo;
class Pipe;
using Pipes = std::vector<Pipe>;
class QueryPlan;
using QueryPlanPtr = std::unique_ptr<QueryPlan>;
struct StorageID;
namespace ClusterProxy
{
using ColumnsDescriptionByShardNum = std::unordered_map<UInt32, ColumnsDescription>;
class SelectStreamFactory final : public IStreamFactory
class SelectStreamFactory
{
public:
struct Shard
{
/// Query and header may be changed depending on shard.
ASTPtr query;
Block header;
Cluster::ShardInfo shard_info;
/// If we connect to replicas lazily.
/// (When there is a local replica with big delay).
bool lazy = false;
UInt32 local_delay = 0;
};
using Shards = std::vector<Shard>;
SelectStreamFactory(
const Block & header_,
const ColumnsDescriptionByShardNum & objects_by_shard_,
@ -31,7 +65,26 @@ public:
ContextPtr context,
std::vector<QueryPlanPtr> & local_plans,
Shards & remote_shards,
UInt32 shard_count) override;
UInt32 shard_count);
struct ShardPlans
{
/// If a shard has local replicas this won't be nullptr
std::unique_ptr<QueryPlan> local_plan;
/// Contains several steps to read from all remote replicas
std::unique_ptr<QueryPlan> remote_plan;
};
ShardPlans createForShardWithParallelReplicas(
const Cluster::ShardInfo & shard_info,
const ASTPtr & query_ast,
const StorageID & main_table,
const ASTPtr & table_function_ptr,
const ThrottlerPtr & throttler,
ContextPtr context,
UInt32 shard_count
);
private:
const Block header;

92
src/Interpreters/ClusterProxy/executeQuery.cpp
View File

@ -20,6 +20,7 @@ namespace DB
namespace ErrorCodes
{
extern const int TOO_LARGE_DISTRIBUTED_DEPTH;
extern const int LOGICAL_ERROR;
}
namespace ClusterProxy
@ -106,21 +107,19 @@ void executeQuery(
QueryProcessingStage::Enum processed_stage,
const StorageID & main_table,
const ASTPtr & table_func_ptr,
IStreamFactory & stream_factory, Poco::Logger * log,
SelectStreamFactory & stream_factory, Poco::Logger * log,
const ASTPtr & query_ast, ContextPtr context, const SelectQueryInfo & query_info,
const ExpressionActionsPtr & sharding_key_expr,
const std::string & sharding_key_column_name,
const ClusterPtr & not_optimized_cluster)
{
assert(log);
const Settings & settings = context->getSettingsRef();
if (settings.max_distributed_depth && context->getClientInfo().distributed_depth >= settings.max_distributed_depth)
throw Exception("Maximum distributed depth exceeded", ErrorCodes::TOO_LARGE_DISTRIBUTED_DEPTH);
std::vector<QueryPlanPtr> plans;
IStreamFactory::Shards remote_shards;
SelectStreamFactory::Shards remote_shards;
auto new_context = updateSettingsForCluster(*query_info.getCluster(), context, settings, log);
@ -213,6 +212,91 @@ void executeQuery(
query_plan.unitePlans(std::move(union_step), std::move(plans));
}
void executeQueryWithParallelReplicas(
QueryPlan & query_plan,
const StorageID & main_table,
const ASTPtr & table_func_ptr,
SelectStreamFactory & stream_factory,
const ASTPtr & query_ast, ContextPtr context, const SelectQueryInfo & query_info,
const ExpressionActionsPtr & sharding_key_expr,
const std::string & sharding_key_column_name,
const ClusterPtr & not_optimized_cluster)
{
const Settings & settings = context->getSettingsRef();
ThrottlerPtr user_level_throttler;
if (auto * process_list_element = context->getProcessListElement())
user_level_throttler = process_list_element->getUserNetworkThrottler();
/// Network bandwidth limit, if needed.
ThrottlerPtr throttler;
if (settings.max_network_bandwidth || settings.max_network_bytes)
{
throttler = std::make_shared<Throttler>(
settings.max_network_bandwidth,
settings.max_network_bytes,
"Limit for bytes to send or receive over network exceeded.",
user_level_throttler);
}
else
throttler = user_level_throttler;
std::vector<QueryPlanPtr> plans;
size_t shards = query_info.getCluster()->getShardCount();
for (const auto & shard_info : query_info.getCluster()->getShardsInfo())
{
ASTPtr query_ast_for_shard;
if (query_info.optimized_cluster && settings.optimize_skip_unused_shards_rewrite_in && shards > 1)
{
query_ast_for_shard = query_ast->clone();
OptimizeShardingKeyRewriteInVisitor::Data visitor_data{
sharding_key_expr,
sharding_key_expr->getSampleBlock().getByPosition(0).type,
sharding_key_column_name,
shard_info,
not_optimized_cluster->getSlotToShard(),
};
OptimizeShardingKeyRewriteInVisitor visitor(visitor_data);
visitor.visit(query_ast_for_shard);
}
else
query_ast_for_shard = query_ast;
auto shard_plans = stream_factory.createForShardWithParallelReplicas(shard_info,
query_ast_for_shard, main_table, table_func_ptr, throttler, context, shards);
if (!shard_plans.local_plan && !shard_plans.remote_plan)
throw Exception(ErrorCodes::LOGICAL_ERROR, "No plans were generated for reading from shard. This is a bug");
if (shard_plans.local_plan)
plans.emplace_back(std::move(shard_plans.local_plan));
if (shard_plans.remote_plan)
plans.emplace_back(std::move(shard_plans.remote_plan));
}
if (plans.empty())
throw Exception(ErrorCodes::LOGICAL_ERROR, "No plans were generated for reading from Distributed. This is a bug");
if (plans.size() == 1)
{
query_plan = std::move(*plans.front());
return;
}
DataStreams input_streams;
input_streams.reserve(plans.size());
for (const auto & plan : plans)
input_streams.emplace_back(plan->getCurrentDataStream());
auto union_step = std::make_unique<UnionStep>(std::move(input_streams));
query_plan.unitePlans(std::move(union_step), std::move(plans));
}
}
}

15
src/Interpreters/ClusterProxy/executeQuery.h
View File

@ -23,7 +23,7 @@ struct StorageID;
namespace ClusterProxy
{
class IStreamFactory;
class SelectStreamFactory;
/// Update settings for Distributed query.
///
@ -46,7 +46,18 @@ void executeQuery(
QueryProcessingStage::Enum processed_stage,
const StorageID & main_table,
const ASTPtr & table_func_ptr,
IStreamFactory & stream_factory, Poco::Logger * log,
SelectStreamFactory & stream_factory, Poco::Logger * log,
const ASTPtr & query_ast, ContextPtr context, const SelectQueryInfo & query_info,
const ExpressionActionsPtr & sharding_key_expr,
const std::string & sharding_key_column_name,
const ClusterPtr & not_optimized_cluster);
void executeQueryWithParallelReplicas(
QueryPlan & query_plan,
const StorageID & main_table,
const ASTPtr & table_func_ptr,
SelectStreamFactory & stream_factory,
const ASTPtr & query_ast, ContextPtr context, const SelectQueryInfo & query_info,
const ExpressionActionsPtr & sharding_key_expr,
const std::string & sharding_key_column_name,

4
src/Interpreters/DatabaseCatalog.cpp
View File

@ -119,7 +119,11 @@ TemporaryTableHolder & TemporaryTableHolder::operator=(TemporaryTableHolder && r
TemporaryTableHolder::~TemporaryTableHolder()
{
if (id != UUIDHelpers::Nil)
{
auto table = getTable();
table->flushAndShutdown();
temporary_tables->dropTable(getContext(), "_tmp_" + toString(id));
}
}
StorageID TemporaryTableHolder::getGlobalTableID() const

1
src/Interpreters/ExpressionAnalyzer.cpp
View File

@ -1999,7 +1999,6 @@ void ExpressionAnalysisResult::checkActions() const
};
check_actions(prewhere_info->prewhere_actions);
check_actions(prewhere_info->alias_actions);
}
}

48
src/Interpreters/HashJoin.cpp
View File

@ -538,6 +538,7 @@ void HashJoin::dataMapInit(MapsVariant & map)
bool HashJoin::overDictionary() const
{
assert(data->type != Type::DICT || table_join->getDictionaryReader());
return data->type == Type::DICT;
}
@ -707,6 +708,13 @@ namespace
void HashJoin::initRightBlockStructure(Block & saved_block_sample)
{
if (isCrossOrComma(kind))
{
/// cross join doesn't have keys, just add all columns
saved_block_sample = sample_block_with_columns_to_add.cloneEmpty();
return;
}
bool multiple_disjuncts = !table_join->oneDisjunct();
/// We could remove key columns for LEFT | INNER HashJoin but we should keep them for JoinSwitcher (if any).
bool save_key_columns = !table_join->forceHashJoin() || isRightOrFull(kind) || multiple_disjuncts;
@ -724,9 +732,7 @@ void HashJoin::initRightBlockStructure(Block & saved_block_sample)
for (auto & column : sample_block_with_columns_to_add)
{
if (!saved_block_sample.findByName(column.name))
{
saved_block_sample.insert(column);
}
}
}
@ -912,6 +918,7 @@ public:
bool is_join_get_)
: join_on_keys(join_on_keys_)
, rows_to_add(block.rows())
, sample_block(saved_block_sample)
, is_join_get(is_join_get_)
{
size_t num_columns_to_add = block_with_columns_to_add.columns();
@ -951,12 +958,46 @@ public:
return ColumnWithTypeAndName(std::move(columns[i]), type_name[i].type, type_name[i].qualified_name);
}
static void assertBlockEqualsStructureUpToLowCard(const Block & lhs_block, const Block & rhs_block)
{
if (lhs_block.columns() != rhs_block.columns())
throw Exception("Different number of columns in blocks", ErrorCodes::LOGICAL_ERROR);
for (size_t i = 0; i < lhs_block.columns(); ++i)
{
const auto & lhs = lhs_block.getByPosition(i);
const auto & rhs = rhs_block.getByPosition(i);
if (lhs.name != rhs.name)
throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "Block structure mismatch: [{}] != [{}]",
lhs_block.dumpStructure(), rhs_block.dumpStructure());
const auto & ltype = recursiveRemoveLowCardinality(lhs.type);
const auto & rtype = recursiveRemoveLowCardinality(rhs.type);
if (!ltype->equals(*rtype))
throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "Block structure mismatch: [{}] != [{}]",
lhs_block.dumpStructure(), rhs_block.dumpStructure());
const auto & lcol = recursiveRemoveLowCardinality(lhs.column);
const auto & rcol = recursiveRemoveLowCardinality(rhs.column);
if (lcol->getDataType() != rcol->getDataType())
throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "Block structure mismatch: [{}] != [{}]",
lhs_block.dumpStructure(), rhs_block.dumpStructure());
}
}
template <bool has_defaults>
void appendFromBlock(const Block & block, size_t row_num)
{
if constexpr (has_defaults)
applyLazyDefaults();
#ifndef NDEBUG
/// Like assertBlocksHaveEqualStructure but doesn't check low cardinality
assertBlockEqualsStructureUpToLowCard(sample_block, block);
#else
UNUSED(assertBlockEqualsStructureUpToLowCard);
#endif
if (is_join_get)
{
/// If it's joinGetOrNull, we need to wrap not-nullable columns in StorageJoin.
@ -1019,6 +1060,7 @@ private:
size_t lazy_defaults_count = 0;
/// for ASOF
const IColumn * left_asof_key = nullptr;
Block sample_block;
bool is_join_get;
@ -1698,7 +1740,7 @@ DataTypePtr HashJoin::joinGetCheckAndGetReturnType(const DataTypes & data_types,
throw Exception("StorageJoin doesn't contain column " + column_name, ErrorCodes::NO_SUCH_COLUMN_IN_TABLE);
auto elem = sample_block_with_columns_to_add.getByName(column_name);
if (or_null)
if (or_null && JoinCommon::canBecomeNullable(elem.type))
elem.type = makeNullable(elem.type);
return elem.type;
}

8
src/Interpreters/IInterpreterUnionOrSelectQuery.h
View File

@ -12,8 +12,13 @@ class IInterpreterUnionOrSelectQuery : public IInterpreter
{
public:
IInterpreterUnionOrSelectQuery(const ASTPtr & query_ptr_, ContextPtr context_, const SelectQueryOptions & options_)
: IInterpreterUnionOrSelectQuery(query_ptr_, Context::createCopy(context_), options_)
{
}
IInterpreterUnionOrSelectQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_, const SelectQueryOptions & options_)
: query_ptr(query_ptr_)
, context(Context::createCopy(context_))
, context(context_)
, options(options_)
, max_streams(context->getSettingsRef().max_threads)
{
@ -60,4 +65,3 @@ protected:
bool uses_view_source = false;
};
}

8
src/Interpreters/InterpreterSelectIntersectExceptQuery.cpp
View File

@ -51,6 +51,14 @@ InterpreterSelectIntersectExceptQuery::InterpreterSelectIntersectExceptQuery(
const ASTPtr & query_ptr_,
ContextPtr context_,
const SelectQueryOptions & options_)
:InterpreterSelectIntersectExceptQuery(query_ptr_, Context::createCopy(context_), options_)
{
}
InterpreterSelectIntersectExceptQuery::InterpreterSelectIntersectExceptQuery(
const ASTPtr & query_ptr_,
ContextMutablePtr context_,
const SelectQueryOptions & options_)
: IInterpreterUnionOrSelectQuery(query_ptr_->clone(), context_, options_)
{
ASTSelectIntersectExceptQuery * ast = query_ptr->as<ASTSelectIntersectExceptQuery>();

5
src/Interpreters/InterpreterSelectIntersectExceptQuery.h
View File

@ -24,6 +24,11 @@ public:
ContextPtr context_,
const SelectQueryOptions & options_);
InterpreterSelectIntersectExceptQuery(
const ASTPtr & query_ptr_,
ContextMutablePtr context_,
const SelectQueryOptions & options_);
BlockIO execute() override;
Block getSampleBlock() { return result_header; }

59
src/Interpreters/InterpreterSelectQuery.cpp
View File

@ -165,6 +165,14 @@ InterpreterSelectQuery::InterpreterSelectQuery(
: InterpreterSelectQuery(query_ptr_, context_, std::nullopt, nullptr, options_, required_result_column_names_)
{}
InterpreterSelectQuery::InterpreterSelectQuery(
const ASTPtr & query_ptr_,
ContextMutablePtr context_,
const SelectQueryOptions & options_,
const Names & required_result_column_names_)
: InterpreterSelectQuery(query_ptr_, context_, std::nullopt, nullptr, options_, required_result_column_names_)
{}
InterpreterSelectQuery::InterpreterSelectQuery(
const ASTPtr & query_ptr_,
ContextPtr context_,
@ -280,6 +288,28 @@ InterpreterSelectQuery::InterpreterSelectQuery(
const StorageMetadataPtr & metadata_snapshot_,
SubqueriesForSets subquery_for_sets_,
PreparedSets prepared_sets_)
: InterpreterSelectQuery(
query_ptr_,
Context::createCopy(context_),
std::move(input_pipe_),
storage_,
options_,
required_result_column_names,
metadata_snapshot_,
std::move(subquery_for_sets_),
std::move(prepared_sets_))
{}
InterpreterSelectQuery::InterpreterSelectQuery(
const ASTPtr & query_ptr_,
ContextMutablePtr context_,
std::optional<Pipe> input_pipe_,
const StoragePtr & storage_,
const SelectQueryOptions & options_,
const Names & required_result_column_names,
const StorageMetadataPtr & metadata_snapshot_,
SubqueriesForSets subquery_for_sets_,
PreparedSets prepared_sets_)
/// NOTE: the query almost always should be cloned because it will be modified during analysis.
: IInterpreterUnionOrSelectQuery(options_.modify_inplace ? query_ptr_ : query_ptr_->clone(), context_, options_)
, storage(storage_)
@ -1624,15 +1654,6 @@ void InterpreterSelectQuery::addEmptySourceToQueryPlan(
{
auto & prewhere_info = *prewhere_info_ptr;
if (prewhere_info.alias_actions)
{
pipe.addSimpleTransform([&](const Block & header)
{
return std::make_shared<ExpressionTransform>(header,
std::make_shared<ExpressionActions>(prewhere_info.alias_actions));
});
}
if (prewhere_info.row_level_filter)
{
pipe.addSimpleTransform([&](const Block & header)
@ -1698,12 +1719,11 @@ void InterpreterSelectQuery::setMergeTreeReadTaskCallbackAndClientInfo(MergeTree
context->setMergeTreeReadTaskCallback(std::move(callback));
}
void InterpreterSelectQuery::setProperClientInfo()
void InterpreterSelectQuery::setProperClientInfo(size_t replica_num, size_t replica_count)
{
context->getClientInfo().query_kind = ClientInfo::QueryKind::SECONDARY_QUERY;
assert(options.shard_count.has_value() && options.shard_num.has_value());
context->getClientInfo().count_participating_replicas = *options.shard_count;
context->getClientInfo().number_of_current_replica = *options.shard_num;
context->getClientInfo().count_participating_replicas = replica_count;
context->getClientInfo().number_of_current_replica = replica_num;
}
bool InterpreterSelectQuery::shouldMoveToPrewhere()
@ -1872,19 +1892,6 @@ void InterpreterSelectQuery::addPrewhereAliasActions()
for (const auto & name : required_columns)
prewhere_info->prewhere_actions->tryRestoreColumn(name);
auto analyzed_result
= TreeRewriter(context).analyze(required_columns_from_prewhere_expr, metadata_snapshot->getColumns().getAllPhysical());
prewhere_info->alias_actions
= ExpressionAnalyzer(required_columns_from_prewhere_expr, analyzed_result, context).getActionsDAG(true, false);
/// Add (physical?) columns required by alias actions.
auto required_columns_from_alias = prewhere_info->alias_actions->getRequiredColumns();
Block prewhere_actions_result = prewhere_info->prewhere_actions->getResultColumns();
for (auto & column : required_columns_from_alias)
if (!prewhere_actions_result.has(column.name))
if (required_columns.end() == std::find(required_columns.begin(), required_columns.end(), column.name))
required_columns.push_back(column.name);
/// Add physical columns required by prewhere actions.
for (const auto & column : required_columns_from_prewhere)
if (!required_aliases_from_prewhere.contains(column))

19
src/Interpreters/InterpreterSelectQuery.h
View File

@ -55,6 +55,12 @@ public:
const SelectQueryOptions &,
const Names & required_result_column_names_ = Names{});
InterpreterSelectQuery(
const ASTPtr & query_ptr_,
ContextMutablePtr context_,
const SelectQueryOptions &,
const Names & required_result_column_names_ = Names{});
/// Read data not from the table specified in the query, but from the prepared pipe `input`.
InterpreterSelectQuery(
const ASTPtr & query_ptr_,
@ -119,7 +125,7 @@ public:
void setMergeTreeReadTaskCallbackAndClientInfo(MergeTreeReadTaskCallback && callback);
/// It will set shard_num and shard_count to the client_info
void setProperClientInfo();
void setProperClientInfo(size_t replica_num, size_t replica_count);
private:
InterpreterSelectQuery(
@ -133,6 +139,17 @@ private:
SubqueriesForSets subquery_for_sets_ = {},
PreparedSets prepared_sets_ = {});
InterpreterSelectQuery(
const ASTPtr & query_ptr_,
ContextMutablePtr context_,
std::optional<Pipe> input_pipe,
const StoragePtr & storage_,
const SelectQueryOptions &,
const Names & required_result_column_names = {},
const StorageMetadataPtr & metadata_snapshot_ = nullptr,
SubqueriesForSets subquery_for_sets_ = {},
PreparedSets prepared_sets_ = {});
ASTSelectQuery & getSelectQuery() { return query_ptr->as<ASTSelectQuery &>(); }
void addPrewhereAliasActions();

9
src/Interpreters/InterpreterSelectWithUnionQuery.cpp
View File

@ -31,8 +31,13 @@ namespace ErrorCodes
}
InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery(
const ASTPtr & query_ptr_, ContextPtr context_,
const SelectQueryOptions & options_, const Names & required_result_column_names)
const ASTPtr & query_ptr_, ContextPtr context_, const SelectQueryOptions & options_, const Names & required_result_column_names)
: InterpreterSelectWithUnionQuery(query_ptr_, Context::createCopy(context_), options_, required_result_column_names)
{
}
InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery(
const ASTPtr & query_ptr_, ContextMutablePtr context_, const SelectQueryOptions & options_, const Names & required_result_column_names)
: IInterpreterUnionOrSelectQuery(query_ptr_, context_, options_)
{
ASTSelectWithUnionQuery * ast = query_ptr->as<ASTSelectWithUnionQuery>();

6
src/Interpreters/InterpreterSelectWithUnionQuery.h
View File

@ -22,6 +22,12 @@ public:
const SelectQueryOptions &,
const Names & required_result_column_names = {});
InterpreterSelectWithUnionQuery(
const ASTPtr & query_ptr_,
ContextMutablePtr context_,
const SelectQueryOptions &,
const Names & required_result_column_names = {});
~InterpreterSelectWithUnionQuery() override;
/// Builds QueryPlan for current query.

32
src/Interpreters/PartLog.cpp
View File

@ -16,6 +16,26 @@
namespace DB
{
namespace ErrorCodes
{
extern const int NOT_IMPLEMENTED;
}
PartLogElement::MergeReasonType PartLogElement::getMergeReasonType(MergeType merge_type)
{
switch (merge_type)
{
case MergeType::Regular:
return REGULAR_MERGE;
case MergeType::TTLDelete:
return TTL_DELETE_MERGE;
case MergeType::TTLRecompress:
return TTL_RECOMPRESS_MERGE;
}
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unknown MergeType {}", static_cast<UInt64>(merge_type));
}
NamesAndTypesList PartLogElement::getNamesAndTypes()
{
auto event_type_datatype = std::make_shared<DataTypeEnum8>(
@ -30,11 +50,22 @@ NamesAndTypesList PartLogElement::getNamesAndTypes()
}
);
auto merge_reason_datatype = std::make_shared<DataTypeEnum8>(
DataTypeEnum8::Values
{
{"NotAMerge", static_cast<Int8>(NOT_A_MERGE)},
{"RegularMerge", static_cast<Int8>(REGULAR_MERGE)},
{"TTLDeleteMerge", static_cast<Int8>(TTL_DELETE_MERGE)},
{"TTLRecompressMerge", static_cast<Int8>(TTL_RECOMPRESS_MERGE)},
}
);
ColumnsWithTypeAndName columns_with_type_and_name;
return {
{"query_id", std::make_shared<DataTypeString>()},
{"event_type", std::move(event_type_datatype)},
{"merge_reason", std::move(merge_reason_datatype)},
{"event_date", std::make_shared<DataTypeDate>()},
{"event_time", std::make_shared<DataTypeDateTime>()},
@ -72,6 +103,7 @@ void PartLogElement::appendToBlock(MutableColumns & columns) const
columns[i++]->insert(query_id);
columns[i++]->insert(event_type);
columns[i++]->insert(merge_reason);
columns[i++]->insert(DateLUT::instance().toDayNum(event_time).toUnderType());
columns[i++]->insert(event_time);
columns[i++]->insert(event_time_microseconds);

15
src/Interpreters/PartLog.h
View File

@ -4,6 +4,7 @@
#include <Interpreters/SystemLog.h>
#include <Core/NamesAndTypes.h>
#include <Core/NamesAndAliases.h>
#include <Storages/MergeTree/MergeType.h>
namespace DB
@ -21,9 +22,22 @@ struct PartLogElement
MOVE_PART = 6,
};
enum MergeReasonType
{
/// merge_reason is relevant only for event_type = 'MERGE_PARTS', in other cases it is NOT_A_MERGE
NOT_A_MERGE = 1,
/// Just regular merge
REGULAR_MERGE = 2,
/// Merge assigned to delete some data from parts (with TTLMergeSelector)
TTL_DELETE_MERGE = 3,
/// Merge with recompression
TTL_RECOMPRESS_MERGE = 4,
};
String query_id;
Type event_type = NEW_PART;
MergeReasonType merge_reason = NOT_A_MERGE;
time_t event_time = 0;
Decimal64 event_time_microseconds = 0;
@ -57,6 +71,7 @@ struct PartLogElement
static std::string name() { return "PartLog"; }
static MergeReasonType getMergeReasonType(MergeType merge_type);
static NamesAndTypesList getNamesAndTypes();
static NamesAndAliases getNamesAndAliases() { return {}; }
void appendToBlock(MutableColumns & columns) const;

3
src/Interpreters/ProcessList.cpp
View File

@ -225,6 +225,8 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as
if (settings.memory_tracker_fault_probability)
thread_group->memory_tracker.setFaultProbability(settings.memory_tracker_fault_probability);
thread_group->memory_tracker.setOvercommitWaitingTime(settings.memory_usage_overcommit_max_wait_microseconds);
/// NOTE: Do not set the limit for thread-level memory tracker since it could show unreal values
/// since allocation and deallocation could happen in different threads
}
@ -244,7 +246,6 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as
user_process_list.user_memory_tracker.setOrRaiseHardLimit(settings.max_memory_usage_for_user);
user_process_list.user_memory_tracker.setSoftLimit(settings.memory_overcommit_ratio_denominator_for_user);
user_process_list.user_memory_tracker.setDescription("(for user)");
user_process_list.user_overcommit_tracker.setMaxWaitTime(settings.memory_usage_overcommit_max_wait_microseconds);
if (!user_process_list.user_throttler)
{

2
src/Interpreters/SelectQueryOptions.h
View File

@ -135,7 +135,7 @@ struct SelectQueryOptions
return *this;
}
SelectQueryOptions & ignoreASTOptimizationsAlias(bool value = true)
SelectQueryOptions & ignoreASTOptimizations(bool value = true)
{
ignore_ast_optimizations = value;
return *this;

7
src/Interpreters/TableJoin.cpp
View File

@ -479,6 +479,13 @@ bool TableJoin::tryInitDictJoin(const Block & sample_block, ContextPtr context)
src_names.push_back(original);
dst_columns.push_back({col.name, col.type});
}
else
{
/// Can't extract column from dictionary table
/// TODO: Sometimes it should be possible to recunstruct required column,
/// e.g. if it's an expression depending on dictionary attributes
return false;
}
}
dictionary_reader = std::make_shared<DictionaryReader>(dict_name, src_names, dst_columns, context);

2
src/Interpreters/TreeRewriter.cpp
View File

@ -1184,7 +1184,7 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect(
if (remove_duplicates)
renameDuplicatedColumns(select_query);
/// Perform it before analyzing JOINs, because it may change number of columns with names unique and break some login inside JOINs
/// Perform it before analyzing JOINs, because it may change number of columns with names unique and break some logic inside JOINs
if (settings.optimize_normalize_count_variants)
TreeOptimizer::optimizeCountConstantAndSumOne(query);

2
src/Interpreters/UserDefinedExecutableFunctionFactory.cpp
View File

@ -43,7 +43,7 @@ public:
size_t getNumberOfArguments() const override { return executable_function->getConfiguration().arguments.size(); }
bool useDefaultImplementationForConstants() const override { return true; }
bool useDefaultImplementationForNulls() const override { return true; }
bool useDefaultImplementationForNulls() const override { return false; }
bool isDeterministic() const override { return false; }
bool isDeterministicInScopeOfQuery() const override { return false; }

14
src/Interpreters/evaluateConstantExpression.cpp
View File

@ -31,11 +31,21 @@ namespace ErrorCodes
extern const int BAD_ARGUMENTS;
}
static std::pair<Field, std::shared_ptr<const IDataType>> getFieldAndDataTypeFromLiteral(ASTLiteral * literal)
{
auto type = applyVisitor(FieldToDataType(), literal->value);
/// In case of Array field nested fields can have different types.
/// Example: Array [1, 2.3] will have 2 fields with types UInt64 and Float64
/// when result type is Array(Float64).
/// So, we need to convert this field to the result type.
Field res = convertFieldToType(literal->value, *type);
return {res, type};
}
std::pair<Field, std::shared_ptr<const IDataType>> evaluateConstantExpression(const ASTPtr & node, ContextPtr context)
{
if (ASTLiteral * literal = node->as<ASTLiteral>())
return std::make_pair(literal->value, applyVisitor(FieldToDataType(), literal->value));
return getFieldAndDataTypeFromLiteral(literal);
NamesAndTypesList source_columns = {{ "_dummy", std::make_shared<DataTypeUInt8>() }};
@ -63,7 +73,7 @@ std::pair<Field, std::shared_ptr<const IDataType>> evaluateConstantExpression(co
/// AST potentially could be transformed to literal during TreeRewriter analyze.
/// For example if we have SQL user defined function that return literal AS subquery.
if (ASTLiteral * literal = ast->as<ASTLiteral>())
return std::make_pair(literal->value, applyVisitor(FieldToDataType(), literal->value));
return getFieldAndDataTypeFromLiteral(literal);
ExpressionActionsPtr expr_for_constant_folding = ExpressionAnalyzer(ast, syntax_result, context).getConstActions();

Some files were not shown because too many files have changed in this diff Show More