Merge remote-tracking branch 'upstream/master' into HEAD

This commit is contained in:
Anton Popov 2022-08-02 12:57:43 +00:00
commit a333cc4146
134 changed files with 1549 additions and 610 deletions

View File

@ -151,8 +151,8 @@ jobs:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr "$TEMP_PATH"
SplitBuildSmokeTest:
needs: [BuilderDebSplitted]
SharedBuildSmokeTest:
needs: [BuilderDebShared]
runs-on: [self-hosted, style-checker]
steps:
- name: Set envs
@ -171,7 +171,7 @@ jobs:
uses: actions/download-artifact@v2
with:
path: ${{ env.REPORTS_PATH }}
- name: Split build check
- name: Shared build check
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
@ -598,7 +598,7 @@ jobs:
##########################################################################################
##################################### SPECIAL BUILDS #####################################
##########################################################################################
BuilderDebSplitted:
BuilderDebShared:
needs: [DockerHubPush]
runs-on: [self-hosted, builder]
steps:
@ -609,7 +609,7 @@ jobs:
IMAGES_PATH=${{runner.temp}}/images_path
REPO_COPY=${{runner.temp}}/build_check/ClickHouse
CACHES_PATH=${{runner.temp}}/../ccaches
BUILD_NAME=binary_splitted
BUILD_NAME=binary_shared
EOF
- name: Download changed images
uses: actions/download-artifact@v2
@ -1012,7 +1012,7 @@ jobs:
# - BuilderBinGCC
- BuilderBinPPC64
- BuilderBinClangTidy
- BuilderDebSplitted
- BuilderDebShared
runs-on: [self-hosted, style-checker]
steps:
- name: Set envs
@ -3153,7 +3153,7 @@ jobs:
- UnitTestsMsan
- UnitTestsUBsan
- UnitTestsReleaseClang
- SplitBuildSmokeTest
- SharedBuildSmokeTest
runs-on: [self-hosted, style-checker]
steps:
- name: Clear repository

View File

@ -216,8 +216,8 @@ jobs:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr "$TEMP_PATH"
SplitBuildSmokeTest:
needs: [BuilderDebSplitted]
SharedBuildSmokeTest:
needs: [BuilderDebShared]
runs-on: [self-hosted, style-checker]
steps:
- name: Set envs
@ -236,7 +236,7 @@ jobs:
uses: actions/download-artifact@v2
with:
path: ${{ env.REPORTS_PATH }}
- name: Split build check
- name: Shared build check
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
@ -620,7 +620,7 @@ jobs:
##########################################################################################
##################################### SPECIAL BUILDS #####################################
##########################################################################################
BuilderDebSplitted:
BuilderDebShared:
needs: [DockerHubPush, FastTest, StyleCheck]
runs-on: [self-hosted, builder]
steps:
@ -631,7 +631,7 @@ jobs:
IMAGES_PATH=${{runner.temp}}/images_path
REPO_COPY=${{runner.temp}}/build_check/ClickHouse
CACHES_PATH=${{runner.temp}}/../ccaches
BUILD_NAME=binary_splitted
BUILD_NAME=binary_shared
EOF
- name: Download changed images
uses: actions/download-artifact@v2
@ -1024,7 +1024,7 @@ jobs:
# - BuilderBinGCC
- BuilderBinPPC64
- BuilderBinClangTidy
- BuilderDebSplitted
- BuilderDebShared
runs-on: [self-hosted, style-checker]
if: ${{ success() || failure() }}
steps:
@ -3416,7 +3416,7 @@ jobs:
- UnitTestsMsan
- UnitTestsUBsan
- UnitTestsReleaseClang
- SplitBuildSmokeTest
- SharedBuildSmokeTest
- CompatibilityCheck
- IntegrationTestsFlakyCheck
- Jepsen

View File

@ -77,10 +77,9 @@ option(USE_STATIC_LIBRARIES "Disable to use shared libraries" ON)
# DEVELOPER ONLY.
# Faster linking if turned on.
option(SPLIT_SHARED_LIBRARIES "Keep all internal libraries as separate .so files" OFF)
option(CLICKHOUSE_SPLIT_BINARY "Make several binaries (clickhouse-server, clickhouse-client etc.) instead of one bundled" OFF)
if (USE_STATIC_LIBRARIES AND (SPLIT_SHARED_LIBRARIES OR CLICKHOUSE_SPLIT_BINARY))
message(FATAL_ERROR "SPLIT_SHARED_LIBRARIES=1 or CLICKHOUSE_SPLIT_BINARY=1 must not be used together with USE_STATIC_LIBRARIES=1")
if (USE_STATIC_LIBRARIES AND SPLIT_SHARED_LIBRARIES)
message(FATAL_ERROR "SPLIT_SHARED_LIBRARIES=1 must not be used together with USE_STATIC_LIBRARIES=1")
endif()
if (NOT USE_STATIC_LIBRARIES AND SPLIT_SHARED_LIBRARIES)
@ -502,7 +501,7 @@ endif ()
message (STATUS
"Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE} ;
USE_STATIC_LIBRARIES=${USE_STATIC_LIBRARIES}
SPLIT_SHARED=${SPLIT_SHARED_LIBRARIES}")
SPLIT_SHARED_LIBRARIES=${SPLIT_SHARED_LIBRARIES}")
include (GNUInstallDirs)

View File

@ -15,4 +15,4 @@ ClickHouse® is an open-source column-oriented database management system that a
* [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any.
## Upcoming events
* **v22.8 Release Webinar** Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release, provide live demos, and share vision into what is coming in the roadmap.
* [**v22.8 Release Webinar**](https://clickhouse.com/company/events/v22-8-release-webinar) Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release, provide live demos, and share vision into what is coming in the roadmap.

View File

@ -100,12 +100,12 @@ def run_docker_image_with_env(
subprocess.check_call(cmd, shell=True)
def is_release_build(build_type, package_type, sanitizer, split_binary):
def is_release_build(build_type, package_type, sanitizer, shared_libraries):
return (
build_type == ""
and package_type == "deb"
and sanitizer == ""
and not split_binary
and not shared_libraries
)
@ -116,7 +116,7 @@ def parse_env_variables(
package_type,
cache,
distcc_hosts,
split_binary,
shared_libraries,
clang_tidy,
version,
author,
@ -202,7 +202,7 @@ def parse_env_variables(
cmake_flags.append("-DCMAKE_INSTALL_PREFIX=/usr")
cmake_flags.append("-DCMAKE_INSTALL_SYSCONFDIR=/etc")
cmake_flags.append("-DCMAKE_INSTALL_LOCALSTATEDIR=/var")
if is_release_build(build_type, package_type, sanitizer, split_binary):
if is_release_build(build_type, package_type, sanitizer, shared_libraries):
cmake_flags.append("-DSPLIT_DEBUG_SYMBOLS=ON")
result.append("WITH_PERFORMANCE=1")
if is_cross_arm:
@ -215,11 +215,11 @@ def parse_env_variables(
cmake_flags.append(f"-DCMAKE_C_COMPILER={cc}")
cmake_flags.append(f"-DCMAKE_CXX_COMPILER={cxx}")
# Create combined output archive for split build and for performance tests.
# Create combined output archive for shared library build and for performance tests.
if package_type == "coverity":
result.append("COMBINED_OUTPUT=coverity")
result.append('COVERITY_TOKEN="$COVERITY_TOKEN"')
elif split_binary:
elif shared_libraries:
result.append("COMBINED_OUTPUT=shared_build")
if sanitizer:
@ -264,14 +264,13 @@ def parse_env_variables(
result.append("BINARY_OUTPUT=tests")
cmake_flags.append("-DENABLE_TESTS=1")
if split_binary:
if shared_libraries:
cmake_flags.append(
"-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1"
"-DCLICKHOUSE_SPLIT_BINARY=1"
)
# We can't always build utils because it requires too much space, but
# we have to build them at least in some way in CI. The split build is
# probably the least heavy disk-wise.
# we have to build them at least in some way in CI. The shared library
# build is probably the least heavy disk-wise.
cmake_flags.append("-DENABLE_UTILS=1")
# utils are not included into clickhouse-bundle, so build everything
build_target = "all"
@ -352,7 +351,7 @@ if __name__ == "__main__":
default="",
)
parser.add_argument("--split-binary", action="store_true")
parser.add_argument("--shared-libraries", action="store_true")
parser.add_argument("--clang-tidy", action="store_true")
parser.add_argument("--cache", choices=("ccache", "distcc", ""), default="")
parser.add_argument(
@ -405,7 +404,7 @@ if __name__ == "__main__":
args.package_type,
args.cache,
args.distcc_hosts,
args.split_binary,
args.shared_libraries,
args.clang_tidy,
args.version,
args.author,

View File

@ -38,6 +38,7 @@ FORMAT_SCHEMA_PATH="$(clickhouse extract-from-config --config-file "$CLICKHOUSE_
# There could be many disks declared in config
readarray -t FILESYSTEM_CACHE_PATHS < <(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key='storage_configuration.disks.*.data_cache_path' || true)
readarray -t DISKS_PATHS < <(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key='storage_configuration.disks.*.path' || true)
CLICKHOUSE_USER="${CLICKHOUSE_USER:-default}"
CLICKHOUSE_PASSWORD="${CLICKHOUSE_PASSWORD:-}"
@ -50,7 +51,8 @@ for dir in "$DATA_DIR" \
"$TMP_DIR" \
"$USER_PATH" \
"$FORMAT_SCHEMA_PATH" \
"${FILESYSTEM_CACHE_PATHS[@]}"
"${FILESYSTEM_CACHE_PATHS[@]}" \
"${DISKS_PATHS[@]}"
do
# check if variable not empty
[ -z "$dir" ] && continue

View File

@ -171,11 +171,11 @@ concurrency-related errors. If it fails:
## Split Build Smoke Test
Checks that the server build in [split build](../development/build.md#split-build)
Checks that the server build in [split build](../development/developer-instruction.md#split-build)
configuration can start and run simple queries. If it fails:
* Fix other test errors first;
* Build the server in [split build](../development/build.md#split-build) configuration
* Build the server in [split build](../development/developer-instruction.md#split-build) configuration
locally and check whether it can start and run `select 1`.

View File

@ -267,19 +267,19 @@ The system will prepare ClickHouse binary builds for your pull request individua
Most probably some of the builds will fail at first times. This is due to the fact that we check builds both with gcc as well as with clang, with almost all of existing warnings (always with the `-Werror` flag) enabled for clang. On that same page, you can find all of the build logs so that you do not have to build ClickHouse in all of the possible ways.
## Browse ClickHouse Source Code {#browse-clickhouse-source-code}
You can use the **Woboq** online code browser available [here](https://clickhouse.com/codebrowser/ClickHouse/src/index.html). It provides code navigation, semantic highlighting, search and indexing. The code snapshot is updated daily.
Also, you can browse sources on [GitHub](https://github.com/ClickHouse/ClickHouse) as usual.
## Faster builds for development: Split build configuration {#split-build}
ClickHouse is normally statically linked into a single static `clickhouse` binary with minimal dependencies. This is convenient for distribution, but it means that for every change the entire binary needs to be re-linked, which is slow and inconvenient for development. As an alternative, you can instead build dynamically linked shared libraries and separate binaries `clickhouse-server`, `clickhouse-client` etc., allowing for faster incremental builds. To use it, add the following flags to your `cmake` invocation:
ClickHouse is normally statically linked into a single static `clickhouse` binary with minimal dependencies. This is convenient for distribution, but it means that for every change the entire binary needs to be re-linked, which is slow and inconvenient for development. As an alternative, you can instead build dynamically linked shared libraries, allowing for faster incremental builds. To use it, add the following flags to your `cmake` invocation:
```
-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1
-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1
```
Note that the split build has several drawbacks:
* There is no single `clickhouse` binary, and you have to run `clickhouse-server`, `clickhouse-client`, etc.
* Risk of segfault if you run any of the programs while rebuilding the project.
* You cannot run the integration tests since they only work a single complete binary.
* You can't easily copy the binaries elsewhere. Instead of moving a single binary you'll need to copy all binaries and libraries.
If you are not interested in functionality provided by third-party libraries, you can further speed up the build using `cmake` options
```
-DENABLE_LIBRARIES=0 -DENABLE_EMBEDDED_COMPILER=0

View File

@ -389,12 +389,6 @@ SETTINGS mutations_sync = 1;
Let's run the same 3 queries.
[Enable](../../operations/settings/settings.md#allow-experimental-projection-optimization) projections for selects:
```sql
SET allow_experimental_projection_optimization = 1;
```
### Query 1. Average Price Per Year {#average-price-projections}
Query:

View File

@ -438,6 +438,18 @@ For more information, see the section “[Configuration files](../../operations/
<include_from>/etc/metrica.xml</include_from>
```
## interserver_listen_host {#interserver-listen-host}
Restriction on hosts that can exchange data between ClickHouse servers.
The default value equals to `listen_host` setting.
Examples:
``` xml
<interserver_listen_host>::ffff:a00:1</interserver_listen_host>
<interserver_listen_host>10.0.0.1</interserver_listen_host>
```
## interserver_http_port {#interserver-http-port}
Port for exchanging data between ClickHouse servers.
@ -970,7 +982,7 @@ Default value: 2.
**Example**
```xml
<background_merges_mutations_concurrency_ratio>3</background_pbackground_merges_mutations_concurrency_ratio>
<background_merges_mutations_concurrency_ratio>3</background_merges_mutations_concurrency_ratio>
```
## background_move_pool_size {#background_move_pool_size}

View File

@ -12,12 +12,13 @@ Reads file as a String. The file content is not parsed, so any information is re
**Syntax**
``` sql
file(path)
file(path[, default])
```
**Arguments**
- `path` — The relative path to the file from [user_files_path](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-user_files_path). Path to file support following wildcards: `*`, `?`, `{abc,def}` and `{N..M}` where `N`, `M` — numbers, `'abc', 'def'` — strings.
- `default` — The value that will be returned in the case when a file does not exist or cannot be accessed. Data types supported: [String](../../sql-reference/data-types/string.md) and [NULL](../../sql-reference/syntax.md#null-literal).
**Example**

View File

@ -170,7 +170,7 @@ sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
В случае использования на разработческой машине старого HDD или SSD, а также при желании использовать меньше места для артефактов сборки можно использовать следующую команду:
```bash
cmake -DUSE_DEBUG_HELPERS=1 -DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1 ..
cmake -DUSE_DEBUG_HELPERS=1 -DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 ..
```
При этом надо учесть, что получаемые в результате сборки исполнимые файлы будут динамически слинкованы с библиотеками, и поэтому фактически станут непереносимыми на другие компьютеры (либо для этого нужно будет предпринять значительно больше усилий по сравнению со статической сборкой). Плюсом же в данном случае является значительно меньшее время сборки (это проявляется не на первой сборке, а на последующих, после внесения изменений в исходный код - тратится меньшее время на линковку по сравнению со статической сборкой) и значительно меньшее использование места на жёстком диске (экономия более, чем в 3 раза по сравнению со статической сборкой). Для целей разработки, когда планируются только отладочные запуски на том же компьютере, где осуществлялась сборка, это может быть наиболее удобным вариантом.
@ -285,3 +285,9 @@ Pull request можно создать, даже если работа над з
Система подготовит сборки ClickHouse специально для вашего pull request. Для их получения, нажмите на ссылку «Details» у проверки «Clickhouse build check». Там вы сможете найти прямые ссылки на собранные .deb пакеты ClickHouse, которые, при желании, вы даже сможете установить на свои продакшен серверы (если не страшно).
Вероятнее всего, часть сборок не будет успешной с первого раза. Ведь мы проверяем сборку кода и gcc и clang, а при сборке с помощью clang включаются почти все существующие в природе warnings (всегда с флагом `-Werror`). На той же странице, вы сможете найти логи сборки - вам не обязательно самому собирать ClickHouse всеми возможными способами.
## Навигация по коду ClickHouse {#navigatsiia-po-kodu-clickhouse}
Для навигации по коду онлайн доступен **Woboq**, он расположен [здесь](https://clickhouse.com/codebrowser/ClickHouse/src/index.html). В нём реализовано удобное перемещение между исходными файлами, семантическая подсветка, подсказки, индексация и поиск. Слепок кода обновляется ежедневно.
Также вы можете просматривать исходники на [GitHub](https://github.com/ClickHouse/ClickHouse).

View File

@ -389,12 +389,6 @@ SETTINGS mutations_sync = 1;
Давайте выполним те же 3 запроса.
[Включите](../../operations/settings/settings.md#allow-experimental-projection-optimization) поддержку проекций:
```sql
SET allow_experimental_projection_optimization = 1;
```
### Запрос 1. Средняя цена за год {#average-price-projections}
Запрос:
@ -647,4 +641,3 @@ no projection: 100 rows in set. Elapsed: 0.069 sec. Processed 26.32 million rows
### Online Playground {#playground}
Этот набор данных доступен в [Online Playground](https://gh-api.clickhouse.tech/play?user=play#U0VMRUNUIHRvd24sIGRpc3RyaWN0LCBjb3VudCgpIEFTIGMsIHJvdW5kKGF2ZyhwcmljZSkpIEFTIHByaWNlLCBiYXIocHJpY2UsIDAsIDUwMDAwMDAsIDEwMCkgRlJPTSB1a19wcmljZV9wYWlkIFdIRVJFIGRhdGUgPj0gJzIwMjAtMDEtMDEnIEdST1VQIEJZIHRvd24sIGRpc3RyaWN0IEhBVklORyBjID49IDEwMCBPUkRFUiBCWSBwcmljZSBERVNDIExJTUlUIDEwMA==).

View File

@ -407,6 +407,18 @@ ClickHouse проверяет условия для `min_part_size` и `min_part
<include_from>/etc/metrica.xml</include_from>
```
## interserver_listen_host {#interserver-listen-host}
Ограничение по хостам, для обмена между серверами ClickHouse.
Значение по умолчанию совпадает со значением параметра listen_host
Примеры:
``` xml
<interserver_listen_host>::ffff:a00:1</interserver_listen_host>
<interserver_listen_host>10.0.0.1</interserver_listen_host>
```
## interserver_http_port {#interserver-http-port}
Порт для обмена между серверами ClickHouse.

View File

@ -12,12 +12,13 @@ sidebar_label: "Функции для работы с файлами"
**Синтаксис**
``` sql
file(path)
file(path[, default])
```
**Аргументы**
- `path` — относительный путь до файла от [user_files_path](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-user_files_path). Путь к файлу может включать следующие символы подстановки и шаблоны: `*`, `?`, `{abc,def}` и `{N..M}`, где `N`, `M` — числа, `'abc', 'def'` — строки.
- `default` — Значение возвращаемое в случае, если указанный файл не существует. Поддерживаемые типы данных: [String](../../sql-reference/data-types/string.md) и [NULL](../../sql-reference/syntax.md#null-literal).
**Примеры**

View File

@ -264,3 +264,9 @@ ClickHouse成员一旦在您的拉取请求上贴上«可以测试»标签
系统将分别为您的拉取请求准备ClickHouse二进制版本。若要检索这些构建信息请在检查列表中单击« ClickHouse构建检查»旁边的«详细信息»链接。在这里您会找到指向ClickHouse的.deb软件包的直接链接此外甚至可以将其部署在生产服务器上如果您不担心
某些构建项很可能会在首次构建时失败。这是因为我们同时检查了基于gcc和clang的构建几乎所有现有的被clang启用的警告总是带有`-Werror`标志。在同一页面上您可以找到所有构建的日志因此不必以所有可能的方式构建ClickHouse。
## 浏览ClickHouse源代码 {#browse-clickhouse-source-code}
您可以使用 **Woboq** 在线代码浏览器 [点击这里](https://clickhouse.com/codebrowser/ClickHouse/src/index.html). 它提供了代码导航和语义突出显示、搜索和索引。 代码快照每天更新。
此外,您还可以像往常一样浏览源代码 [GitHub](https://github.com/ClickHouse/ClickHouse)

View File

@ -18,11 +18,7 @@ option (ENABLE_CLICKHOUSE_SERVER "Server mode (main mode)" ${ENABLE_CLICKHOUSE_A
option (ENABLE_CLICKHOUSE_CLIENT "Client mode (interactive tui/shell that connects to the server)"
${ENABLE_CLICKHOUSE_ALL})
if (CLICKHOUSE_SPLIT_BINARY)
option (ENABLE_CLICKHOUSE_SELF_EXTRACTING "Self-extracting executable" OFF)
else ()
option (ENABLE_CLICKHOUSE_SELF_EXTRACTING "Self-extracting executable" ON)
endif ()
# https://clickhouse.com/docs/en/operations/utilities/clickhouse-local/
option (ENABLE_CLICKHOUSE_LOCAL "Local files fast processing mode" ${ENABLE_CLICKHOUSE_ALL})
@ -80,12 +76,7 @@ if (NOT ENABLE_NURAFT)
set(ENABLE_CLICKHOUSE_KEEPER_CONVERTER OFF)
endif()
if (CLICKHOUSE_SPLIT_BINARY)
option(ENABLE_CLICKHOUSE_INSTALL "Install ClickHouse without .deb/.rpm/.tgz packages (having the binary only)" OFF)
else ()
option(ENABLE_CLICKHOUSE_INSTALL "Install ClickHouse without .deb/.rpm/.tgz packages (having the binary only)"
${ENABLE_CLICKHOUSE_ALL})
endif ()
option(ENABLE_CLICKHOUSE_INSTALL "Install ClickHouse without .deb/.rpm/.tgz packages (having the binary only)" ${ENABLE_CLICKHOUSE_ALL})
message(STATUS "ClickHouse modes:")
@ -211,10 +202,6 @@ macro(clickhouse_target_link_split_lib target name)
endif()
endmacro()
macro(clickhouse_program_link_split_binary name)
clickhouse_target_link_split_lib(clickhouse-${name} ${name})
endmacro()
macro(clickhouse_program_add_library name)
string(TOUPPER ${name} name_uc)
string(REPLACE "-" "_" name_uc ${name_uc})
@ -239,17 +226,8 @@ macro(clickhouse_program_add_library name)
endif()
endmacro()
macro(clickhouse_program_add_executable name)
if(CLICKHOUSE_SPLIT_BINARY)
clickhouse_add_executable(clickhouse-${name} clickhouse-${name}.cpp)
clickhouse_program_link_split_binary(${name})
install(TARGETS clickhouse-${name} ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
endif()
endmacro()
macro(clickhouse_program_add name)
clickhouse_program_add_library(${name})
clickhouse_program_add_executable(${name})
endmacro()
add_subdirectory (server)
@ -342,50 +320,14 @@ if (CLICKHOUSE_ONE_SHARED)
install (TARGETS clickhouse-lib LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT clickhouse)
endif()
if (CLICKHOUSE_SPLIT_BINARY)
set (CLICKHOUSE_ALL_TARGETS
clickhouse-server
clickhouse-client
clickhouse-local
clickhouse-benchmark
clickhouse-extract-from-config
clickhouse-compressor
clickhouse-format
clickhouse-obfuscator
clickhouse-git-import
clickhouse-copier
clickhouse-static-files-disk-uploader
clickhouse-disks)
if (ENABLE_CLICKHOUSE_ODBC_BRIDGE)
list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-odbc-bridge)
endif ()
if (ENABLE_CLICKHOUSE_LIBRARY_BRIDGE)
list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-library-bridge)
endif ()
if (ENABLE_CLICKHOUSE_KEEPER)
list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-keeper)
endif ()
if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER)
list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-keeper-converter)
endif ()
if (ENABLE_CLICKHOUSE_SU)
list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-su)
endif ()
set_target_properties(${CLICKHOUSE_ALL_TARGETS} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..)
add_custom_target (clickhouse-bundle ALL DEPENDS ${CLICKHOUSE_ALL_TARGETS})
add_custom_target (clickhouse ALL DEPENDS clickhouse-bundle)
install(PROGRAMS clickhouse-split-helper DESTINATION ${CMAKE_INSTALL_BINDIR} RENAME clickhouse COMPONENT clickhouse)
else ()
clickhouse_add_executable (clickhouse main.cpp)
if (NOT USE_STATIC_LIBRARIES AND SPLIT_SHARED_LIBRARIES)
# Shared split (dev) build: In CI, the server is run with custom LD_LIBRARY_PATH. This makes the harmful env check re-execute the
# process in a clean environment but as in CI the containing directory is not included in DT_RUNPATH/DT_RPATH, the server won't come up.
target_compile_definitions(clickhouse PRIVATE DISABLE_HARMFUL_ENV_VAR_CHECK)
endif ()
# A library that prevent usage of several functions from libc.
if (ARCH_AMD64 AND OS_LINUX AND NOT OS_ANDROID)
set (HARMFUL_LIB harmful)
@ -546,7 +488,6 @@ else ()
clickhouse_make_empty_debug_info_for_nfpm(TARGET clickhouse DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/${SPLITTED_DEBUG_SYMBOLS_DIR})
install (TARGETS clickhouse RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
endif()
endif()
if (ENABLE_TESTS)
set (CLICKHOUSE_UNIT_TESTS_TARGETS unit_tests_dbms)

View File

@ -352,6 +352,9 @@ try
{
UseSSL use_ssl;
ThreadStatus thread_status;
StackTrace::setShowAddresses(config().getBool("show_addresses_in_stack_traces", true));
setupSignalHandler();
std::cout << std::fixed << std::setprecision(3);

View File

@ -345,6 +345,7 @@ struct Checker
;
#ifndef DISABLE_HARMFUL_ENV_VAR_CHECK
/// NOTE: We will migrate to full static linking or our own dynamic loader to make this code obsolete.
void checkHarmfulEnvironmentVariables(char ** argv)
{
@ -396,6 +397,7 @@ void checkHarmfulEnvironmentVariables(char ** argv)
_exit(error);
}
}
#endif
}
@ -422,7 +424,9 @@ int main(int argc_, char ** argv_)
/// will work only after additional call of this function.
updatePHDRCache();
#ifndef DISABLE_HARMFUL_ENV_VAR_CHECK
checkHarmfulEnvironmentVariables(argv_);
#endif
/// Reset new handler to default (that throws std::bad_alloc)
/// It is needed because LLVM library clobbers it.

View File

@ -46,6 +46,7 @@
#include <boost/algorithm/string.hpp>
#include <boost/container/flat_map.hpp>
#include <Common/TerminalSize.h>
#include <bit>
static const char * documentation = R"(
@ -186,7 +187,7 @@ static UInt64 transform(UInt64 x, UInt64 seed)
if (x == 2 || x == 3)
return x ^ (seed & 1);
size_t num_leading_zeros = __builtin_clzll(x);
size_t num_leading_zeros = std::countl_zero(x);
return feistelNetwork(x, 64 - num_leading_zeros - 1, seed);
}

View File

@ -367,7 +367,7 @@ Poco::Net::SocketAddress Server::socketBindListen(
return address;
}
std::vector<std::string> getListenHosts(const Poco::Util::AbstractConfiguration & config)
Strings getListenHosts(const Poco::Util::AbstractConfiguration & config)
{
auto listen_hosts = DB::getMultipleValuesFromConfig(config, "", "listen_host");
if (listen_hosts.empty())
@ -378,6 +378,16 @@ std::vector<std::string> getListenHosts(const Poco::Util::AbstractConfiguration
return listen_hosts;
}
Strings getInterserverListenHosts(const Poco::Util::AbstractConfiguration & config)
{
auto interserver_listen_hosts = DB::getMultipleValuesFromConfig(config, "", "interserver_listen_host");
if (!interserver_listen_hosts.empty())
return interserver_listen_hosts;
/// Use more general restriction in case of emptiness
return getListenHosts(config);
}
bool getListenTry(const Poco::Util::AbstractConfiguration & config)
{
bool listen_try = config.getBool("listen_try", false);
@ -627,6 +637,8 @@ int Server::main(const std::vector<std::string> & /*args*/)
MainThreadStatus::getInstance();
StackTrace::setShowAddresses(config().getBool("show_addresses_in_stack_traces", true));
registerFunctions();
registerAggregateFunctions();
registerTableFunctions();
@ -1232,6 +1244,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
/* already_loaded = */ false); /// Reload it right now (initial loading)
const auto listen_hosts = getListenHosts(config());
const auto interserver_listen_hosts = getInterserverListenHosts(config());
const auto listen_try = getListenTry(config());
if (config().has("keeper_server"))
@ -1627,7 +1640,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
{
std::lock_guard lock(servers_lock);
createServers(config(), listen_hosts, listen_try, server_pool, async_metrics, servers);
createServers(config(), listen_hosts, interserver_listen_hosts, listen_try, server_pool, async_metrics, servers);
if (servers.empty())
throw Exception(
"No servers started (add valid listen_host and 'tcp_port' or 'http_port' to configuration file.)",
@ -1809,7 +1822,8 @@ int Server::main(const std::vector<std::string> & /*args*/)
void Server::createServers(
Poco::Util::AbstractConfiguration & config,
const std::vector<std::string> & listen_hosts,
const Strings & listen_hosts,
const Strings & interserver_listen_hosts,
bool listen_try,
Poco::ThreadPool & server_pool,
AsynchronousMetrics & async_metrics,
@ -1927,51 +1941,6 @@ void Server::createServers(
#endif
});
/// Interserver IO HTTP
port_name = "interserver_http_port";
createServer(config, listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter
{
Poco::Net::ServerSocket socket;
auto address = socketBindListen(config, socket, listen_host, port);
socket.setReceiveTimeout(settings.http_receive_timeout);
socket.setSendTimeout(settings.http_send_timeout);
return ProtocolServerAdapter(
listen_host,
port_name,
"replica communication (interserver): http://" + address.toString(),
std::make_unique<HTTPServer>(
context(),
createHandlerFactory(*this, async_metrics, "InterserverIOHTTPHandler-factory"),
server_pool,
socket,
http_params));
});
port_name = "interserver_https_port";
createServer(config, listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter
{
#if USE_SSL
Poco::Net::SecureServerSocket socket;
auto address = socketBindListen(config, socket, listen_host, port, /* secure = */ true);
socket.setReceiveTimeout(settings.http_receive_timeout);
socket.setSendTimeout(settings.http_send_timeout);
return ProtocolServerAdapter(
listen_host,
port_name,
"secure replica communication (interserver): https://" + address.toString(),
std::make_unique<HTTPServer>(
context(),
createHandlerFactory(*this, async_metrics, "InterserverIOHTTPSHandler-factory"),
server_pool,
socket,
http_params));
#else
UNUSED(port);
throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.",
ErrorCodes::SUPPORT_IS_DISABLED};
#endif
});
port_name = "mysql_port";
createServer(config, listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter
{
@ -2030,6 +1999,55 @@ void Server::createServers(
});
}
/// Now iterate over interserver_listen_hosts
for (const auto & interserver_listen_host : interserver_listen_hosts)
{
/// Interserver IO HTTP
const char * port_name = "interserver_http_port";
createServer(config, interserver_listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter
{
Poco::Net::ServerSocket socket;
auto address = socketBindListen(config, socket, interserver_listen_host, port);
socket.setReceiveTimeout(settings.http_receive_timeout);
socket.setSendTimeout(settings.http_send_timeout);
return ProtocolServerAdapter(
interserver_listen_host,
port_name,
"replica communication (interserver): http://" + address.toString(),
std::make_unique<HTTPServer>(
context(),
createHandlerFactory(*this, async_metrics, "InterserverIOHTTPHandler-factory"),
server_pool,
socket,
http_params));
});
port_name = "interserver_https_port";
createServer(config, interserver_listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter
{
#if USE_SSL
Poco::Net::SecureServerSocket socket;
auto address = socketBindListen(config, socket, interserver_listen_host, port, /* secure = */ true);
socket.setReceiveTimeout(settings.http_receive_timeout);
socket.setSendTimeout(settings.http_send_timeout);
return ProtocolServerAdapter(
interserver_listen_host,
port_name,
"secure replica communication (interserver): https://" + address.toString(),
std::make_unique<HTTPServer>(
context(),
createHandlerFactory(*this, async_metrics, "InterserverIOHTTPSHandler-factory"),
server_pool,
socket,
http_params));
#else
UNUSED(port);
throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.",
ErrorCodes::SUPPORT_IS_DISABLED};
#endif
});
}
}
void Server::updateServers(
@ -2041,6 +2059,7 @@ void Server::updateServers(
Poco::Logger * log = &logger();
const auto listen_hosts = getListenHosts(config);
const auto interserver_listen_hosts = getInterserverListenHosts(config);
const auto listen_try = getListenTry(config);
/// Remove servers once all their connections are closed
@ -2073,7 +2092,7 @@ void Server::updateServers(
}
}
createServers(config, listen_hosts, listen_try, server_pool, async_metrics, servers, /* start_servers= */ true);
createServers(config, listen_hosts, interserver_listen_hosts, listen_try, server_pool, async_metrics, servers, /* start_servers= */ true);
std::erase_if(servers, std::bind_front(check_server, ""));
}

View File

@ -86,7 +86,8 @@ private:
void createServers(
Poco::Util::AbstractConfiguration & config,
const std::vector<std::string> & listen_hosts,
const Strings & listen_hosts,
const Strings & interserver_listen_hosts,
bool listen_try,
Poco::ThreadPool & server_pool,
AsynchronousMetrics & async_metrics,

View File

@ -188,6 +188,10 @@
<listen_host>127.0.0.1</listen_host>
-->
<!-- <interserver_listen_host>::</interserver_listen_host> -->
<!-- Listen host for communication between replicas. Used for data exchange -->
<!-- Default values - equal to listen_host -->
<!-- Don't exit if IPv6 or IPv4 networks are unavailable while trying to listen. -->
<!-- <listen_try>0</listen_try> -->
@ -1386,4 +1390,13 @@
<lru_cache_size>268435456</lru_cache_size>
<continue_if_corrupted>true</continue_if_corrupted>
</merge_tree_metadata_cache-->
<!-- This allows to disable exposing addresses in stack traces for security reasons.
Please be aware that it does not improve security much, but makes debugging much harder.
The addresses that are small offsets from zero will be displayed nevertheless to show nullptr dereferences.
Regardless of this configuration, the addresses are visible in the system.stack_trace and system.trace_log tables
if the user has access to these tables.
I don't recommend to change this setting.
-->
<show_addresses_in_stack_traces>false</show_addresses_in_stack_traces>
</clickhouse>

View File

@ -152,8 +152,8 @@ template <typename Data>
class AggregateFunctionDistinct : public IAggregateFunctionDataHelper<Data, AggregateFunctionDistinct<Data>>
{
private:
static constexpr auto prefix_size = sizeof(Data);
AggregateFunctionPtr nested_func;
size_t prefix_size;
size_t arguments_num;
AggregateDataPtr getNestedPlace(AggregateDataPtr __restrict place) const noexcept
@ -170,7 +170,11 @@ public:
AggregateFunctionDistinct(AggregateFunctionPtr nested_func_, const DataTypes & arguments, const Array & params_)
: IAggregateFunctionDataHelper<Data, AggregateFunctionDistinct>(arguments, params_)
, nested_func(nested_func_)
, arguments_num(arguments.size()) {}
, arguments_num(arguments.size())
{
size_t nested_size = nested_func->alignOfData();
prefix_size = (sizeof(Data) + nested_size - 1) / nested_size * nested_size;
}
void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena * arena) const override
{

View File

@ -583,9 +583,14 @@ try
if (has_vertical_output_suffix)
current_format = "Vertical";
/// It is not clear how to write progress intermixed with data with parallel formatting.
bool logs_into_stdout = server_logs_file == "-";
bool extras_into_stdout = need_render_progress || logs_into_stdout;
bool select_only_into_file = select_into_file && !select_into_file_and_stdout;
/// It is not clear how to write progress and logs
/// intermixed with data with parallel formatting.
/// It may increase code complexity significantly.
if (!need_render_progress || (select_into_file && !select_into_file_and_stdout))
if (!extras_into_stdout || select_only_into_file)
output_format = global_context->getOutputFormatParallelIfPossible(
current_format, out_file_buf ? *out_file_buf : *out_buf, block);
else

View File

@ -298,7 +298,7 @@ ColumnPtr ColumnDecimal<T>::filter(const IColumn::Filter & filt, ssize_t result_
{
while (mask)
{
size_t index = __builtin_ctzll(mask);
size_t index = std::countr_zero(mask);
res_data.push_back(data_pos[index]);
#ifdef __BMI__
mask = _blsr_u64(mask);

View File

@ -240,7 +240,7 @@ ColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t result
size_t res_chars_size = res->chars.size();
while (mask)
{
size_t index = __builtin_ctzll(mask);
size_t index = std::countr_zero(mask);
res->chars.resize(res_chars_size + n);
memcpySmallAllowReadWriteOverflow15(&res->chars[res_chars_size], data_pos + index * n, n);
res_chars_size += n;

View File

@ -508,7 +508,7 @@ ColumnPtr ColumnVector<T>::filter(const IColumn::Filter & filt, ssize_t result_s
{
while (mask)
{
size_t index = __builtin_ctzll(mask);
size_t index = std::countr_zero(mask);
res_data.push_back(data_pos[index]);
#ifdef __BMI__
mask = _blsr_u64(mask);

View File

@ -2,13 +2,14 @@
#include <Columns/ColumnVector.h>
#include <Common/typeid_cast.h>
#include <Common/HashTable/HashSet.h>
#include <bit>
#include "ColumnsCommon.h"
namespace DB
{
#if defined(__SSE2__) && defined(__POPCNT__)
#if defined(__SSE2__)
/// Transform 64-byte mask to 64-bit mask.
static UInt64 toBits64(const Int8 * bytes64)
{
@ -41,11 +42,11 @@ size_t countBytesInFilter(const UInt8 * filt, size_t start, size_t end)
const Int8 * end_pos = pos + (end - start);
#if defined(__SSE2__) && defined(__POPCNT__)
#if defined(__SSE2__)
const Int8 * end_pos64 = pos + (end - start) / 64 * 64;
for (; pos < end_pos64; pos += 64)
count += __builtin_popcountll(toBits64(pos));
count += std::popcount(toBits64(pos));
/// TODO Add duff device for tail?
#endif
@ -74,11 +75,11 @@ size_t countBytesInFilterWithNull(const IColumn::Filter & filt, const UInt8 * nu
const Int8 * pos2 = reinterpret_cast<const Int8 *>(null_map) + start;
const Int8 * end_pos = pos + (end - start);
#if defined(__SSE2__) && defined(__POPCNT__)
#if defined(__SSE2__)
const Int8 * end_pos64 = pos + (end - start) / 64 * 64;
for (; pos < end_pos64; pos += 64, pos2 += 64)
count += __builtin_popcountll(toBits64(pos) & ~toBits64(pos2));
count += std::popcount(toBits64(pos) & ~toBits64(pos2));
/// TODO Add duff device for tail?
#endif
@ -259,7 +260,7 @@ namespace
{
while (mask)
{
size_t index = __builtin_ctzll(mask);
size_t index = std::countr_zero(mask);
copy_array(offsets_pos + index);
#ifdef __BMI__
mask = _blsr_u64(mask);

View File

@ -36,7 +36,7 @@ inline UInt64 bytes64MaskToBits64Mask(const UInt8 * bytes64)
_mm256_loadu_si256(reinterpret_cast<const __m256i *>(bytes64)), zero32))) & 0xffffffff)
| (static_cast<UInt64>(_mm256_movemask_epi8(_mm256_cmpeq_epi8(
_mm256_loadu_si256(reinterpret_cast<const __m256i *>(bytes64+32)), zero32))) << 32);
#elif defined(__SSE2__) && defined(__POPCNT__)
#elif defined(__SSE2__)
static const __m128i zero16 = _mm_setzero_si128();
UInt64 res =
(static_cast<UInt64>(_mm_movemask_epi8(_mm_cmpeq_epi8(

View File

@ -3,6 +3,7 @@
#include <Common/HashTable/HashMap.h>
#include <Common/HashTable/HashTable.h>
#include <bit>
#include <new>
#include <variant>
@ -21,17 +22,17 @@ struct StringKey24
inline StringRef ALWAYS_INLINE toStringRef(const StringKey8 & n)
{
assert(n != 0);
return {reinterpret_cast<const char *>(&n), 8ul - (__builtin_clzll(n) >> 3)};
return {reinterpret_cast<const char *>(&n), 8ul - (std::countl_zero(n) >> 3)};
}
inline StringRef ALWAYS_INLINE toStringRef(const StringKey16 & n)
{
assert(n.items[1] != 0);
return {reinterpret_cast<const char *>(&n), 16ul - (__builtin_clzll(n.items[1]) >> 3)};
return {reinterpret_cast<const char *>(&n), 16ul - (std::countl_zero(n.items[1]) >> 3)};
}
inline StringRef ALWAYS_INLINE toStringRef(const StringKey24 & n)
{
assert(n.c != 0);
return {reinterpret_cast<const char *>(&n), 24ul - (__builtin_clzll(n.c) >> 3)};
return {reinterpret_cast<const char *>(&n), 24ul - (std::countl_zero(n.c) >> 3)};
}
struct StringHashTableHash

View File

@ -11,6 +11,7 @@
#include <IO/WriteHelpers.h>
#include <Core/Defines.h>
#include <bit>
#include <cmath>
#include <cstring>
@ -205,7 +206,7 @@ struct TrailingZerosCounter<UInt32>
{
static int apply(UInt32 val)
{
return __builtin_ctz(val);
return std::countr_zero(val);
}
};
@ -214,7 +215,7 @@ struct TrailingZerosCounter<UInt64>
{
static int apply(UInt64 val)
{
return __builtin_ctzll(val);
return std::countr_zero(val);
}
};

View File

@ -5,6 +5,7 @@
#include <Common/formatIPv6.h>
#include <cstring>
#include <bit>
namespace DB
@ -89,7 +90,7 @@ bool matchIPv6Subnet(const uint8_t * addr, const uint8_t * cidr_addr, UInt8 pref
if (mask)
{
auto offset = __builtin_ctz(mask);
auto offset = std::countr_zero(mask);
if (prefix / 8 != offset)
return prefix / 8 < offset;

View File

@ -25,6 +25,10 @@
M(WriteBufferFromFileDescriptorWrite, "Number of writes (write/pwrite) to a file descriptor. Does not include sockets.") \
M(WriteBufferFromFileDescriptorWriteFailed, "Number of times the write (write/pwrite) to a file descriptor have failed.") \
M(WriteBufferFromFileDescriptorWriteBytes, "Number of bytes written to file descriptors. If the file is compressed, this will show compressed data size.") \
M(FileSync, "Number of times the F_FULLFSYNC/fsync/fdatasync function was called for files.") \
M(DirectorySync, "Number of times the F_FULLFSYNC/fsync/fdatasync function was called for directories.") \
M(FileSyncElapsedMicroseconds, "Total time spent waiting for F_FULLFSYNC/fsync/fdatasync syscall for files.") \
M(DirectorySyncElapsedMicroseconds, "Total time spent waiting for F_FULLFSYNC/fsync/fdatasync syscall for directories.") \
M(ReadCompressedBytes, "Number of bytes (the number of bytes before decompression) read from compressed sources (files, network).") \
M(CompressedReadBufferBlocks, "Number of compressed blocks (the blocks of data that are compressed independent of each other) read from compressed sources (files, network).") \
M(CompressedReadBufferBytes, "Number of uncompressed bytes (the number of bytes after decompression) read from compressed sources (files, network).") \

View File

@ -78,7 +78,7 @@ private:
constexpr uint64_t nextAlphaSize(uint64_t x)
{
constexpr uint64_t alpha_map_elements_per_counter = 6;
return 1ULL << (sizeof(uint64_t) * 8 - __builtin_clzll(x * alpha_map_elements_per_counter));
return 1ULL << (sizeof(uint64_t) * 8 - std::countl_zero(x * alpha_map_elements_per_counter));
}
public:

View File

@ -1,6 +1,5 @@
#include <Common/StackTrace.h>
#include <Core/Defines.h>
#include <Common/Dwarf.h>
#include <Common/Elf.h>
#include <Common/SymbolIndex.h>
@ -8,6 +7,7 @@
#include <base/CachedFn.h>
#include <base/demangle.h>
#include <atomic>
#include <cstring>
#include <filesystem>
#include <sstream>
@ -19,6 +19,32 @@
# include <libunwind.h>
#endif
namespace
{
/// Currently this variable is set up once on server startup.
/// But we use atomic just in case, so it is possible to be modified at runtime.
std::atomic<bool> show_addresses = true;
bool shouldShowAddress(const void * addr)
{
/// If the address is less than 4096, most likely it is a nullptr dereference with offset,
/// and showing this offset is secure nevertheless.
/// NOTE: 4096 is the page size on x86 and it can be different on other systems,
/// but for the purpose of this branch, it does not matter.
if (reinterpret_cast<uintptr_t>(addr) < 4096)
return true;
return show_addresses.load(std::memory_order_relaxed);
}
}
void StackTrace::setShowAddresses(bool show)
{
show_addresses.store(show, std::memory_order_relaxed);
}
std::string signalToErrorMessage(int sig, const siginfo_t & info, [[maybe_unused]] const ucontext_t & context)
{
std::stringstream error; // STYLE_CHECK_ALLOW_STD_STRING_STREAM
@ -30,7 +56,7 @@ std::string signalToErrorMessage(int sig, const siginfo_t & info, [[maybe_unused
/// Print info about address and reason.
if (nullptr == info.si_addr)
error << "Address: NULL pointer.";
else
else if (shouldShowAddress(info.si_addr))
error << "Address: " << info.si_addr;
#if defined(__x86_64__) && !defined(OS_FREEBSD) && !defined(OS_DARWIN) && !defined(__arm__) && !defined(__powerpc__)
@ -372,7 +398,9 @@ static void toStringEveryLineImpl(
else
out << "?";
if (shouldShowAddress(physical_addr))
out << " @ " << physical_addr;
out << " in " << (object ? object->name : "?");
for (size_t j = 0; j < inline_frames.size(); ++j)
@ -393,11 +421,14 @@ static void toStringEveryLineImpl(
for (size_t i = offset; i < size; ++i)
{
const void * addr = frame_pointers[i];
if (shouldShowAddress(addr))
{
out << i << ". " << addr;
callback(out.str());
out.str({});
}
}
#endif
}

View File

@ -67,6 +67,11 @@ public:
void toStringEveryLine(std::function<void(const std::string &)> callback) const;
/// Displaying the addresses can be disabled for security reasons.
/// If you turn off addresses, it will be more secure, but we will be unable to help you with debugging.
/// Please note: addresses are also available in the system.stack_trace and system.trace_log tables.
static void setShowAddresses(bool show);
protected:
void tryCapture();

View File

@ -2,6 +2,7 @@
#include <Common/StringUtils/StringUtils.h>
#include <widechar_width.h>
#include <bit>
namespace DB
@ -124,7 +125,7 @@ size_t computeWidthImpl(const UInt8 * data, size_t size, size_t prefix, size_t l
if (non_regular_width_mask)
{
auto num_regular_chars = __builtin_ctz(non_regular_width_mask);
auto num_regular_chars = std::countr_zero(non_regular_width_mask);
width += num_regular_chars;
i += num_regular_chars;
break;

View File

@ -83,7 +83,7 @@ inline size_t countCodePoints(const UInt8 * data, size_t size)
const auto threshold = vdupq_n_s8(0xBF);
for (; data < src_end_sse; data += bytes_sse)
res += __builtin_popcountll(get_nibble_mask(vcgtq_s8(vld1q_s8(reinterpret_cast<const int8_t *>(data)), threshold)));
res += std::popcount(get_nibble_mask(vcgtq_s8(vld1q_s8(reinterpret_cast<const int8_t *>(data)), threshold)));
res >>= 2;
#endif

View File

@ -18,7 +18,6 @@
#cmakedefine01 USE_DATASKETCHES
#cmakedefine01 USE_YAML_CPP
#cmakedefine01 CLICKHOUSE_SPLIT_BINARY
#cmakedefine01 USE_BZIP2
#cmakedefine01 USE_MINIZIP
#cmakedefine01 USE_SNAPPY

View File

@ -1,5 +1,6 @@
#include <iostream>
#include <string>
#include <bit>
#include <fmt/format.h>
@ -561,7 +562,7 @@ int main(int argc, char ** argv)
/// Fill source data
for (size_t i = 0; i < size; ++i)
{
keys[i] = __builtin_ctz(i + 1); /// Make keys to have just slightly more realistic distribution.
keys[i] = std::countr_zero(i + 1); /// Make keys to have just slightly more realistic distribution.
values[i] = 1234.5; /// The distribution of values does not affect execution speed.
}

View File

@ -1,6 +1,7 @@
#pragma once
#include <algorithm>
#include <bit>
#include <cstdint>
#include <Core/Defines.h>
@ -50,7 +51,7 @@ inline int memcmpSmallAllowOverflow15(const Char * a, size_t a_size, const Char
if (mask)
{
offset += __builtin_ctz(mask);
offset += std::countr_zero(mask);
if (offset >= min_size)
break;
@ -82,7 +83,7 @@ inline int memcmpSmallLikeZeroPaddedAllowOverflow15(const Char * a, size_t a_siz
if (mask)
{
offset += __builtin_ctz(mask);
offset += std::countr_zero(mask);
if (offset >= min_size)
break;
@ -123,7 +124,7 @@ inline int memcmpSmallLikeZeroPaddedAllowOverflow15(const Char * a, size_t a_siz
if (mask)
{
offset += __builtin_ctz(mask);
offset += std::countr_zero(mask);
if (offset >= max_size)
return 0;
@ -150,7 +151,7 @@ inline int memcmpSmallAllowOverflow15(const Char * a, const Char * b, size_t siz
if (mask)
{
offset += __builtin_ctz(mask);
offset += std::countr_zero(mask);
if (offset >= size)
return 0;
@ -180,7 +181,7 @@ inline bool memequalSmallAllowOverflow15(const Char * a, size_t a_size, const Ch
if (mask)
{
offset += __builtin_ctz(mask);
offset += std::countr_zero(mask);
return offset >= a_size;
}
}
@ -203,7 +204,7 @@ inline int memcmpSmallMultipleOf16(const Char * a, const Char * b, size_t size)
if (mask)
{
offset += __builtin_ctz(mask);
offset += std::countr_zero(mask);
return detail::cmp(a[offset], b[offset]);
}
}
@ -222,7 +223,7 @@ inline int memcmp16(const Char * a, const Char * b)
if (mask)
{
auto offset = __builtin_ctz(mask);
auto offset = std::countr_zero(mask);
return detail::cmp(a[offset], b[offset]);
}
@ -252,7 +253,7 @@ inline bool memoryIsZeroSmallAllowOverflow15(const void * data, size_t size)
if (mask)
{
offset += __builtin_ctz(mask);
offset += std::countr_zero(mask);
return offset >= size;
}
}
@ -285,7 +286,7 @@ inline int memcmpSmallAllowOverflow15(const Char * a, size_t a_size, const Char
if (mask)
{
offset += __builtin_ctz(mask);
offset += std::countr_zero(mask);
if (offset >= min_size)
break;
@ -317,7 +318,7 @@ inline int memcmpSmallLikeZeroPaddedAllowOverflow15(const Char * a, size_t a_siz
if (mask)
{
offset += __builtin_ctz(mask);
offset += std::countr_zero(mask);
if (offset >= min_size)
break;
@ -359,7 +360,7 @@ inline int memcmpSmallLikeZeroPaddedAllowOverflow15(const Char * a, size_t a_siz
if (mask)
{
offset += __builtin_ctz(mask);
offset += std::countr_zero(mask);
if (offset >= max_size)
return 0;
@ -386,7 +387,7 @@ inline int memcmpSmallAllowOverflow15(const Char * a, const Char * b, size_t siz
if (mask)
{
offset += __builtin_ctz(mask);
offset += std::countr_zero(mask);
if (offset >= size)
return 0;
@ -416,7 +417,7 @@ inline bool memequalSmallAllowOverflow15(const Char * a, size_t a_size, const Ch
if (mask)
{
offset += __builtin_ctz(mask);
offset += std::countr_zero(mask);
return offset >= a_size;
}
}
@ -439,7 +440,7 @@ inline int memcmpSmallMultipleOf16(const Char * a, const Char * b, size_t size)
if (mask)
{
offset += __builtin_ctz(mask);
offset += std::countr_zero(mask);
return detail::cmp(a[offset], b[offset]);
}
}
@ -459,7 +460,7 @@ inline int memcmp16(const Char * a, const Char * b)
if (mask)
{
auto offset = __builtin_ctz(mask);
auto offset = std::countr_zero(mask);
return detail::cmp(a[offset], b[offset]);
}
@ -490,7 +491,7 @@ inline bool memoryIsZeroSmallAllowOverflow15(const void * data, size_t size)
if (mask)
{
offset += __builtin_ctz(mask);
offset += std::countr_zero(mask);
return offset >= size;
}
}
@ -523,7 +524,7 @@ inline int memcmpSmallAllowOverflow15(const Char * a, size_t a_size, const Char
if (mask)
{
offset += __builtin_ctzll(mask) >> 2;
offset += std::countr_zero(mask) >> 2;
if (offset >= min_size)
break;
@ -548,7 +549,7 @@ inline int memcmpSmallLikeZeroPaddedAllowOverflow15(const Char * a, size_t a_siz
if (mask)
{
offset += __builtin_ctzll(mask) >> 2;
offset += std::countr_zero(mask) >> 2;
if (offset >= min_size)
break;
@ -589,7 +590,7 @@ inline int memcmpSmallLikeZeroPaddedAllowOverflow15(const Char * a, size_t a_siz
if (mask)
{
offset += __builtin_ctzll(mask) >> 2;
offset += std::countr_zero(mask) >> 2;
if (offset >= max_size)
return 0;
@ -611,7 +612,7 @@ inline int memcmpSmallAllowOverflow15(const Char * a, const Char * b, size_t siz
if (mask)
{
offset += __builtin_ctzll(mask) >> 2;
offset += std::countr_zero(mask) >> 2;
if (offset >= size)
return 0;
@ -637,7 +638,7 @@ inline bool memequalSmallAllowOverflow15(const Char * a, size_t a_size, const Ch
if (mask)
{
offset += __builtin_ctzll(mask) >> 2;
offset += std::countr_zero(mask) >> 2;
return offset >= a_size;
}
}
@ -656,7 +657,7 @@ inline int memcmpSmallMultipleOf16(const Char * a, const Char * b, size_t size)
if (mask)
{
offset += __builtin_ctzll(mask) >> 2;
offset += std::countr_zero(mask) >> 2;
return detail::cmp(a[offset], b[offset]);
}
}
@ -672,7 +673,7 @@ inline int memcmp16(const Char * a, const Char * b)
mask = ~mask;
if (mask)
{
auto offset = __builtin_ctzll(mask) >> 2;
auto offset = std::countr_zero(mask) >> 2;
return detail::cmp(a[offset], b[offset]);
}
return 0;
@ -694,7 +695,7 @@ inline bool memoryIsZeroSmallAllowOverflow15(const void * data, size_t size)
if (mask)
{
offset += __builtin_ctzll(mask) >> 2;
offset += std::countr_zero(mask) >> 2;
return offset >= size;
}
}

View File

@ -1,5 +1,6 @@
#include "CompressedReadBufferBase.h"
#include <bit>
#include <cstring>
#include <cassert>
#include <city.h>
@ -93,8 +94,8 @@ static void validateChecksum(char * data, size_t size, const Checksum expected_c
}
/// Check if the difference caused by single bit flip in stored checksum.
size_t difference = __builtin_popcountll(expected_checksum.first ^ calculated_checksum.first)
+ __builtin_popcountll(expected_checksum.second ^ calculated_checksum.second);
size_t difference = std::popcount(expected_checksum.first ^ calculated_checksum.first)
+ std::popcount(expected_checksum.second ^ calculated_checksum.second);
if (difference == 1)
{

View File

@ -8,6 +8,7 @@
#include <Parsers/ASTFunction.h>
#include <IO/WriteHelpers.h>
#include <Core/Types.h>
#include <bit>
namespace DB
@ -413,7 +414,7 @@ UInt32 getValuableBitsNumber(UInt64 min, UInt64 max)
{
UInt64 diff_bits = min ^ max;
if (diff_bits)
return 64 - __builtin_clzll(diff_bits);
return 64 - std::countl_zero(diff_bits);
return 0;
}

View File

@ -34,6 +34,8 @@ namespace ProfileEvents
extern const Event AIOWriteBytes;
extern const Event AIORead;
extern const Event AIOReadBytes;
extern const Event FileSync;
extern const Event FileSyncElapsedMicroseconds;
}
namespace DB
@ -544,6 +546,9 @@ public:
file_path,
std::to_string(bytes_written));
ProfileEvents::increment(ProfileEvents::FileSync);
Stopwatch watch;
#if defined(OS_DARWIN)
if (::fsync(file.fd) < 0)
throwFromErrnoWithPath("Cannot fsync " + file_path, file_path, ErrorCodes::CANNOT_FSYNC);
@ -551,6 +556,7 @@ public:
if (::fdatasync(file.fd) < 0)
throwFromErrnoWithPath("Cannot fdatasync " + file_path, file_path, ErrorCodes::CANNOT_FSYNC);
#endif
ProfileEvents::increment(ProfileEvents::FileSyncElapsedMicroseconds, watch.elapsedMicroseconds());
current_block_index += buffer_size_in_blocks;

View File

@ -1,6 +1,8 @@
#include <Disks/LocalDirectorySyncGuard.h>
#include <Common/ProfileEvents.h>
#include <Common/Exception.h>
#include <Disks/IDisk.h>
#include <Common/Stopwatch.h>
#include <fcntl.h> // O_RDWR
/// OSX does not have O_DIRECTORY
@ -8,6 +10,12 @@
#define O_DIRECTORY O_RDWR
#endif
namespace ProfileEvents
{
extern const Event DirectorySync;
extern const Event DirectorySyncElapsedMicroseconds;
}
namespace DB
{
@ -29,8 +37,12 @@ LocalDirectorySyncGuard::LocalDirectorySyncGuard(const String & full_path)
LocalDirectorySyncGuard::~LocalDirectorySyncGuard()
{
ProfileEvents::increment(ProfileEvents::DirectorySync);
try
{
Stopwatch watch;
#if defined(OS_DARWIN)
if (fcntl(fd, F_FULLFSYNC, 0))
throwFromErrno("Cannot fcntl(F_FULLFSYNC)", ErrorCodes::CANNOT_FSYNC);
@ -40,6 +52,8 @@ LocalDirectorySyncGuard::~LocalDirectorySyncGuard()
#endif
if (-1 == ::close(fd))
throw Exception("Cannot close file", ErrorCodes::CANNOT_CLOSE_FILE);
ProfileEvents::increment(ProfileEvents::DirectorySyncElapsedMicroseconds, watch.elapsedMicroseconds());
}
catch (...)
{

View File

@ -529,7 +529,12 @@ namespace JSONUtils
writeObjectStart(out, 2);
writeTitle<true>("name", out, 3);
writeDoubleQuoted(fields[i].name, out);
/// The field names are pre-escaped to be put into JSON string literal.
writeChar('"', out);
writeString(fields[i].name, out);
writeChar('"', out);
writeFieldDelimiter(out);
writeTitle<true>("type", out, 3);
writeJSONString(fields[i].type->getName(), out, settings);

View File

@ -1,7 +1,10 @@
#include <Columns/ColumnNullable.h>
#include <Columns/ColumnString.h>
#include <Columns/ColumnConst.h>
#include <Columns/IColumn.h>
#include <Functions/FunctionFactory.h>
#include <DataTypes/DataTypeString.h>
#include <DataTypes/DataTypeNullable.h>
#include <IO/ReadBufferFromFile.h>
#include <IO/WriteBufferFromVector.h>
#include <IO/copyData.h>
@ -19,6 +22,7 @@ namespace ErrorCodes
{
extern const int ILLEGAL_COLUMN;
extern const int NOT_IMPLEMENTED;
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int DATABASE_ACCESS_DENIED;
}
@ -30,21 +34,41 @@ public:
static FunctionPtr create(ContextPtr context_) { return std::make_shared<FunctionFile>(context_); }
explicit FunctionFile(ContextPtr context_) : WithContext(context_) {}
bool isVariadic() const override { return true; }
String getName() const override { return name; }
size_t getNumberOfArguments() const override { return 1; }
size_t getNumberOfArguments() const override { return 0; }
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
{
if (arguments.empty() || arguments.size() > 2)
throw Exception(
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
"Number of arguments for function {} doesn't match: passed {}, should be 1 or 2",
getName(), toString(arguments.size()));
if (!isString(arguments[0].type))
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "{} is only implemented for type String", getName());
if (arguments.size() == 2)
{
if (arguments[1].type->onlyNull())
return makeNullable(std::make_shared<DataTypeString>());
if (!isString(arguments[1].type))
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "{} only accepts String or Null as second argument", getName());
}
return std::make_shared<DataTypeString>();
}
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; }
bool useDefaultImplementationForNulls() const override { return false; }
bool useDefaultImplementationForConstants() const override { return true; }
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
{
const ColumnPtr column = arguments[0].column;
const ColumnString * column_src = checkAndGetColumn<ColumnString>(column.get());
@ -53,6 +77,31 @@ public:
fmt::format("Illegal column {} of argument of function {}", arguments[0].column->getName(), getName()),
ErrorCodes::ILLEGAL_COLUMN);
String default_result;
ColumnUInt8::MutablePtr col_null_map_to;
ColumnUInt8::Container * vec_null_map_to [[maybe_unused]] = nullptr;
if (arguments.size() == 2)
{
if (result_type->isNullable())
{
col_null_map_to = ColumnUInt8::create(input_rows_count, false);
vec_null_map_to = &col_null_map_to->getData();
}
else
{
const auto & default_column = arguments[1].column;
const ColumnConst * default_col = checkAndGetColumn<ColumnConst>(default_column.get());
if (!default_col)
throw Exception(
"Illegal column " + arguments[1].column->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN);
default_result = default_col->getValue<String>();
}
}
auto result = ColumnString::create();
auto & res_chars = result->getChars();
auto & res_offsets = result->getOffsets();
@ -77,6 +126,8 @@ public:
/// Otherwise it will not allow to work with symlinks in `user_files_path` directory.
file_path = fs::absolute(file_path).lexically_normal();
try
{
if (need_check && file_path.string().find(user_files_absolute_path_string) != 0)
throw Exception(ErrorCodes::DATABASE_ACCESS_DENIED, "File is not inside {}", user_files_absolute_path.string());
@ -84,11 +135,25 @@ public:
WriteBufferFromVector out(res_chars, AppendModeTag{});
copyData(in, out);
out.finalize();
}
catch (...)
{
if (arguments.size() == 1)
throw;
if (vec_null_map_to)
(*vec_null_map_to)[row] = true;
else
res_chars.insert(default_result.data(), default_result.data() + default_result.size());
}
res_chars.push_back(0);
res_offsets[row] = res_chars.size();
}
if (vec_null_map_to)
return ColumnNullable::create(std::move(result), std::move(col_null_map_to));
return result;
}
};

View File

@ -8,6 +8,7 @@
#include <Functions/IFunction.h>
#include <IO/WriteBufferFromVector.h>
#include <IO/WriteHelpers.h>
#include <bit>
namespace DB
@ -285,7 +286,7 @@ public:
{
while (x)
{
result_array_values_data.push_back(getTrailingZeroBitsUnsafe(x));
result_array_values_data.push_back(std::countr_zero(x));
x &= (x - 1);
}
}

View File

@ -1,5 +1,6 @@
#include <Functions/FunctionBinaryArithmetic.h>
#include <Functions/FunctionFactory.h>
#include <bit>
namespace DB
{
@ -14,7 +15,7 @@ struct BitHammingDistanceImpl
static inline NO_SANITIZE_UNDEFINED Result apply(A a, B b)
{
UInt64 res = static_cast<UInt64>(a) ^ static_cast<UInt64>(b);
return __builtin_popcountll(res);
return std::popcount(res);
}
#if USE_EMBEDDED_COMPILER

View File

@ -8,9 +8,7 @@
#include <Common/DateLUT.h>
#include <Common/ClickHouseRevision.h>
#if defined(OS_LINUX)
#include <Poco/Environment.h>
#endif
#include <Common/config_version.h>
@ -109,7 +107,6 @@ namespace
static FunctionPtr create(ContextPtr context) { return std::make_shared<FunctionZooKeeperSessionUptime>(context); }
};
#if defined(OS_LINUX)
class FunctionGetOSKernelVersion : public FunctionConstantBase<FunctionGetOSKernelVersion, String, DataTypeString>
{
public:
@ -117,7 +114,6 @@ namespace
explicit FunctionGetOSKernelVersion(ContextPtr context) : FunctionConstantBase(Poco::Environment::osName() + " " + Poco::Environment::osVersion(), context->isDistributed()) {}
static FunctionPtr create(ContextPtr context) { return std::make_shared<FunctionGetOSKernelVersion>(context); }
};
#endif
}
@ -171,12 +167,10 @@ REGISTER_FUNCTION(ZooKeeperSessionUptime)
}
#if defined(OS_LINUX)
REGISTER_FUNCTION(GetOSKernelVersion)
{
factory.registerFunction<FunctionGetOSKernelVersion>();
}
#endif
}

View File

@ -10,6 +10,7 @@
#include <IO/Operators.h>
#include <base/find_symbols.h>
#include <cstdlib>
#include <bit>
#ifdef __SSE2__
#include <emmintrin.h>
@ -698,7 +699,7 @@ void readCSVStringInto(Vector & s, ReadBuffer & buf, const FormatSettings::CSV &
uint16_t bit_mask = _mm_movemask_epi8(eq);
if (bit_mask)
{
next_pos += __builtin_ctz(bit_mask);
next_pos += std::countr_zero(bit_mask);
return;
}
}
@ -716,7 +717,7 @@ void readCSVStringInto(Vector & s, ReadBuffer & buf, const FormatSettings::CSV &
uint64_t bit_mask = get_nibble_mask(eq);
if (bit_mask)
{
next_pos += __builtin_ctzll(bit_mask) >> 2;
next_pos += std::countr_zero(bit_mask) >> 2;
return;
}
}

View File

@ -18,6 +18,8 @@ namespace ProfileEvents
extern const Event WriteBufferFromFileDescriptorWriteFailed;
extern const Event WriteBufferFromFileDescriptorWriteBytes;
extern const Event DiskWriteElapsedMicroseconds;
extern const Event FileSync;
extern const Event FileSyncElapsedMicroseconds;
}
namespace CurrentMetrics
@ -113,12 +115,18 @@ void WriteBufferFromFileDescriptor::sync()
/// If buffer has pending data - write it.
next();
ProfileEvents::increment(ProfileEvents::FileSync);
Stopwatch watch;
/// Request OS to sync data with storage medium.
#if defined(OS_DARWIN)
int res = ::fsync(fd);
#else
int res = ::fdatasync(fd);
#endif
ProfileEvents::increment(ProfileEvents::FileSyncElapsedMicroseconds, watch.elapsedMicroseconds());
if (-1 == res)
throwFromErrnoWithPath("Cannot fsync " + getFileName(), getFileName(), ErrorCodes::CANNOT_FSYNC);
}

View File

@ -1513,8 +1513,7 @@ ActionsDAG::SplitResult ActionsDAG::splitActionsBeforeArrayJoin(const NameSet &
}
auto res = split(split_nodes);
/// Do not remove array joined columns if they are not used.
/// res.first->project_input = false;
res.second->project_input = project_input;
return res;
}

View File

@ -33,7 +33,7 @@ static UInt32 toPowerOfTwo(UInt32 x)
{
if (x <= 1)
return 1;
return static_cast<UInt32>(1) << (32 - __builtin_clz(x - 1));
return static_cast<UInt32>(1) << (32 - std::countl_zero(x - 1));
}
ConcurrentHashJoin::ConcurrentHashJoin(ContextPtr context_, std::shared_ptr<TableJoin> table_join_, size_t slots_, const Block & right_sample_block, bool any_take_last_row_)

View File

@ -30,6 +30,7 @@
#include <Interpreters/Set.h>
#include <Interpreters/TableJoin.h>
#include <Interpreters/FullSortingMergeJoin.h>
#include <Interpreters/replaceForPositionalArguments.h>
#include <Processors/QueryPlan/ExpressionStep.h>
@ -112,79 +113,6 @@ bool allowEarlyConstantFolding(const ActionsDAG & actions, const Settings & sett
return true;
}
bool checkPositionalArguments(ASTPtr & argument, const ASTSelectQuery * select_query, ASTSelectQuery::Expression expression)
{
auto columns = select_query->select()->children;
const auto * expr_with_alias = dynamic_cast<const ASTWithAlias *>(argument.get());
if (expr_with_alias && !expr_with_alias->alias.empty())
return false;
const auto * ast_literal = typeid_cast<const ASTLiteral *>(argument.get());
if (!ast_literal)
return false;
auto which = ast_literal->value.getType();
if (which != Field::Types::UInt64)
return false;
auto pos = ast_literal->value.get<UInt64>();
if (!pos || pos > columns.size())
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Positional argument out of bounds: {} (exprected in range [1, {}]",
pos, columns.size());
const auto & column = columns[--pos];
if (typeid_cast<const ASTIdentifier *>(column.get()) || typeid_cast<const ASTLiteral *>(column.get()))
{
argument = column->clone();
}
else if (typeid_cast<const ASTFunction *>(column.get()))
{
std::function<void(ASTPtr)> throw_if_aggregate_function = [&](ASTPtr node)
{
if (const auto * function = typeid_cast<const ASTFunction *>(node.get()))
{
auto is_aggregate_function = AggregateUtils::isAggregateFunction(*function);
if (is_aggregate_function)
{
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal value (aggregate function) for positional argument in {}",
ASTSelectQuery::expressionToString(expression));
}
else
{
if (function->arguments)
{
for (const auto & arg : function->arguments->children)
throw_if_aggregate_function(arg);
}
}
}
};
if (expression == ASTSelectQuery::Expression::GROUP_BY)
throw_if_aggregate_function(column);
argument = column->clone();
}
else
{
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal value for positional argument in {}",
ASTSelectQuery::expressionToString(expression));
}
return true;
}
void replaceForPositionalArguments(ASTPtr & argument, const ASTSelectQuery * select_query, ASTSelectQuery::Expression expression)
{
auto argument_with_replacement = argument->clone();
if (checkPositionalArguments(argument_with_replacement, select_query, expression))
argument = argument_with_replacement;
}
}
bool sanitizeBlock(Block & block, bool throw_if_cannot_create_column)
@ -1594,10 +1522,8 @@ ActionsDAGPtr SelectQueryExpressionAnalyzer::appendOrderBy(ExpressionActionsChai
throw Exception("Bad ORDER BY expression AST", ErrorCodes::UNKNOWN_TYPE_OF_AST_NODE);
if (getContext()->getSettingsRef().enable_positional_arguments)
{
replaceForPositionalArguments(ast->children.at(0), select_query, ASTSelectQuery::Expression::ORDER_BY);
}
}
getRootActions(select_query->orderBy(), only_types, step.actions());

View File

@ -4,6 +4,7 @@
#include <Common/StringUtils/StringUtils.h>
#include <Common/UTF8Helpers.h>
#include <bit>
#if defined(__SSE2__)
#include <emmintrin.h>
@ -122,7 +123,7 @@ bool SplitTokenExtractor::nextInStringPadded(const char * data, size_t length, s
const auto alnum_chars_ranges = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0,
'\xFF', '\x80', 'z', 'a', 'Z', 'A', '9', '0');
// Every bit represents if `haystack` character is in the ranges (1) or not (0)
const int result_bitmask = _mm_cvtsi128_si32(_mm_cmpestrm(alnum_chars_ranges, 8, haystack, haystack_length, _SIDD_CMP_RANGES));
const unsigned result_bitmask = _mm_cvtsi128_si32(_mm_cmpestrm(alnum_chars_ranges, 8, haystack, haystack_length, _SIDD_CMP_RANGES));
#else
// NOTE: -1 and +1 required since SSE2 has no `>=` and `<=` instructions on packed 8-bit integers (epi8).
const auto number_begin = _mm_set1_epi8('0' - 1);
@ -136,7 +137,7 @@ bool SplitTokenExtractor::nextInStringPadded(const char * data, size_t length, s
// every bit represents if `haystack` character `c` satisfies condition:
// (c < 0) || (c > '0' - 1 && c < '9' + 1) || (c > 'a' - 1 && c < 'z' + 1) || (c > 'A' - 1 && c < 'Z' + 1)
// < 0 since _mm_cmplt_epi8 threats chars as SIGNED, and so all chars > 0x80 are negative.
const int result_bitmask = _mm_movemask_epi8(_mm_or_si128(_mm_or_si128(_mm_or_si128(
const unsigned result_bitmask = _mm_movemask_epi8(_mm_or_si128(_mm_or_si128(_mm_or_si128(
_mm_cmplt_epi8(haystack, zero),
_mm_and_si128(_mm_cmpgt_epi8(haystack, number_begin), _mm_cmplt_epi8(haystack, number_end))),
_mm_and_si128(_mm_cmpgt_epi8(haystack, alpha_lower_begin), _mm_cmplt_epi8(haystack, alpha_lower_end))),
@ -152,7 +153,7 @@ bool SplitTokenExtractor::nextInStringPadded(const char * data, size_t length, s
continue;
}
const auto token_start_pos_in_current_haystack = getTrailingZeroBitsUnsafe(result_bitmask);
const auto token_start_pos_in_current_haystack = std::countr_zero(result_bitmask);
if (*token_length == 0)
// new token
*token_start = *pos + token_start_pos_in_current_haystack;
@ -160,7 +161,7 @@ bool SplitTokenExtractor::nextInStringPadded(const char * data, size_t length, s
// end of token starting in one of previous haystacks
return true;
const auto token_bytes_in_current_haystack = getTrailingZeroBitsUnsafe(~(result_bitmask >> token_start_pos_in_current_haystack));
const auto token_bytes_in_current_haystack = std::countr_zero(~(result_bitmask >> token_start_pos_in_current_haystack));
*token_length += token_bytes_in_current_haystack;
*pos += token_start_pos_in_current_haystack + token_bytes_in_current_haystack;

View File

@ -748,13 +748,20 @@ BlockIO InterpreterSelectQuery::execute()
Block InterpreterSelectQuery::getSampleBlockImpl()
{
auto & select_query = getSelectQuery();
query_info.query = query_ptr;
/// NOTE: this is required for getQueryProcessingStage(), so should be initialized before ExpressionAnalysisResult.
query_info.has_window = query_analyzer->hasWindow();
/// NOTE: this is required only for IStorage::read(), and to be precise MergeTreeData::read(), in case of projections.
query_info.has_order_by = select_query.orderBy() != nullptr;
query_info.need_aggregate = query_analyzer->hasAggregation();
if (storage && !options.only_analyze)
{
auto & query = getSelectQuery();
query_analyzer->makeSetsForIndex(query.where());
query_analyzer->makeSetsForIndex(query.prewhere());
query_analyzer->makeSetsForIndex(select_query.where());
query_analyzer->makeSetsForIndex(select_query.prewhere());
query_info.sets = std::move(query_analyzer->getPreparedSets());
query_info.subquery_for_sets = std::move(query_analyzer->getSubqueriesForSets());

View File

@ -430,8 +430,9 @@ MergeTreeSetIndex::MergeTreeSetIndex(const Columns & set_elements, std::vector<K
SortDescription sort_description;
for (size_t i = 0; i < tuple_size; ++i)
{
block_to_sort.insert({ordered_set[i], nullptr, ordered_set[i]->getName()});
sort_description.emplace_back(ordered_set[i]->getName(), 1, 1);
String column_name = "_" + toString(i);
block_to_sort.insert({ordered_set[i], nullptr, column_name});
sort_description.emplace_back(column_name, 1, 1);
}
sortBlock(block_to_sort, sort_description);

View File

@ -29,6 +29,7 @@
#include <Interpreters/evaluateConstantExpression.h>
#include <Interpreters/PredicateExpressionsOptimizer.h>
#include <Interpreters/RewriteOrderByVisitor.hpp>
#include <Interpreters/replaceForPositionalArguments.h>
#include <Parsers/IAST_fwd.h>
#include <Parsers/ASTExpressionList.h>
@ -1252,6 +1253,26 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect(
normalize(query, result.aliases, all_source_columns_set, select_options.ignore_alias, settings, /* allow_self_aliases = */ true, getContext());
if (getContext()->getSettingsRef().enable_positional_arguments)
{
if (select_query->groupBy())
{
for (auto & expr : select_query->groupBy()->children)
replaceForPositionalArguments(expr, select_query, ASTSelectQuery::Expression::GROUP_BY);
}
if (select_query->orderBy())
{
for (auto & expr : select_query->orderBy()->children)
replaceForPositionalArguments(expr, select_query, ASTSelectQuery::Expression::ORDER_BY);
}
if (select_query->limitBy())
{
for (auto & expr : select_query->limitBy()->children)
replaceForPositionalArguments(expr, select_query, ASTSelectQuery::Expression::LIMIT_BY);
}
}
/// Remove unneeded columns according to 'required_result_columns'.
/// Leave all selected columns in case of DISTINCT; columns that contain arrayJoin function inside.
/// Must be after 'normalizeTree' (after expanding aliases, for aliases not get lost)

View File

@ -0,0 +1,83 @@
#include <Interpreters/replaceForPositionalArguments.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTLiteral.h>
#include <AggregateFunctions/AggregateFunctionFactory.h>
namespace DB
{
namespace ErrorCodes
{
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
}
bool replaceForPositionalArguments(ASTPtr & argument, const ASTSelectQuery * select_query, ASTSelectQuery::Expression expression)
{
auto columns = select_query->select()->children;
const auto * expr_with_alias = dynamic_cast<const ASTWithAlias *>(argument.get());
if (expr_with_alias && !expr_with_alias->alias.empty())
return false;
const auto * ast_literal = typeid_cast<const ASTLiteral *>(argument.get());
if (!ast_literal)
return false;
auto which = ast_literal->value.getType();
if (which != Field::Types::UInt64)
return false;
auto pos = ast_literal->value.get<UInt64>();
if (!pos || pos > columns.size())
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Positional argument out of bounds: {} (exprected in range [1, {}]",
pos, columns.size());
const auto & column = columns[--pos];
if (typeid_cast<const ASTIdentifier *>(column.get()) || typeid_cast<const ASTLiteral *>(column.get()))
{
argument = column->clone();
}
else if (typeid_cast<const ASTFunction *>(column.get()))
{
std::function<void(ASTPtr)> throw_if_aggregate_function = [&](ASTPtr node)
{
if (const auto * function = typeid_cast<const ASTFunction *>(node.get()))
{
auto is_aggregate_function = AggregateUtils::isAggregateFunction(*function);
if (is_aggregate_function)
{
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal value (aggregate function) for positional argument in {}",
ASTSelectQuery::expressionToString(expression));
}
else
{
if (function->arguments)
{
for (const auto & arg : function->arguments->children)
throw_if_aggregate_function(arg);
}
}
}
};
if (expression == ASTSelectQuery::Expression::GROUP_BY)
throw_if_aggregate_function(column);
argument = column->clone();
}
else
{
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal value for positional argument in {}",
ASTSelectQuery::expressionToString(expression));
}
return true;
}
}

View File

@ -0,0 +1,9 @@
#pragma once
#include <Parsers/ASTSelectQuery.h>
namespace DB
{
bool replaceForPositionalArguments(ASTPtr & argument, const ASTSelectQuery * select_query, ASTSelectQuery::Expression expression);
}

View File

@ -1,6 +1,7 @@
#include <string_view>
#include <Parsers/ExpressionListParsers.h>
#include <Parsers/ParserSetQuery.h>
#include <Parsers/ASTAsterisk.h>
#include <Parsers/ASTExpressionList.h>
@ -9,6 +10,7 @@
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTSelectQuery.h>
#include <Parsers/ASTSetQuery.h>
#include <Parsers/ASTSelectWithUnionQuery.h>
#include <Parsers/ASTSubquery.h>
#include <Parsers/ASTTablesInSelectQuery.h>
@ -603,6 +605,13 @@ bool ParserTableFunctionExpression::parseImpl(Pos & pos, ASTPtr & node, Expected
{
if (ParserTableFunctionView().parse(pos, node, expected))
return true;
ParserKeyword s_settings("SETTINGS");
if (s_settings.ignore(pos, expected))
{
ParserSetQuery parser_settings(true);
if (parser_settings.parse(pos, node, expected))
return true;
}
return elem_parser.parse(pos, node, expected);
}

View File

@ -26,6 +26,39 @@ const char * IAST::hilite_substitution = "\033[1;36m";
const char * IAST::hilite_none = "\033[0m";
IAST::~IAST()
{
/// If deleter was set, move our children to it.
/// Will avoid recursive destruction.
if (deleter)
{
deleter->push_back(std::move(children));
return;
}
std::list<ASTs> queue;
queue.push_back(std::move(children));
while (!queue.empty())
{
for (auto & node : queue.front())
{
/// If two threads remove ASTPtr concurrently,
/// it is possible that neither thead will see use_count == 1.
/// It is ok. Will need one more extra stack frame in this case.
if (node.use_count() == 1)
{
/// Deleter is only set when current thread is the single owner.
/// No extra synchronisation is needed.
ASTPtr to_delete;
node.swap(to_delete);
to_delete->deleter = &queue;
}
}
queue.pop_front();
}
}
size_t IAST::size() const
{
size_t res = 1;

View File

@ -9,6 +9,7 @@
#include <algorithm>
#include <set>
#include <list>
class SipHash;
@ -34,7 +35,7 @@ class IAST : public std::enable_shared_from_this<IAST>, public TypePromotion<IAS
public:
ASTs children;
virtual ~IAST() = default;
virtual ~IAST();
IAST() = default;
IAST(const IAST &) = default;
IAST & operator=(const IAST &) = default;
@ -273,6 +274,9 @@ public:
private:
size_t checkDepthImpl(size_t max_depth, size_t level) const;
/// This deleter is used in ~IAST to avoid possible stack overflow in destructor.
std::list<ASTs> * deleter = nullptr;
};
template <typename AstArray>

View File

@ -0,0 +1,147 @@
#include <gtest/gtest.h>
#include <Parsers/IAST.h>
#include <Common/StackTrace.h>
struct StackDecrementer
{
size_t & depth;
explicit StackDecrementer(size_t & depth_) : depth(depth_) {}
virtual ~StackDecrementer()
{
--depth;
}
};
struct ASTCounting : public StackDecrementer, public DB::IAST
{
explicit ASTCounting(size_t & depth_) : StackDecrementer(depth_) {}
String getID(char) const override { return ""; }
DB::ASTPtr clone() const override { return nullptr; }
};
class ASTWithMembers : public ASTCounting
{
public:
explicit ASTWithMembers(DB::ASTs members_, size_t & depth_) : ASTCounting(depth_), members(std::move(members_))
{
children = members;
}
DB::ASTs members;
};
class ASTWithDecrementer : public ASTWithMembers
{
public:
ASTWithDecrementer(DB::ASTs members_, size_t & depth_, size_t max_depth_)
: ASTWithMembers(std::move(members_), depth_), max_depth(max_depth_)
{
}
size_t max_depth;
~ASTWithDecrementer() override
{
++depth;
if (depth == max_depth)
std::cout << StackTrace().toString() << std::endl;
if (depth > max_depth)
EXPECT_LE(depth, max_depth);
}
};
TEST(ASTDeleter, SimpleChain)
{
size_t depth = 0;
size_t max_depth = 5;
size_t chain_length = 10;
{
DB::ASTPtr ast = std::make_shared<ASTWithDecrementer>(DB::ASTs{}, depth, max_depth);
for (size_t i = 0; i < chain_length; ++i)
ast = std::make_shared<ASTWithDecrementer>(DB::ASTs{std::move(ast)}, depth, max_depth);
}
}
TEST(ASTDeleter, SimpleChainLong)
{
size_t depth = 0;
size_t max_depth = 5;
size_t chain_length = 100000;
{
DB::ASTPtr ast = std::make_shared<ASTWithDecrementer>(DB::ASTs{}, depth, max_depth);
for (size_t i = 0; i < chain_length; ++i)
ast = std::make_shared<ASTWithDecrementer>(DB::ASTs{std::move(ast)}, depth, max_depth);
}
}
TEST(ASTDeleter, ChainWithExtraMember)
{
size_t depth = 0;
size_t max_depth = 5;
size_t member_depth = 10;
size_t chain_length = 100;
{
DB::ASTPtr ast = std::make_shared<ASTWithDecrementer>(DB::ASTs{}, depth, max_depth);
for (size_t i = 0; i < chain_length; ++i)
{
ast = std::make_shared<ASTWithDecrementer>(DB::ASTs{std::move(ast)}, depth, max_depth);
if (i > member_depth)
{
DB::ASTPtr member = ast;
for (size_t j = 0; j < member_depth; ++j)
member = member->children.front();
ast->as<ASTWithDecrementer>()->members.push_back(std::move(member));
}
}
}
}
TEST(ASTDeleter, DoubleChain)
{
size_t depth = 0;
size_t max_depth = 5;
size_t chain_length = 10;
{
DB::ASTPtr ast1 = std::make_shared<ASTWithDecrementer>(DB::ASTs{}, depth, max_depth);
DB::ASTPtr ast2 = std::make_shared<ASTWithDecrementer>(DB::ASTs{}, depth, max_depth);
for (size_t i = 0; i < chain_length; ++i)
{
ast1 = std::make_shared<ASTWithDecrementer>(DB::ASTs{std::move(ast1)}, depth, max_depth);
ast2 = std::make_shared<ASTWithDecrementer>(DB::ASTs{std::move(ast2)}, depth, max_depth);
ast1->as<ASTWithDecrementer>()->members.push_back(ast2->children.front());
ast2->as<ASTWithDecrementer>()->members.push_back(ast1->children.front());
}
}
}
TEST(ASTDeleter, DoubleChainLong)
{
size_t depth = 0;
size_t max_depth = 5;
size_t chain_length = 100000;
{
DB::ASTPtr ast1 = std::make_shared<ASTWithDecrementer>(DB::ASTs{}, depth, max_depth);
DB::ASTPtr ast2 = std::make_shared<ASTWithDecrementer>(DB::ASTs{}, depth, max_depth);
for (size_t i = 0; i < chain_length; ++i)
{
ast1 = std::make_shared<ASTWithDecrementer>(DB::ASTs{std::move(ast1)}, depth, max_depth);
ast2 = std::make_shared<ASTWithDecrementer>(DB::ASTs{std::move(ast2)}, depth, max_depth);
ast1->as<ASTWithDecrementer>()->members.push_back(ast2->children.front());
ast2->as<ASTWithDecrementer>()->members.push_back(ast1->children.front());
}
}
}

View File

@ -47,7 +47,8 @@
M(INT64, arrow::Int64Type) \
M(FLOAT, arrow::FloatType) \
M(DOUBLE, arrow::DoubleType) \
M(BINARY, arrow::BinaryType)
M(BINARY, arrow::BinaryType) \
M(STRING, arrow::StringType)
namespace DB
{

View File

@ -72,7 +72,7 @@ protected:
size_t field_number = 0;
size_t row_count = 0;
NamesAndTypes fields;
NamesAndTypes fields; /// The field names are pre-escaped to be put into JSON string literal.
Statistics statistics;
FormatSettings settings;

View File

@ -282,7 +282,7 @@ public:
*
* SelectQueryInfo is required since the stage can depends on the query
* (see Distributed() engine and optimize_skip_unused_shards,
* see also MergeTree engine and allow_experimental_projection_optimization).
* see also MergeTree engine and projection optimization).
* And to store optimized cluster (after optimize_skip_unused_shards).
* It will also store needed stuff for projection query pipeline.
*

View File

@ -8,13 +8,16 @@
#include <Storages/MergeTree/MergeTreeIndexReader.h>
#include <Storages/MergeTree/KeyCondition.h>
#include <Storages/MergeTree/MergeTreeDataPartUUID.h>
#include <Storages/MergeTree/StorageFromMergeTreeDataPart.h>
#include <Storages/ReadInOrderOptimizer.h>
#include <Storages/VirtualColumnUtils.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTSampleRatio.h>
#include <Parsers/parseIdentifierOrStringLiteral.h>
#include <Interpreters/ExpressionAnalyzer.h>
#include <Interpreters/InterpreterSelectQuery.h>
#include <Interpreters/Context.h>
#include <Processors/ConcatProcessor.h>
#include <Processors/QueryPlan/QueryPlan.h>
@ -26,7 +29,9 @@
#include <Processors/QueryPlan/UnionStep.h>
#include <Processors/QueryPlan/QueryIdHolder.h>
#include <Processors/QueryPlan/AggregatingStep.h>
#include <Processors/QueryPlan/SortingStep.h>
#include <Processors/Sources/SourceFromSingleChunk.h>
#include <Processors/Transforms/AggregatingTransform.h>
#include <Core/UUID.h>
#include <DataTypes/DataTypeDate.h>
@ -35,12 +40,7 @@
#include <DataTypes/DataTypeTuple.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeArray.h>
#include <Storages/VirtualColumnUtils.h>
#include <Interpreters/InterpreterSelectQuery.h>
#include <Processors/Transforms/AggregatingTransform.h>
#include <Storages/MergeTree/StorageFromMergeTreeDataPart.h>
#include <IO/WriteBufferFromOStream.h>
namespace DB
@ -184,6 +184,7 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read(
query_info.projection->desc->type,
query_info.projection->desc->name);
const ASTSelectQuery & select_query = query_info.query->as<ASTSelectQuery &>();
QueryPlanResourceHolder resources;
auto projection_plan = std::make_unique<QueryPlan>();
@ -230,6 +231,25 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read(
expression_before_aggregation->setStepDescription("Before GROUP BY");
projection_plan->addStep(std::move(expression_before_aggregation));
}
/// NOTE: input_order_info (for projection and not) is set only if projection is complete
if (query_info.has_order_by && !query_info.need_aggregate && query_info.projection->input_order_info)
{
chassert(query_info.projection->complete);
SortDescription output_order_descr = InterpreterSelectQuery::getSortDescription(select_query, context);
UInt64 limit = InterpreterSelectQuery::getLimitForSorting(select_query, context);
auto sorting_step = std::make_unique<SortingStep>(
projection_plan->getCurrentDataStream(),
query_info.projection->input_order_info->order_key_prefix_descr,
output_order_descr,
settings.max_block_size,
limit);
sorting_step->setStepDescription("ORDER BY for projections");
projection_plan->addStep(std::move(sorting_step));
}
}
auto ordinary_query_plan = std::make_unique<QueryPlan>();
@ -365,7 +385,7 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read(
InputOrderInfoPtr group_by_info = query_info.projection->input_order_info;
SortDescription group_by_sort_description;
if (group_by_info && settings.optimize_aggregation_in_order)
group_by_sort_description = getSortDescriptionFromGroupBy(query_info.query->as<ASTSelectQuery &>());
group_by_sort_description = getSortDescriptionFromGroupBy(select_query);
else
group_by_info = nullptr;

View File

@ -8,6 +8,7 @@
#include <base/range.h>
#include <Interpreters/castColumn.h>
#include <DataTypes/DataTypeNothing.h>
#include <bit>
#ifdef __SSE2__
#include <emmintrin.h>
@ -473,7 +474,7 @@ size_t numZerosInTail(const UInt8 * begin, const UInt8 * end)
count += 64;
else
{
count += __builtin_clzll(val);
count += std::countl_zero(val);
return count;
}
}
@ -507,7 +508,7 @@ size_t numZerosInTail(const UInt8 * begin, const UInt8 * end)
count += 64;
else
{
count += __builtin_clzll(val);
count += std::countl_zero(val);
return count;
}
}
@ -531,7 +532,7 @@ size_t MergeTreeRangeReader::ReadResult::numZerosInTail(const UInt8 * begin, con
size_t count = 0;
#if defined(__SSE2__) && defined(__POPCNT__)
#if defined(__SSE2__)
const __m128i zero16 = _mm_setzero_si128();
while (end - begin >= 64)
{
@ -555,7 +556,7 @@ size_t MergeTreeRangeReader::ReadResult::numZerosInTail(const UInt8 * begin, con
count += 64;
else
{
count += __builtin_clzll(val);
count += std::countl_zero(val);
return count;
}
}
@ -583,7 +584,7 @@ size_t MergeTreeRangeReader::ReadResult::numZerosInTail(const UInt8 * begin, con
count += 64;
else
{
count += __builtin_clzll(val);
count += std::countl_zero(val);
return count;
}
}

View File

@ -26,6 +26,7 @@
#include <Interpreters/PreparedSets.h>
#include <Interpreters/TreeRewriter.h>
#include <Interpreters/convertFieldToType.h>
#include <Interpreters/evaluateConstantExpression.h>
#include <Poco/Logger.h>
#include <Poco/Util/AbstractConfiguration.h>
@ -76,7 +77,7 @@ static RocksDBOptions getOptionsFromConfig(const Poco::Util::AbstractConfigurati
// returns keys may be filter by condition
static bool traverseASTFilter(
const String & primary_key, const DataTypePtr & primary_key_type, const ASTPtr & elem, const PreparedSets & sets, FieldVectorPtr & res)
const String & primary_key, const DataTypePtr & primary_key_type, const ASTPtr & elem, const PreparedSets & sets, const ContextPtr & context, FieldVectorPtr & res)
{
const auto * function = elem->as<ASTFunction>();
if (!function)
@ -86,7 +87,7 @@ static bool traverseASTFilter(
{
// one child has the key filter condition is ok
for (const auto & child : function->arguments->children)
if (traverseASTFilter(primary_key, primary_key_type, child, sets, res))
if (traverseASTFilter(primary_key, primary_key_type, child, sets, context, res))
return true;
return false;
}
@ -94,7 +95,7 @@ static bool traverseASTFilter(
{
// make sure every child has the key filter condition
for (const auto & child : function->arguments->children)
if (!traverseASTFilter(primary_key, primary_key_type, child, sets, res))
if (!traverseASTFilter(primary_key, primary_key_type, child, sets, context, res))
return false;
return true;
}
@ -102,7 +103,7 @@ static bool traverseASTFilter(
{
const auto & args = function->arguments->as<ASTExpressionList &>();
const ASTIdentifier * ident;
const IAST * value;
std::shared_ptr<IAST> value;
if (args.children.size() != 2)
return false;
@ -115,7 +116,7 @@ static bool traverseASTFilter(
if (ident->name() != primary_key)
return false;
value = args.children.at(1).get();
value = args.children.at(1);
PreparedSetKey set_key;
if ((value->as<ASTSubquery>() || value->as<ASTIdentifier>()))
@ -140,17 +141,18 @@ static bool traverseASTFilter(
else
{
if ((ident = args.children.at(0)->as<ASTIdentifier>()))
value = args.children.at(1).get();
value = args.children.at(1);
else if ((ident = args.children.at(1)->as<ASTIdentifier>()))
value = args.children.at(0).get();
value = args.children.at(0);
else
return false;
if (ident->name() != primary_key)
return false;
const auto node = evaluateConstantExpressionAsLiteral(value, context);
/// function->name == "equals"
if (const auto * literal = value->as<ASTLiteral>())
if (const auto * literal = node->as<ASTLiteral>())
{
auto converted_field = convertFieldToType(literal->value, *primary_key_type);
if (!converted_field.isNull())
@ -166,14 +168,14 @@ static bool traverseASTFilter(
* TODO support key like search
*/
static std::pair<FieldVectorPtr, bool> getFilterKeys(
const String & primary_key, const DataTypePtr & primary_key_type, const SelectQueryInfo & query_info)
const String & primary_key, const DataTypePtr & primary_key_type, const SelectQueryInfo & query_info, const ContextPtr & context)
{
const auto & select = query_info.query->as<ASTSelectQuery &>();
if (!select.where())
return {{}, true};
FieldVectorPtr res = std::make_shared<FieldVector>();
auto matched_keys = traverseASTFilter(primary_key, primary_key_type, select.where(), query_info.sets, res);
auto matched_keys = traverseASTFilter(primary_key, primary_key_type, select.where(), query_info.sets, context, res);
return std::make_pair(res, !matched_keys);
}
@ -461,7 +463,7 @@ Pipe StorageEmbeddedRocksDB::read(
const Names & column_names,
const StorageSnapshotPtr & storage_snapshot,
SelectQueryInfo & query_info,
ContextPtr /*context*/,
ContextPtr context_,
QueryProcessingStage::Enum /*processed_stage*/,
size_t max_block_size,
unsigned num_streams)
@ -473,7 +475,7 @@ Pipe StorageEmbeddedRocksDB::read(
Block sample_block = storage_snapshot->metadata->getSampleBlock();
auto primary_key_data_type = sample_block.getByName(primary_key).type;
std::tie(keys, all_scan) = getFilterKeys(primary_key, primary_key_data_type, query_info);
std::tie(keys, all_scan) = getFilterKeys(primary_key, primary_key_data_type, query_info, context_);
if (all_scan)
{
auto iterator = std::unique_ptr<rocksdb::Iterator>(rocksdb_ptr->NewIterator(rocksdb::ReadOptions()));

View File

@ -154,8 +154,6 @@ struct SelectQueryInfoBase
TreeRewriterResultPtr syntax_analyzer_result;
PrewhereInfoPtr prewhere_info;
/// This is an additional filer applied to current table.
/// It is needed only for additional PK filtering.
ASTPtr additional_filter_ast;
@ -168,8 +166,11 @@ struct SelectQueryInfoBase
/// Example: x IN (1, 2, 3)
PreparedSets sets;
/// Cached value of ExpressionAnalysisResult::has_window
/// Cached value of ExpressionAnalysisResult
bool has_window = false;
bool has_order_by = false;
bool need_aggregate = false;
PrewhereInfoPtr prewhere_info;
ClusterPtr getCluster() const { return !optimized_cluster ? cluster : optimized_cluster; }

View File

@ -21,7 +21,6 @@ const char * auto_config_build[]
"LINK_FLAGS", "@FULL_EXE_LINKER_FLAGS_NORMALIZED@",
"BUILD_COMPILE_DEFINITIONS", "@BUILD_COMPILE_DEFINITIONS@",
"STATIC", "@USE_STATIC_LIBRARIES@",
"SPLIT_BINARY", "@CLICKHOUSE_SPLIT_BINARY@",
"USE_EMBEDDED_COMPILER", "@USE_EMBEDDED_COMPILER@",
"USE_GLIBC_COMPATIBILITY", "@GLIBC_COMPATIBILITY@",
"USE_JEMALLOC", "@ENABLE_JEMALLOC@",
@ -59,7 +58,6 @@ const char * auto_config_build[]
"USE_AWS_S3", "@USE_AWS_S3@",
"USE_CASSANDRA", "@USE_CASSANDRA@",
"USE_YAML_CPP", "@USE_YAML_CPP@",
"CLICKHOUSE_SPLIT_BINARY", "@CLICKHOUSE_SPLIT_BINARY@",
"USE_SENTRY", "@USE_SENTRY@",
"USE_DATASKETCHES", "@USE_DATASKETCHES@",
"USE_AVRO", "@USE_AVRO@",

View File

@ -4,7 +4,10 @@
#include <TableFunctions/TableFunctionFactory.h>
#include <TableFunctions/parseColumnsListForTableFunction.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTSelectWithUnionQuery.h>
#include <Parsers/ASTSetQuery.h>
#include <Parsers/parseQuery.h>
#include <Storages/checkAndGetLiteralArgument.h>
#include <Storages/StorageExecutable.h>
#include <DataTypes/DataTypeFactory.h>
@ -48,23 +51,35 @@ void TableFunctionExecutable::parseArguments(const ASTPtr & ast_function, Contex
std::vector<String> script_name_with_arguments;
boost::split(script_name_with_arguments, script_name_with_arguments_value, [](char c){ return c == ' '; });
script_name = script_name_with_arguments[0];
script_name = std::move(script_name_with_arguments[0]);
script_name_with_arguments.erase(script_name_with_arguments.begin());
arguments = std::move(script_name_with_arguments);
format = checkAndGetLiteralArgument<String>(args[1], "format");
structure = checkAndGetLiteralArgument<String>(args[2], "structure");
for (size_t i = 3; i < args.size(); ++i)
{
if (args[i]->as<ASTSetQuery>())
{
settings_query = std::move(args[i]);
}
else
{
ASTPtr query = args[i]->children.at(0);
if (!query->as<ASTSelectWithUnionQuery>())
throw Exception(ErrorCodes::UNSUPPORTED_METHOD,
"Table function '{}' argument is invalid input query {}",
getName(),
query->formatForErrorMessage());
if (query->as<ASTSelectWithUnionQuery>())
{
input_queries.emplace_back(std::move(query));
}
else
{
throw Exception(
ErrorCodes::UNSUPPORTED_METHOD,
"Table function '{}' argument is invalid {}",
getName(),
args[i]->formatForErrorMessage());
}
}
}
}
ColumnsDescription TableFunctionExecutable::getActualTableStructure(ContextPtr context) const
@ -79,6 +94,8 @@ StoragePtr TableFunctionExecutable::executeImpl(const ASTPtr & /*ast_function*/,
ExecutableSettings settings;
settings.script_name = script_name;
settings.script_arguments = arguments;
if (settings_query != nullptr)
settings.applyChanges(settings_query->as<ASTSetQuery>()->changes);
auto storage = std::make_shared<StorageExecutable>(storage_id, format, settings, input_queries, getActualTableStructure(context), ConstraintsDescription{});
storage->startup();

View File

@ -6,6 +6,7 @@ namespace DB
{
class Context;
class ASTSetQuery;
/* executable(script_name_optional_arguments, format, structure, input_query) - creates a temporary storage from executable file
*
@ -32,5 +33,6 @@ private:
String format;
String structure;
std::vector<ASTPtr> input_queries;
ASTPtr settings_query = nullptr;
};
}

View File

@ -39,7 +39,7 @@ def _can_export_binaries(build_config: BuildConfig) -> bool:
return False
if build_config["bundled"] != "bundled":
return False
if build_config["splitted"] == "splitted":
if build_config["libraries"] == "shared":
return False
if build_config["sanitizer"] != "":
return True
@ -68,8 +68,8 @@ def get_packager_cmd(
cmd += f" --build-type={build_config['build_type']}"
if build_config["sanitizer"]:
cmd += f" --sanitizer={build_config['sanitizer']}"
if build_config["splitted"] == "splitted":
cmd += " --split-binary"
if build_config["libraries"] == "shared":
cmd += " --shared-libraries"
if build_config["tidy"] == "enable":
cmd += " --clang-tidy"

View File

@ -37,7 +37,7 @@ class BuildResult:
build_type,
sanitizer,
bundled,
splitted,
libraries,
status,
elapsed_seconds,
with_coverage,
@ -46,7 +46,7 @@ class BuildResult:
self.build_type = build_type
self.sanitizer = sanitizer
self.bundled = bundled
self.splitted = splitted
self.libraries = libraries
self.status = status
self.elapsed_seconds = elapsed_seconds
self.with_coverage = with_coverage
@ -91,7 +91,7 @@ def get_failed_report(
build_type="unknown",
sanitizer="unknown",
bundled="unknown",
splitted="unknown",
libraries="unknown",
status=message,
elapsed_seconds=0,
with_coverage=False,
@ -108,7 +108,7 @@ def process_report(
build_type=build_config["build_type"],
sanitizer=build_config["sanitizer"],
bundled=build_config["bundled"],
splitted=build_config["splitted"],
libraries=build_config["libraries"],
status="success" if build_report["status"] else "failure",
elapsed_seconds=build_report["elapsed_seconds"],
with_coverage=False,

View File

@ -206,6 +206,7 @@ Merge it only if you intend to backport changes to the target branch, otherwise
)
self.cherrypick_pr.add_to_labels(Labels.LABEL_CHERRYPICK)
self.cherrypick_pr.add_to_labels(Labels.LABEL_DO_NOT_TEST)
if self.pr.assignee is not None:
self.cherrypick_pr.add_to_assignees(self.pr.assignee)
self.cherrypick_pr.add_to_assignees(self.pr.user)
@ -238,7 +239,8 @@ Merge it only if you intend to backport changes to the target branch, otherwise
head=self.backport_branch,
)
self.backport_pr.add_to_labels(Labels.LABEL_BACKPORT)
self.backport_pr.add_to_assignees(self.pr.assignee)
if self.pr.assignee is not None:
self.cherrypick_pr.add_to_assignees(self.pr.assignee)
self.backport_pr.add_to_assignees(self.pr.user)
@property

View File

@ -14,7 +14,7 @@ CI_CONFIG = {
"package_type": "deb",
"static_binary_name": "amd64",
"bundled": "bundled",
"splitted": "unsplitted",
"libraries": "static",
"additional_pkgs": True,
"tidy": "disable",
"with_coverage": False,
@ -25,7 +25,7 @@ CI_CONFIG = {
"sanitizer": "",
"package_type": "coverity",
"bundled": "bundled",
"splitted": "unsplitted",
"libraries": "static",
"tidy": "disable",
"with_coverage": False,
"official": False,
@ -37,7 +37,7 @@ CI_CONFIG = {
"package_type": "deb",
"static_binary_name": "aarch64",
"bundled": "bundled",
"splitted": "unsplitted",
"libraries": "static",
"additional_pkgs": True,
"tidy": "disable",
"with_coverage": False,
@ -48,7 +48,7 @@ CI_CONFIG = {
"sanitizer": "address",
"package_type": "deb",
"bundled": "bundled",
"splitted": "unsplitted",
"libraries": "static",
"tidy": "disable",
"with_coverage": False,
},
@ -58,7 +58,7 @@ CI_CONFIG = {
"sanitizer": "undefined",
"package_type": "deb",
"bundled": "bundled",
"splitted": "unsplitted",
"libraries": "static",
"tidy": "disable",
"with_coverage": False,
},
@ -68,7 +68,7 @@ CI_CONFIG = {
"sanitizer": "thread",
"package_type": "deb",
"bundled": "bundled",
"splitted": "unsplitted",
"libraries": "static",
"tidy": "disable",
"with_coverage": False,
},
@ -78,7 +78,7 @@ CI_CONFIG = {
"sanitizer": "memory",
"package_type": "deb",
"bundled": "bundled",
"splitted": "unsplitted",
"libraries": "static",
"tidy": "disable",
"with_coverage": False,
},
@ -88,7 +88,7 @@ CI_CONFIG = {
"sanitizer": "",
"package_type": "deb",
"bundled": "bundled",
"splitted": "unsplitted",
"libraries": "static",
"tidy": "disable",
"with_coverage": False,
},
@ -98,7 +98,7 @@ CI_CONFIG = {
"sanitizer": "",
"package_type": "binary",
"bundled": "bundled",
"splitted": "unsplitted",
"libraries": "static",
"tidy": "disable",
"with_coverage": False,
},
@ -109,17 +109,17 @@ CI_CONFIG = {
"package_type": "binary",
"static_binary_name": "debug-amd64",
"bundled": "bundled",
"splitted": "unsplitted",
"libraries": "static",
"tidy": "enable",
"with_coverage": False,
},
"binary_splitted": {
"binary_shared": {
"compiler": "clang-14",
"build_type": "",
"sanitizer": "",
"package_type": "binary",
"bundled": "bundled",
"splitted": "splitted",
"libraries": "shared",
"tidy": "disable",
"with_coverage": False,
},
@ -130,7 +130,7 @@ CI_CONFIG = {
"package_type": "binary",
"static_binary_name": "macos",
"bundled": "bundled",
"splitted": "unsplitted",
"libraries": "static",
"tidy": "disable",
"with_coverage": False,
},
@ -140,7 +140,7 @@ CI_CONFIG = {
"sanitizer": "",
"package_type": "binary",
"bundled": "bundled",
"splitted": "unsplitted",
"libraries": "static",
"tidy": "disable",
"with_coverage": False,
},
@ -151,7 +151,7 @@ CI_CONFIG = {
"package_type": "binary",
"static_binary_name": "freebsd",
"bundled": "bundled",
"splitted": "unsplitted",
"libraries": "static",
"tidy": "disable",
"with_coverage": False,
},
@ -162,7 +162,7 @@ CI_CONFIG = {
"package_type": "binary",
"static_binary_name": "macos-aarch64",
"bundled": "bundled",
"splitted": "unsplitted",
"libraries": "static",
"tidy": "disable",
"with_coverage": False,
},
@ -173,7 +173,7 @@ CI_CONFIG = {
"package_type": "binary",
"static_binary_name": "powerpc64le",
"bundled": "bundled",
"splitted": "unsplitted",
"libraries": "static",
"tidy": "disable",
"with_coverage": False,
},
@ -192,7 +192,7 @@ CI_CONFIG = {
],
"ClickHouse special build check": [
"binary_tidy",
"binary_splitted",
"binary_shared",
"binary_darwin",
"binary_aarch64",
"binary_freebsd",
@ -297,7 +297,7 @@ CI_CONFIG = {
"required_build": "package_release",
},
"Split build smoke test": {
"required_build": "binary_splitted",
"required_build": "binary_shared",
},
"Unit tests (release-clang)": {
"required_build": "binary_release",

View File

@ -32,7 +32,18 @@ class ClickHouseHelper:
}
for i in range(5):
response = requests.post(url, params=params, data=json_str, headers=auth)
try:
response = requests.post(
url, params=params, data=json_str, headers=auth
)
except Exception as e:
logging.warning(
"Received exception while sending data to %s on %s attempt: %s",
url,
i,
e,
)
continue
logging.info("Response content '%s'", response.content)

View File

@ -290,7 +290,7 @@ tr:hover td {{filter: brightness(95%);}}
<th>Build type</th>
<th>Sanitizer</th>
<th>Bundled</th>
<th>Splitted</th>
<th>Libraries</th>
<th>Status</th>
<th>Build log</th>
<th>Build time</th>
@ -335,7 +335,7 @@ def create_build_html_report(
row += "<td>{}</td>".format("none")
row += "<td>{}</td>".format(build_result.bundled)
row += "<td>{}</td>".format(build_result.splitted)
row += "<td>{}</td>".format(build_result.libraries)
if build_result.status:
style = _get_status_style(build_result.status)

View File

@ -1,8 +1,4 @@
if(CLICKHOUSE_SPLIT_BINARY)
set (TEST_USE_BINARIES CLICKHOUSE_TESTS_SERVER_BIN_PATH=${ClickHouse_BINARY_DIR}/programs/clickhouse-server CLICKHOUSE_TESTS_CLIENT_BIN_PATH=${ClickHouse_BINARY_DIR}/programs/clickhouse-client)
else()
set (TEST_USE_BINARIES CLICKHOUSE_TESTS_SERVER_BIN_PATH=${ClickHouse_BINARY_DIR}/programs/clickhouse CLICKHOUSE_TESTS_CLIENT_BIN_PATH=${ClickHouse_BINARY_DIR}/programs/clickhouse)
endif()
find_program(DOCKER_CMD docker)
find_program(DOCKER_COMPOSE_CMD docker-compose)

View File

@ -46,5 +46,19 @@
"test_storage_s3/test.py::test_url_reconnect_in_the_middle",
"test_system_metrics/test.py::test_readonly_metrics",
"test_system_replicated_fetches/test.py::test_system_replicated_fetches",
"test_zookeeper_config_load_balancing/test.py::test_round_robin"
"test_zookeeper_config_load_balancing/test.py::test_round_robin",
"test_tlsv1_3/test.py::test_https",
"test_tlsv1_3/test.py::test_https_wrong_cert",
"test_tlsv1_3/test.py::test_https_non_ssl_auth",
"test_tlsv1_3/test.py::test_create_user",
"test_user_ip_restrictions/test.py::test_ipv4",
"test_user_ip_restrictions/test.py::test_ipv6",
"test_ssl_cert_authentication/test.py::test_https",
"test_ssl_cert_authentication/test.py::test_https_wrong_cert",
"test_ssl_cert_authentication/test.py::test_https_non_ssl_auth",
"test_ssl_cert_authentication/test.py::test_create_user",
"test_grpc_protocol_ssl/test.py::test_secure_channel",
"test_grpc_protocol_ssl/test.py::test_insecure_channel",
"test_grpc_protocol_ssl/test.py::test_wrong_client_certificate"
]

View File

@ -163,6 +163,19 @@ def test_executable_function_input_multiple_pipes_python(started_cluster):
assert actual == expected
def test_executable_function_input_slow_python_timeout_increased(started_cluster):
skip_test_msan(node)
query = "SELECT * FROM executable('input_slow.py', 'TabSeparated', 'value String', {source}, SETTINGS {settings})"
settings = "command_termination_timeout = 26, command_read_timeout = 26000, command_write_timeout = 26000"
assert node.query(query.format(source="(SELECT 1)", settings=settings)) == "Key 1\n"
assert (
node.query(
query.format(source="(SELECT id FROM test_data_table)", settings=settings)
)
== "Key 0\nKey 1\nKey 2\n"
)
def test_executable_storage_no_input_bash(started_cluster):
skip_test_msan(node)
node.query("DROP TABLE IF EXISTS test_table")

View File

@ -5,7 +5,8 @@ import grpc
from helpers.cluster import ClickHouseCluster, run_and_check
GRPC_PORT = 9100
NODE_IP = "10.5.172.77" # It's important for the node to work at this IP because 'server-cert.pem' requires that (see server-ext.cnf).
# It's important for the node to work at this IP because 'server-cert.pem' requires that (see server-ext.cnf).
NODE_IP = "10.5.172.77" # Never copy-paste this line
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
DEFAULT_ENCODING = "utf-8"

View File

@ -33,7 +33,7 @@ def test_different_user(started_cluster):
node.start_clickhouse(start_wait_sec=3)
log = node.grep_in_log("Effective")
expected_message = "Effective user of the process \(.*\) does not match the owner of the data \(.*\)\. Run under 'sudo -u .*'\."
expected_message = "Effective user of the process \\(.+?\\) does not match the owner of the data \\(.+?\\)\\. Run under 'sudo -u .*'\\."
if re.search(expected_message, log) is None:
pytest.fail(
'Expected the server to fail with a message "{}", but the last message is "{}"'.format(

View File

@ -165,10 +165,7 @@ def test_deduplication_while_move(started_cluster):
assert TSV(
n.query(
"SELECT count() FROM test_deduplication_d",
settings={
"allow_experimental_query_deduplication": 1,
"allow_experimental_projection_optimization": 1,
},
settings={"allow_experimental_query_deduplication": 1},
)
) == TSV("2")

View File

@ -5,7 +5,8 @@ import ssl
import os.path
HTTPS_PORT = 8443
NODE_IP = "10.5.172.77" # It's important for the node to work at this IP because 'server-cert.pem' requires that (see server-ext.cnf).
# It's important for the node to work at this IP because 'server-cert.pem' requires that (see server-ext.cnf).
NODE_IP = "10.5.172.77" # Never copy-paste this line
NODE_IP_WITH_HTTPS_PORT = NODE_IP + ":" + str(HTTPS_PORT)
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))

View File

@ -0,0 +1,3 @@
<clickhouse>
<interserver_listen_host replace="replace">10.0.0.10</interserver_listen_host>
</clickhouse>

View File

@ -0,0 +1,3 @@
<clickhouse>
<interserver_listen_host remove="remove"></interserver_listen_host>
</clickhouse>

View File

@ -0,0 +1,55 @@
"""Test Interserver responses on configured IP."""
from pathlib import Path
import pytest
from helpers.cluster import ClickHouseCluster
import requests
import socket
import time
cluster = ClickHouseCluster(__file__)
INTERSERVER_LISTEN_HOST = "10.0.0.10"
INTERSERVER_HTTP_PORT = 9009
node_with_interserver_listen_host = cluster.add_instance(
"node_with_interserver_listen_host",
main_configs=["configs/config.d/interserver-listen-host.xml"],
ipv4_address=INTERSERVER_LISTEN_HOST, # used to configure acc. interface in test container
ipv6_address="2001:3984:3989::1:1000",
)
node_without_interserver_listen_host = cluster.add_instance(
"node_without_interserver_listen_host",
main_configs=["configs/config.d/no-interserver-listen-host.xml"],
ipv6_address="2001:3984:3989::2:1000",
)
@pytest.fixture(scope="module")
def start_cluster():
try:
cluster.start()
yield cluster
finally:
cluster.shutdown()
def test_request_to_node_with_interserver_listen_host(start_cluster):
time.sleep(5) # waiting for interserver listener to start
response_interserver = requests.get(
f"http://{INTERSERVER_LISTEN_HOST}:{INTERSERVER_HTTP_PORT}"
)
response_client = requests.get(
f"http://{node_without_interserver_listen_host.ip_address}:8123"
)
assert response_interserver.status_code == 200
assert "Ok." in response_interserver.text
assert response_client.status_code == 200
def test_request_to_node_without_interserver_listen_host(start_cluster):
response = requests.get(
f"http://{node_without_interserver_listen_host.ip_address}:{INTERSERVER_HTTP_PORT}"
)
assert response.status_code == 200

View File

@ -5,7 +5,8 @@ import ssl
import os.path
HTTPS_PORT = 8443
NODE_IP = "10.5.172.77" # It's important for the node to work at this IP because 'server-cert.pem' requires that (see server-ext.cnf).
# It's important for the node to work at this IP because 'server-cert.pem' requires that (see server-ext.cnf).
NODE_IP = "10.5.172.77" # Never copy-paste this line
NODE_IP_WITH_HTTPS_PORT = NODE_IP + ":" + str(HTTPS_PORT)
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))

View File

@ -8,47 +8,56 @@ node_ipv4 = cluster.add_instance(
"node_ipv4",
main_configs=[],
user_configs=["configs/users_ipv4.xml"],
ipv4_address="10.5.172.77",
ipv4_address="10.5.172.77", # Never copy-paste this line
)
client_ipv4_ok = cluster.add_instance(
"client_ipv4_ok", main_configs=[], user_configs=[], ipv4_address="10.5.172.10"
"client_ipv4_ok",
main_configs=[],
user_configs=[],
ipv4_address="10.5.172.10", # Never copy-paste this line
)
client_ipv4_ok_direct = cluster.add_instance(
"client_ipv4_ok_direct", main_configs=[], user_configs=[], ipv4_address="10.5.173.1"
"client_ipv4_ok_direct",
main_configs=[],
user_configs=[],
ipv4_address="10.5.173.1", # Never copy-paste this line
)
client_ipv4_ok_full_mask = cluster.add_instance(
"client_ipv4_ok_full_mask",
main_configs=[],
user_configs=[],
ipv4_address="10.5.175.77",
ipv4_address="10.5.175.77", # Never copy-paste this line
)
client_ipv4_bad = cluster.add_instance(
"client_ipv4_bad", main_configs=[], user_configs=[], ipv4_address="10.5.173.10"
"client_ipv4_bad",
main_configs=[],
user_configs=[],
ipv4_address="10.5.173.10", # Never copy-paste this line
)
node_ipv6 = cluster.add_instance(
"node_ipv6",
main_configs=["configs/config_ipv6.xml"],
user_configs=["configs/users_ipv6.xml"],
ipv6_address="2001:3984:3989::1:1000",
ipv6_address="2001:3984:3989::1:1000", # Never copy-paste this line
)
client_ipv6_ok = cluster.add_instance(
"client_ipv6_ok",
main_configs=[],
user_configs=[],
ipv6_address="2001:3984:3989::5555",
ipv6_address="2001:3984:3989::5555", # Never copy-paste this line
)
client_ipv6_ok_direct = cluster.add_instance(
"client_ipv6_ok_direct",
main_configs=[],
user_configs=[],
ipv6_address="2001:3984:3989::1:1111",
ipv6_address="2001:3984:3989::1:1111", # Never copy-paste this line
)
client_ipv6_bad = cluster.add_instance(
"client_ipv6_bad",
main_configs=[],
user_configs=[],
ipv6_address="2001:3984:3989::1:1112",
ipv6_address="2001:3984:3989::1:1112", # Never copy-paste this line
)

View File

@ -11,6 +11,7 @@
[zookeeper :as zk])
(:import (org.apache.zookeeper ZooKeeper KeeperException KeeperException$BadVersionException)))
(def root-path "/queue")
(defn enqueue [val _ _] {:type :invoke, :f :enqueue :value val})
(defn dequeue [_ _] {:type :invoke, :f :dequeue})
@ -22,18 +23,20 @@
:conn (zk-connect node 9181 30000))
:nodename node))
(setup! [this test])
(setup! [this test]
(exec-with-retries 30 (fn []
(zk-create-if-not-exists conn root-path ""))))
(invoke! [this test op]
(case (:f op)
:enqueue (try
(do
(zk-create-if-not-exists conn (str "/" (:value op)) "")
(zk-create conn (concat-path root-path (:value op)) "")
(assoc op :type :ok))
(catch Exception _ (assoc op :type :info, :error :connect-error)))
:dequeue
(try
(let [result (zk-multi-delete-first-child conn "/")]
(let [result (zk-multi-delete-first-child conn root-path)]
(if (not (nil? result))
(assoc op :type :ok :value result)
(assoc op :type :fail :value result)))
@ -42,7 +45,7 @@
; drain via delete is to long, just list all nodes
(exec-with-retries 30 (fn []
(zk-sync conn)
(assoc op :type :ok :value (into #{} (map #(str %1) (zk-list conn "/"))))))))
(assoc op :type :ok :value (into #{} (map #(str %1) (zk-list conn root-path))))))))
(teardown! [_ test])

View File

@ -88,10 +88,14 @@
new-set (conj current-set elem)]
(zk-set conn path (pr-str new-set) (:version (:stat current-value)))))
(defn zk-create-if-not-exists
(defn zk-create
[conn path data]
(zk/create conn path :data (data/to-bytes (str data)) :persistent? true))
(defn zk-create-if-not-exists
[conn path data]
(if-not (zk/exists conn path) (zk-create conn path data)))
(defn zk-create-sequential
[conn path-prefix data]
(zk/create conn path-prefix :data (data/to-bytes (str data)) :persistent? true :sequential? true))
@ -119,6 +123,10 @@
(subs path 0 rslash_pos)
"/")))
(defn concat-path
[parent child]
(str parent (if (= parent "/") "" "/") child))
(defn zk-multi-delete-first-child
[conn path]
(let [{children :children stat :stat} (zk-list-with-stat conn path)
@ -128,7 +136,7 @@
(try
(do (.check txn path (:version stat))
(.setData txn path (data/to-bytes "") -1) ; I'm just checking multitransactions
(.delete txn (str path first-child) -1)
(.delete txn (concat-path path first-child) -1)
(.commit txn)
first-child)
(catch KeeperException$BadVersionException _ nil)

View File

@ -1,5 +1,7 @@
-- Tags: shard
set enable_positional_arguments=0;
select 40 as z from (select * from system.numbers limit 3) group by z;
select 41 as z from remote('127.0.0.{2,3}', system.one) group by z;
select count(), 42 AS z from remote('127.0.0.{2,3}', system.one) group by z;

View File

@ -1,6 +1,7 @@
drop table if exists t;
create table t(n int, a Int64, s String) engine = MergeTree() order by a;
set enable_positional_arguments=0;
set optimize_trivial_insert_select=1;
-- due to aggregate functions, optimize_trivial_insert_select will not be applied

View File

@ -8,6 +8,7 @@ CREATE TABLE test
ENGINE = MergeTree()
ORDER BY tuple();
SET enable_positional_arguments=0;
SET optimize_move_functions_out_of_any = 1;
SELECT any(arrayFilter((c, d) -> (4 = d), `Source.C1`, `Source.C2`)[1]) AS x

Some files were not shown because too many files have changed in this diff Show More