thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-10 09:32:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into improve-recovery-of-replicated-access-storage

Browse Source
This commit is contained in:
Vitaly Baranov 2022-09-19 15:26:21 +02:00 committed by GitHub
commit 052ecaa211
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
274 changed files with 4571 additions and 2901 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
.github/workflows/backport_branches.yml vendored
View File

@ -112,10 +112,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
#########################################################################################
#################################### ORDINARY BUILDS ####################################
@ -162,10 +160,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
BuilderDebAarch64:
needs: [DockerHubPush]
@ -209,10 +205,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
BuilderDebAsan:
needs: [DockerHubPush]
@ -254,10 +248,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
BuilderDebTsan:
needs: [DockerHubPush]
@ -299,10 +291,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
BuilderDebDebug:
needs: [DockerHubPush]
@ -344,10 +334,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
BuilderBinDarwin:
needs: [DockerHubPush]
@ -391,10 +379,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
BuilderBinDarwinAarch64:
needs: [DockerHubPush]
@ -438,10 +424,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
############################################################################################
##################################### Docker images #######################################
@ -468,10 +452,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
############################################################################################
##################################### BUILD REPORTER #######################################
@ -514,10 +496,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
BuilderSpecialReport:
needs:
@ -554,10 +534,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
##############################################################################################
########################### FUNCTIONAl STATELESS TESTS #######################################
@ -594,10 +572,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
##############################################################################################
############################ FUNCTIONAl STATEFUL TESTS #######################################
@ -634,10 +610,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
##############################################################################################
######################################### STRESS TESTS #######################################
@ -677,10 +651,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
#############################################################################################
############################# INTEGRATION TESTS #############################################
@ -716,10 +688,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FinishCheck:
needs:

6
.github/workflows/cherry_pick.yml vendored
View File

@ -40,8 +40,6 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"

12
.github/workflows/docs_check.yml vendored
View File

@ -125,10 +125,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
DocsCheck:
needs: DockerHubPush
@ -158,10 +156,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FinishCheck:
needs:

6
.github/workflows/docs_release.yml vendored
View File

@ -116,8 +116,6 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"

6
.github/workflows/jepsen.yml vendored
View File

@ -36,8 +36,6 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"

501
.github/workflows/master.yml vendored
View File

@ -112,10 +112,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
CompatibilityCheck:
needs: [BuilderDebRelease]
@ -146,10 +144,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
SharedBuildSmokeTest:
needs: [BuilderDebShared]
@ -180,10 +176,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
#########################################################################################
#################################### ORDINARY BUILDS ####################################
@ -230,10 +224,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
BuilderDebAarch64:
needs: [DockerHubPush]
@ -273,10 +265,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
BuilderBinRelease:
needs: [DockerHubPush]
@ -320,56 +310,9 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
# BuilderBinGCC:
# needs: [DockerHubPush]
# runs-on: [self-hosted, builder]
# steps:
# - name: Set envs
# run: |
# cat >> "$GITHUB_ENV" << 'EOF'
# TEMP_PATH=${{runner.temp}}/build_check
# IMAGES_PATH=${{runner.temp}}/images_path
# REPO_COPY=${{runner.temp}}/build_check/ClickHouse
# CACHES_PATH=${{runner.temp}}/../ccaches
# BUILD_NAME=binary_gcc
# EOF
# - name: Download changed images
# uses: actions/download-artifact@v2
# with:
# name: changed_images
# path: ${{ env.IMAGES_PATH }}
# - name: Clear repository
# run: |
# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
# - name: Check out repository code
# uses: actions/checkout@v2
# - name: Build
# run: |
# git -C "$GITHUB_WORKSPACE" submodule sync --recursive
# git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
# sudo rm -fr "$TEMP_PATH"
# mkdir -p "$TEMP_PATH"
# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
# - name: Upload build URLs to artifacts
# if: ${{ success() || failure() }}
# uses: actions/upload-artifact@v2
# with:
# name: ${{ env.BUILD_URLS }}
# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
# - name: Cleanup
# if: always()
# run: |
# # shellcheck disable=SC2046
# docker kill $(docker ps -q) ||:
# # shellcheck disable=SC2046
# docker rm -f $(docker ps -a -q) ||:
# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
BuilderDebAsan:
needs: [DockerHubPush]
runs-on: [self-hosted, builder]
@ -410,10 +353,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
BuilderDebUBsan:
needs: [DockerHubPush]
@ -455,10 +396,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
BuilderDebTsan:
needs: [DockerHubPush]
@ -500,10 +439,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
BuilderDebMsan:
needs: [DockerHubPush]
@ -545,10 +482,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
BuilderDebDebug:
needs: [DockerHubPush]
@ -590,10 +525,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
##########################################################################################
##################################### SPECIAL BUILDS #####################################
@ -638,10 +571,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
BuilderBinClangTidy:
needs: [DockerHubPush]
@ -683,10 +614,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
BuilderBinDarwin:
needs: [DockerHubPush]
@ -730,10 +659,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
BuilderBinAarch64:
needs: [DockerHubPush]
@ -777,10 +704,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
BuilderBinFreeBSD:
needs: [DockerHubPush]
@ -824,10 +749,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
BuilderBinDarwinAarch64:
needs: [DockerHubPush]
@ -871,10 +794,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
BuilderBinPPC64:
needs: [DockerHubPush]
@ -918,10 +839,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
BuilderBinAmd64SSE2:
needs: [DockerHubPush]
@ -965,10 +884,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
############################################################################################
##################################### Docker images #######################################
@ -995,10 +912,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
############################################################################################
##################################### BUILD REPORTER #######################################
@ -1045,10 +960,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
BuilderSpecialReport:
needs:
@ -1092,10 +1005,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
##############################################################################################
########################### FUNCTIONAl STATELESS TESTS #######################################
@ -1132,10 +1043,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestReleaseDatabaseOrdinary:
needs: [BuilderDebRelease]
@ -1169,10 +1078,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestReleaseDatabaseReplicated0:
needs: [BuilderDebRelease]
@ -1208,10 +1115,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestReleaseDatabaseReplicated1:
needs: [BuilderDebRelease]
@ -1247,10 +1152,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestReleaseS3:
needs: [BuilderDebRelease]
@ -1284,10 +1187,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestAarch64:
needs: [BuilderDebAarch64]
@ -1321,10 +1222,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestAsan0:
needs: [BuilderDebAsan]
@ -1360,10 +1259,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestAsan1:
needs: [BuilderDebAsan]
@ -1399,10 +1296,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestTsan0:
needs: [BuilderDebTsan]
@ -1438,10 +1333,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestTsan1:
needs: [BuilderDebTsan]
@ -1477,10 +1370,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestTsan2:
needs: [BuilderDebTsan]
@ -1516,10 +1407,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestUBsan:
needs: [BuilderDebUBsan]
@ -1553,10 +1442,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestMsan0:
needs: [BuilderDebMsan]
@ -1592,10 +1479,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestMsan1:
needs: [BuilderDebMsan]
@ -1631,10 +1516,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestMsan2:
needs: [BuilderDebMsan]
@ -1670,10 +1553,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestDebug0:
needs: [BuilderDebDebug]
@ -1709,10 +1590,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestDebug1:
needs: [BuilderDebDebug]
@ -1748,10 +1627,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestDebug2:
needs: [BuilderDebDebug]
@ -1787,10 +1664,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
##############################################################################################
############################ FUNCTIONAl STATEFUL TESTS #######################################
@ -1827,10 +1702,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatefulTestAarch64:
needs: [BuilderDebAarch64]
@ -1864,10 +1737,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatefulTestAsan:
needs: [BuilderDebAsan]
@ -1901,10 +1772,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatefulTestTsan:
needs: [BuilderDebTsan]
@ -1938,10 +1807,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatefulTestMsan:
needs: [BuilderDebMsan]
@ -1975,10 +1842,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatefulTestUBsan:
needs: [BuilderDebUBsan]
@ -2012,10 +1877,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatefulTestDebug:
needs: [BuilderDebDebug]
@ -2049,10 +1912,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
##############################################################################################
######################################### STRESS TESTS #######################################
@ -2088,10 +1949,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
StressTestTsan:
needs: [BuilderDebTsan]
@ -2128,10 +1987,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
StressTestMsan:
needs: [BuilderDebMsan]
@ -2164,10 +2021,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
StressTestUBsan:
needs: [BuilderDebUBsan]
@ -2200,10 +2055,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
StressTestDebug:
needs: [BuilderDebDebug]
@ -2236,10 +2089,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
#############################################################################################
############################# INTEGRATION TESTS #############################################
@ -2277,10 +2128,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
IntegrationTestsAsan1:
needs: [BuilderDebAsan]
@ -2315,10 +2164,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
IntegrationTestsAsan2:
needs: [BuilderDebAsan]
@ -2353,10 +2200,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
IntegrationTestsTsan0:
needs: [BuilderDebTsan]
@ -2391,10 +2236,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
IntegrationTestsTsan1:
needs: [BuilderDebTsan]
@ -2429,10 +2272,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
IntegrationTestsTsan2:
needs: [BuilderDebTsan]
@ -2467,10 +2308,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
IntegrationTestsTsan3:
needs: [BuilderDebTsan]
@ -2505,10 +2344,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
IntegrationTestsRelease0:
needs: [BuilderDebRelease]
@ -2543,10 +2380,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
IntegrationTestsRelease1:
needs: [BuilderDebRelease]
@ -2581,10 +2416,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
##############################################################################################
##################################### AST FUZZERS ############################################
@ -2620,10 +2453,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
ASTFuzzerTestTsan:
needs: [BuilderDebTsan]
@ -2656,10 +2487,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
ASTFuzzerTestUBSan:
needs: [BuilderDebUBsan]
@ -2692,10 +2521,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
ASTFuzzerTestMSan:
needs: [BuilderDebMsan]
@ -2728,10 +2555,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
ASTFuzzerTestDebug:
needs: [BuilderDebDebug]
@ -2764,10 +2589,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
#############################################################################################
#################################### UNIT TESTS #############################################
@ -2803,10 +2626,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
UnitTestsReleaseClang:
needs: [BuilderBinRelease]
@ -2839,10 +2660,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
# UnitTestsReleaseGCC:
# needs: [BuilderBinGCC]
@ -2875,10 +2694,8 @@ jobs:
# - name: Cleanup
# if: always()
# run: |
# # shellcheck disable=SC2046
# docker kill $(docker ps -q) ||:
# # shellcheck disable=SC2046
# docker rm -f $(docker ps -a -q) ||:
# docker ps --quiet | xargs --no-run-if-empty docker kill ||:
# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
# sudo rm -fr "$TEMP_PATH"
UnitTestsTsan:
needs: [BuilderDebTsan]
@ -2911,10 +2728,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
UnitTestsMsan:
needs: [BuilderDebMsan]
@ -2947,10 +2762,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
UnitTestsUBsan:
needs: [BuilderDebUBsan]
@ -2983,10 +2796,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
#############################################################################################
#################################### PERFORMANCE TESTS ######################################
@ -3024,10 +2835,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
PerformanceComparisonX86-1:
needs: [BuilderDebRelease]
@ -3062,10 +2871,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
PerformanceComparisonX86-2:
needs: [BuilderDebRelease]
@ -3100,10 +2907,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
PerformanceComparisonX86-3:
needs: [BuilderDebRelease]
@ -3138,10 +2943,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FinishCheck:
needs:

6
.github/workflows/nightly.yml vendored
View File

@ -119,8 +119,6 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"

522
.github/workflows/pull_request.yml vendored
View File

File diff suppressed because it is too large Load Diff

6
.github/workflows/release.yml vendored
View File

@ -66,8 +66,6 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"

288
.github/workflows/release_branches.yml vendored
View File

@ -103,10 +103,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
#########################################################################################
#################################### ORDINARY BUILDS ####################################
@ -153,10 +151,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
BuilderDebAarch64:
needs: [DockerHubPush]
@ -196,10 +192,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
BuilderDebAsan:
needs: [DockerHubPush]
@ -241,10 +235,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
BuilderDebUBsan:
needs: [DockerHubPush]
@ -286,10 +278,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
BuilderDebTsan:
needs: [DockerHubPush]
@ -331,10 +321,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
BuilderDebMsan:
needs: [DockerHubPush]
@ -376,10 +364,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
BuilderDebDebug:
needs: [DockerHubPush]
@ -421,10 +407,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
BuilderBinDarwin:
needs: [DockerHubPush]
@ -468,10 +452,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
BuilderBinDarwinAarch64:
needs: [DockerHubPush]
@ -515,10 +497,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
############################################################################################
##################################### Docker images #######################################
@ -545,10 +525,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
############################################################################################
##################################### BUILD REPORTER #######################################
@ -594,10 +572,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
BuilderSpecialReport:
needs:
@ -634,10 +610,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
##############################################################################################
########################### FUNCTIONAl STATELESS TESTS #######################################
@ -674,10 +648,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestAarch64:
needs: [BuilderDebAarch64]
@ -711,10 +683,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestAsan0:
needs: [BuilderDebAsan]
@ -750,10 +720,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestAsan1:
needs: [BuilderDebAsan]
@ -789,10 +757,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestTsan0:
needs: [BuilderDebTsan]
@ -828,10 +794,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestTsan1:
needs: [BuilderDebTsan]
@ -867,10 +831,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestTsan2:
needs: [BuilderDebTsan]
@ -906,10 +868,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestUBsan:
needs: [BuilderDebUBsan]
@ -943,10 +903,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestMsan0:
needs: [BuilderDebMsan]
@ -982,10 +940,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestMsan1:
needs: [BuilderDebMsan]
@ -1021,10 +977,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestMsan2:
needs: [BuilderDebMsan]
@ -1060,10 +1014,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestDebug0:
needs: [BuilderDebDebug]
@ -1099,10 +1051,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestDebug1:
needs: [BuilderDebDebug]
@ -1138,10 +1088,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestDebug2:
needs: [BuilderDebDebug]
@ -1177,10 +1125,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
##############################################################################################
############################ FUNCTIONAl STATEFUL TESTS #######################################
@ -1217,10 +1163,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatefulTestAarch64:
needs: [BuilderDebAarch64]
@ -1254,10 +1198,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatefulTestAsan:
needs: [BuilderDebAsan]
@ -1291,10 +1233,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatefulTestTsan:
needs: [BuilderDebTsan]
@ -1328,10 +1268,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatefulTestMsan:
needs: [BuilderDebMsan]
@ -1365,10 +1303,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatefulTestUBsan:
needs: [BuilderDebUBsan]
@ -1402,10 +1338,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatefulTestDebug:
needs: [BuilderDebDebug]
@ -1439,10 +1373,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
##############################################################################################
######################################### STRESS TESTS #######################################
@ -1478,10 +1410,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
StressTestTsan:
needs: [BuilderDebTsan]
@ -1518,10 +1448,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
StressTestMsan:
needs: [BuilderDebMsan]
@ -1554,10 +1482,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
StressTestUBsan:
needs: [BuilderDebUBsan]
@ -1590,10 +1516,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
StressTestDebug:
needs: [BuilderDebDebug]
@ -1626,10 +1550,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
#############################################################################################
############################# INTEGRATION TESTS #############################################
@ -1667,10 +1589,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
IntegrationTestsAsan1:
needs: [BuilderDebAsan]
@ -1705,10 +1625,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
IntegrationTestsAsan2:
needs: [BuilderDebAsan]
@ -1743,10 +1661,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
IntegrationTestsTsan0:
needs: [BuilderDebTsan]
@ -1781,10 +1697,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
IntegrationTestsTsan1:
needs: [BuilderDebTsan]
@ -1819,10 +1733,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
IntegrationTestsTsan2:
needs: [BuilderDebTsan]
@ -1857,10 +1769,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
IntegrationTestsTsan3:
needs: [BuilderDebTsan]
@ -1895,10 +1805,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
IntegrationTestsRelease0:
needs: [BuilderDebRelease]
@ -1933,10 +1841,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
IntegrationTestsRelease1:
needs: [BuilderDebRelease]
@ -1971,10 +1877,8 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FinishCheck:
needs:

6
.github/workflows/woboq.yml vendored
View File

@ -37,8 +37,6 @@ jobs:
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"

15
base/base/bit_cast.h
View File

@ -5,8 +5,9 @@
#include <type_traits>
/** \brief Returns value `from` converted to type `To` while retaining bit representation.
* `To` and `From` must satisfy `CopyConstructible`.
/** Returns value `from` converted to type `To` while retaining bit representation.
* `To` and `From` must satisfy `CopyConstructible`.
* In contrast to std::bit_cast can cast types of different width.
*/
template <typename To, typename From>
std::decay_t<To> bit_cast(const From & from)
@ -15,13 +16,3 @@ std::decay_t<To> bit_cast(const From & from)
memcpy(static_cast<void*>(&res), &from, std::min(sizeof(res), sizeof(from)));
return res;
}
/** \brief Returns value `from` converted to type `To` while retaining bit representation.
* `To` and `From` must satisfy `CopyConstructible`.
*/
template <typename To, typename From>
std::decay_t<To> safe_bit_cast(const From & from)
{
static_assert(sizeof(To) == sizeof(From), "bit cast on types of different width");
return bit_cast<To, From>(from);
}

1
base/base/phdr_cache.h
View File

@ -8,6 +8,7 @@
* As a drawback, this only works if no dynamic object unloading happens after this point.
* This function is thread-safe. You should call it to update cache after loading new shared libraries.
* Otherwise exception handling from dlopened libraries won't work (will call std::terminate immediately).
* NOTE: dlopen is forbidden in our code.
*
* NOTE: It is disabled with Thread Sanitizer because TSan can only use original "dl_iterate_phdr" function.
*/

2
contrib/NuRaft vendored

@ -1 +1 @@
Subproject commit e15858f8ad0ce8aba85cf74e3763874c76bf927c
Subproject commit 1be805e7cb2494aa8170015493474379b0362dfc

2
contrib/krb5 vendored

@ -1 +1 @@
Subproject commit d879821c7a4c70b0c3ad739d9951d1a2b1903df7
Subproject commit b89e20367b074bd02dd118a6534099b21e88b3c3

2
contrib/replxx vendored

@ -1 +1 @@
Subproject commit 3fd0e3c9364a589447453d9906d854ebd8d385c5
Subproject commit 5d04501f93a4fb7f0bb8b73b8f614bc986f9e25b

6
contrib/zlib-ng-cmake/CMakeLists.txt
View File

@ -2,8 +2,10 @@ set (SOURCE_DIR ${CMAKE_SOURCE_DIR}/contrib/zlib-ng)
add_definitions(-DZLIB_COMPAT)
add_definitions(-DWITH_GZFILEOP)
add_definitions(-DUNALIGNED_OK)
add_definitions(-DUNALIGNED64_OK)
if(NOT ARCH_S390X)
add_definitions(-DUNALIGNED_OK)
add_definitions(-DUNALIGNED64_OK)
endif()
set (HAVE_UNISTD_H 1)
add_definitions(-D_LARGEFILE64_SOURCE=1 -D__USE_LARGEFILE64)

5
docker/test/integration/runner/dockerd-entrypoint.sh
View File

@ -28,10 +28,9 @@ done
set -e
# cleanup for retry run if volume is not recreated
# shellcheck disable=SC2046
{
docker ps -aq | xargs -r docker kill || true
docker ps -aq | xargs -r docker rm || true
docker ps --all --quiet | xargs --no-run-if-empty docker kill || true
docker ps --all --quiet | xargs --no-run-if-empty docker rm || true
}
echo "Start tests"

3
docs/en/interfaces/formats.md
View File

@ -2,10 +2,9 @@
slug: /en/interfaces/formats
sidebar_position: 21
sidebar_label: Input and Output Formats
title: Formats for Input and Output Data
---
# Formats for Input and Output Data
ClickHouse can accept and return data in various formats. A format supported for input can be used to parse the data provided to `INSERT`s, to perform `SELECT`s from a file-backed table such as File, URL or HDFS, or to read an external dictionary. A format supported for output can be used to arrange the
results of a `SELECT`, and to perform `INSERT`s into a file-backed table.

15
docs/en/operations/settings/settings.md
View File

@ -3145,6 +3145,17 @@ Result:
└─────┴─────┴───────┘
```
## enable_extended_results_for_datetime_functions {#enable-extended-results-for-datetime-functions}
Enables or disables returning results of type `Date32` with extended range (compared to type `Date`) for functions [toStartOfYear](../../sql-reference/functions/date-time-functions.md#tostartofyear), [toStartOfISOYear](../../sql-reference/functions/date-time-functions.md#tostartofisoyear), [toStartOfQuarter](../../sql-reference/functions/date-time-functions.md#tostartofquarter), [toStartOfMonth](../../sql-reference/functions/date-time-functions.md#tostartofmonth), [toStartOfWeek](../../sql-reference/functions/date-time-functions.md#tostartofweek), [toMonday](../../sql-reference/functions/date-time-functions.md#tomonday) and [toLastDayOfMonth](../../sql-reference/functions/date-time-functions.md#tolastdayofmonth).
Possible values:
- 0 — Functions return `Date` for all types of arguments.
- 1 — Functions return `Date32` for `Date32` or `DateTime64` arguments and `Date` otherwise.
Default value: `0`.
## optimize_move_to_prewhere {#optimize_move_to_prewhere}
Enables or disables automatic [PREWHERE](../../sql-reference/statements/select/prewhere.md) optimization in [SELECT](../../sql-reference/statements/select/index.md) queries.
@ -3530,8 +3541,8 @@ desc format(JSONEachRow, '{"x" : 1, "y" : "String", "z" : "0.0.0.0" }') settings
Result:
```sql
x UInt8
y Nullable(String)
x UInt8
y Nullable(String)
z IPv4
```

2
docs/en/operations/storing-data.md
View File

@ -155,7 +155,7 @@ Example of configuration for versions earlier than 22.8:
<endpoint>...</endpoint>
... s3 configuration ...
<data_cache_enabled>1</data_cache_enabled>
<data_cache_size>10000000</data_cache_size>
<data_cache_max_size>10000000</data_cache_max_size>
</s3>
</disks>
<policies>

9
docs/en/sql-reference/data-types/geo.md
View File

@ -7,13 +7,8 @@ title: "Geo Data Types"
ClickHouse supports data types for representing geographical objects — locations, lands, etc.
:::warning
Currently geo data types are an experimental feature. To work with them you must set `allow_experimental_geo_types = 1`.
:::
**See Also**
- [Representing simple geographical features](https://en.wikipedia.org/wiki/GeoJSON).
- [allow_experimental_geo_types](../../operations/settings/settings.md#allow-experimental-geo-types) setting.
## Point
@ -24,7 +19,6 @@ Currently geo data types are an experimental feature. To work with them you must
Query:
```sql
SET allow_experimental_geo_types = 1;
CREATE TABLE geo_point (p Point) ENGINE = Memory();
INSERT INTO geo_point VALUES((10, 10));
SELECT p, toTypeName(p) FROM geo_point;
@ -46,7 +40,6 @@ Result:
Query:
```sql
SET allow_experimental_geo_types = 1;
CREATE TABLE geo_ring (r Ring) ENGINE = Memory();
INSERT INTO geo_ring VALUES([(0, 0), (10, 0), (10, 10), (0, 10)]);
SELECT r, toTypeName(r) FROM geo_ring;
@ -68,7 +61,6 @@ Result:
This is a polygon with one hole:
```sql
SET allow_experimental_geo_types = 1;
CREATE TABLE geo_polygon (pg Polygon) ENGINE = Memory();
INSERT INTO geo_polygon VALUES([[(20, 20), (50, 20), (50, 50), (20, 50)], [(30, 30), (50, 50), (50, 30)]]);
SELECT pg, toTypeName(pg) FROM geo_polygon;
@ -91,7 +83,6 @@ Result:
This multipolygon consists of two separate polygons — the first one without holes, and the second with one hole:
```sql
SET allow_experimental_geo_types = 1;
CREATE TABLE geo_multipolygon (mpg MultiPolygon) ENGINE = Memory();
INSERT INTO geo_multipolygon VALUES([[[(0, 0), (10, 0), (10, 10), (0, 10)]], [[(20, 20), (50, 20), (50, 50), (20, 50)],[(30, 30), (50, 50), (50, 30)]]]);
SELECT mpg, toTypeName(mpg) FROM geo_multipolygon;

10
docs/en/sql-reference/formats.mdx Normal file
View File

@ -0,0 +1,10 @@
---
slug: /en/sql-reference/formats
sidebar_position: 50
sidebar_label: Input and Output Formats
title: Formats for Input and Output Data
---
import Content from '@site/docs/en/interfaces/formats.md';
<Content />

18
docs/en/sql-reference/functions/date-time-functions.md
View File

@ -268,13 +268,15 @@ Result:
```
:::note
The return type of `toStartOf*`, `toLastDayOfMonth`, `toMonday` functions described below is `Date` or `DateTime`.
Though these functions can take values of the extended types `Date32` and `DateTime64` as an argument, passing them a time outside the normal range (year 1970 to 2149 for `Date` / 2106 for `DateTime`) will produce wrong results.
In case argument is out of normal range:
The return type of `toStartOf*`, `toLastDayOfMonth`, `toMonday` functions described below is determined by the configuration parameter [enable_extended_results_for_datetime_functions](../../operations/settings/settings#enable-extended-results-for-datetime-functions) which is `0` by default.
Behavior for
* `enable_extended_results_for_datetime_functions = 0`: Functions `toStartOf*`, `toLastDayOfMonth`, `toMonday` return `Date` or `DateTime`. Though these functions can take values of the extended types `Date32` and `DateTime64` as an argument, passing them a time outside the normal range (year 1970 to 2149 for `Date` / 2106 for `DateTime`) will produce wrong results. In case argument is out of normal range:
* If the argument is smaller than 1970, the result will be calculated from the argument `1970-01-01 (00:00:00)` instead.
* If the return type is `DateTime` and the argument is larger than `2106-02-07 08:28:15`, the result will be calculated from the argument `2106-02-07 08:28:15` instead.
* If the return type is `Date` and the argument is larger than `2149-06-06`, the result will be calculated from the argument `2149-06-06` instead.
* If `toLastDayOfMonth` is called with an argument greater then `2149-05-31`, the result will be calculated from the argument `2149-05-31` instead.
* `enable_extended_results_for_datetime_functions = 1`: Functions `toStartOfYear`, `toStartOfISOYear`, `toStartOfQuarter`, `toStartOfMonth`, `toStartOfWeek`, `toLastDayOfMonth`, `toMonday` return `Date` or `DateTime` if their argument is a `Date` or `DateTime`, and they return `Date32` or `DateTime64` if their argument is a `Date32` or `DateTime64`.
:::
## toStartOfYear
@ -303,6 +305,8 @@ Returns the date.
Rounds up a date or date with time to the last day of the month.
Returns the date.
If `toLastDayOfMonth` is called with an argument of type `Date` greater then 2149-05-31, the result will be calculated from the argument 2149-05-31 instead.
## toMonday
Rounds down a date or date with time to the nearest Monday.
@ -640,7 +644,7 @@ Result:
## date\_diff
Returns the difference between two dates or dates with time values.
Returns the difference between two dates or dates with time values.
The difference is calculated using relative units, e.g. the difference between `2022-01-01` and `2021-12-29` is 3 days for day unit (see [toRelativeDayNum](#torelativedaynum)), 1 month for month unit (see [toRelativeMonthNum](#torelativemonthnum)), 1 year for year unit (see [toRelativeYearNum](#torelativeyearnum)).
**Syntax**
@ -1059,9 +1063,9 @@ SELECT
## timeSlots(StartTime, Duration,\[, Size\])
For a time interval starting at ‘StartTime’ and continuing for ‘Duration’ seconds, it returns an array of moments in time, consisting of points from this interval rounded down to the ‘Size’ in seconds. ‘Size’ is an optional parameter set to 1800 (30 minutes) by default.
This is necessary, for example, when searching for pageviews in the corresponding session.
Accepts DateTime and DateTime64 as ’StartTime’ argument. For DateTime, ’Duration’ and ’Size’ arguments must be `UInt32`. For ’DateTime64’ they must be `Decimal64`.
For a time interval starting at ‘StartTime’ and continuing for ‘Duration’ seconds, it returns an array of moments in time, consisting of points from this interval rounded down to the ‘Size’ in seconds. ‘Size’ is an optional parameter set to 1800 (30 minutes) by default.
This is necessary, for example, when searching for pageviews in the corresponding session.
Accepts DateTime and DateTime64 as ’StartTime’ argument. For DateTime, ’Duration’ and ’Size’ arguments must be `UInt32`. For ’DateTime64’ they must be `Decimal64`.
Returns an array of DateTime/DateTime64 (return type matches the type of ’StartTime’). For DateTime64, the return value's scale can differ from the scale of ’StartTime’ --- the highest scale among all given arguments is taken.
Example:

5
docs/en/sql-reference/statements/select/group-by.md
View File

@ -213,9 +213,10 @@ If the `WITH TOTALS` modifier is specified, another row will be calculated. This
This extra row is only produced in `JSON*`, `TabSeparated*`, and `Pretty*` formats, separately from the other rows:
- In `JSON*` formats, this row is output as a separate ‘totals’ field.
- In `TabSeparated*` formats, the row comes after the main result, preceded by an empty row (after the other data).
- In `XML` and `JSON*` formats, this row is output as a separate ‘totals’ field.
- In `TabSeparated*`, `CSV*` and `Vertical` formats, the row comes after the main result, preceded by an empty row (after the other data).
- In `Pretty*` formats, the row is output as a separate table after the main result.
- In `Template` format, the row is output according to specified template.
- In the other formats it is not available.
:::note

4
docs/en/sql-reference/statements/select/index.md
View File

@ -135,9 +135,9 @@ In all other cases, we do not recommend using the asterisk, since it only gives
In addition to results, you can also get minimum and maximum values for the results columns. To do this, set the **extremes** setting to 1. Minimums and maximums are calculated for numeric types, dates, and dates with times. For other columns, the default values are output.
An extra two rows are calculated – the minimums and maximums, respectively. These extra two rows are output in `JSON*`, `TabSeparated*`, and `Pretty*` [formats](../../../interfaces/formats.md), separate from the other rows. They are not output for other formats.
An extra two rows are calculated – the minimums and maximums, respectively. These extra two rows are output in `XML`, `JSON*`, `TabSeparated*`, `CSV*`, `Vertical`, `Template` and `Pretty*` [formats](../../../interfaces/formats.md), separate from the other rows. They are not output for other formats.
In `JSON*` formats, the extreme values are output in a separate ‘extremes’ field. In `TabSeparated*` formats, the row comes after the main result, and after ‘totals’ if present. It is preceded by an empty row (after the other data). In `Pretty*` formats, the row is output as a separate table after the main result, and after `totals` if present.
In `JSON*` and `XML` formats, the extreme values are output in a separate ‘extremes’ field. In `TabSeparated*`, `CSV*` and `Vertical` formats, the row comes after the main result, and after ‘totals’ if present. It is preceded by an empty row (after the other data). In `Pretty*` formats, the row is output as a separate table after the main result, and after `totals` if present. In `Template` format the extreme values are output according to specified template.
Extreme values are calculated for rows before `LIMIT`, but after `LIMIT BY`. However, when using `LIMIT offset, size`, the rows before `offset` are included in `extremes`. In stream requests, the result may also include a small number of rows that passed through `LIMIT`.

11
docs/ru/operations/settings/settings.md
View File

@ -3799,6 +3799,17 @@ Exception: Total regexp lengths too large.
Значение по умолчанию: `1`.
## enable_extended_results_for_datetime_functions {#enable-extended-results-for-datetime-functions}
Включает или отключает возвращение результатов типа `Date32` с расширенным диапазоном (по сравнению с типом `Date`) для функций [toStartOfYear](../../sql-reference/functions/date-time-functions.md#tostartofyear), [toStartOfISOYear](../../sql-reference/functions/date-time-functions.md#tostartofisoyear), [toStartOfQuarter](../../sql-reference/functions/date-time-functions.md#tostartofquarter), [toStartOfMonth](../../sql-reference/functions/date-time-functions.md#tostartofmonth), [toStartOfWeek](../../sql-reference/functions/date-time-functions.md#tostartofweek), [toMonday](../../sql-reference/functions/date-time-functions.md#tomonday) и [toLastDayOfMonth](../../sql-reference/functions/date-time-functions.md#tolastdayofmonth).
Возможные значения:
- 0 — Функции возвращают результаты типа `Date` для всех типов аргументов.
- 1 — Функции возвращают результаты типа `Date32` для аргументов типа `Date32` или `DateTime64` и возвращают `Date` в других случаях.
Значение по умолчанию: `0`.
**Пример**
Запрос:

30
docs/ru/sql-reference/functions/date-time-functions.md
View File

@ -268,24 +268,18 @@ SELECT toUnixTimestamp('2017-11-05 08:07:47', 'Asia/Tokyo') AS unix_timestamp;
```
:::note
Тип возвращаемого описанными далее функциями `toStartOf*`, `toMonday` значения - `Date` или `DateTime`.
Хотя эти функции могут принимать значения типа `Date32` или `DateTime64` в качестве аргумента, при обработке аргумента вне нормального диапазона значений (`1970` - `2148` для `Date` и `1970-01-01 00:00:00`-`2106-02-07 08:28:15` для `DateTime`) будет получен некорректный результат.
Возвращаемые значения для значений вне нормального диапазона:
* `1970-01-01 (00:00:00)` будет возвращён для моментов времени до 1970 года,
* `2106-02-07 08:28:15` будет взят в качестве аргумента, если полученный аргумент превосходит данное значение и возвращаемый тип - `DateTime`,
* `2149-06-06` будет взят в качестве аргумента, если полученный аргумент превосходит данное значение и возвращаемый тип - `Date`,
* `2149-05-31` будет результатом функции `toLastDayOfMonth` при обработке аргумента больше `2149-05-31`.
Тип возвращаемого значения описанными далее функциями `toStartOf*`, `toLastDayOfMonth`, `toMonday` определяется конфигурационным параметром [enable_extended_results_for_datetime_functions](../../operations/settings/settings#enable-extended-results-for-datetime-functions) имеющим по умолчанию значение `0`.
Поведение для
* `enable_extended_results_for_datetime_functions = 0`: Функции `toStartOf*`, `toLastDayOfMonth`, `toMonday` возвращают `Date` или `DateTime`. Хотя эти функции могут принимать значения типа `Date32` или `DateTime64` в качестве аргумента, при обработке аргумента вне нормального диапазона значений (`1970` - `2148` для `Date` и `1970-01-01 00:00:00`-`2106-02-07 08:28:15` для `DateTime`) будет получен некорректный результат.
В случае если значение аргумента вне нормального диапазона:
* `1970-01-01 (00:00:00)` будет возвращён для моментов времени до 1970 года,
* `2106-02-07 08:28:15` будет взят в качестве аргумента, если полученный аргумент превосходит данное значение и возвращаемый тип - `DateTime`,
* `2149-06-06` будет взят в качестве аргумента, если полученный аргумент превосходит данное значение и возвращаемый тип - `Date`,
* `2149-05-31` будет результатом функции `toLastDayOfMonth` при обработке аргумента больше `2149-05-31`.
* `enable_extended_results_for_datetime_functions = 1`: Функции `toStartOfYear`, `toStartOfISOYear`, `toStartOfQuarter`, `toStartOfMonth`, `toStartOfWeek`, `toLastDayOfMonth`, `toMonday` возвращают `Date` или `DateTime` если их аргумент `Date` или `DateTime` и они возвращают `Date32` или `DateTime64` если их аргумент `Date32` или `DateTime64`.
:::
:::note
Тип возвращаемого описанными далее функциями `toStartOf*`, `toLastDayOfMonth`, `toMonday` значения - `Date` или `DateTime`.
Хотя эти функции могут принимать значения типа `Date32` или `DateTime64` в качестве аргумента, при обработке аргумента вне нормального диапазона значений (`1970` - `2148` для `Date` и `1970-01-01 00:00:00`-`2106-02-07 08:28:15` для `DateTime`) будет получен некорректный результат.
Возвращаемые значения для значений вне нормального диапазона:
* `1970-01-01 (00:00:00)` будет возвращён для моментов времени до 1970 года,
* `2106-02-07 08:28:15` будет взят в качестве аргумента, если полученный аргумент превосходит данное значение и возвращаемый тип - `DateTime`,
* `2149-06-06` будет взят в качестве аргумента, если полученный аргумент превосходит данное значение и возвращаемый тип - `Date`.
:::
*
## toStartOfYear {#tostartofyear}
Округляет дату или дату-с-временем вниз до первого дня года.
@ -324,6 +318,8 @@ SELECT toStartOfISOYear(toDate('2017-01-01')) AS ISOYear20170101;
Округляет дату или дату-с-временем до последнего числа месяца.
Возвращается дата.
Если `toLastDayOfMonth` вызывается с аргументом типа `Date` большим чем 2149-05-31, то результат будет вычислен от аргумента 2149-05-31.
## toMonday {#tomonday}
Округляет дату или дату-с-временем вниз до ближайшего понедельника.
@ -977,7 +973,7 @@ SELECT now('Europe/Moscow');
## timeSlots(StartTime, Duration,\[, Size\]) {#timeslotsstarttime-duration-size}
Для интервала, начинающегося в `StartTime` и длящегося `Duration` секунд, возвращает массив моментов времени, кратных `Size`. Параметр `Size` указывать необязательно, по умолчанию он равен 1800 секундам (30 минутам) - необязательный параметр.
Данная функция может использоваться, например, для анализа количества просмотров страницы за соответствующую сессию.
Аргумент `StartTime` может иметь тип `DateTime` или `DateTime64`. В случае, если используется `DateTime`, аргументы `Duration` и `Size` должны иметь тип `UInt32`; Для DateTime64 они должны быть типа `Decimal64`.
Аргумент `StartTime` может иметь тип `DateTime` или `DateTime64`. В случае, если используется `DateTime`, аргументы `Duration` и `Size` должны иметь тип `UInt32`; Для DateTime64 они должны быть типа `Decimal64`.
Возвращает массив DateTime/DateTime64 (тип будет совпадать с типом параметра ’StartTime’). Для DateTime64 масштаб(scale) возвращаемой величины может отличаться от масштаба фргумента ’StartTime’ --- результат будет иметь наибольший масштаб среди всех данных аргументов.
Пример использования:

1
programs/benchmark/clickhouse-benchmark.cpp
View File

@ -1,2 +1 @@
extern int mainEntryClickHouseBenchmark(int argc, char ** argv);
int main(int argc_, char ** argv_) { return mainEntryClickHouseBenchmark(argc_, argv_); }

2
programs/client/Client.cpp
View File

@ -150,7 +150,7 @@ std::vector<String> Client::loadWarningMessages()
size_t rows = packet.block.rows();
for (size_t i = 0; i < rows; ++i)
messages.emplace_back(column.getDataAt(i).toString());
messages.emplace_back(column[i].get<String>());
}
continue;

1
programs/client/clickhouse-client.cpp
View File

@ -1,2 +1 @@
int mainEntryClickHouseClient(int argc, char ** argv);
int main(int argc_, char ** argv_) { return mainEntryClickHouseClient(argc_, argv_); }

1
programs/compressor/clickhouse-compressor.cpp
View File

@ -1,2 +1 @@
int mainEntryClickHouseCompressor(int argc, char ** argv);
int main(int argc_, char ** argv_) { return mainEntryClickHouseCompressor(argc_, argv_); }

1
programs/copier/clickhouse-copier.cpp
View File

@ -1,2 +1 @@
int mainEntryClickHouseClusterCopier(int argc, char ** argv);
int main(int argc_, char ** argv_) { return mainEntryClickHouseClusterCopier(argc_, argv_); }

1
programs/disks/clickhouse-disks.cpp
View File

@ -1,2 +1 @@
int mainEntryClickHouseDisks(int argc, char ** argv);
int main(int argc_, char ** argv_) { return mainEntryClickHouseDisks(argc_, argv_); }

1
programs/extract-from-config/clickhouse-extract-from-config.cpp
View File

@ -1,2 +1 @@
int mainEntryClickHouseExtractFromConfig(int argc, char ** argv);
int main(int argc_, char ** argv_) { return mainEntryClickHouseExtractFromConfig(argc_, argv_); }

1
programs/format/clickhouse-format.cpp
View File

@ -1,2 +1 @@
int mainEntryClickHouseFormat(int argc, char ** argv);
int main(int argc_, char ** argv_) { return mainEntryClickHouseFormat(argc_, argv_); }

1
programs/git-import/clickhouse-git-import.cpp
View File

@ -1,2 +1 @@
int mainEntryClickHouseGitImport(int argc, char ** argv);
int main(int argc_, char ** argv_) { return mainEntryClickHouseGitImport(argc_, argv_); }

1
programs/keeper-converter/clickhouse-keeper-converter.cpp
View File

@ -1,2 +1 @@
int mainEntryClickHouseKeeperConverter(int argc, char ** argv);
int main(int argc_, char ** argv_) { return mainEntryClickHouseKeeperConverter(argc_, argv_); }

1
programs/library-bridge/CMakeLists.txt
View File

@ -9,6 +9,7 @@ set (CLICKHOUSE_LIBRARY_BRIDGE_SOURCES
LibraryBridge.cpp
LibraryBridgeHandlerFactory.cpp
LibraryBridgeHandlers.cpp
SharedLibrary.cpp
library-bridge.cpp
)

2
programs/library-bridge/CatBoostLibraryHandler.cpp
View File

@ -98,7 +98,7 @@ void placeStringColumn(const ColumnString & column, const char ** buffer, size_t
size_t size = column.size();
for (size_t i = 0; i < size; ++i)
{
*buffer = const_cast<char *>(column.getDataAtWithTerminatingZero(i).data);
*buffer = const_cast<char *>(column.getDataAt(i).data);
buffer += features_count;
}
}

3
programs/library-bridge/CatBoostLibraryHandler.h
View File

@ -7,12 +7,13 @@
#include <Columns/ColumnVector.h>
#include <Columns/ColumnsNumber.h>
#include <Columns/IColumn.h>
#include <Common/SharedLibrary.h>
#include <base/defines.h>
#include "SharedLibrary.h"
#include <chrono>
#include <mutex>
namespace DB
{

2
programs/library-bridge/ExternalDictionaryLibraryHandler.h
View File

@ -1,6 +1,6 @@
#pragma once
#include <Common/SharedLibrary.h>
#include "SharedLibrary.h"
#include <Common/logger_useful.h>
#include "ExternalDictionaryLibraryUtils.h"

3
src/Common/SharedLibrary.cpp → programs/library-bridge/SharedLibrary.cpp
View File

@ -1,8 +1,7 @@
#include "SharedLibrary.h"
#include <string>
#include <boost/core/noncopyable.hpp>
#include <base/phdr_cache.h>
#include "Exception.h"
#include <Common/Exception.h>
namespace DB

0
src/Common/SharedLibrary.h → programs/library-bridge/SharedLibrary.h
View File

1
programs/local/clickhouse-local.cpp
View File

@ -1,2 +1 @@
int mainEntryClickHouseLocal(int argc, char ** argv);
int main(int argc_, char ** argv_) { return mainEntryClickHouseLocal(argc_, argv_); }

31
programs/main.cpp
View File

@ -402,6 +402,36 @@ void checkHarmfulEnvironmentVariables(char ** argv)
}
/// Don't allow dlopen in the main ClickHouse binary, because it is harmful and insecure.
/// We don't use it. But it can be used by some libraries for implementation of "plugins".
/// We absolutely discourage the ancient technique of loading
/// 3rd-party uncontrolled dangerous libraries into the process address space,
/// because it is insane.
extern "C"
{
void * dlopen(const char *, int)
{
return nullptr;
}
void * dlmopen(long, const char *, int) // NOLINT
{
return nullptr;
}
int dlclose(void *)
{
return 0;
}
const char * dlerror()
{
return "ClickHouse does not allow dynamic library loading";
}
}
/// This allows to implement assert to forbid initialization of a class in static constructors.
/// Usage:
///
@ -422,6 +452,7 @@ int main(int argc_, char ** argv_)
/// PHDR cache is required for query profiler to work reliably
/// It also speed up exception handling, but exceptions from dynamically loaded libraries (dlopen)
/// will work only after additional call of this function.
/// Note: we forbid dlopen in our code.
updatePHDRCache();
#ifndef DISABLE_HARMFUL_ENV_VAR_CHECK

2
programs/obfuscator/clickhouse-obfuscator.cpp
View File

@ -1,3 +1 @@
int mainEntryClickHouseObfuscator(int argc, char ** argv);
int main(int argc_, char ** argv_) { return mainEntryClickHouseObfuscator(argc_, argv_); }

23
programs/server/clickhouse-server.cpp
View File

@ -1,24 +1 @@
#include <new>
#include <base/phdr_cache.h>
int mainEntryClickHouseServer(int argc, char ** argv);
/**
* This is the entry-point for the split build server. The initialization
* is copied from single-binary entry point in main.cpp.
*/
int main(int argc_, char ** argv_)
{
/// Reset new handler to default (that throws std::bad_alloc)
/// It is needed because LLVM library clobbers it.
std::set_new_handler(nullptr);
/// PHDR cache is required for query profiler to work reliably
/// It also speed up exception handling, but exceptions from dynamically loaded libraries (dlopen)
/// will work only after additional call of this function.
updatePHDRCache();
return mainEntryClickHouseServer(argc_, argv_);
}

1
programs/static-files-disk-uploader/clickhouse-static-files-disk-uploader.cpp
View File

@ -1,2 +1 @@
int mainEntryClickHouseStaticFilesDiskUploader(int argc, char ** argv);
int main(int argc_, char ** argv_) { return mainEntryClickHouseStaticFilesDiskUploader(argc_, argv_); }

1
programs/su/clickhouse-su.cpp
View File

@ -1,2 +1 @@
int mainEntryClickHouseSU(int argc, char ** argv);
int main(int argc_, char ** argv_) { return mainEntryClickHouseSU(argc_, argv_); }

10
src/AggregateFunctions/AggregateFunctionArray.h
View File

@ -64,6 +64,11 @@ public:
return nested_func->isVersioned();
}
size_t getVersionFromRevision(size_t revision) const override
{
return nested_func->getVersionFromRevision(revision);
}
size_t getDefaultVersion() const override
{
return nested_func->getDefaultVersion();
@ -79,6 +84,11 @@ public:
nested_func->destroy(place);
}
void destroyUpToState(AggregateDataPtr __restrict place) const noexcept override
{
nested_func->destroyUpToState(place);
}
bool hasTrivialDestructor() const override
{
return nested_func->hasTrivialDestructor();

149
src/AggregateFunctions/AggregateFunctionCategoricalInformationValue.cpp
View File

@ -1,12 +1,18 @@
#include <AggregateFunctions/AggregateFunctionCategoricalInformationValue.h>
#include <AggregateFunctions/IAggregateFunction.h>
#include <AggregateFunctions/AggregateFunctionFactory.h>
#include <AggregateFunctions/FactoryHelpers.h>
#include <AggregateFunctions/Helpers.h>
#include <Columns/ColumnArray.h>
#include <Columns/ColumnsNumber.h>
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypesNumber.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
namespace DB
{
struct Settings;
namespace ErrorCodes
@ -15,6 +21,136 @@ namespace ErrorCodes
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
}
/** The function takes arguments x1, x2, ... xn, y. All arguments are bool.
* x arguments represents the fact that some category is true.
*
* It calculates how many times y was true and how many times y was false when every n-th category was true
* and the total number of times y was true and false.
*
* So, the size of the state is (n + 1) * 2 cells.
*/
class AggregateFunctionCategoricalIV final : public IAggregateFunctionHelper<AggregateFunctionCategoricalIV>
{
private:
using Counter = UInt64;
size_t category_count;
static Counter & counter(AggregateDataPtr __restrict place, size_t i, bool what)
{
return reinterpret_cast<Counter *>(place)[i * 2 + (what ? 1 : 0)];
}
static const Counter & counter(ConstAggregateDataPtr __restrict place, size_t i, bool what)
{
return reinterpret_cast<const Counter *>(place)[i * 2 + (what ? 1 : 0)];
}
public:
AggregateFunctionCategoricalIV(const DataTypes & arguments_, const Array & params_) :
IAggregateFunctionHelper<AggregateFunctionCategoricalIV>{arguments_, params_},
category_count{arguments_.size() - 1}
{
// notice: argument types has been checked before
}
String getName() const override
{
return "categoricalInformationValue";
}
bool allocatesMemoryInArena() const override { return false; }
void create(AggregateDataPtr __restrict place) const override
{
memset(place, 0, sizeOfData());
}
void destroy(AggregateDataPtr __restrict) const noexcept override
{
// nothing
}
bool hasTrivialDestructor() const override
{
return true;
}
size_t sizeOfData() const override
{
return sizeof(Counter) * (category_count + 1) * 2;
}
size_t alignOfData() const override
{
return alignof(Counter);
}
void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena *) const override
{
const auto * y_col = static_cast<const ColumnUInt8 *>(columns[category_count]);
bool y = y_col->getData()[row_num];
for (size_t i = 0; i < category_count; ++i)
{
const auto * x_col = static_cast<const ColumnUInt8 *>(columns[i]);
bool x = x_col->getData()[row_num];
if (x)
++counter(place, i, y);
}
++counter(place, category_count, y);
}
void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena *) const override
{
for (size_t i = 0; i <= category_count; ++i)
{
counter(place, i, false) += counter(rhs, i, false);
counter(place, i, true) += counter(rhs, i, true);
}
}
void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional<size_t> /* version */) const override
{
buf.write(place, sizeOfData());
}
void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional<size_t> /* version */, Arena *) const override
{
buf.read(place, sizeOfData());
}
DataTypePtr getReturnType() const override
{
return std::make_shared<DataTypeArray>(
std::make_shared<DataTypeNumber<Float64>>());
}
void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override /// NOLINT
{
auto & col = static_cast<ColumnArray &>(to);
auto & data_col = static_cast<ColumnFloat64 &>(col.getData());
auto & offset_col = static_cast<ColumnArray::ColumnOffsets &>(col.getOffsetsColumn());
data_col.reserve(data_col.size() + category_count);
Float64 sum_no = static_cast<Float64>(counter(place, category_count, false));
Float64 sum_yes = static_cast<Float64>(counter(place, category_count, true));
for (size_t i = 0; i < category_count; ++i)
{
Float64 no = static_cast<Float64>(counter(place, i, false));
Float64 yes = static_cast<Float64>(counter(place, i, true));
data_col.insertValue((no / sum_no - yes / sum_yes) * (log((no / sum_no) / (yes / sum_yes))));
}
offset_col.insertValue(data_col.size());
}
};
namespace
{
@ -39,16 +175,15 @@ AggregateFunctionPtr createAggregateFunctionCategoricalIV(
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
}
return std::make_shared<AggregateFunctionCategoricalIV<>>(arguments, params);
return std::make_shared<AggregateFunctionCategoricalIV>(arguments, params);
}
}
void registerAggregateFunctionCategoricalIV(
AggregateFunctionFactory & factory
)
void registerAggregateFunctionCategoricalIV(AggregateFunctionFactory & factory)
{
factory.registerFunction("categoricalInformationValue", createAggregateFunctionCategoricalIV);
AggregateFunctionProperties properties = { .returns_default_when_only_null = true };
factory.registerFunction("categoricalInformationValue", { createAggregateFunctionCategoricalIV, properties });
}
}

135
src/AggregateFunctions/AggregateFunctionCategoricalInformationValue.h
View File

@ -1,135 +0,0 @@
#pragma once
#include <AggregateFunctions/IAggregateFunction.h>
#include <Columns/ColumnArray.h>
#include <Columns/ColumnsNumber.h>
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypesNumber.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#include <base/range.h>
namespace DB
{
struct Settings;
template <typename T = UInt64>
class AggregateFunctionCategoricalIV final : public IAggregateFunctionHelper<AggregateFunctionCategoricalIV<T>>
{
private:
size_t category_count;
public:
AggregateFunctionCategoricalIV(const DataTypes & arguments_, const Array & params_) :
IAggregateFunctionHelper<AggregateFunctionCategoricalIV<T>> {arguments_, params_},
category_count {arguments_.size() - 1}
{
// notice: argument types has been checked before
}
String getName() const override
{
return "categoricalInformationValue";
}
bool allocatesMemoryInArena() const override { return false; }
void create(AggregateDataPtr __restrict place) const override
{
memset(place, 0, sizeOfData());
}
void destroy(AggregateDataPtr __restrict) const noexcept override
{
// nothing
}
bool hasTrivialDestructor() const override
{
return true;
}
size_t sizeOfData() const override
{
return sizeof(T) * (category_count + 1) * 2;
}
size_t alignOfData() const override
{
return alignof(T);
}
void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena *) const override
{
const auto * y_col = static_cast<const ColumnUInt8 *>(columns[category_count]);
bool y = y_col->getData()[row_num];
for (size_t i : collections::range(0, category_count))
{
const auto * x_col = static_cast<const ColumnUInt8 *>(columns[i]);
bool x = x_col->getData()[row_num];
if (x)
reinterpret_cast<T *>(place)[i * 2 + size_t(y)] += 1;
}
reinterpret_cast<T *>(place)[category_count * 2 + size_t(y)] += 1;
}
void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena *) const override
{
for (size_t i : collections::range(0, category_count + 1))
{
reinterpret_cast<T *>(place)[i * 2] += reinterpret_cast<const T *>(rhs)[i * 2];
reinterpret_cast<T *>(place)[i * 2 + 1] += reinterpret_cast<const T *>(rhs)[i * 2 + 1];
}
}
void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional<size_t> /* version */) const override
{
buf.write(place, sizeOfData());
}
void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional<size_t> /* version */, Arena *) const override
{
buf.read(place, sizeOfData());
}
DataTypePtr getReturnType() const override
{
return std::make_shared<DataTypeArray>(
std::make_shared<DataTypeNumber<Float64>>()
);
}
void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override /// NOLINT
{
auto & col = static_cast<ColumnArray &>(to);
auto & data_col = static_cast<ColumnFloat64 &>(col.getData());
auto & offset_col = static_cast<ColumnArray::ColumnOffsets &>(
col.getOffsetsColumn()
);
data_col.reserve(data_col.size() + category_count);
T sum_no = reinterpret_cast<const T *>(place)[category_count * 2];
T sum_yes = reinterpret_cast<const T *>(place)[category_count * 2 + 1];
Float64 rev_no = 1. / sum_no;
Float64 rev_yes = 1. / sum_yes;
for (size_t i : collections::range(0, category_count))
{
T no = reinterpret_cast<const T *>(place)[i * 2];
T yes = reinterpret_cast<const T *>(place)[i * 2 + 1];
data_col.insertValue((no * rev_no - yes * rev_yes) * (log(no * rev_no) - log(yes * rev_yes)));
}
offset_col.insertValue(data_col.size());
}
};
}

21
src/AggregateFunctions/AggregateFunctionDistinct.h
View File

@ -225,6 +225,12 @@ public:
nested_func->destroy(getNestedPlace(place));
}
void destroyUpToState(AggregateDataPtr __restrict place) const noexcept override
{
this->data(place).~Data();
nested_func->destroyUpToState(getNestedPlace(place));
}
String getName() const override
{
return nested_func->getName() + "Distinct";
@ -245,6 +251,21 @@ public:
return nested_func->isState();
}
bool isVersioned() const override
{
return nested_func->isVersioned();
}
size_t getVersionFromRevision(size_t revision) const override
{
return nested_func->getVersionFromRevision(revision);
}
size_t getDefaultVersion() const override
{
return nested_func->getDefaultVersion();
}
AggregateFunctionPtr getNestedFunction() const override { return nested_func; }
};

25
src/AggregateFunctions/AggregateFunctionForEach.h
View File

@ -66,6 +66,7 @@ private:
if (old_size < new_size)
{
char * old_state = state.array_of_aggregate_datas;
char * new_state = arena.alignedAlloc(
new_size * nested_size_of_data,
nested_func->alignOfData());
@ -134,23 +135,43 @@ public:
return nested_func->isVersioned();
}
size_t getVersionFromRevision(size_t revision) const override
{
return nested_func->getVersionFromRevision(revision);
}
size_t getDefaultVersion() const override
{
return nested_func->getDefaultVersion();
}
void destroy(AggregateDataPtr __restrict place) const noexcept override
template <bool up_to_state>
void destroyImpl(AggregateDataPtr __restrict place) const noexcept
{
AggregateFunctionForEachData & state = data(place);
char * nested_state = state.array_of_aggregate_datas;
for (size_t i = 0; i < state.dynamic_array_size; ++i)
{
nested_func->destroy(nested_state);
if constexpr (up_to_state)
nested_func->destroyUpToState(nested_state);
else
nested_func->destroy(nested_state);
nested_state += nested_size_of_data;
}
}
void destroy(AggregateDataPtr __restrict place) const noexcept override
{
destroyImpl<false>(place);
}
void destroyUpToState(AggregateDataPtr __restrict place) const noexcept override
{
destroyImpl<true>(place);
}
bool hasTrivialDestructor() const override
{
return nested_func->hasTrivialDestructor();

10
src/AggregateFunctions/AggregateFunctionIf.h
View File

@ -71,6 +71,11 @@ public:
return nested_func->isVersioned();
}
size_t getVersionFromRevision(size_t revision) const override
{
return nested_func->getVersionFromRevision(revision);
}
size_t getDefaultVersion() const override
{
return nested_func->getDefaultVersion();
@ -86,6 +91,11 @@ public:
nested_func->destroy(place);
}
void destroyUpToState(AggregateDataPtr __restrict place) const noexcept override
{
nested_func->destroyUpToState(place);
}
bool hasTrivialDestructor() const override
{
return nested_func->hasTrivialDestructor();

46
src/AggregateFunctions/AggregateFunctionMap.h
View File

@ -84,6 +84,26 @@ private:
using Base = IAggregateFunctionDataHelper<Data, AggregateFunctionMap<KeyType>>;
public:
bool isState() const override
{
return nested_func->isState();
}
bool isVersioned() const override
{
return nested_func->isVersioned();
}
size_t getVersionFromRevision(size_t revision) const override
{
return nested_func->getVersionFromRevision(revision);
}
size_t getDefaultVersion() const override
{
return nested_func->getDefaultVersion();
}
AggregateFunctionMap(AggregateFunctionPtr nested, const DataTypes & types) : Base(types, nested->getParameters()), nested_func(nested)
{
if (types.empty())
@ -187,6 +207,32 @@ public:
}
}
template <bool up_to_state>
void destroyImpl(AggregateDataPtr __restrict place) const noexcept
{
AggregateFunctionMapCombinatorData<KeyType> & state = Base::data(place);
for (const auto & [key, nested_place] : state.merged_maps)
{
if constexpr (up_to_state)
nested_func->destroyUpToState(nested_place);
else
nested_func->destroy(nested_place);
}
state.~Data();
}
void destroy(AggregateDataPtr __restrict place) const noexcept override
{
destroyImpl<false>(place);
}
void destroyUpToState(AggregateDataPtr __restrict place) const noexcept override
{
destroyImpl<true>(place);
}
void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional<size_t> /* version */) const override
{
auto & merged_maps = this->data(place).merged_maps;

10
src/AggregateFunctions/AggregateFunctionMerge.h
View File

@ -80,6 +80,11 @@ public:
nested_func->destroy(place);
}
void destroyUpToState(AggregateDataPtr __restrict place) const noexcept override
{
nested_func->destroyUpToState(place);
}
bool hasTrivialDestructor() const override
{
return nested_func->hasTrivialDestructor();
@ -126,6 +131,11 @@ public:
}
AggregateFunctionPtr getNestedFunction() const override { return nested_func; }
bool isState() const override
{
return nested_func->isState();
}
};
}

10
src/AggregateFunctions/AggregateFunctionMinMaxAny.h
View File

@ -492,7 +492,7 @@ public:
void insertResultInto(IColumn & to) const
{
if (has())
assert_cast<ColumnString &>(to).insertDataWithTerminatingZero(getData(), size);
assert_cast<ColumnString &>(to).insertData(getData(), size);
else
assert_cast<ColumnString &>(to).insertDefault();
}
@ -569,7 +569,7 @@ public:
void change(const IColumn & column, size_t row_num, Arena * arena)
{
changeImpl(assert_cast<const ColumnString &>(column).getDataAtWithTerminatingZero(row_num), arena);
changeImpl(assert_cast<const ColumnString &>(column).getDataAt(row_num), arena);
}
void change(const Self & to, Arena * arena)
@ -618,7 +618,7 @@ public:
bool changeIfLess(const IColumn & column, size_t row_num, Arena * arena)
{
if (!has() || assert_cast<const ColumnString &>(column).getDataAtWithTerminatingZero(row_num) < getStringRef())
if (!has() || assert_cast<const ColumnString &>(column).getDataAt(row_num) < getStringRef())
{
change(column, row_num, arena);
return true;
@ -640,7 +640,7 @@ public:
bool changeIfGreater(const IColumn & column, size_t row_num, Arena * arena)
{
if (!has() || assert_cast<const ColumnString &>(column).getDataAtWithTerminatingZero(row_num) > getStringRef())
if (!has() || assert_cast<const ColumnString &>(column).getDataAt(row_num) > getStringRef())
{
change(column, row_num, arena);
return true;
@ -667,7 +667,7 @@ public:
bool isEqualTo(const IColumn & column, size_t row_num) const
{
return has() && assert_cast<const ColumnString &>(column).getDataAtWithTerminatingZero(row_num) == getStringRef();
return has() && assert_cast<const ColumnString &>(column).getDataAt(row_num) == getStringRef();
}
static bool allocatesMemoryInArena()

20
src/AggregateFunctions/AggregateFunctionNull.h
View File

@ -114,6 +114,11 @@ public:
nested_function->destroy(nestedPlace(place));
}
void destroyUpToState(AggregateDataPtr __restrict place) const noexcept override
{
nested_function->destroyUpToState(nestedPlace(place));
}
bool hasTrivialDestructor() const override
{
return nested_function->hasTrivialDestructor();
@ -189,6 +194,21 @@ public:
return nested_function->isState();
}
bool isVersioned() const override
{
return nested_function->isVersioned();
}
size_t getVersionFromRevision(size_t revision) const override
{
return nested_function->getVersionFromRevision(revision);
}
size_t getDefaultVersion() const override
{
return nested_function->getDefaultVersion();
}
AggregateFunctionPtr getNestedFunction() const override { return nested_function; }
#if USE_EMBEDDED_COMPILER

5
src/AggregateFunctions/AggregateFunctionOrFill.h
View File

@ -98,6 +98,11 @@ public:
nested_function->destroy(place);
}
void destroyUpToState(AggregateDataPtr __restrict place) const noexcept override
{
nested_function->destroyUpToState(place);
}
void add(
AggregateDataPtr __restrict place,
const IColumn ** columns,

21
src/AggregateFunctions/AggregateFunctionResample.h
View File

@ -91,6 +91,21 @@ public:
return nested_function->isState();
}
bool isVersioned() const override
{
return nested_function->isVersioned();
}
size_t getVersionFromRevision(size_t revision) const override
{
return nested_function->getVersionFromRevision(revision);
}
size_t getDefaultVersion() const override
{
return nested_function->getDefaultVersion();
}
bool allocatesMemoryInArena() const override
{
return nested_function->allocatesMemoryInArena();
@ -134,6 +149,12 @@ public:
nested_function->destroy(place + i * size_of_data);
}
void destroyUpToState(AggregateDataPtr __restrict place) const noexcept override
{
for (size_t i = 0; i < total; ++i)
nested_function->destroyUpToState(place + i * size_of_data);
}
void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena * arena) const override
{
Key key;

12
src/AggregateFunctions/AggregateFunctionSimpleState.h
View File

@ -56,10 +56,22 @@ public:
return nested_func->getDefaultVersion();
}
bool isState() const override
{
return nested_func->isState();
}
size_t getVersionFromRevision(size_t revision) const override
{
return nested_func->getVersionFromRevision(revision);
}
void create(AggregateDataPtr __restrict place) const override { nested_func->create(place); }
void destroy(AggregateDataPtr __restrict place) const noexcept override { nested_func->destroy(place); }
void destroyUpToState(AggregateDataPtr __restrict place) const noexcept override { nested_func->destroyUpToState(place); }
bool hasTrivialDestructor() const override { return nested_func->hasTrivialDestructor(); }
size_t sizeOfData() const override { return nested_func->sizeOfData(); }

2
src/AggregateFunctions/AggregateFunctionState.h
View File

@ -69,6 +69,8 @@ public:
nested_func->destroy(place);
}
void destroyUpToState(AggregateDataPtr __restrict) const noexcept override {}
bool hasTrivialDestructor() const override
{
return nested_func->hasTrivialDestructor();

9
src/AggregateFunctions/AggregateFunctionUniqUpTo.h
View File

@ -18,11 +18,6 @@
#include <IO/WriteHelpers.h>
#if !defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Warray-bounds"
#endif
namespace DB
{
struct Settings;
@ -291,7 +286,3 @@ public:
}
#if !defined(__clang__)
#pragma GCC diagnostic pop
#endif

26
src/AggregateFunctions/IAggregateFunction.h
View File

@ -113,6 +113,17 @@ public:
/// Delete data for aggregation.
virtual void destroy(AggregateDataPtr __restrict place) const noexcept = 0;
/// Delete all combinator states that were used after combinator -State.
/// For example for uniqArrayStateForEachMap(...) it will destroy
/// states that were created by combinators Map and ForEach.
/// It's needed because ColumnAggregateFunction in the result will be
/// responsible only for destruction of states that were created
/// by aggregate function and all combinators before -State combinator.
virtual void destroyUpToState(AggregateDataPtr __restrict place) const noexcept
{
destroy(place);
}
/// It is not necessary to delete data.
virtual bool hasTrivialDestructor() const = 0;
@ -277,8 +288,7 @@ public:
Arena * arena) const = 0;
/** Insert result of aggregate function into result column with batch size.
* If destroy_place_after_insert is true. Then implementation of this method
* must destroy aggregate place if insert state into result column was successful.
* The implementation of this method will destroy aggregate place up to -State if insert state into result column was successful.
* All places that were not inserted must be destroyed if there was exception during insert into result column.
*/
virtual void insertResultIntoBatch(
@ -287,8 +297,7 @@ public:
AggregateDataPtr * places,
size_t place_offset,
IColumn & to,
Arena * arena,
bool destroy_place_after_insert) const = 0;
Arena * arena) const = 0;
/** Destroy batch of aggregate places.
*/
@ -612,8 +621,7 @@ public:
AggregateDataPtr * places,
size_t place_offset,
IColumn & to,
Arena * arena,
bool destroy_place_after_insert) const override
Arena * arena) const override
{
size_t batch_index = row_begin;
@ -622,9 +630,9 @@ public:
for (; batch_index < row_end; ++batch_index)
{
static_cast<const Derived *>(this)->insertResultInto(places[batch_index] + place_offset, to, arena);
if (destroy_place_after_insert)
static_cast<const Derived *>(this)->destroy(places[batch_index] + place_offset);
/// For State AggregateFunction ownership of aggregate place is passed to result column after insert,
/// so we need to destroy all states up to state of -State combinator.
static_cast<const Derived *>(this)->destroyUpToState(places[batch_index] + place_offset);
}
}
catch (...)

9
src/AggregateFunctions/ReservoirSamplerDeterministic.h
View File

@ -165,11 +165,6 @@ public:
sorted = false;
}
#if !defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wclass-memaccess"
#endif
void write(DB::WriteBuffer & buf) const
{
size_t size = samples.size();
@ -193,10 +188,6 @@ public:
}
}
#if !defined(__clang__)
#pragma GCC diagnostic pop
#endif
private:
/// We allocate some memory on the stack to avoid allocations when there are many objects with a small number of elements.
using Element = std::pair<T, UInt32>;

36
src/Client/QueryFuzzer.cpp
View File

@ -137,9 +137,41 @@ Field QueryFuzzer::fuzzField(Field field)
break;
}
}
else if (type == Field::Types::Array || type == Field::Types::Tuple)
else if (type == Field::Types::Array)
{
auto & arr = field.reinterpret<FieldVector>();
auto & arr = field.get<Array>();
if (fuzz_rand() % 5 == 0 && !arr.empty())
{
size_t pos = fuzz_rand() % arr.size();
arr.erase(arr.begin() + pos);
std::cerr << "erased\n";
}
if (fuzz_rand() % 5 == 0)
{
if (!arr.empty())
{
size_t pos = fuzz_rand() % arr.size();
arr.insert(arr.begin() + pos, fuzzField(arr[pos]));
std::cerr << fmt::format("inserted (pos {})\n", pos);
}
else
{
arr.insert(arr.begin(), getRandomField(0));
std::cerr << "inserted (0)\n";
}
}
for (auto & element : arr)
{
element = fuzzField(element);
}
}
else if (type == Field::Types::Tuple)
{
auto & arr = field.get<Tuple>();
if (fuzz_rand() % 5 == 0 && !arr.empty())
{

5
src/Client/Suggest.cpp
View File

@ -187,9 +187,8 @@ void Suggest::fillWordsFromBlock(const Block & block)
Words new_words;
new_words.reserve(rows);
for (size_t i = 0; i < rows; ++i)
{
new_words.emplace_back(column.getDataAt(i).toString());
}
new_words.emplace_back(column[i].get<String>());
addWords(std::move(new_words));
}

2
src/Columns/ColumnAggregateFunction.cpp
View File

@ -162,7 +162,7 @@ MutableColumnPtr ColumnAggregateFunction::convertToValues(MutableColumnPtr colum
};
callback(res);
res->forEachSubcolumn(callback);
res->forEachSubcolumnRecursively(callback);
for (auto * val : data)
func->insertResultInto(val, *res, &column_aggregate_func.createOrGetArena());

19
src/Columns/ColumnArray.cpp
View File

@ -151,23 +151,24 @@ void ColumnArray::get(size_t n, Field & res) const
StringRef ColumnArray::getDataAt(size_t n) const
{
assert(n < size());
/** Returns the range of memory that covers all elements of the array.
* Works for arrays of fixed length values.
* For arrays of strings and arrays of arrays, the resulting chunk of memory may not be one-to-one correspondence with the elements,
* since it contains only the data laid in succession, but not the offsets.
*/
size_t offset_of_first_elem = offsetAt(n);
StringRef first = getData().getDataAtWithTerminatingZero(offset_of_first_elem);
/// We are using pointer arithmetic on the addresses of the array elements.
if (!data->isFixedAndContiguous())
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getDataAt is not supported for {}", getName());
size_t array_size = sizeAt(n);
if (array_size == 0)
return StringRef(first.data, 0);
return StringRef(nullptr, 0);
size_t offset_of_last_elem = getOffsets()[n] - 1;
StringRef last = getData().getDataAtWithTerminatingZero(offset_of_last_elem);
size_t offset_of_first_elem = offsetAt(n);
StringRef first = getData().getDataAt(offset_of_first_elem);
return StringRef(first.data, last.data + last.size - first.data);
return StringRef(first.data, first.size * array_size);
}
@ -183,7 +184,7 @@ void ColumnArray::insertData(const char * pos, size_t length)
/** Similarly - only for arrays of fixed length values.
*/
if (!data->isFixedAndContiguous())
throw Exception("Method insertData is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED);
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method insertData is not supported for {}", getName());
size_t field_size = data->sizeOfValueIfFixed();

8
src/Columns/ColumnArray.h
View File

@ -157,6 +157,14 @@ public:
callback(data);
}
void forEachSubcolumnRecursively(ColumnCallback callback) override
{
callback(offsets);
offsets->forEachSubcolumnRecursively(callback);
callback(data);
data->forEachSubcolumnRecursively(callback);
}
bool structureEquals(const IColumn & rhs) const override
{
if (const auto * rhs_concrete = typeid_cast<const ColumnArray *>(&rhs))

11
src/Columns/ColumnConst.h
View File

@ -81,11 +81,6 @@ public:
return data->getDataAt(0);
}
StringRef getDataAtWithTerminatingZero(size_t) const override
{
return data->getDataAtWithTerminatingZero(0);
}
UInt64 get64(size_t) const override
{
return data->get64(0);
@ -240,6 +235,12 @@ public:
callback(data);
}
void forEachSubcolumnRecursively(ColumnCallback callback) override
{
callback(data);
data->forEachSubcolumnRecursively(callback);
}
bool structureEquals(const IColumn & rhs) const override
{
if (const auto * rhs_concrete = typeid_cast<const ColumnConst *>(&rhs))

17
src/Columns/ColumnLowCardinality.h
View File

@ -59,10 +59,6 @@ public:
void get(size_t n, Field & res) const override { getDictionary().get(getIndexes().getUInt(n), res); }
StringRef getDataAt(size_t n) const override { return getDictionary().getDataAt(getIndexes().getUInt(n)); }
StringRef getDataAtWithTerminatingZero(size_t n) const override
{
return getDictionary().getDataAtWithTerminatingZero(getIndexes().getUInt(n));
}
bool isDefaultAt(size_t n) const override { return getDictionary().isDefaultAt(getIndexes().getUInt(n)); }
UInt64 get64(size_t n) const override { return getDictionary().get64(getIndexes().getUInt(n)); }
@ -177,6 +173,19 @@ public:
callback(dictionary.getColumnUniquePtr());
}
void forEachSubcolumnRecursively(ColumnCallback callback) override
{
callback(idx.getPositionsPtr());
idx.getPositionsPtr()->forEachSubcolumnRecursively(callback);
/// Column doesn't own dictionary if it's shared.
if (!dictionary.isShared())
{
callback(dictionary.getColumnUniquePtr());
dictionary.getColumnUniquePtr()->forEachSubcolumnRecursively(callback);
}
}
bool structureEquals(const IColumn & rhs) const override
{
if (const auto * rhs_low_cardinality = typeid_cast<const ColumnLowCardinality *>(&rhs))

6
src/Columns/ColumnMap.cpp
View File

@ -278,6 +278,12 @@ void ColumnMap::forEachSubcolumn(ColumnCallback callback)
callback(nested);
}
void ColumnMap::forEachSubcolumnRecursively(ColumnCallback callback)
{
callback(nested);
nested->forEachSubcolumnRecursively(callback);
}
bool ColumnMap::structureEquals(const IColumn & rhs) const
{
if (const auto * rhs_map = typeid_cast<const ColumnMap *>(&rhs))

1
src/Columns/ColumnMap.h
View File

@ -89,6 +89,7 @@ public:
size_t allocatedBytes() const override;
void protect() override;
void forEachSubcolumn(ColumnCallback callback) override;
void forEachSubcolumnRecursively(ColumnCallback callback) override;
bool structureEquals(const IColumn & rhs) const override;
double getRatioOfDefaultRows(double sample_ratio) const override;
void getIndicesOfNonDefaultRows(Offsets & indices, size_t from, size_t limit) const override;

8
src/Columns/ColumnNullable.h
View File

@ -136,6 +136,14 @@ public:
callback(null_map);
}
void forEachSubcolumnRecursively(ColumnCallback callback) override
{
callback(nested_column);
nested_column->forEachSubcolumnRecursively(callback);
callback(null_map);
null_map->forEachSubcolumnRecursively(callback);
}
bool structureEquals(const IColumn & rhs) const override
{
if (const auto * rhs_nullable = typeid_cast<const ColumnNullable *>(&rhs))

12
src/Columns/ColumnObject.cpp
View File

@ -671,6 +671,18 @@ void ColumnObject::forEachSubcolumn(ColumnCallback callback)
callback(part);
}
void ColumnObject::forEachSubcolumnRecursively(ColumnCallback callback)
{
for (auto & entry : subcolumns)
{
for (auto & part : entry->data.data)
{
callback(part);
part->forEachSubcolumnRecursively(callback);
}
}
}
void ColumnObject::insert(const Field & field)
{
const auto & object = field.get<const Object &>();

1
src/Columns/ColumnObject.h
View File

@ -211,6 +211,7 @@ public:
size_t byteSize() const override;
size_t allocatedBytes() const override;
void forEachSubcolumn(ColumnCallback callback) override;
void forEachSubcolumnRecursively(ColumnCallback callback) override;
void insert(const Field & field) override;
void insertDefault() override;
void insertFrom(const IColumn & src, size_t n) override;

8
src/Columns/ColumnSparse.cpp
View File

@ -750,6 +750,14 @@ void ColumnSparse::forEachSubcolumn(ColumnCallback callback)
callback(offsets);
}
void ColumnSparse::forEachSubcolumnRecursively(ColumnCallback callback)
{
callback(values);
values->forEachSubcolumnRecursively(callback);
callback(offsets);
offsets->forEachSubcolumnRecursively(callback);
}
const IColumn::Offsets & ColumnSparse::getOffsetsData() const
{
return assert_cast<const ColumnUInt64 &>(*offsets).getData();

1
src/Columns/ColumnSparse.h
View File

@ -140,6 +140,7 @@ public:
ColumnPtr compress() const override;
void forEachSubcolumn(ColumnCallback callback) override;
void forEachSubcolumnRecursively(ColumnCallback callback) override;
bool structureEquals(const IColumn & rhs) const override;

27
src/Columns/ColumnString.h
View File

@ -108,24 +108,12 @@ public:
return StringRef(&chars[offsetAt(n)], sizeAt(n) - 1);
}
StringRef getDataAtWithTerminatingZero(size_t n) const override
{
assert(n < size());
return StringRef(&chars[offsetAt(n)], sizeAt(n));
}
bool isDefaultAt(size_t n) const override
{
assert(n < size());
return sizeAt(n) == 1;
}
/// Suppress gcc 7.3.1 warning: '*((void*)&<anonymous> +8)' may be used uninitialized in this function
#if !defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif
void insert(const Field & x) override
{
const String & s = x.get<const String &>();
@ -138,10 +126,6 @@ public:
offsets.push_back(new_size);
}
#if !defined(__clang__)
#pragma GCC diagnostic pop
#endif
void insertFrom(const IColumn & src_, size_t n) override
{
const ColumnString & src = assert_cast<const ColumnString &>(src_);
@ -177,17 +161,6 @@ public:
offsets.push_back(new_size);
}
/// Like getData, but inserting data should be zero-ending (i.e. length is 1 byte greater than real string size).
void insertDataWithTerminatingZero(const char * pos, size_t length)
{
const size_t old_size = chars.size();
const size_t new_size = old_size + length;
chars.resize(new_size);
memcpy(chars.data() + old_size, pos, length);
offsets.push_back(new_size);
}
void popBack(size_t n) override
{
size_t nested_n = offsets.back() - offsetAt(offsets.size() - n);

9
src/Columns/ColumnTuple.cpp
View File

@ -501,6 +501,15 @@ void ColumnTuple::forEachSubcolumn(ColumnCallback callback)
callback(column);
}
void ColumnTuple::forEachSubcolumnRecursively(ColumnCallback callback)
{
for (auto & column : columns)
{
callback(column);
column->forEachSubcolumnRecursively(callback);
}
}
bool ColumnTuple::structureEquals(const IColumn & rhs) const
{
if (const auto * rhs_tuple = typeid_cast<const ColumnTuple *>(&rhs))

1
src/Columns/ColumnTuple.h
View File

@ -97,6 +97,7 @@ public:
size_t allocatedBytes() const override;
void protect() override;
void forEachSubcolumn(ColumnCallback callback) override;
void forEachSubcolumnRecursively(ColumnCallback callback) override;
bool structureEquals(const IColumn & rhs) const override;
bool isCollationSupported() const override;
ColumnPtr compress() const override;

64
src/Columns/ColumnUnique.h
View File

@ -1,4 +1,5 @@
#pragma once
#include <Columns/IColumnUnique.h>
#include <Columns/IColumnImpl.h>
#include <Columns/ReverseIndex.h>
@ -7,16 +8,17 @@
#include <Columns/ColumnNullable.h>
#include <Columns/ColumnString.h>
#include <Columns/ColumnFixedString.h>
#include <Columns/ColumnConst.h>
#include <DataTypes/DataTypeNullable.h>
#include <DataTypes/NumberTraits.h>
#include <Common/typeid_cast.h>
#include <Common/assert_cast.h>
#include <base/range.h>
#include <Common/FieldVisitors.h>
#include <base/range.h>
#include <base/unaligned.h>
#include "Columns/ColumnConst.h"
namespace DB
@ -70,10 +72,6 @@ public:
void get(size_t n, Field & res) const override { getNestedColumn()->get(n, res); }
bool isDefaultAt(size_t n) const override { return n == 0; }
StringRef getDataAt(size_t n) const override { return getNestedColumn()->getDataAt(n); }
StringRef getDataAtWithTerminatingZero(size_t n) const override
{
return getNestedColumn()->getDataAtWithTerminatingZero(n);
}
UInt64 get64(size_t n) const override { return getNestedColumn()->get64(n); }
UInt64 getUInt(size_t n) const override { return getNestedColumn()->getUInt(n); }
Int64 getInt(size_t n) const override { return getNestedColumn()->getInt(n); }
@ -115,6 +113,15 @@ public:
nested_column_nullable = ColumnNullable::create(column_holder, nested_null_mask);
}
void forEachSubcolumnRecursively(IColumn::ColumnCallback callback) override
{
callback(column_holder);
column_holder->forEachSubcolumnRecursively(callback);
reverse_index.setColumn(getRawColumnPtr());
if (is_nullable)
nested_column_nullable = ColumnNullable::create(column_holder, nested_null_mask);
}
bool structureEquals(const IColumn & rhs) const override
{
if (auto rhs_concrete = typeid_cast<const ColumnUnique *>(&rhs))
@ -309,17 +316,52 @@ size_t ColumnUnique<ColumnType>::getNullValueIndex() const
return 0;
}
namespace
{
class FieldVisitorGetData : public StaticVisitor<>
{
public:
StringRef res;
[[noreturn]] static void throwUnsupported()
{
throw Exception("Unsupported field type", ErrorCodes::LOGICAL_ERROR);
}
[[noreturn]] void operator() (const Null &) { throwUnsupported(); }
[[noreturn]] void operator() (const Array &) { throwUnsupported(); }
[[noreturn]] void operator() (const Tuple &) { throwUnsupported(); }
[[noreturn]] void operator() (const Map &) { throwUnsupported(); }
[[noreturn]] void operator() (const Object &) { throwUnsupported(); }
[[noreturn]] void operator() (const AggregateFunctionStateData &) { throwUnsupported(); }
void operator() (const String & x) { res = {x.data(), x.size()}; }
void operator() (const UInt64 & x) { res = {reinterpret_cast<const char *>(&x), sizeof(x)}; }
void operator() (const UInt128 & x) { res = {reinterpret_cast<const char *>(&x), sizeof(x)}; }
void operator() (const UInt256 & x) { res = {reinterpret_cast<const char *>(&x), sizeof(x)}; }
void operator() (const Int64 & x) { res = {reinterpret_cast<const char *>(&x), sizeof(x)}; }
void operator() (const Int128 & x) { res = {reinterpret_cast<const char *>(&x), sizeof(x)}; }
void operator() (const Int256 & x) { res = {reinterpret_cast<const char *>(&x), sizeof(x)}; }
void operator() (const UUID & x) { res = {reinterpret_cast<const char *>(&x), sizeof(x)}; }
void operator() (const Float64 & x) { res = {reinterpret_cast<const char *>(&x), sizeof(x)}; }
void operator() (const DecimalField<Decimal32> & x) { res = {reinterpret_cast<const char *>(&x), sizeof(x)}; }
void operator() (const DecimalField<Decimal64> & x) { res = {reinterpret_cast<const char *>(&x), sizeof(x)}; }
void operator() (const DecimalField<Decimal128> & x) { res = {reinterpret_cast<const char *>(&x), sizeof(x)}; }
void operator() (const DecimalField<Decimal256> & x) { res = {reinterpret_cast<const char *>(&x), sizeof(x)}; }
void operator() (const bool & x) { res = {reinterpret_cast<const char *>(&x), sizeof(x)}; }
};
}
template <typename ColumnType>
size_t ColumnUnique<ColumnType>::uniqueInsert(const Field & x)
{
if (x.isNull())
return getNullValueIndex();
if (valuesHaveFixedSize())
return uniqueInsertData(&x.reinterpret<char>(), size_of_value_if_fixed);
const auto & val = x.get<String>();
return uniqueInsertData(val.data(), val.size());
FieldVisitorGetData visitor;
applyVisitor(visitor, x);
return uniqueInsertData(visitor.res.data, visitor.res.size);
}
template <typename ColumnType>

10
src/Columns/IColumn.h
View File

@ -106,13 +106,6 @@ public:
/// Is used to optimize some computations (in aggregation, for example).
[[nodiscard]] virtual StringRef getDataAt(size_t n) const = 0;
/// Like getData, but has special behavior for columns that contain variable-length strings.
/// Returns zero-ending memory chunk (i.e. its size is 1 byte longer).
[[nodiscard]] virtual StringRef getDataAtWithTerminatingZero(size_t n) const
{
return getDataAt(n);
}
/// If column stores integers, it returns n-th element transformed to UInt64 using static_cast.
/// If column stores floating point numbers, bits of n-th elements are copied to lower bits of UInt64, the remaining bits are zeros.
/// Is used to optimize some computations (in aggregation, for example).
@ -421,6 +414,9 @@ public:
using ColumnCallback = std::function<void(WrappedPtr&)>;
virtual void forEachSubcolumn(ColumnCallback) {}
/// Similar to forEachSubcolumn but it also do recursive calls.
virtual void forEachSubcolumnRecursively(ColumnCallback) {}
/// Columns have equal structure.
/// If true - you can use "compareAt", "insertFrom", etc. methods.
[[nodiscard]] virtual bool structureEquals(const IColumn &) const

36
src/Columns/tests/gtest_column_object.cpp
View File

@ -1,8 +1,11 @@
#include <Common/FieldVisitorsAccurateComparison.h>
#include <DataTypes/getLeastSupertype.h>
#include <DataTypes/ObjectUtils.h>
#include <DataTypes/DataTypeTuple.h>
#include <Interpreters/castColumn.h>
#include <Interpreters/convertFieldToType.h>
#include <Columns/ColumnObject.h>
#include <Columns/ColumnTuple.h>
#include <Common/FieldVisitorToString.h>
#include <Common/randomSeed.h>
@ -118,3 +121,36 @@ TEST(ColumnObject, InsertRangeFrom)
checkFieldsAreEqual(subcolumn_dst, fields_dst);
}
}
TEST(ColumnObject, Unflatten)
{
auto check_empty_tuple = [](const auto & type, const auto & column)
{
const auto & type_tuple = assert_cast<const DataTypeTuple &>(*type);
const auto & column_tuple = assert_cast<const ColumnTuple &>(*column);
ASSERT_EQ(type_tuple.getElements().size(), 1);
ASSERT_EQ(type_tuple.getElements()[0]->getName(), "UInt8");
ASSERT_EQ(type_tuple.getElementNames()[0], ColumnObject::COLUMN_NAME_DUMMY);
ASSERT_EQ(column_tuple.getColumns().size(), 1);
ASSERT_EQ(column_tuple.getColumns()[0]->getName(), "UInt8");
};
{
auto column_object = ColumnObject::create(false);
auto [column, type] = unflattenObjectToTuple(*column_object);
check_empty_tuple(type, column);
ASSERT_EQ(column->size(), 0);
}
{
auto column_object = ColumnObject::create(false);
column_object->insertManyDefaults(5);
auto [column, type] = unflattenObjectToTuple(*column_object);
check_empty_tuple(type, column);
ASSERT_EQ(column->size(), 5);
}
}

12
src/Common/Allocator.h
View File

@ -281,14 +281,6 @@ private:
#endif
};
/** When using AllocatorWithStackMemory, located on the stack,
* GCC 4.9 mistakenly assumes that we can call `free` from a pointer to the stack.
* In fact, the combination of conditions inside AllocatorWithStackMemory does not allow this.
*/
#if !defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wfree-nonheap-object"
#endif
/** Allocator with optimization to place small memory ranges in automatic memory.
*/
@ -366,7 +358,3 @@ extern template class Allocator<false, false>;
extern template class Allocator<true, false>;
extern template class Allocator<false, true>;
extern template class Allocator<true, true>;
#if !defined(__clang__)
#pragma GCC diagnostic pop
#endif

13
src/Common/DateLUTImpl.h
View File

@ -29,13 +29,6 @@
#define DATE_LUT_ADD ((1970 - DATE_LUT_MIN_YEAR) * 366L * 86400)
#if defined(__PPC__)
#if !defined(__clang__)
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif
#endif
/// Flags for toYearWeek() function.
enum class WeekModeFlag : UInt8
{
@ -1445,9 +1438,3 @@ public:
return s;
}
};
#if defined(__PPC__)
#if !defined(__clang__)
#pragma GCC diagnostic pop
#endif
#endif

991
src/Common/Dwarf.cpp
View File

File diff suppressed because it is too large Load Diff

141
src/Common/Dwarf.h
View File

@ -19,6 +19,7 @@
*/
/** This file was edited for ClickHouse.
* Original is from folly library.
*/
#include <functional>
@ -113,8 +114,8 @@ public:
// seems as the same path can be represented in multiple ways
private:
std::string_view baseDir_; /// NOLINT
std::string_view subDir_; /// NOLINT
std::string_view file_; /// NOLINT
std::string_view subDir_; /// NOLINT
std::string_view file_; /// NOLINT
};
// Indicates inline function `name` is called at `line@file`.
@ -171,8 +172,6 @@ public:
private:
static bool findDebugInfoOffset(uintptr_t address, std::string_view aranges, uint64_t & offset);
void init();
std::shared_ptr<const Elf> elf_; /// NOLINT
// DWARF section made up of chunks, each prefixed with a length header.
@ -228,6 +227,7 @@ private:
{
uint64_t name = 0;
uint64_t form = 0;
int64_t implicitConst = 0; // only set when form=DW_FORM_implicit_const
explicit operator bool() const { return name != 0 || form != 0; }
};
@ -239,25 +239,43 @@ private:
std::variant<uint64_t, std::string_view> attr_value;
};
enum
{
DW_UT_compile = 0x01,
DW_UT_skeleton = 0x04,
};
struct CompilationUnit
{
bool is64Bit; /// NOLINT
uint8_t version;
uint8_t addr_size;
bool is64Bit = false; /// NOLINT
uint8_t version = 0;
uint8_t unit_type = DW_UT_compile; // DW_UT_compile or DW_UT_skeleton
uint8_t addr_size = 0;
// Offset in .debug_info of this compilation unit.
uint32_t offset;
uint32_t size;
uint32_t offset = 0;
uint32_t size = 0;
// Offset in .debug_info for the first DIE in this compilation unit.
uint32_t first_die;
uint64_t abbrev_offset;
uint32_t first_die = 0;
uint64_t abbrev_offset = 0;
// The beginning of the CU's contribution to .debug_addr
std::optional<uint64_t> addr_base; // DW_AT_addr_base (DWARF 5)
// The beginning of the offsets table (immediately following the
// header) of the CU's contribution to .debug_loclists
std::optional<uint64_t> loclists_base; // DW_AT_loclists_base (DWARF 5)
// The beginning of the offsets table (immediately following the
// header) of the CU's contribution to .debug_rnglists
std::optional<uint64_t> rnglists_base; // DW_AT_rnglists_base (DWARF 5)
// Points to the first string offset of the compilation unit’s
// contribution to the .debug_str_offsets (or .debug_str_offsets.dwo) section.
std::optional<uint64_t> str_offsets_base; // DW_AT_str_offsets_base (DWARF 5)
// Only the CompilationUnit that contains the caller functions needs this cache.
// Indexed by (abbr.code - 1) if (abbr.code - 1) < abbrCache.size();
std::vector<DIEAbbreviation> abbr_cache;
};
static CompilationUnit getCompilationUnit(std::string_view info, uint64_t offset);
/** cu must exist during the life cycle of created detail::Die. */
/** cu must exist during the life cycle of created Die. */
Die getDieAtOffset(const CompilationUnit & cu, uint64_t offset) const;
bool findLocation(
@ -278,16 +296,16 @@ private:
class LineNumberVM
{
public:
LineNumberVM(std::string_view data, std::string_view compilationDirectory);
LineNumberVM(
std::string_view data,
std::string_view compilationDirectory,
std::string_view debugStr,
std::string_view debugLineStr);
bool findAddress(uintptr_t target, Path & file, uint64_t & line);
/** Gets full file name at given index including directory. */
Path getFullFileName(uint64_t index) const
{
auto fn = getFileName(index);
return Path({}, getIncludeDirectory(fn.directoryIndex), fn.relativeName);
}
Path getFullFileName(uint64_t index) const;
private:
void init();
@ -327,24 +345,42 @@ private:
bool nextDefineFile(std::string_view & program, FileName & fn) const;
// Initialization
bool is64Bit_; /// NOLINT
std::string_view data_; /// NOLINT
std::string_view compilationDirectory_; /// NOLINT
bool is64Bit_; /// NOLINT
std::string_view data_; /// NOLINT
std::string_view compilationDirectory_; /// NOLINT
std::string_view debugStr_; // needed for DWARF 5 /// NOLINT
std::string_view debugLineStr_; // DWARF 5 /// NOLINT
// Header
uint16_t version_; /// NOLINT
uint8_t minLength_; /// NOLINT
uint16_t version_; /// NOLINT
uint8_t minLength_; /// NOLINT
bool defaultIsStmt_; /// NOLINT
int8_t lineBase_; /// NOLINT
uint8_t lineRange_; /// NOLINT
int8_t lineBase_; /// NOLINT
uint8_t lineRange_; /// NOLINT
uint8_t opcodeBase_; /// NOLINT
const uint8_t * standardOpcodeLengths_; /// NOLINT
std::string_view includeDirectories_; /// NOLINT
size_t includeDirectoryCount_; /// NOLINT
// 6.2.4 The Line Number Program Header.
struct
{
size_t includeDirectoryCount;
std::string_view includeDirectories;
size_t fileNameCount;
std::string_view fileNames;
} v4_;
std::string_view fileNames_; /// NOLINT
size_t fileNameCount_; /// NOLINT
struct
{
uint8_t directoryEntryFormatCount;
std::string_view directoryEntryFormat;
uint64_t directoriesCount;
std::string_view directories;
uint8_t fileNameEntryFormatCount;
std::string_view fileNameEntryFormat;
uint64_t fileNamesCount;
std::string_view fileNames;
} v5_;
// State machine registers
uint64_t address_; /// NOLINT
@ -397,20 +433,26 @@ private:
*/
size_t forEachAttribute(const CompilationUnit & cu, const Die & die, std::function<bool(const Attribute & die)> f) const;
Attribute readAttribute(const Die & die, AttributeSpec spec, std::string_view & info) const;
Attribute readAttribute(
const CompilationUnit & cu,
const Die & die,
AttributeSpec spec,
std::string_view & info) const;
// Read one attribute <name, form> pair, remove_prefix sp; returns <0, 0> at end.
static AttributeSpec readAttributeSpec(std::string_view & sp);
// Read one attribute value, remove_prefix sp
using AttributeValue = std::variant<uint64_t, std::string_view>;
AttributeValue readAttributeValue(std::string_view & sp, uint64_t form, bool is64Bit) const;
AttributeValue readAttributeValue(std::string_view & sp, uint64_t form, bool is64_bit) const;
// Get an ELF section by name, return true if found
bool getSection(const char * name, std::string_view * section) const;
std::string_view getSection(const char * name) const;
CompilationUnit getCompilationUnit(uint64_t offset) const;
// Finds the Compilation Unit starting at offset.
CompilationUnit findCompilationUnit(uint64_t targetOffset) const;
// Get a string from the .debug_str section
std::string_view getStringFromStringSection(uint64_t offset) const;
template <class T>
std::optional<T> getAttribute(const CompilationUnit & cu, const Die & die, uint64_t attr_name) const
@ -429,17 +471,24 @@ private:
}
// Check if the given address is in the range list at the given offset in .debug_ranges.
bool isAddrInRangeList(uint64_t address, std::optional<uint64_t> base_addr, size_t offset, uint8_t addr_size) const;
bool isAddrInRangeList(
const CompilationUnit & cu,
uint64_t address,
std::optional<uint64_t> base_addr,
size_t offset,
uint8_t addr_size) const;
// Finds the Compilation Unit starting at offset.
static CompilationUnit findCompilationUnit(std::string_view info, uint64_t targetOffset);
std::string_view info_; // .debug_info /// NOLINT
std::string_view abbrev_; // .debug_abbrev /// NOLINT
std::string_view aranges_; // .debug_aranges /// NOLINT
std::string_view line_; // .debug_line /// NOLINT
std::string_view strings_; // .debug_str /// NOLINT
std::string_view ranges_; // .debug_ranges /// NOLINT
std::string_view abbrev_; // .debug_abbrev /// NOLINT
std::string_view addr_; // .debug_addr (DWARF 5) /// NOLINT
std::string_view aranges_; // .debug_aranges /// NOLINT
std::string_view info_; // .debug_info /// NOLINT
std::string_view line_; // .debug_line /// NOLINT
std::string_view line_str_; // .debug_line_str (DWARF 5) /// NOLINT
std::string_view loclists_; // .debug_loclists (DWARF 5) /// NOLINT
std::string_view ranges_; // .debug_ranges /// NOLINT
std::string_view rnglists_; // .debug_rnglists (DWARF 5) /// NOLINT
std::string_view str_; // .debug_str /// NOLINT
std::string_view str_offsets_; // .debug_str_offsets (DWARF 5) /// NOLINT
};
}

17
src/Common/FieldVisitorConvertToNumber.h
View File

@ -94,21 +94,7 @@ public:
T operator() (const DecimalField<U> & x) const
{
if constexpr (std::is_floating_point_v<T>)
return x.getValue(). template convertTo<T>() / x.getScaleMultiplier(). template convertTo<T>();
else if constexpr (std::is_same_v<T, UInt128>)
{
if constexpr (sizeof(U) < 16)
{
return UInt128(0, (x.getValue() / x.getScaleMultiplier()).value);
}
else if constexpr (sizeof(U) == 16)
{
auto tmp = (x.getValue() / x.getScaleMultiplier()).value;
return UInt128(tmp >> 64, UInt64(tmp));
}
else
throw Exception("No conversion to old UInt128 from " + demangle(typeid(U).name()), ErrorCodes::NOT_IMPLEMENTED);
}
return x.getValue().template convertTo<T>() / x.getScaleMultiplier().template convertTo<T>();
else
return (x.getValue() / x.getScaleMultiplier()). template convertTo<T>();
}
@ -134,4 +120,3 @@ public:
};
}

2
src/Common/FieldVisitorSum.cpp
View File

@ -15,7 +15,7 @@ FieldVisitorSum::FieldVisitorSum(const Field & rhs_) : rhs(rhs_) {}
bool FieldVisitorSum::operator() (Int64 & x) const { return this->operator()(reinterpret_cast<UInt64 &>(x)); }
bool FieldVisitorSum::operator() (UInt64 & x) const
{
x += rhs.reinterpret<UInt64>();
x += applyVisitor(FieldVisitorConvertToNumber<UInt64>(), rhs);
return x != 0;
}

3
src/Common/FieldVisitorSum.h
View File

@ -1,6 +1,7 @@
#pragma once
#include <Common/FieldVisitors.h>
#include <Common/FieldVisitorConvertToNumber.h>
namespace DB
@ -41,7 +42,7 @@ public:
requires is_big_int_v<T>
bool operator() (T & x) const
{
x += rhs.reinterpret<T>();
x += applyVisitor(FieldVisitorConvertToNumber<T>(), rhs);
return x != T(0);
}
};

10
src/Common/PODArray.h
View File

@ -115,7 +115,13 @@ protected:
}
/// Minimum amount of memory to allocate for num_elements, including padding.
static size_t minimum_memory_for_elements(size_t num_elements) { return byte_size(num_elements) + pad_right + pad_left; } /// NOLINT
static size_t minimum_memory_for_elements(size_t num_elements)
{
size_t amount;
if (__builtin_add_overflow(byte_size(num_elements), pad_left + pad_right, &amount))
throw Exception("Amount of memory requested to allocate is more than allowed", ErrorCodes::CANNOT_ALLOCATE_MEMORY);
return amount;
}
void alloc_for_num_elements(size_t num_elements) /// NOLINT
{
@ -225,9 +231,7 @@ public:
void clear() { c_end = c_start; }
template <typename ... TAllocatorParams>
#if defined(__clang__)
ALWAYS_INLINE /// Better performance in clang build, worse performance in gcc build.
#endif
void reserve(size_t n, TAllocatorParams &&... allocator_params)
{
if (n > capacity())

10
src/Common/examples/compact_array.cpp
View File

@ -1,9 +1,3 @@
/// Bug in GCC: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=59124
#if !defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Warray-bounds"
#endif
#include <Common/CompactArray.h>
#include <IO/WriteBufferFromFile.h>
#include <IO/ReadBufferFromFile.h>
@ -262,7 +256,3 @@ int main()
runTests();
return 0;
}
#if !defined(__clang__)
#pragma GCC diagnostic pop
#endif

9
src/Common/examples/parallel_aggregation.cpp
View File

@ -69,11 +69,6 @@ static void aggregate1(Map & map, Source::const_iterator begin, Source::const_it
++map[*it];
}
#if !defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif
static void aggregate12(Map & map, Source::const_iterator begin, Source::const_iterator end)
{
Map::LookupResult found = nullptr;
@ -122,10 +117,6 @@ static void aggregate22(MapTwoLevel & map, Source::const_iterator begin, Source:
}
}
#if !defined(__clang__)
#pragma GCC diagnostic pop
#endif
static void merge2(MapTwoLevel * maps, size_t num_threads, size_t bucket)
{
for (size_t i = 1; i < num_threads; ++i)

19
src/Common/examples/parallel_aggregation2.cpp
View File

@ -62,11 +62,6 @@ struct AggregateIndependent
}
};
#if !defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif
template <typename Map>
struct AggregateIndependentWithSequentialKeysOptimization
{
@ -115,11 +110,6 @@ struct AggregateIndependentWithSequentialKeysOptimization
}
};
#if !defined(__clang__)
#pragma GCC diagnostic pop
#endif
template <typename Map>
struct MergeSequential
{
@ -265,20 +255,11 @@ struct Creator
void operator()(Value &) const {}
};
#if !defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif
struct Updater
{
void operator()(Value & x) const { ++x; }
};
#if !defined(__clang__)
#pragma GCC diagnostic pop
#endif
struct Merger
{
void operator()(Value & dst, const Value & src) const { dst += src; }

3
src/Coordination/CoordinationSettings.cpp
View File

@ -189,9 +189,6 @@ KeeperConfigurationAndSettings::loadFromConfig(const Poco::Util::AbstractConfigu
ret->coordination_settings->loadFromConfig("keeper_server.coordination_settings", config);
if (ret->coordination_settings->quorum_reads)
LOG_WARNING(&Poco::Logger::get("KeeperConfigurationAndSettings"), "Setting 'quorum_reads' is deprecated. Please use 'read_mode'");
return ret;
}

6
src/Coordination/CoordinationSettings.h
View File

@ -26,7 +26,6 @@ struct Settings;
M(Milliseconds, heart_beat_interval_ms, 500, "Heartbeat interval between quorum nodes", 0) \
M(Milliseconds, election_timeout_lower_bound_ms, 1000, "Lower bound of election timer (avoid too often leader elections)", 0) \
M(Milliseconds, election_timeout_upper_bound_ms, 2000, "Upper bound of election timer (avoid too often leader elections)", 0) \
M(Milliseconds, leadership_expiry, 0, "How often will leader node check if it still has majority. Set it lower or equal to election_timeout_lower_bound_ms to have linearizable reads.", 0) \
M(UInt64, reserved_log_items, 100000, "How many log items to store (don't remove during compaction)", 0) \
M(UInt64, snapshot_distance, 100000, "How many log items we have to collect to write new snapshot", 0) \
M(Bool, auto_forwarding, true, "Allow to forward write requests from followers to leader", 0) \
@ -39,12 +38,11 @@ struct Settings;
M(UInt64, stale_log_gap, 10000, "When node became stale and should receive snapshots from leader", 0) \
M(UInt64, fresh_log_gap, 200, "When node became fresh", 0) \
M(UInt64, max_requests_batch_size, 100, "Max size of batch in requests count before it will be sent to RAFT", 0) \
M(Bool, quorum_reads, false, "Deprecated - use read_mode. Execute read requests as writes through whole RAFT consesus with similar speed", 0) \
M(Bool, quorum_reads, false, "Execute read requests as writes through whole RAFT consesus with similar speed", 0) \
M(Bool, force_sync, true, "Call fsync on each change in RAFT changelog", 0) \
M(Bool, compress_logs, true, "Write compressed coordination logs in ZSTD format", 0) \
M(Bool, compress_snapshots_with_zstd_format, true, "Write compressed snapshots in ZSTD format (instead of custom LZ4)", 0) \
M(UInt64, configuration_change_tries_count, 20, "How many times we will try to apply configuration change (add/remove server) to the cluster", 0) \
M(String, read_mode, "nonlinear", "How should reads be processed. Valid values: 'nonlinear', 'fastlinear', 'quorum'. 'nonlinear' is the fastest option because there are no consistency requirements", 0)
M(UInt64, configuration_change_tries_count, 20, "How many times we will try to apply configuration change (add/remove server) to the cluster", 0)
DECLARE_SETTINGS_TRAITS(CoordinationSettingsTraits, LIST_OF_COORDINATION_SETTINGS)

Some files were not shown because too many files have changed in this diff Show More