thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-12-12 01:12:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into keeper-map-backup-restore

Browse Source
This commit is contained in:
Antonio Andelic 2023-11-14 09:47:54 +00:00
commit 29f54db917
125 changed files with 2575 additions and 10315 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/actions/common_setup/action.yml vendored
View File

@ -19,6 +19,8 @@ runs:
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/${{inputs.job_type}}
REPO_COPY=${{runner.temp}}/${{inputs.job_type}}/git-repo-copy
IMAGES_PATH=${{runner.temp}}/images_path
REPORTS_PATH=${{runner.temp}}/reports_dir
EOF
if [ -z "${{env.GITHUB_JOB_OVERRIDDEN}}" ] && [ "true" == "${{inputs.nested_job}}" ]; then
echo "The GITHUB_JOB_OVERRIDDEN ENV is unset, and must be set for the nested jobs"

347
.github/workflows/backport_branches.yml vendored
View File

@ -105,66 +105,22 @@ jobs:
path: ${{ runner.temp }}/changed_images.json
CompatibilityCheckX86:
needs: [BuilderDebRelease]
runs-on: [self-hosted, style-checker]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/compatibility_check
REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse
REPORTS_PATH=${{runner.temp}}/reports_dir
EOF
- name: Check out repository code
uses: ClickHouse/checkout@v1
uses: ./.github/workflows/reusable_test.yml
with:
clear-repository: true
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: CompatibilityCheckX86
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
test_name: Compatibility check X86
runner_type: style-checker
run_command: |
cd "$REPO_COPY/tests/ci"
python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions
CompatibilityCheckAarch64:
needs: [BuilderDebAarch64]
runs-on: [self-hosted, style-checker]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/compatibility_check
REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse
REPORTS_PATH=${{runner.temp}}/reports_dir
EOF
- name: Check out repository code
uses: ClickHouse/checkout@v1
uses: ./.github/workflows/reusable_test.yml
with:
clear-repository: true
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: CompatibilityCheckAarch64
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
test_name: Compatibility check X86
runner_type: style-checker
run_command: |
cd "$REPO_COPY/tests/ci"
python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc
#########################################################################################
#################################### ORDINARY BUILDS ####################################
#########################################################################################
@ -239,303 +195,114 @@ jobs:
##################################### BUILD REPORTER #######################################
############################################################################################
BuilderReport:
if: ${{ success() || failure() }}
needs:
- BuilderDebRelease
- BuilderDebAarch64
- BuilderDebAsan
- BuilderDebTsan
- BuilderDebDebug
runs-on: [self-hosted, style-checker]
if: ${{ success() || failure() }}
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
CHECK_NAME=ClickHouse build check
REPORTS_PATH=${{runner.temp}}/reports_dir
TEMP_PATH=${{runner.temp}}/report_check
NEEDS_DATA_PATH=${{runner.temp}}/needs.json
EOF
- name: Download json reports
uses: actions/download-artifact@v3
uses: ./.github/workflows/reusable_test.yml
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Report Builder
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cat > "$NEEDS_DATA_PATH" << 'EOF'
test_name: ClickHouse build check
runner_type: style-checker
additional_envs: |
NEEDS_DATA<<NDENV
${{ toJSON(needs) }}
EOF
NDENV
run_command: |
cd "$GITHUB_WORKSPACE/tests/ci"
python3 build_report_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
BuilderSpecialReport:
if: ${{ success() || failure() }}
needs:
- BuilderBinDarwin
- BuilderBinDarwinAarch64
runs-on: [self-hosted, style-checker]
if: ${{ success() || failure() }}
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/report_check
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=ClickHouse special build check
NEEDS_DATA_PATH=${{runner.temp}}/needs.json
EOF
- name: Download json reports
uses: actions/download-artifact@v3
uses: ./.github/workflows/reusable_test.yml
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Report Builder
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cat > "$NEEDS_DATA_PATH" << 'EOF'
test_name: ClickHouse special build check
runner_type: style-checker
additional_envs: |
NEEDS_DATA<<NDENV
${{ toJSON(needs) }}
EOF
NDENV
run_command: |
cd "$GITHUB_WORKSPACE/tests/ci"
python3 build_report_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
############################################################################################
#################################### INSTALL PACKAGES ######################################
############################################################################################
InstallPackagesTestRelease:
needs: [BuilderDebRelease]
runs-on: [self-hosted, style-checker]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/test_install
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Install packages (amd64)
REPO_COPY=${{runner.temp}}/test_install/ClickHouse
EOF
- name: Download json reports
uses: actions/download-artifact@v3
uses: ./.github/workflows/reusable_test.yml
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Test packages installation
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
test_name: Install packages (amd64)
runner_type: style-checker
run_command: |
cd "$REPO_COPY/tests/ci"
python3 install_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
InstallPackagesTestAarch64:
needs: [BuilderDebAarch64]
runs-on: [self-hosted, style-checker-aarch64]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/test_install
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Install packages (arm64)
REPO_COPY=${{runner.temp}}/test_install/ClickHouse
EOF
- name: Download json reports
uses: actions/download-artifact@v3
uses: ./.github/workflows/reusable_test.yml
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Test packages installation
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
test_name: Install packages (arm64)
runner_type: style-checker-aarch64
run_command: |
cd "$REPO_COPY/tests/ci"
python3 install_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
##############################################################################################
########################### FUNCTIONAl STATELESS TESTS #######################################
##############################################################################################
FunctionalStatelessTestAsan:
needs: [BuilderDebAsan]
runs-on: [self-hosted, func-tester]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_debug
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (asan)
REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
uses: ./.github/workflows/reusable_test.yml
with:
test_name: Stateless tests (asan)
runner_type: func-tester
additional_envs: |
KILL_TIMEOUT=10800
EOF
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Functional test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
run_command: |
cd "$REPO_COPY/tests/ci"
python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
##############################################################################################
############################ FUNCTIONAl STATEFUL TESTS #######################################
##############################################################################################
FunctionalStatefulTestDebug:
needs: [BuilderDebDebug]
runs-on: [self-hosted, func-tester]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateful_debug
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateful tests (debug)
REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse
uses: ./.github/workflows/reusable_test.yml
with:
test_name: Stateful tests (debug)
runner_type: func-tester
additional_envs: |
KILL_TIMEOUT=3600
EOF
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Functional test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
run_command: |
cd "$REPO_COPY/tests/ci"
python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
##############################################################################################
######################################### STRESS TESTS #######################################
##############################################################################################
StressTestTsan:
needs: [BuilderDebTsan]
# func testers have 16 cores + 128 GB memory
# while stress testers have 36 cores + 72 memory
# It would be better to have something like 32 + 128,
# but such servers almost unavailable as spot instances.
runs-on: [self-hosted, func-tester]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stress_thread
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stress test (tsan)
REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse
EOF
- name: Download json reports
uses: actions/download-artifact@v3
uses: ./.github/workflows/reusable_test.yml
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Stress test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
test_name: Stress test (tsan)
runner_type: stress-tester
run_command: |
cd "$REPO_COPY/tests/ci"
python3 stress_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
#############################################################################################
############################# INTEGRATION TESTS #############################################
#############################################################################################
IntegrationTestsRelease:
needs: [BuilderDebRelease]
runs-on: [self-hosted, stress-tester]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_release
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (release)
REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse
EOF
- name: Download json reports
uses: actions/download-artifact@v3
uses: ./.github/workflows/reusable_test.yml
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Integration test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
test_name: Integration tests (release)
runner_type: stress-tester
batches: 4
run_command: |
cd "$REPO_COPY/tests/ci"
python3 integration_test_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FinishCheck:
needs:
- DockerHubPush

70
.github/workflows/docs_check.yml vendored
View File

@ -96,68 +96,30 @@ jobs:
path: ${{ runner.temp }}/changed_images.json
StyleCheck:
needs: DockerHubPush
runs-on: [self-hosted, style-checker]
if: ${{ success() || failure() }}
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{ runner.temp }}/style_check
# We need additional `&& ! cancelled()` to have the job being able to cancel
if: ${{ success() || failure() || ( always() && ! cancelled() ) }}
uses: ./.github/workflows/reusable_test.yml
with:
test_name: Style check
runner_type: style-checker
run_command: |
cd "$REPO_COPY/tests/ci"
python3 style_check.py
secrets:
secret_envs: |
ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
RCSK
EOF
- name: Download changed images
# even if artifact does not exist, e.g. on `do not test` label or failed Docker job
continue-on-error: true
uses: actions/download-artifact@v3
with:
name: changed_images
path: ${{ env.TEMP_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Style Check
run: |
cd "$GITHUB_WORKSPACE/tests/ci"
python3 style_check.py
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
DocsCheck:
needs: DockerHubPush
runs-on: [self-hosted, func-tester-aarch64]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/docs_check
REPO_COPY=${{runner.temp}}/docs_check/ClickHouse
EOF
- name: Download changed images
uses: actions/download-artifact@v3
uses: ./.github/workflows/reusable_test.yml
with:
name: changed_images
path: ${{ env.TEMP_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Docs Check
run: |
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
test_name: Docs check
runner_type: func-tester-aarch64
additional_envs: |
run_command: |
cd "$REPO_COPY/tests/ci"
python3 docs_check.py
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FinishCheck:
needs:
- StyleCheck

57
.github/workflows/jepsen.yml vendored
View File

@ -11,60 +11,19 @@ on: # yamllint disable-line rule:truthy
workflow_call:
jobs:
KeeperJepsenRelease:
runs-on: [self-hosted, style-checker]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/keeper_jepsen
REPO_COPY=${{runner.temp}}/keeper_jepsen/ClickHouse
EOF
- name: Check out repository code
uses: ClickHouse/checkout@v1
uses: ./.github/workflows/reusable_test.yml
with:
clear-repository: true
fetch-depth: 0
filter: tree:0
- name: Jepsen Test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
test_name: Jepsen keeper check
runner_type: style-checker
run_command: |
cd "$REPO_COPY/tests/ci"
python3 jepsen_check.py keeper
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
# ServerJepsenRelease:
# runs-on: [self-hosted, style-checker]
# if: ${{ always() }}
# needs: [KeeperJepsenRelease]
# steps:
# - name: Set envs
# run: |
# cat >> "$GITHUB_ENV" << 'EOF'
# TEMP_PATH=${{runner.temp}}/server_jepsen
# REPO_COPY=${{runner.temp}}/server_jepsen/ClickHouse
# EOF
# - name: Check out repository code
# uses: ClickHouse/checkout@v1
# uses: ./.github/workflows/reusable_test.yml
# with:
# clear-repository: true
# fetch-depth: 0
# filter: tree:0
# - name: Jepsen Test
# run: |
# sudo rm -fr "$TEMP_PATH"
# mkdir -p "$TEMP_PATH"
# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
# test_name: Jepsen server check
# runner_type: style-checker
# run_command: |
# cd "$REPO_COPY/tests/ci"
# python3 jepsen_check.py server
# - name: Cleanup
# if: always()
# run: |
# docker ps --quiet | xargs --no-run-if-empty docker kill ||:
# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
# sudo rm -fr "$TEMP_PATH"

85
.github/workflows/libfuzzer.yml vendored
View File

@ -10,86 +10,17 @@ on: # yamllint disable-line rule:truthy
workflow_call:
jobs:
BuilderFuzzers:
runs-on: [self-hosted, builder]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/build_check
IMAGES_PATH=${{runner.temp}}/images_path
REPO_COPY=${{runner.temp}}/build_check/ClickHouse
CACHES_PATH=${{runner.temp}}/../ccaches
BUILD_NAME=fuzzers
EOF
- name: Download changed images
# even if artifact does not exist, e.g. on `do not test` label or failed Docker job
continue-on-error: true
uses: actions/download-artifact@v3
uses: ./.github/workflows/reusable_build.yml
with:
name: changed_images
path: ${{ env.IMAGES_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
submodules: true
ref: ${{github.ref}}
- name: Build
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v3
with:
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
build_name: fuzzers
libFuzzerTest:
needs: [BuilderFuzzers]
runs-on: [self-hosted, func-tester]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/libfuzzer
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=libFuzzer tests
REPO_COPY=${{runner.temp}}/libfuzzer/ClickHouse
uses: ./.github/workflows/reusable_test.yml
with:
test_name: libFuzzer tests
runner_type: func-tester
additional_envs: |
KILL_TIMEOUT=10800
EOF
- name: Download changed images
# even if artifact does not exist, e.g. on `do not test` label or failed Docker job
continue-on-error: true
uses: actions/download-artifact@v3
with:
name: changed_images
path: ${{ env.TEMP_PATH }}
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: libFuzzer test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
run_command: |
cd "$REPO_COPY/tests/ci"
python3 libfuzzer_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"

3339
.github/workflows/master.yml vendored
View File

File diff suppressed because it is too large Load Diff

3
.github/workflows/nightly.yml vendored
View File

@ -74,9 +74,6 @@ jobs:
with:
name: changed_images
path: ${{ runner.temp }}/changed_images.json
Codebrowser:
needs: [DockerHubPush]
uses: ./.github/workflows/woboq.yml
SonarCloud:
runs-on: [self-hosted, builder]
env:

4378
.github/workflows/pull_request.yml vendored
View File

File diff suppressed because it is too large Load Diff

1486
.github/workflows/release_branches.yml vendored
View File

File diff suppressed because it is too large Load Diff

9
.github/workflows/reusable_build.yml vendored
View File

@ -1,6 +1,10 @@
### For the pure soul wishes to move it to another place
# https://github.com/orgs/community/discussions/9050
env:
# Force the stdout and stderr streams to be unbuffered
PYTHONUNBUFFERED: 1
name: Build ClickHouse
'on':
workflow_call:
@ -25,6 +29,8 @@ name: Build ClickHouse
jobs:
Build:
name: Build-${{inputs.build_name}}
env:
GITHUB_JOB_OVERRIDDEN: Build-${{inputs.build_name}}
runs-on: [self-hosted, '${{inputs.runner_type}}']
steps:
- name: Check out repository code
@ -37,8 +43,6 @@ jobs:
- name: Set build envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
IMAGES_PATH=${{runner.temp}}/images_path
GITHUB_JOB_OVERRIDDEN=Build-${{inputs.build_name}}
${{inputs.additional_envs}}
EOF
python3 "$GITHUB_WORKSPACE"/tests/ci/ci_config.py --build-name "${{inputs.build_name}}" >> "$GITHUB_ENV"
@ -71,4 +75,5 @@ jobs:
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Clean
if: always()
uses: ./.github/actions/clean

113
.github/workflows/reusable_test.yml vendored Normal file
View File

@ -0,0 +1,113 @@
### For the pure soul wishes to move it to another place
# https://github.com/orgs/community/discussions/9050
name: Testing workflow
'on':
workflow_call:
inputs:
test_name:
description: the value of test type from tests/ci/ci_config.py, ends up as $CHECK_NAME ENV
required: true
type: string
runner_type:
description: the label of runner to use
required: true
type: string
run_command:
description: the command to launch the check. Usually starts with `cd '$REPO_COPY/tests/ci'`
required: true
type: string
batches:
description: how many batches for the test will be launched
default: 1
type: number
checkout_depth:
description: the value of the git shallow checkout
required: false
type: number
default: 1
submodules:
description: if the submodules should be checked out
required: false
type: boolean
default: false
additional_envs:
description: additional ENV variables to setup the job
type: string
secrets:
secret_envs:
description: if given, it's passed to the environments
required: false
env:
# Force the stdout and stderr streams to be unbuffered
PYTHONUNBUFFERED: 1
CHECK_NAME: ${{inputs.test_name}}
jobs:
PrepareStrategy:
# batches < 1 is misconfiguration,
# and we need this step only for batches > 1
if: ${{ inputs.batches > 1 }}
runs-on: [self-hosted, style-checker-aarch64]
outputs:
batches: ${{steps.batches.outputs.batches}}
steps:
- name: Calculate batches
id: batches
run: |
batches_output=$(python3 -c 'import json; print(json.dumps(list(range(${{inputs.batches}}))))')
echo "batches=${batches_output}" >> "$GITHUB_OUTPUT"
Test:
# If PrepareStrategy is skipped for batches == 1,
# we still need to launch the test.
# `! failure()` is mandatory here to launch on skipped Job
# `&& !cancelled()` to allow the be cancelable
if: ${{ ( !failure() && !cancelled() ) && inputs.batches > 0 }}
# Do not add `-0` to the end, if there's only one batch
name: ${{inputs.test_name}}${{ inputs.batches > 1 && format('-{0}',matrix.batch) || '' }}
env:
GITHUB_JOB_OVERRIDDEN: ${{inputs.test_name}}${{ inputs.batches > 1 && format('-{0}',matrix.batch) || '' }}
runs-on: [self-hosted, '${{inputs.runner_type}}']
needs: [PrepareStrategy]
strategy:
fail-fast: false # we always wait for entire matrix
matrix:
# if PrepareStrategy does not have batches, we use 0
batch: ${{ needs.PrepareStrategy.outputs.batches
&& fromJson(needs.PrepareStrategy.outputs.batches)
|| fromJson('[0]')}}
steps:
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
submodules: ${{inputs.submodules}}
fetch-depth: ${{inputs.checkout_depth}}
filter: tree:0
- name: Set build envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
${{inputs.additional_envs}}
${{secrets.secret_envs}}
EOF
- name: Common setup
uses: ./.github/actions/common_setup
with:
job_type: test
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: Setup batch
if: ${{ inputs.batches > 1}}
run: |
cat >> "$GITHUB_ENV" << 'EOF'
RUN_BY_HASH_NUM=${{matrix.batch}}
RUN_BY_HASH_TOTAL=${{inputs.batches}}
EOF
- name: Run test
run: ${{inputs.run_command}}
- name: Clean
if: always()
uses: ./.github/actions/clean

44
.github/workflows/woboq.yml vendored
View File

@ -1,44 +0,0 @@
name: WoboqBuilder
env:
# Force the stdout and stderr streams to be unbuffered
PYTHONUNBUFFERED: 1
concurrency:
group: woboq
on: # yamllint disable-line rule:truthy
workflow_dispatch:
workflow_call:
jobs:
# don't use dockerhub push because this image updates so rarely
WoboqCodebrowser:
runs-on: [self-hosted, style-checker]
timeout-minutes: 420 # the task is pretty heavy, so there's an additional hour
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/codebrowser
REPO_COPY=${{runner.temp}}/codebrowser/ClickHouse
IMAGES_PATH=${{runner.temp}}/images_path
EOF
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
submodules: 'true'
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.IMAGES_PATH }}
- name: Codebrowser
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci" && python3 codebrowser_check.py
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"

7
CMakeLists.txt
View File

@ -554,6 +554,13 @@ if (ENABLE_RUST)
endif()
endif()
if (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" AND NOT SANITIZE AND OS_LINUX AND (ARCH_AMD64 OR ARCH_AARCH64))
set(CHECK_LARGE_OBJECT_SIZES_DEFAULT ON)
else ()
set(CHECK_LARGE_OBJECT_SIZES_DEFAULT OFF)
endif ()
option(CHECK_LARGE_OBJECT_SIZES "Check that there are no large object files after build." ${CHECK_LARGE_OBJECT_SIZES_DEFAULT})
add_subdirectory (base)
add_subdirectory (src)
add_subdirectory (programs)

10
base/base/wide_integer_impl.h
View File

@ -65,7 +65,7 @@ class IsTupleLike
static void check(...);
public:
static constexpr const bool value = !std::is_void<decltype(check<T>(nullptr))>::value;
static constexpr const bool value = !std::is_void_v<decltype(check<T>(nullptr))>;
};
}
@ -79,7 +79,7 @@ class numeric_limits<wide::integer<Bits, Signed>>
{
public:
static constexpr bool is_specialized = true;
static constexpr bool is_signed = is_same<Signed, signed>::value;
static constexpr bool is_signed = is_same_v<Signed, signed>;
static constexpr bool is_integer = true;
static constexpr bool is_exact = true;
static constexpr bool has_infinity = false;
@ -91,7 +91,7 @@ public:
static constexpr bool is_iec559 = false;
static constexpr bool is_bounded = true;
static constexpr bool is_modulo = true;
static constexpr int digits = Bits - (is_same<Signed, signed>::value ? 1 : 0);
static constexpr int digits = Bits - (is_same_v<Signed, signed> ? 1 : 0);
static constexpr int digits10 = digits * 0.30103 /*std::log10(2)*/;
static constexpr int max_digits10 = 0;
static constexpr int radix = 2;
@ -104,7 +104,7 @@ public:
static constexpr wide::integer<Bits, Signed> min() noexcept
{
if (is_same<Signed, signed>::value)
if constexpr (is_same_v<Signed, signed>)
{
using T = wide::integer<Bits, signed>;
T res{};
@ -118,7 +118,7 @@ public:
{
using T = wide::integer<Bits, Signed>;
T res{};
res.items[T::_impl::big(0)] = is_same<Signed, signed>::value
res.items[T::_impl::big(0)] = is_same_v<Signed, signed>
? std::numeric_limits<typename wide::integer<Bits, Signed>::signed_base_type>::max()
: std::numeric_limits<typename wide::integer<Bits, Signed>::base_type>::max();
for (unsigned i = 1; i < wide::integer<Bits, Signed>::_impl::item_count; ++i)

8
base/glibc-compatibility/CMakeLists.txt
View File

@ -5,9 +5,6 @@ if (GLIBC_COMPATIBILITY)
endif()
enable_language(ASM)
include(CheckIncludeFile)
check_include_file("sys/random.h" HAVE_SYS_RANDOM_H)
add_headers_and_sources(glibc_compatibility .)
add_headers_and_sources(glibc_compatibility musl)
@ -21,11 +18,6 @@ if (GLIBC_COMPATIBILITY)
message (FATAL_ERROR "glibc_compatibility can only be used on x86_64 or aarch64.")
endif ()
list(REMOVE_ITEM glibc_compatibility_sources musl/getentropy.c)
if(HAVE_SYS_RANDOM_H)
list(APPEND glibc_compatibility_sources musl/getentropy.c)
endif()
# Need to omit frame pointers to match the performance of glibc
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fomit-frame-pointer")

180
cmake/cpu_features.cmake
View File

@ -1,10 +1,5 @@
# https://software.intel.com/sites/landingpage/IntrinsicsGuide/
include (CheckCXXSourceCompiles)
include (CMakePushCheckState)
cmake_push_check_state ()
# The variables HAVE_* determine if compiler has support for the flag to use the corresponding instruction set.
# The options ENABLE_* determine if we will tell compiler to actually use the corresponding instruction set if compiler can do it.
@ -137,178 +132,53 @@ elseif (ARCH_AMD64)
endif()
# ClickHouse can be cross-compiled (e.g. on an ARM host for x86) but it is also possible to build ClickHouse on x86 w/o AVX for x86 w/
# AVX. We only check that the compiler can emit certain SIMD instructions, we don't care if the host system is able to run the binary.
# Therefore, use check_cxx_source_compiles (= does the code compile+link?) instead of check_cxx_source_runs (= does the code
# compile+link+run).
# AVX. We only assume that the compiler can emit certain SIMD instructions, we don't care if the host system is able to run the binary.
SET (HAVE_SSSE3 1)
SET (HAVE_SSE41 1)
SET (HAVE_SSE42 1)
SET (HAVE_PCLMULQDQ 1)
SET (HAVE_POPCNT 1)
SET (HAVE_AVX 1)
SET (HAVE_AVX2 1)
SET (HAVE_AVX512 1)
SET (HAVE_AVX512_VBMI 1)
SET (HAVE_BMI 1)
SET (HAVE_BMI2 1)
set (TEST_FLAG "-mssse3")
set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
check_cxx_source_compiles("
#include <tmmintrin.h>
int main() {
__m64 a = _mm_abs_pi8(__m64());
(void)a;
return 0;
}
" HAVE_SSSE3)
if (HAVE_SSSE3 AND ENABLE_SSSE3)
set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
set (COMPILER_FLAGS "${COMPILER_FLAGS} -mssse3")
endif ()
set (TEST_FLAG "-msse4.1")
set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
check_cxx_source_compiles("
#include <smmintrin.h>
int main() {
auto a = _mm_insert_epi8(__m128i(), 0, 0);
(void)a;
return 0;
}
" HAVE_SSE41)
if (HAVE_SSE41 AND ENABLE_SSE41)
set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
set (COMPILER_FLAGS "${COMPILER_FLAGS} -msse4.1")
endif ()
set (TEST_FLAG "-msse4.2")
set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
check_cxx_source_compiles("
#include <nmmintrin.h>
int main() {
auto a = _mm_crc32_u64(0, 0);
(void)a;
return 0;
}
" HAVE_SSE42)
if (HAVE_SSE42 AND ENABLE_SSE42)
set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
set (COMPILER_FLAGS "${COMPILER_FLAGS} -msse4.2")
endif ()
set (TEST_FLAG "-mpclmul")
set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
check_cxx_source_compiles("
#include <wmmintrin.h>
int main() {
auto a = _mm_clmulepi64_si128(__m128i(), __m128i(), 0);
(void)a;
return 0;
}
" HAVE_PCLMULQDQ)
if (HAVE_PCLMULQDQ AND ENABLE_PCLMULQDQ)
set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
set (COMPILER_FLAGS "${COMPILER_FLAGS} -mpclmul")
endif ()
set (TEST_FLAG "-mpopcnt")
set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
check_cxx_source_compiles("
int main() {
auto a = __builtin_popcountll(0);
(void)a;
return 0;
}
" HAVE_POPCNT)
if (HAVE_POPCNT AND ENABLE_POPCNT)
set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
set (COMPILER_FLAGS "${COMPILER_FLAGS} -mpopcnt")
endif ()
set (TEST_FLAG "-mavx")
set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
check_cxx_source_compiles("
#include <immintrin.h>
int main() {
auto a = _mm256_insert_epi8(__m256i(), 0, 0);
(void)a;
return 0;
}
" HAVE_AVX)
if (HAVE_AVX AND ENABLE_AVX)
set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
set (COMPILER_FLAGS "${COMPILER_FLAGS} -mavx")
endif ()
set (TEST_FLAG "-mavx2")
set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
check_cxx_source_compiles("
#include <immintrin.h>
int main() {
auto a = _mm256_add_epi16(__m256i(), __m256i());
(void)a;
return 0;
}
" HAVE_AVX2)
if (HAVE_AVX2 AND ENABLE_AVX2)
set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
set (COMPILER_FLAGS "${COMPILER_FLAGS} -mavx2")
endif ()
set (TEST_FLAG "-mavx512f -mavx512bw -mavx512vl")
set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
check_cxx_source_compiles("
#include <immintrin.h>
int main() {
auto a = _mm512_setzero_epi32();
(void)a;
auto b = _mm512_add_epi16(__m512i(), __m512i());
(void)b;
auto c = _mm_cmp_epi8_mask(__m128i(), __m128i(), 0);
(void)c;
return 0;
}
" HAVE_AVX512)
if (HAVE_AVX512 AND ENABLE_AVX512)
set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
set (COMPILER_FLAGS "${COMPILER_FLAGS} -mavx512f -mavx512bw -mavx512vl")
endif ()
set (TEST_FLAG "-mavx512vbmi")
set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
check_cxx_source_compiles("
#include <immintrin.h>
int main() {
auto a = _mm512_permutexvar_epi8(__m512i(), __m512i());
(void)a;
return 0;
}
" HAVE_AVX512_VBMI)
if (HAVE_AVX512 AND ENABLE_AVX512 AND HAVE_AVX512_VBMI AND ENABLE_AVX512_VBMI)
set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
set (COMPILER_FLAGS "${COMPILER_FLAGS} -mavx512vbmi")
endif ()
set (TEST_FLAG "-mbmi")
set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
check_cxx_source_compiles("
#include <immintrin.h>
int main() {
auto a = _blsr_u32(0);
(void)a;
return 0;
}
" HAVE_BMI)
if (HAVE_BMI AND ENABLE_BMI)
set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
set (COMPILER_FLAGS "${COMPILER_FLAGS} -mbmi")
endif ()
set (TEST_FLAG "-mbmi2")
set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
check_cxx_source_compiles("
#include <immintrin.h>
int main() {
auto a = _pdep_u64(0, 0);
(void)a;
return 0;
}
" HAVE_BMI2)
if (HAVE_BMI2 AND HAVE_AVX2 AND ENABLE_AVX2 AND ENABLE_BMI2)
set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
set (COMPILER_FLAGS "${COMPILER_FLAGS} -mbmi2")
endif ()
# Limit avx2/avx512 flag for specific source build
set (X86_INTRINSICS_FLAGS "")
if (ENABLE_AVX2_FOR_SPEC_OP)
if (HAVE_BMI)
set (X86_INTRINSICS_FLAGS "${X86_INTRINSICS_FLAGS} -mbmi")
endif ()
if (HAVE_AVX AND HAVE_AVX2)
set (X86_INTRINSICS_FLAGS "${X86_INTRINSICS_FLAGS} -mavx -mavx2")
endif ()
endif ()
if (ENABLE_AVX512_FOR_SPEC_OP)
set (X86_INTRINSICS_FLAGS "")
if (HAVE_BMI)
@ -321,5 +191,3 @@ elseif (ARCH_AMD64)
else ()
# RISC-V + exotic platforms
endif ()
cmake_pop_check_state ()

2
contrib/arrow vendored

@ -1 +1 @@
Subproject commit 1d93838f69a802639ca144ea5704a98e2481810d
Subproject commit ba5c67934e8274d649befcffab56731632dc5253

60
contrib/arrow-cmake/CMakeLists.txt
View File

@ -109,7 +109,6 @@ set (ORC_CXX_HAS_CSTDINT 1)
set (ORC_CXX_HAS_THREAD_LOCAL 1)
include(orc_check.cmake)
configure_file("${ORC_INCLUDE_DIR}/orc/orc-config.hh.in" "${ORC_BUILD_INCLUDE_DIR}/orc/orc-config.hh")
configure_file("${ORC_SOURCE_SRC_DIR}/Adaptor.hh.in" "${ORC_BUILD_INCLUDE_DIR}/Adaptor.hh")
@ -198,7 +197,9 @@ target_link_libraries(_orc PRIVATE
ch_contrib::snappy
ch_contrib::zlib
ch_contrib::zstd)
target_include_directories(_orc SYSTEM BEFORE PUBLIC ${ORC_INCLUDE_DIR})
target_include_directories(_orc SYSTEM BEFORE PUBLIC
${ORC_INCLUDE_DIR}
"${ClickHouse_SOURCE_DIR}/contrib/arrow-cmake/cpp/src/orc/c++/include")
target_include_directories(_orc SYSTEM BEFORE PUBLIC ${ORC_BUILD_INCLUDE_DIR})
target_include_directories(_orc SYSTEM PRIVATE
${ORC_SOURCE_SRC_DIR}
@ -212,8 +213,6 @@ target_include_directories(_orc SYSTEM PRIVATE
set(LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src/arrow")
configure_file("${LIBRARY_DIR}/util/config.h.cmake" "${CMAKE_CURRENT_BINARY_DIR}/cpp/src/arrow/util/config.h")
# arrow/cpp/src/arrow/CMakeLists.txt (ARROW_SRCS + ARROW_COMPUTE + ARROW_IPC)
set(ARROW_SRCS
"${LIBRARY_DIR}/array/array_base.cc"
@ -230,6 +229,8 @@ set(ARROW_SRCS
"${LIBRARY_DIR}/array/builder_nested.cc"
"${LIBRARY_DIR}/array/builder_primitive.cc"
"${LIBRARY_DIR}/array/builder_union.cc"
"${LIBRARY_DIR}/array/builder_run_end.cc"
"${LIBRARY_DIR}/array/array_run_end.cc"
"${LIBRARY_DIR}/array/concatenate.cc"
"${LIBRARY_DIR}/array/data.cc"
"${LIBRARY_DIR}/array/diff.cc"
@ -309,9 +310,12 @@ set(ARROW_SRCS
"${LIBRARY_DIR}/util/debug.cc"
"${LIBRARY_DIR}/util/tracing.cc"
"${LIBRARY_DIR}/util/atfork_internal.cc"
"${LIBRARY_DIR}/util/crc32.cc"
"${LIBRARY_DIR}/util/hashing.cc"
"${LIBRARY_DIR}/util/ree_util.cc"
"${LIBRARY_DIR}/util/union_util.cc"
"${LIBRARY_DIR}/vendored/base64.cpp"
"${LIBRARY_DIR}/vendored/datetime/tz.cpp"
"${LIBRARY_DIR}/vendored/musl/strptime.c"
"${LIBRARY_DIR}/vendored/uriparser/UriCommon.c"
"${LIBRARY_DIR}/vendored/uriparser/UriCompare.c"
@ -328,39 +332,20 @@ set(ARROW_SRCS
"${LIBRARY_DIR}/vendored/uriparser/UriRecompose.c"
"${LIBRARY_DIR}/vendored/uriparser/UriResolve.c"
"${LIBRARY_DIR}/vendored/uriparser/UriShorten.c"
"${LIBRARY_DIR}/vendored/double-conversion/bignum.cc"
"${LIBRARY_DIR}/vendored/double-conversion/bignum-dtoa.cc"
"${LIBRARY_DIR}/vendored/double-conversion/cached-powers.cc"
"${LIBRARY_DIR}/vendored/double-conversion/double-to-string.cc"
"${LIBRARY_DIR}/vendored/double-conversion/fast-dtoa.cc"
"${LIBRARY_DIR}/vendored/double-conversion/fixed-dtoa.cc"
"${LIBRARY_DIR}/vendored/double-conversion/string-to-double.cc"
"${LIBRARY_DIR}/vendored/double-conversion/strtod.cc"
"${LIBRARY_DIR}/compute/api_aggregate.cc"
"${LIBRARY_DIR}/compute/api_scalar.cc"
"${LIBRARY_DIR}/compute/api_vector.cc"
"${LIBRARY_DIR}/compute/cast.cc"
"${LIBRARY_DIR}/compute/exec.cc"
"${LIBRARY_DIR}/compute/exec/accumulation_queue.cc"
"${LIBRARY_DIR}/compute/exec/accumulation_queue.h"
"${LIBRARY_DIR}/compute/exec/aggregate.cc"
"${LIBRARY_DIR}/compute/exec/aggregate_node.cc"
"${LIBRARY_DIR}/compute/exec/asof_join_node.cc"
"${LIBRARY_DIR}/compute/exec/bloom_filter.cc"
"${LIBRARY_DIR}/compute/exec/exec_plan.cc"
"${LIBRARY_DIR}/compute/exec/expression.cc"
"${LIBRARY_DIR}/compute/exec/filter_node.cc"
"${LIBRARY_DIR}/compute/exec/hash_join.cc"
"${LIBRARY_DIR}/compute/exec/hash_join_dict.cc"
"${LIBRARY_DIR}/compute/exec/hash_join_node.cc"
"${LIBRARY_DIR}/compute/exec/key_hash.cc"
"${LIBRARY_DIR}/compute/exec/key_map.cc"
"${LIBRARY_DIR}/compute/exec/map_node.cc"
"${LIBRARY_DIR}/compute/exec/options.cc"
"${LIBRARY_DIR}/compute/exec/order_by_impl.cc"
"${LIBRARY_DIR}/compute/exec/partition_util.cc"
"${LIBRARY_DIR}/compute/exec/project_node.cc"
"${LIBRARY_DIR}/compute/exec/query_context.cc"
"${LIBRARY_DIR}/compute/exec/sink_node.cc"
"${LIBRARY_DIR}/compute/exec/source_node.cc"
"${LIBRARY_DIR}/compute/exec/swiss_join.cc"
"${LIBRARY_DIR}/compute/exec/task_util.cc"
"${LIBRARY_DIR}/compute/exec/tpch_node.cc"
"${LIBRARY_DIR}/compute/exec/union_node.cc"
"${LIBRARY_DIR}/compute/exec/util.cc"
"${LIBRARY_DIR}/compute/function.cc"
"${LIBRARY_DIR}/compute/function_internal.cc"
"${LIBRARY_DIR}/compute/kernel.cc"
@ -403,8 +388,13 @@ set(ARROW_SRCS
"${LIBRARY_DIR}/compute/kernels/vector_select_k.cc"
"${LIBRARY_DIR}/compute/kernels/vector_selection.cc"
"${LIBRARY_DIR}/compute/kernels/vector_sort.cc"
"${LIBRARY_DIR}/compute/kernels/vector_selection_internal.cc"
"${LIBRARY_DIR}/compute/kernels/vector_selection_filter_internal.cc"
"${LIBRARY_DIR}/compute/kernels/vector_selection_take_internal.cc"
"${LIBRARY_DIR}/compute/light_array.cc"
"${LIBRARY_DIR}/compute/registry.cc"
"${LIBRARY_DIR}/compute/expression.cc"
"${LIBRARY_DIR}/compute/ordering.cc"
"${LIBRARY_DIR}/compute/row/compare_internal.cc"
"${LIBRARY_DIR}/compute/row/encode_internal.cc"
"${LIBRARY_DIR}/compute/row/grouper.cc"
@ -459,7 +449,7 @@ target_link_libraries(_arrow PUBLIC _orc)
add_dependencies(_arrow protoc)
target_include_directories(_arrow SYSTEM BEFORE PUBLIC ${ARROW_SRC_DIR})
target_include_directories(_arrow SYSTEM BEFORE PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/cpp/src")
target_include_directories(_arrow SYSTEM BEFORE PUBLIC "${ClickHouse_SOURCE_DIR}/contrib/arrow-cmake/cpp/src")
target_include_directories(_arrow SYSTEM PRIVATE ${ARROW_SRC_DIR})
target_include_directories(_arrow SYSTEM PRIVATE ${HDFS_INCLUDE_DIR})
@ -488,10 +478,10 @@ set(PARQUET_SRCS
"${LIBRARY_DIR}/exception.cc"
"${LIBRARY_DIR}/file_reader.cc"
"${LIBRARY_DIR}/file_writer.cc"
"${LIBRARY_DIR}/page_index.cc"
"${LIBRARY_DIR}/level_conversion.cc"
"${LIBRARY_DIR}/level_comparison.cc"
"${LIBRARY_DIR}/metadata.cc"
"${LIBRARY_DIR}/murmur3.cc"
"${LIBRARY_DIR}/platform.cc"
"${LIBRARY_DIR}/printer.cc"
"${LIBRARY_DIR}/properties.cc"
@ -500,6 +490,8 @@ set(PARQUET_SRCS
"${LIBRARY_DIR}/stream_reader.cc"
"${LIBRARY_DIR}/stream_writer.cc"
"${LIBRARY_DIR}/types.cc"
"${LIBRARY_DIR}/bloom_filter_reader.cc"
"${LIBRARY_DIR}/xxhasher.cc"
"${GEN_LIBRARY_DIR}/parquet_constants.cpp"
"${GEN_LIBRARY_DIR}/parquet_types.cpp"

61
contrib/arrow-cmake/cpp/src/arrow/util/config.h Normal file
View File

@ -0,0 +1,61 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#define ARROW_VERSION_MAJOR 11
#define ARROW_VERSION_MINOR 0
#define ARROW_VERSION_PATCH 0
#define ARROW_VERSION ((ARROW_VERSION_MAJOR * 1000) + ARROW_VERSION_MINOR) * 1000 + ARROW_VERSION_PATCH
#define ARROW_VERSION_STRING "11.0.0"
#define ARROW_SO_VERSION "1100"
#define ARROW_FULL_SO_VERSION "1100.0.0"
#define ARROW_CXX_COMPILER_ID "Clang"
#define ARROW_CXX_COMPILER_VERSION "ClickHouse"
#define ARROW_CXX_COMPILER_FLAGS ""
#define ARROW_BUILD_TYPE ""
#define ARROW_GIT_ID ""
#define ARROW_GIT_DESCRIPTION ""
#define ARROW_PACKAGE_KIND ""
/* #undef ARROW_COMPUTE */
/* #undef ARROW_CSV */
/* #undef ARROW_CUDA */
/* #undef ARROW_DATASET */
/* #undef ARROW_FILESYSTEM */
/* #undef ARROW_FLIGHT */
/* #undef ARROW_FLIGHT_SQL */
/* #undef ARROW_IPC */
/* #undef ARROW_JEMALLOC */
/* #undef ARROW_JEMALLOC_VENDORED */
/* #undef ARROW_JSON */
/* #undef ARROW_ORC */
/* #undef ARROW_PARQUET */
/* #undef ARROW_SUBSTRAIT */
/* #undef ARROW_GCS */
/* #undef ARROW_S3 */
/* #undef ARROW_USE_NATIVE_INT128 */
/* #undef ARROW_WITH_MUSL */
/* #undef ARROW_WITH_OPENTELEMETRY */
/* #undef ARROW_WITH_UCX */
/* #undef GRPCPP_PP_INCLUDE */

38
contrib/arrow-cmake/cpp/src/orc/c++/include/orc/orc-config.hh Normal file
View File

@ -0,0 +1,38 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef ORC_CONFIG_HH
#define ORC_CONFIG_HH
#define ORC_VERSION ""
#define ORC_CXX_HAS_CSTDINT
#ifdef ORC_CXX_HAS_CSTDINT
#include <cstdint>
#else
#include <stdint.h>
#endif
// Following MACROS should be keeped for backward compatibility.
#define ORC_NOEXCEPT noexcept
#define ORC_NULLPTR nullptr
#define ORC_OVERRIDE override
#define ORC_UNIQUE_PTR std::unique_ptr
#endif

113
contrib/aws-cmake/AwsFeatureTests.cmake
View File

@ -1,114 +1,13 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0.
include(CheckCSourceRuns)
option(USE_CPU_EXTENSIONS "Whenever possible, use functions optimized for CPUs with specific extensions (ex: SSE, AVX)." ON)
# In the current (11/2/21) state of mingw64, the packaged gcc is not capable of emitting properly aligned avx2 instructions under certain circumstances.
# This leads to crashes for windows builds using mingw64 when invoking the avx2-enabled versions of certain functions. Until we can find a better
# work-around, disable avx2 (and all other extensions) in mingw builds.
#
# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=54412
#
if (MINGW)
message(STATUS "MINGW detected! Disabling avx2 and other CPU extensions")
set(USE_CPU_EXTENSIONS OFF)
if (ARCH_AMD64)
set (AWS_ARCH_INTEL 1)
elseif (ARCH_AARCH64)
set (AWS_ARCH_ARM64 1)
endif ()
if(NOT CMAKE_CROSSCOMPILING)
check_c_source_runs("
#include <stdbool.h>
bool foo(int a, int b, int *c) {
return __builtin_mul_overflow(a, b, c);
}
int main() {
int out;
if (foo(1, 2, &out)) {
return 0;
}
return 0;
}" AWS_HAVE_GCC_OVERFLOW_MATH_EXTENSIONS)
if (USE_CPU_EXTENSIONS)
check_c_source_runs("
int main() {
int foo = 42;
_mulx_u32(1, 2, &foo);
return foo != 2;
}" AWS_HAVE_MSVC_MULX)
endif()
endif()
check_c_source_compiles("
#include <Windows.h>
#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP)
int main() {
return 0;
}
#else
it's not windows desktop
#endif
" AWS_HAVE_WINAPI_DESKTOP)
check_c_source_compiles("
int main() {
#if !(defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86))
# error \"not intel\"
#endif
return 0;
}
" AWS_ARCH_INTEL)
check_c_source_compiles("
int main() {
#if !(defined(__aarch64__) || defined(_M_ARM64))
# error \"not arm64\"
#endif
return 0;
}
" AWS_ARCH_ARM64)
check_c_source_compiles("
int main() {
#if !(defined(__arm__) || defined(_M_ARM))
# error \"not arm\"
#endif
return 0;
}
" AWS_ARCH_ARM32)
check_c_source_compiles("
int main() {
int foo = 42, bar = 24;
__asm__ __volatile__(\"\":\"=r\"(foo):\"r\"(bar):\"memory\");
}" AWS_HAVE_GCC_INLINE_ASM)
check_c_source_compiles("
#include <sys/auxv.h>
int main() {
#ifdef __linux__
getauxval(AT_HWCAP);
getauxval(AT_HWCAP2);
#endif
return 0;
}" AWS_HAVE_AUXV)
string(REGEX MATCH "^(aarch64|arm)" ARM_CPU "${CMAKE_SYSTEM_PROCESSOR}")
if(NOT LEGACY_COMPILER_SUPPORT OR ARM_CPU)
check_c_source_compiles("
#include <execinfo.h>
int main() {
backtrace(NULL, 0);
return 0;
}" AWS_HAVE_EXECINFO)
endif()
check_c_source_compiles("
#include <linux/if_link.h>
int main() {
return 1;
}" AWS_HAVE_LINUX_IF_LINK_H)
set (AWS_HAVE_GCC_INLINE_ASM 1)
set (AWS_HAVE_AUXV 1)

47
contrib/aws-cmake/AwsSIMD.cmake
View File

@ -1,55 +1,14 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0.
include(CheckCCompilerFlag)
include(CheckIncludeFile)
if (USE_CPU_EXTENSIONS)
if (MSVC)
check_c_compiler_flag("/arch:AVX2" HAVE_M_AVX2_FLAG)
if (HAVE_M_AVX2_FLAG)
set(AVX2_CFLAGS "/arch:AVX2")
endif()
else()
check_c_compiler_flag(-mavx2 HAVE_M_AVX2_FLAG)
if (HAVE_M_AVX2_FLAG)
if (HAVE_AVX2)
set (AVX2_CFLAGS "-mavx -mavx2")
set (HAVE_AVX2_INTRINSICS 1)
set (HAVE_MM256_EXTRACT_EPI64 1)
endif()
endif()
cmake_push_check_state()
set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${AVX2_CFLAGS}")
check_c_source_compiles("
#include <immintrin.h>
#include <emmintrin.h>
#include <string.h>
int main() {
__m256i vec;
memset(&vec, 0, sizeof(vec));
_mm256_shuffle_epi8(vec, vec);
_mm256_set_epi32(1,2,3,4,5,6,7,8);
_mm256_permutevar8x32_epi32(vec, vec);
return 0;
}" HAVE_AVX2_INTRINSICS)
check_c_source_compiles("
#include <immintrin.h>
#include <string.h>
int main() {
__m256i vec;
memset(&vec, 0, sizeof(vec));
return (int)_mm256_extract_epi64(vec, 2);
}" HAVE_MM256_EXTRACT_EPI64)
cmake_pop_check_state()
endif() # USE_CPU_EXTENSIONS
macro(simd_add_definition_if target definition)
if(${definition})
target_compile_definitions(${target} PRIVATE -D${definition})

45
contrib/aws-cmake/AwsThreadAffinity.cmake
View File

@ -1,50 +1,9 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0.
include(CheckSymbolExists)
# Check if the platform supports setting thread affinity
# (important for hitting full NIC entitlement on NUMA architectures)
function(aws_set_thread_affinity_method target)
# Non-POSIX, Android, and Apple platforms do not support thread affinity.
if (NOT UNIX OR ANDROID OR APPLE)
target_compile_definitions(${target} PRIVATE
-DAWS_AFFINITY_METHOD=AWS_AFFINITY_METHOD_NONE)
return()
endif()
cmake_push_check_state()
list(APPEND CMAKE_REQUIRED_DEFINITIONS -D_GNU_SOURCE)
list(APPEND CMAKE_REQUIRED_LIBRARIES pthread)
set(headers "pthread.h")
# BSDs put nonportable pthread declarations in a separate header.
if(CMAKE_SYSTEM_NAME MATCHES BSD)
set(headers "${headers};pthread_np.h")
endif()
# Using pthread attrs is the preferred method, but is glibc-specific.
check_symbol_exists(pthread_attr_setaffinity_np "${headers}" USE_PTHREAD_ATTR_SETAFFINITY)
if (USE_PTHREAD_ATTR_SETAFFINITY)
target_compile_definitions(${target} PRIVATE
-DAWS_AFFINITY_METHOD=AWS_AFFINITY_METHOD_PTHREAD_ATTR)
return()
endif()
# This method is still nonportable, but is supported by musl and BSDs.
check_symbol_exists(pthread_setaffinity_np "${headers}" USE_PTHREAD_SETAFFINITY)
if (USE_PTHREAD_SETAFFINITY)
target_compile_definitions(${target} PRIVATE
-DAWS_AFFINITY_METHOD=AWS_AFFINITY_METHOD_PTHREAD)
return()
endif()
# If we got here, we expected thread affinity support but didn't find it.
# We still build with degraded NUMA performance, but show a warning.
message(WARNING "No supported method for setting thread affinity")
target_compile_definitions(${target} PRIVATE
-DAWS_AFFINITY_METHOD=AWS_AFFINITY_METHOD_NONE)
cmake_pop_check_state()
# This code has been cut, because I don't care about it.
target_compile_definitions(${target} PRIVATE -DAWS_AFFINITY_METHOD=AWS_AFFINITY_METHOD_NONE)
endfunction()

50
contrib/aws-cmake/AwsThreadName.cmake
View File

@ -1,61 +1,13 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0.
include(CheckSymbolExists)
# Check how the platform supports setting thread name
function(aws_set_thread_name_method target)
if (WINDOWS)
# On Windows we do a runtime check, instead of compile-time check
return()
elseif (APPLE)
if (APPLE)
# All Apple platforms we support have the same function, so no need for compile-time check.
return()
endif()
cmake_push_check_state()
list(APPEND CMAKE_REQUIRED_DEFINITIONS -D_GNU_SOURCE)
list(APPEND CMAKE_REQUIRED_LIBRARIES pthread)
# The start of the test program
set(c_source_start "
#define _GNU_SOURCE
#include <pthread.h>
#if defined(__FreeBSD__) || defined(__NETBSD__)
#include <pthread_np.h>
#endif
int main() {
pthread_t thread_id;
")
# The end of the test program
set(c_source_end "}")
# pthread_setname_np() usually takes 2 args
check_c_source_compiles("
${c_source_start}
pthread_setname_np(thread_id, \"asdf\");
${c_source_end}"
PTHREAD_SETNAME_TAKES_2ARGS)
if (PTHREAD_SETNAME_TAKES_2ARGS)
target_compile_definitions(${target} PRIVATE -DAWS_PTHREAD_SETNAME_TAKES_2ARGS)
return()
endif()
# But on NetBSD it takes 3!
check_c_source_compiles("
${c_source_start}
pthread_setname_np(thread_id, \"asdf\", NULL);
${c_source_end}
" PTHREAD_SETNAME_TAKES_3ARGS)
if (PTHREAD_SETNAME_TAKES_3ARGS)
target_compile_definitions(${target} PRIVATE -DAWS_PTHREAD_SETNAME_TAKES_3ARGS)
return()
endif()
# And on many older/weirder platforms it's just not supported
cmake_pop_check_state()
endfunction()

2
contrib/grpc vendored

@ -1 +1 @@
Subproject commit b723ecae0991bb873fe87a595dfb187178733fde
Subproject commit 6e5e645de7cb0604e3ad4ba55abff2eca38c1803

2
contrib/libcxx-cmake/CMakeLists.txt
View File

@ -1,5 +1,3 @@
include(CheckCXXCompilerFlag)
set(LIBCXX_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/llvm-project/libcxx")
set(SRCS

19
contrib/libunwind-cmake/CMakeLists.txt
View File

@ -1,6 +1,3 @@
include(CheckCCompilerFlag)
include(CheckCXXCompilerFlag)
set(LIBUNWIND_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libunwind")
set(LIBUNWIND_CXX_SOURCES
@ -48,27 +45,11 @@ target_compile_definitions(unwind PRIVATE -D_LIBUNWIND_NO_HEAP=1 -D_DEBUG -D_LIB
# and disable sanitizers (otherwise infinite loop may happen)
target_compile_options(unwind PRIVATE -O3 -fno-exceptions -funwind-tables -fno-sanitize=all $<$<COMPILE_LANGUAGE:CXX>:-nostdinc++ -fno-rtti>)
check_c_compiler_flag(-Wunused-but-set-variable HAVE_WARNING_UNUSED_BUT_SET_VARIABLE)
if (HAVE_WARNING_UNUSED_BUT_SET_VARIABLE)
target_compile_options(unwind PRIVATE -Wno-unused-but-set-variable)
endif ()
check_cxx_compiler_flag(-Wmissing-attributes HAVE_WARNING_MISSING_ATTRIBUTES)
if (HAVE_WARNING_MISSING_ATTRIBUTES)
target_compile_options(unwind PRIVATE -Wno-missing-attributes)
endif ()
check_cxx_compiler_flag(-Wmaybe-uninitialized HAVE_WARNING_MAYBE_UNINITIALIZED)
if (HAVE_WARNING_MAYBE_UNINITIALIZED)
target_compile_options(unwind PRIVATE -Wno-maybe-uninitialized)
endif ()
# The library is using register variables that are bound to specific registers
# Example: DwarfInstructions.hpp: register unsigned long long x16 __asm("x16") = cfa;
check_cxx_compiler_flag(-Wregister HAVE_WARNING_REGISTER)
if (HAVE_WARNING_REGISTER)
target_compile_options(unwind PRIVATE "$<$<COMPILE_LANGUAGE:CXX>:-Wno-register>")
endif ()
install(
TARGETS unwind

74
contrib/rocksdb-cmake/CMakeLists.txt
View File

@ -76,7 +76,6 @@ else()
endif()
endif()
include(CheckCCompilerFlag)
if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64")
if(POWER9)
set(HAS_POWER9 1)
@ -88,21 +87,12 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64")
endif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64")
if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64|arm64|ARM64")
CHECK_C_COMPILER_FLAG("-march=armv8-a+crc+crypto" HAS_ARMV8_CRC)
if(HAS_ARMV8_CRC)
message(STATUS " HAS_ARMV8_CRC yes")
set(HAS_ARMV8_CRC 1)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=armv8-a+crc+crypto -Wno-unused-function")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=armv8-a+crc+crypto -Wno-unused-function")
endif(HAS_ARMV8_CRC)
endif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64|arm64|ARM64")
include(CheckCXXSourceCompiles)
if(NOT MSVC)
set(CMAKE_REQUIRED_FLAGS "-msse4.2 -mpclmul")
endif()
unset(CMAKE_REQUIRED_FLAGS)
if(HAVE_SSE42)
add_definitions(-DHAVE_SSE42)
add_definitions(-DHAVE_PCLMUL)
@ -121,75 +111,18 @@ elseif(CMAKE_SYSTEM_NAME MATCHES "Linux")
add_definitions(-DOS_LINUX)
elseif(CMAKE_SYSTEM_NAME MATCHES "SunOS")
add_definitions(-DOS_SOLARIS)
elseif(CMAKE_SYSTEM_NAME MATCHES "kFreeBSD")
add_definitions(-DOS_GNU_KFREEBSD)
elseif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
add_definitions(-DOS_FREEBSD)
elseif(CMAKE_SYSTEM_NAME MATCHES "NetBSD")
add_definitions(-DOS_NETBSD)
elseif(CMAKE_SYSTEM_NAME MATCHES "OpenBSD")
add_definitions(-DOS_OPENBSD)
elseif(CMAKE_SYSTEM_NAME MATCHES "DragonFly")
add_definitions(-DOS_DRAGONFLYBSD)
elseif(CMAKE_SYSTEM_NAME MATCHES "Android")
add_definitions(-DOS_ANDROID)
elseif(CMAKE_SYSTEM_NAME MATCHES "Windows")
add_definitions(-DWIN32 -DOS_WIN -D_MBCS -DWIN64 -DNOMINMAX)
if(MINGW)
add_definitions(-D_WIN32_WINNT=_WIN32_WINNT_VISTA)
endif()
endif()
if(NOT WIN32)
add_definitions(-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX)
endif()
option(WITH_FALLOCATE "build with fallocate" ON)
if(WITH_FALLOCATE)
CHECK_C_SOURCE_COMPILES("
#include <fcntl.h>
#include <linux/falloc.h>
int main() {
int fd = open(\"/dev/null\", 0);
fallocate(fd, FALLOC_FL_KEEP_SIZE, 0, 1024);
}
" HAVE_FALLOCATE)
if(HAVE_FALLOCATE)
add_definitions(-DROCKSDB_FALLOCATE_PRESENT)
endif()
endif()
CHECK_C_SOURCE_COMPILES("
#include <fcntl.h>
int main() {
int fd = open(\"/dev/null\", 0);
sync_file_range(fd, 0, 1024, SYNC_FILE_RANGE_WRITE);
}
" HAVE_SYNC_FILE_RANGE_WRITE)
if(HAVE_SYNC_FILE_RANGE_WRITE)
add_definitions(-DROCKSDB_RANGESYNC_PRESENT)
endif()
CHECK_C_SOURCE_COMPILES("
#include <pthread.h>
int main() {
(void) PTHREAD_MUTEX_ADAPTIVE_NP;
}
" HAVE_PTHREAD_MUTEX_ADAPTIVE_NP)
if(HAVE_PTHREAD_MUTEX_ADAPTIVE_NP)
if (OS_LINUX OR OS_FREEBSD)
add_definitions(-DROCKSDB_PTHREAD_ADAPTIVE_MUTEX)
endif()
include(CheckCXXSymbolExists)
if (OS_FREEBSD)
check_cxx_symbol_exists(malloc_usable_size "${ROCKSDB_SOURCE_DIR}/malloc_np.h" HAVE_MALLOC_USABLE_SIZE)
else()
check_cxx_symbol_exists(malloc_usable_size "${ROCKSDB_SOURCE_DIR}/malloc.h" HAVE_MALLOC_USABLE_SIZE)
endif()
if(HAVE_MALLOC_USABLE_SIZE)
add_definitions(-DROCKSDB_MALLOC_USABLE_SIZE)
endif()
if (OS_LINUX)
add_definitions(-DROCKSDB_SCHED_GETCPU_PRESENT)
add_definitions(-DROCKSDB_AUXV_SYSAUXV_PRESENT)
@ -204,7 +137,6 @@ include_directories("${ROCKSDB_SOURCE_DIR}/include")
if(WITH_FOLLY_DISTRIBUTED_MUTEX)
include_directories("${ROCKSDB_SOURCE_DIR}/third-party/folly")
endif()
find_package(Threads REQUIRED)
# Main library source code
@ -497,7 +429,7 @@ set(SOURCES
${ROCKSDB_SOURCE_DIR}/utilities/transactions/lock/range/range_tree/lib/util/memarena.cc
rocksdb_build_version.cc)
if(HAVE_SSE42 AND NOT MSVC)
if(HAVE_SSE42)
set_source_files_properties(
"${ROCKSDB_SOURCE_DIR}/util/crc32c.cc"
PROPERTIES COMPILE_FLAGS "-msse4.2 -mpclmul")

2
contrib/xz-cmake/CMakeLists.txt
View File

@ -98,8 +98,6 @@ if (ARCH_S390X)
add_compile_definitions(WORDS_BIGENDIAN)
endif ()
find_package(Threads REQUIRED)
add_library(_liblzma
${SRC_DIR}/src/common/mythread.h

4
docker/docs/builder/Dockerfile
View File

@ -4,8 +4,8 @@ FROM node:16-alpine
RUN apk add --no-cache git openssh bash
# At this point we want to really update /opt/clickhouse-docs
# despite the cached images
# At this point we want to really update /opt/clickhouse-docs directory
# So we reset the cache
ARG CACHE_INVALIDATOR=0
RUN git clone https://github.com/ClickHouse/clickhouse-docs.git \

2
docker/test/stateless/stress_tests.lib
View File

@ -74,8 +74,6 @@ function configure()
randomize_config_boolean_value use_compression zookeeper
randomize_config_boolean_value allow_experimental_block_number_column merge_tree_settings
# for clickhouse-server (via service)
echo "ASAN_OPTIONS='malloc_context_size=10 verbosity=1 allocator_release_to_os_interval_ms=10000'" >> /etc/environment
# for clickhouse-client

7
docs/en/operations/query-cache.md
View File

@ -169,7 +169,12 @@ Also, results of queries with non-deterministic functions are not cached by defa
[`getMacro()`](../sql-reference/functions/other-functions.md#getMacro) etc.
To force caching of results of queries with non-deterministic functions regardless, use setting
[query_cache_store_results_of_queries_with_nondeterministic_functions](settings/settings.md#query-cache-store-results-of-queries-with-nondeterministic-functions).
[query_cache_nondeterministic_function_handling](settings/settings.md#query-cache-nondeterministic-function-handling).
:::note
Prior to ClickHouse v23.11, setting 'query_cache_store_results_of_queries_with_nondeterministic_functions = 0 / 1' controlled whether
results of queries with non-deterministic results were cached. In newer ClickHouse versions, this setting is obsolete and has no effect.
:::
Finally, entries in the query cache are not shared between users due to security reasons. For example, user A must not be able to bypass a
row policy on a table by running the same query as another user B for whom no such policy exists. However, if necessary, cache entries can

22
docs/en/operations/settings/query-complexity.md
View File

@ -172,27 +172,7 @@ If you set `timeout_before_checking_execution_speed `to 0, ClickHouse will use c
## timeout_overflow_mode {#timeout-overflow-mode}
What to do if the query is run longer than `max_execution_time`: `throw` or `break`. By default, `throw`.
# max_execution_time_leaf
Similar semantic to `max_execution_time` but only apply on leaf node for distributed or remote queries.
For example, if we want to limit execution time on leaf node to `10s` but no limit on the initial node, instead of having `max_execution_time` in the nested subquery settings:
``` sql
SELECT count() FROM cluster(cluster, view(SELECT * FROM t SETTINGS max_execution_time = 10));
```
We can use `max_execution_time_leaf` as the query settings:
``` sql
SELECT count() FROM cluster(cluster, view(SELECT * FROM t)) SETTINGS max_execution_time_leaf = 10;
```
# timeout_overflow_mode_leaf
What to do when the query in leaf node run longer than `max_execution_time_leaf`: `throw` or `break`. By default, `throw`.
What to do if the query is run longer than ‘max_execution_time’: ‘throw’ or ‘break’. By default, throw.
## min_execution_speed {#min-execution-speed}

11
docs/en/operations/settings/settings.md
View File

@ -1657,16 +1657,17 @@ Possible values:
Default value: `1`.
## query_cache_store_results_of_queries_with_nondeterministic_functions {#query-cache-store-results-of-queries-with-nondeterministic-functions}
## query_cache_nondeterministic_function_handling {#query-cache-nondeterministic-function-handling}
If turned on, then results of `SELECT` queries with non-deterministic functions (e.g. `rand()`, `now()`) can be cached in the [query cache](../query-cache.md).
Controls how the [query cache](../query-cache.md) handles `SELECT` queries with non-deterministic functions like `rand()` or `now()`.
Possible values:
- 0 - Disabled
- 1 - Enabled
- `'throw'` - Throw an exception and don't cache the query result.
- `'save'` - Cache the query result.
- `'ignore'` - Don't cache the query result and don't throw an exception.
Default value: `0`.
Default value: `throw`.
## query_cache_min_query_runs {#query-cache-min-query-runs}

4
docs/en/sql-reference/dictionaries/index.md
View File

@ -1769,7 +1769,7 @@ Example of settings:
<password>qwerty123</password>
<keyspase>database_name</keyspase>
<column_family>table_name</column_family>
<allow_filering>1</allow_filering>
<allow_filtering>1</allow_filtering>
<partition_key_prefix>1</partition_key_prefix>
<consistency>One</consistency>
<where>"SomeColumn" = 42</where>
@ -1787,7 +1787,7 @@ Setting fields:
- `password` – Password of the Cassandra user.
- `keyspace` – Name of the keyspace (database).
- `column_family` – Name of the column family (table).
- `allow_filering` – Flag to allow or not potentially expensive conditions on clustering key columns. Default value is 1.
- `allow_filtering` – Flag to allow or not potentially expensive conditions on clustering key columns. Default value is 1.
- `partition_key_prefix` – Number of partition key columns in primary key of the Cassandra table. Required for compose key dictionaries. Order of key columns in the dictionary definition must be the same as in Cassandra. Default value is 1 (the first key column is a partition key and other key columns are clustering key).
- `consistency` – Consistency level. Possible values: `One`, `Two`, `Three`, `All`, `EachQuorum`, `Quorum`, `LocalQuorum`, `LocalOne`, `Serial`, `LocalSerial`. Default value is `One`.
- `where` – Optional selection criteria.

2
docs/en/sql-reference/functions/date-time-functions.md
View File

@ -1381,7 +1381,7 @@ toStartOfFifteenMinutes(toDateTime('2023-04-21 10:20:00')): 2023-04-21 10:15:00
toStartOfFifteenMinutes(toDateTime('2023-04-21 10:23:00')): 2023-04-21 10:15:00
```
## toStartOfInterval(time_or_data, INTERVAL x unit \[, time_zone\])
## toStartOfInterval(date_or_date_with_time, INTERVAL x unit \[, time_zone\])
This function generalizes other `toStartOf*()` functions. For example,
- `toStartOfInterval(t, INTERVAL 1 year)` returns the same as `toStartOfYear(t)`,

16
docs/en/sql-reference/window-functions/index.md
View File

@ -86,14 +86,14 @@ WINDOW window_name as ([[PARTITION BY grouping_column] [ORDER BY sorting_column]
These functions can be used only as a window function.
`row_number()` - Number the current row within its partition starting from 1.
`first_value(x)` - Return the first non-NULL value evaluated within its ordered frame.
`last_value(x)` - Return the last non-NULL value evaluated within its ordered frame.
`nth_value(x, offset)` - Return the first non-NULL value evaluated against the nth row (offset) in its ordered frame.
`rank()` - Rank the current row within its partition with gaps.
`dense_rank()` - Rank the current row within its partition without gaps.
`lagInFrame(x)` - Return a value evaluated at the row that is at a specified physical offset row before the current row within the ordered frame.
`leadInFrame(x)` - Return a value evaluated at the row that is offset rows after the current row within the ordered frame.
- `row_number()` - Number the current row within its partition starting from 1.
- `first_value(x)` - Return the first non-NULL value evaluated within its ordered frame.
- `last_value(x)` - Return the last non-NULL value evaluated within its ordered frame.
- `nth_value(x, offset)` - Return the first non-NULL value evaluated against the nth row (offset) in its ordered frame.
- `rank()` - Rank the current row within its partition with gaps.
- `dense_rank()` - Rank the current row within its partition without gaps.
- `lagInFrame(x)` - Return a value evaluated at the row that is at a specified physical offset row before the current row within the ordered frame.
- `leadInFrame(x)` - Return a value evaluated at the row that is offset rows after the current row within the ordered frame.
```text
PARTITION

4
docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md
View File

@ -723,7 +723,7 @@ SOURCE(REDIS(
<password>qwerty123</password>
<keyspase>database_name</keyspase>
<column_family>table_name</column_family>
<allow_filering>1</allow_filering>
<allow_filtering>1</allow_filtering>
<partition_key_prefix>1</partition_key_prefix>
<consistency>One</consistency>
<where>"SomeColumn" = 42</where>
@ -741,7 +741,7 @@ SOURCE(REDIS(
- `password` – пароль для соединения с Cassandra.
- `keyspace` – имя keyspace (база данных).
- `column_family` – имя семейства столбцов (таблица).
- `allow_filering` – флаг, разрешающий или не разрешающий потенциально дорогостоящие условия на кластеризации ключевых столбцов. Значение по умолчанию: 1.
- `allow_filtering` – флаг, разрешающий или не разрешающий потенциально дорогостоящие условия на кластеризации ключевых столбцов. Значение по умолчанию: 1.
- `partition_key_prefix` – количество партиций ключевых столбцов в первичном ключе таблицы Cassandra.
Необходимо для составления ключей словаря. Порядок ключевых столбцов в определении словаря должен быть таким же, как в Cassandra.
Значение по умолчанию: 1 (первый ключевой столбец - это ключ партицирования, остальные ключевые столбцы - ключи кластеризации).

5
programs/CMakeLists.txt
View File

@ -432,6 +432,11 @@ if (USE_BINARY_HASH)
add_custom_command(TARGET clickhouse POST_BUILD COMMAND ./clickhouse hash-binary > hash && ${OBJCOPY_PATH} --add-section .clickhouse.hash=hash clickhouse COMMENT "Adding section '.clickhouse.hash' to clickhouse binary" VERBATIM)
endif()
if (CHECK_LARGE_OBJECT_SIZES)
add_custom_command(TARGET clickhouse POST_BUILD
COMMAND "${CMAKE_SOURCE_DIR}/utils/check-style/check-large-objects.sh" "${CMAKE_BINARY_DIR}")
endif ()
if (SPLIT_DEBUG_SYMBOLS)
clickhouse_split_debug_symbols(TARGET clickhouse DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/${SPLITTED_DEBUG_SYMBOLS_DIR} BINARY_PATH clickhouse)
else()

5
programs/client/Client.cpp
View File

@ -17,18 +17,15 @@
#include "Core/Protocol.h"
#include "Parsers/formatAST.h"
#include <base/find_symbols.h>
#include <Access/AccessControl.h>
#include "config_version.h"
#include <Common/config_version.h>
#include <Common/Exception.h>
#include <Common/formatReadable.h>
#include <Common/TerminalSize.h>
#include <Common/Config/ConfigProcessor.h>
#include <Common/Config/getClientConfigPath.h>
#include <Core/QueryProcessingStage.h>
#include <Columns/ColumnString.h>
#include <Poco/Util/Application.h>

2
programs/keeper/Keeper.cpp
View File

@ -35,7 +35,7 @@
#include "Core/Defines.h"
#include "config.h"
#include "config_version.h"
#include <Common/config_version.h>
#include "config_tools.h"

2
programs/server/Server.cpp
View File

@ -98,7 +98,7 @@
#include <unordered_set>
#include "config.h"
#include "config_version.h"
#include <Common/config_version.h>
#if defined(OS_LINUX)
# include <cstdlib>

2
src/Access/KerberosInit.cpp
View File

@ -44,7 +44,7 @@ private:
krb5_ccache defcache = nullptr;
krb5_get_init_creds_opt * options = nullptr;
// Credentials structure including ticket, session key, and lifetime info.
krb5_creds my_creds;
krb5_creds my_creds {};
krb5_keytab keytab = nullptr;
krb5_principal defcache_princ = nullptr;
String fmtError(krb5_error_code code) const;

1
src/CMakeLists.txt
View File

@ -18,7 +18,6 @@ include (../cmake/version.cmake)
message (STATUS "Will build ${VERSION_FULL} revision ${VERSION_REVISION} ${VERSION_OFFICIAL}")
include (configure_config.cmake)
configure_file (Common/config.h.in ${CONFIG_INCLUDE_PATH}/config.h)
configure_file (Common/config_version.h.in ${CONFIG_INCLUDE_PATH}/config_version.h)
configure_file (Common/config_version.cpp.in ${CONFIG_INCLUDE_PATH}/config_version.cpp)
if (USE_DEBUG_HELPERS)

26
src/Client/ClientBase.cpp
View File

@ -77,7 +77,7 @@
#include <memory>
#include <unordered_map>
#include "config_version.h"
#include <Common/config_version.h>
#include "config.h"
namespace fs = std::filesystem;
@ -108,6 +108,7 @@ namespace ErrorCodes
extern const int FILE_ALREADY_EXISTS;
extern const int USER_SESSION_LIMIT_EXCEEDED;
extern const int NOT_IMPLEMENTED;
extern const int CANNOT_READ_FROM_FILE_DESCRIPTOR;
}
}
@ -1384,6 +1385,23 @@ void ClientBase::addMultiquery(std::string_view query, Arguments & common_argume
common_arguments.emplace_back(query);
}
namespace
{
bool isStdinNotEmptyAndValid(ReadBufferFromFileDescriptor & std_in)
{
try
{
return !std_in.eof();
}
catch (const Exception & e)
{
if (e.code() == ErrorCodes::CANNOT_READ_FROM_FILE_DESCRIPTOR)
return false;
throw;
}
}
}
void ClientBase::processInsertQuery(const String & query_to_execute, ASTPtr parsed_query)
{
@ -1403,7 +1421,7 @@ void ClientBase::processInsertQuery(const String & query_to_execute, ASTPtr pars
/// Process the query that requires transferring data blocks to the server.
const auto & parsed_insert_query = parsed_query->as<ASTInsertQuery &>();
if ((!parsed_insert_query.data && !parsed_insert_query.infile) && (is_interactive || (!stdin_is_a_tty && std_in.eof())))
if ((!parsed_insert_query.data && !parsed_insert_query.infile) && (is_interactive || (!stdin_is_a_tty && !isStdinNotEmptyAndValid(std_in))))
{
const auto & settings = global_context->getSettingsRef();
if (settings.throw_if_no_data_to_insert)
@ -1460,7 +1478,7 @@ void ClientBase::sendData(Block & sample, const ColumnsDescription & columns_des
if (!parsed_insert_query)
return;
bool have_data_in_stdin = !is_interactive && !stdin_is_a_tty && !std_in.eof();
bool have_data_in_stdin = !is_interactive && !stdin_is_a_tty && isStdinNotEmptyAndValid(std_in);
if (need_render_progress)
{
@ -1851,7 +1869,7 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin
if (is_async_insert_with_inlined_data)
{
bool have_data_in_stdin = !is_interactive && !stdin_is_a_tty && !std_in.eof();
bool have_data_in_stdin = !is_interactive && !stdin_is_a_tty && isStdinNotEmptyAndValid(std_in);
bool have_external_data = have_data_in_stdin || insert->infile;
if (have_external_data)

4
src/Client/Connection.cpp
View File

@ -35,7 +35,7 @@
#include <pcg_random.hpp>
#include <base/scope_guard.h>
#include "config_version.h"
#include <Common/config_version.h>
#include "config.h"
#if USE_SSL
@ -296,7 +296,7 @@ void Connection::sendHello()
"Parameters 'default_database', 'user' and 'password' must not contain ASCII control characters");
writeVarUInt(Protocol::Client::Hello, *out);
writeStringBinary((VERSION_NAME " ") + client_name, *out);
writeStringBinary(std::string(VERSION_NAME) + " " + client_name, *out);
writeVarUInt(VERSION_MAJOR, *out);
writeVarUInt(VERSION_MINOR, *out);
// NOTE For backward compatibility of the protocol, client cannot send its version_patch.

2
src/Common/ClickHouseRevision.cpp
View File

@ -1,5 +1,5 @@
#include <Common/ClickHouseRevision.h>
#include "config_version.h"
#include <Common/config_version.h>
namespace ClickHouseRevision
{

2
src/Common/ErrorCodes.cpp
View File

@ -583,7 +583,7 @@
M(701, CLUSTER_DOESNT_EXIST) \
M(702, CLIENT_INFO_DOES_NOT_MATCH) \
M(703, INVALID_IDENTIFIER) \
M(704, CANNOT_USE_QUERY_CACHE_WITH_NONDETERMINISTIC_FUNCTIONS) \
M(704, QUERY_CACHE_USED_WITH_NONDETERMINISTIC_FUNCTIONS) \
M(705, TABLE_NOT_EMPTY) \
M(706, LIBSSH_ERROR) \
M(999, KEEPER_EXCEPTION) \

2
src/Common/Exception.cpp
View File

@ -21,7 +21,7 @@
#include <Common/LockMemoryExceptionInThread.h>
#include <filesystem>
#include "config_version.h"
#include <Common/config_version.h>
namespace fs = std::filesystem;

10
src/Common/Exception.h
View File

@ -73,8 +73,8 @@ protected:
struct MessageMasked
{
std::string msg;
MessageMasked(const std::string & msg_);
MessageMasked(std::string && msg_);
explicit MessageMasked(const std::string & msg_);
explicit MessageMasked(std::string && msg_);
};
Exception(const MessageMasked & msg_masked, int code, bool remote_);
@ -123,7 +123,7 @@ public:
Exception(CreateFromSTDTag, const std::exception & exc);
Exception * clone() const override { return new Exception(*this); }
void rethrow() const override { throw *this; }
void rethrow() const override { throw *this; } // NOLINT
const char * name() const noexcept override { return "DB::Exception"; }
const char * what() const noexcept override { return message().data(); }
@ -181,7 +181,7 @@ public:
: Exception(msg, code), saved_errno(saved_errno_), path(path_) {}
ErrnoException * clone() const override { return new ErrnoException(*this); }
void rethrow() const override { throw *this; }
void rethrow() const override { throw *this; } // NOLINT
int getErrno() const { return saved_errno; }
std::optional<std::string> getPath() const { return path; }
@ -219,7 +219,7 @@ public:
void setFileName(const String & file_name_) { file_name = file_name_; }
Exception * clone() const override { return new ParsingException(*this); }
void rethrow() const override { throw *this; }
void rethrow() const override { throw *this; } // NOLINT
private:
ssize_t line_number{-1};

8
src/Common/LoggingFormatStringHelpers.h
View File

@ -106,8 +106,8 @@ template <typename T> constexpr std::string_view tryGetStaticFormatString(T && x
/// Most likely it was a string literal.
/// Unfortunately, there's no good way to check if something is a string literal.
/// But fmtlib requires a format string to be compile-time constant unless fmt::runtime is used.
static_assert(std::is_nothrow_convertible<T, const char * const>::value);
static_assert(!std::is_pointer<T>::value);
static_assert(std::is_nothrow_convertible_v<T, const char * const>);
static_assert(!std::is_pointer_v<T>);
return std::string_view(x);
}
}
@ -127,8 +127,8 @@ template<> struct ConstexprIfsAreNotIfdefs<true>
{
/// See tryGetStaticFormatString(...)
static_assert(!std::is_same_v<std::string, std::decay_t<T>>);
static_assert(std::is_nothrow_convertible<T, const char * const>::value);
static_assert(!std::is_pointer<T>::value);
static_assert(std::is_nothrow_convertible_v<T, const char * const>);
static_assert(!std::is_pointer_v<T>);
return std::string_view(x);
}

6
src/Common/NamedCollections/NamedCollectionUtils.cpp
View File

@ -229,6 +229,12 @@ public:
create_query.changes.emplace_back(name, value);
create_query.overridability = std::move(result_overridability_map);
if (create_query.changes.empty())
throw Exception(
ErrorCodes::BAD_ARGUMENTS,
"Named collection cannot be empty (collection name: {})",
query.collection_name);
writeCreateQueryToMetadata(
create_query,
getMetadataPath(query.collection_name),

1
src/Common/ProfileEvents.cpp
View File

@ -317,6 +317,7 @@ The server successfully detected this situation and will download merged part fr
\
M(CannotWriteToWriteBufferDiscard, "Number of stack traces dropped by query profiler or signal handler because pipe is full or cannot write to pipe.") \
M(QueryProfilerSignalOverruns, "Number of times we drop processing of a query profiler signal due to overrun plus the number of signals that OS has not delivered due to overrun.") \
M(QueryProfilerConcurrencyOverruns, "Number of times we drop processing of a query profiler signal due to too many concurrent query profilers in other threads, which may indicate overload.") \
M(QueryProfilerRuns, "Number of times QueryProfiler had been run.") \
\
M(CreatedLogEntryForMerge, "Successfully created log entry to merge parts in ReplicatedMergeTree.") \

12
src/Common/QueryProfiler.cpp
View File

@ -22,6 +22,7 @@ namespace CurrentMetrics
namespace ProfileEvents
{
extern const Event QueryProfilerSignalOverruns;
extern const Event QueryProfilerConcurrencyOverruns;
extern const Event QueryProfilerRuns;
}
@ -40,8 +41,19 @@ namespace
/// to ignore delivered signals after timer_delete().
thread_local bool signal_handler_disarmed = true;
/// Don't permit too many threads be busy inside profiler,
/// which could slow down the system in some environments.
std::atomic<Int64> concurrent_invocations = 0;
void writeTraceInfo(TraceType trace_type, int /* sig */, siginfo_t * info, void * context)
{
SCOPE_EXIT({ concurrent_invocations.fetch_sub(1, std::memory_order_relaxed); });
if (concurrent_invocations.fetch_add(1, std::memory_order_relaxed) > 100)
{
ProfileEvents::incrementNoTrace(ProfileEvents::QueryProfilerConcurrencyOverruns);
return;
}
auto saved_errno = errno; /// We must restore previous value of errno in signal handler.
#if defined(OS_LINUX)

14
src/Common/config_version.cpp.in
View File

@ -1,3 +1,17 @@
/// This file was autogenerated by CMake
#include <Common/config_version.h>
const unsigned VERSION_REVISION = @VERSION_REVISION@;
const char * VERSION_NAME = "@VERSION_NAME@";
const unsigned VERSION_MAJOR = @VERSION_MAJOR@;
const unsigned VERSION_MINOR = @VERSION_MINOR@;
const unsigned VERSION_PATCH = @VERSION_PATCH@;
const char * VERSION_STRING = "@VERSION_STRING@";
const char * VERSION_STRING_SHORT = "@VERSION_STRING_SHORT@";
const char * VERSION_OFFICIAL = "@VERSION_OFFICIAL@";
const char * VERSION_FULL = "@VERSION_FULL@";
const char * VERSION_DESCRIBE = "@VERSION_DESCRIBE@";
const unsigned VERSION_INTEGER = @VERSION_INTEGER@;
const char * VERSION_GITHASH = "@VERSION_GITHASH@";

21
src/Common/config_version.h Normal file
View File

@ -0,0 +1,21 @@
#pragma once
/// These fields are changing only on every release, but we still don't want to have them in the header file,
/// because it will make build cache less efficient.
// NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION,
// only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes.
extern const unsigned VERSION_REVISION;
extern const char * VERSION_NAME;
extern const unsigned VERSION_MAJOR;
extern const unsigned VERSION_MINOR;
extern const unsigned VERSION_PATCH;
extern const char * VERSION_STRING;
extern const char * VERSION_STRING_SHORT;
extern const char * VERSION_OFFICIAL;
extern const char * VERSION_FULL;
extern const char * VERSION_DESCRIBE;
extern const unsigned VERSION_INTEGER;
/// These fields are frequently changing and we don't want to have them in the header file to allow caching.
extern const char * VERSION_GITHASH;

24
src/Common/config_version.h.in
View File

@ -1,24 +0,0 @@
/// This file was autogenerated by CMake
#pragma once
// NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION,
// only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes.
#cmakedefine VERSION_REVISION @VERSION_REVISION@
#cmakedefine VERSION_NAME "@VERSION_NAME@"
#cmakedefine VERSION_MAJOR @VERSION_MAJOR@
#cmakedefine VERSION_MINOR @VERSION_MINOR@
#cmakedefine VERSION_PATCH @VERSION_PATCH@
#cmakedefine VERSION_STRING "@VERSION_STRING@"
#cmakedefine VERSION_STRING_SHORT "@VERSION_STRING_SHORT@"
#cmakedefine VERSION_OFFICIAL "@VERSION_OFFICIAL@"
#cmakedefine VERSION_FULL "@VERSION_FULL@"
#cmakedefine VERSION_DESCRIBE "@VERSION_DESCRIBE@"
#cmakedefine VERSION_INTEGER @VERSION_INTEGER@
/// These fields are frequently changing and we don't want to have them in the header file to allow caching.
extern const char * VERSION_GITHASH;
#if !defined(VERSION_OFFICIAL)
# define VERSION_OFFICIAL ""
#endif

2
src/Common/formatIPv6.h
View File

@ -45,7 +45,7 @@ void formatIPv6(const unsigned char * src, char *& dst, uint8_t zeroed_tail_byte
* @return - true if parsed successfully, false otherwise.
*/
template <typename T, typename EOFfunction>
requires (std::is_same<typename std::remove_cv<T>::type, char>::value)
requires (std::is_same_v<std::remove_cv_t<T>, char>)
inline bool parseIPv4(T * &src, EOFfunction eof, unsigned char * dst, int32_t first_octet = -1)
{
if (src == nullptr || first_octet > 255)

2
src/Coordination/FourLetterCommand.h
View File

@ -7,7 +7,7 @@
#include <Coordination/KeeperDispatcher.h>
#include <IO/WriteBufferFromString.h>
#include "config_version.h"
#include <Common/config_version.h>
namespace DB

2
src/Core/PostgreSQL/PoolWithFailover.h
View File

@ -14,7 +14,7 @@
static constexpr inline auto POSTGRESQL_POOL_DEFAULT_SIZE = 16;
static constexpr inline auto POSTGRESQL_POOL_WAIT_TIMEOUT = 5000;
static constexpr inline auto POSTGRESQL_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES = 5;
static constexpr inline auto POSTGRESQL_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES = 2;
namespace postgres
{

2
src/Core/PostgreSQL/Utils.cpp
View File

@ -16,7 +16,7 @@ ConnectionInfo formatConnectionString(String dbname, String host, UInt16 port, S
<< " port=" << port
<< " user=" << DB::quote << user
<< " password=" << DB::quote << password
<< " connect_timeout=10";
<< " connect_timeout=2";
return {out.str(), host + ':' + DB::toString(port)};
}

19
src/Core/Settings.h
View File

@ -364,16 +364,16 @@ class IColumn;
M(UInt64, max_bytes_to_read, 0, "Limit on read bytes (after decompression) from the most 'deep' sources. That is, only in the deepest subquery. When reading from a remote server, it is only checked on a remote server.", 0) \
M(OverflowMode, read_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \
\
M(UInt64, max_rows_to_read_leaf, 0, "Limit on read rows on the leaf nodes for distributed queries. Limit is applied for local reads only, excluding the final merge stage on the root node. Note, the setting is unstable with prefer_localhost_replica=1.", 0) \
M(UInt64, max_bytes_to_read_leaf, 0, "Limit on read bytes (after decompression) on the leaf nodes for distributed queries. Limit is applied for local reads only, excluding the final merge stage on the root node. Note, the setting is unstable with prefer_localhost_replica=1.", 0) \
M(UInt64, max_rows_to_read_leaf, 0, "Limit on read rows on the leaf nodes for distributed queries. Limit is applied for local reads only excluding the final merge stage on the root node. Note, the setting is unstable with prefer_localhost_replica=1.", 0) \
M(UInt64, max_bytes_to_read_leaf, 0, "Limit on read bytes (after decompression) on the leaf nodes for distributed queries. Limit is applied for local reads only excluding the final merge stage on the root node. Note, the setting is unstable with prefer_localhost_replica=1.", 0) \
M(OverflowMode, read_overflow_mode_leaf, OverflowMode::THROW, "What to do when the leaf limit is exceeded.", 0) \
\
M(UInt64, max_rows_to_group_by, 0, "If aggregation during GROUP BY is generating more than the specified number of rows (unique GROUP BY keys), the behavior will be determined by the 'group_by_overflow_mode' which by default is - throw an exception, but can be also switched to an approximate GROUP BY mode.", 0) \
M(UInt64, max_rows_to_group_by, 0, "If aggregation during GROUP BY is generating more than specified number of rows (unique GROUP BY keys), the behavior will be determined by the 'group_by_overflow_mode' which by default is - throw an exception, but can be also switched to an approximate GROUP BY mode.", 0) \
M(OverflowModeGroupBy, group_by_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \
M(UInt64, max_bytes_before_external_group_by, 0, "If memory usage during GROUP BY operation is exceeding this threshold in bytes, activate the 'external aggregation' mode (spill data to disk). Recommended value is half of available system memory.", 0) \
\
M(UInt64, max_rows_to_sort, 0, "If more than the specified amount of records have to be processed for ORDER BY operation, the behavior will be determined by the 'sort_overflow_mode' which by default is - throw an exception", 0) \
M(UInt64, max_bytes_to_sort, 0, "If more than the specified amount of (uncompressed) bytes have to be processed for ORDER BY operation, the behavior will be determined by the 'sort_overflow_mode' which by default is - throw an exception", 0) \
M(UInt64, max_rows_to_sort, 0, "If more than specified amount of records have to be processed for ORDER BY operation, the behavior will be determined by the 'sort_overflow_mode' which by default is - throw an exception", 0) \
M(UInt64, max_bytes_to_sort, 0, "If more than specified amount of (uncompressed) bytes have to be processed for ORDER BY operation, the behavior will be determined by the 'sort_overflow_mode' which by default is - throw an exception", 0) \
M(OverflowMode, sort_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \
M(UInt64, max_bytes_before_external_sort, 0, "If memory usage during ORDER BY operation is exceeding this threshold in bytes, activate the 'external sorting' mode (spill data to disk). Recommended value is half of available system memory.", 0) \
M(UInt64, max_bytes_before_remerge_sort, 1000000000, "In case of ORDER BY with LIMIT, when memory usage is higher than specified threshold, perform additional steps of merging blocks before final merge to keep just top LIMIT rows.", 0) \
@ -384,10 +384,8 @@ class IColumn;
M(OverflowMode, result_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \
\
/* TODO: Check also when merging and finalizing aggregate functions. */ \
M(Seconds, max_execution_time, 0, "If query runtime exceeds the specified number of seconds, the behavior will be determined by the 'timeout_overflow_mode', which by default is - throw an exception. Note that the timeout is checked and query can stop only in designated places during data processing. It currently cannot stop during merging of aggregation states or during query analysis, and the actual run time will be higher than the value of this setting.", 0) \
M(Seconds, max_execution_time, 0, "If query run time exceeded the specified number of seconds, the behavior will be determined by the 'timeout_overflow_mode' which by default is - throw an exception. Note that the timeout is checked and query can stop only in designated places during data processing. It currently cannot stop during merging of aggregation states or during query analysis, and the actual run time will be higher than the value of this setting.", 0) \
M(OverflowMode, timeout_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \
M(Seconds, max_execution_time_leaf, 0, "Similar semantic to max_execution_time but only apply on leaf node for distributed queries, the time out behavior will be determined by 'timeout_overflow_mode_leaf' which by default is - throw an exception", 0) \
M(OverflowMode, timeout_overflow_mode_leaf, OverflowMode::THROW, "What to do when the leaf limit is exceeded.", 0) \
\
M(UInt64, min_execution_speed, 0, "Minimum number of execution rows per second.", 0) \
M(UInt64, max_execution_speed, 0, "Maximum number of execution rows per second.", 0) \
@ -401,7 +399,7 @@ class IColumn;
\
M(UInt64, max_sessions_for_user, 0, "Maximum number of simultaneous sessions for a user.", 0) \
\
M(UInt64, max_subquery_depth, 100, "If a query has more than the specified number of nested subqueries, throw an exception. This allows you to have a sanity check to protect the users of your cluster from going insane with their queries.", 0) \
M(UInt64, max_subquery_depth, 100, "If a query has more than specified number of nested subqueries, throw an exception. This allows you to have a sanity check to protect the users of your cluster from going insane with their queries.", 0) \
M(UInt64, max_analyze_depth, 5000, "Maximum number of analyses performed by interpreter.", 0) \
M(UInt64, max_ast_depth, 1000, "Maximum depth of query syntax tree. Checked after parsing.", 0) \
M(UInt64, max_ast_elements, 50000, "Maximum size of query syntax tree in number of nodes. Checked after parsing.", 0) \
@ -620,7 +618,7 @@ class IColumn;
M(Bool, use_query_cache, false, "Enable the query cache", 0) \
M(Bool, enable_writes_to_query_cache, true, "Enable storing results of SELECT queries in the query cache", 0) \
M(Bool, enable_reads_from_query_cache, true, "Enable reading results of SELECT queries from the query cache", 0) \
M(Bool, query_cache_store_results_of_queries_with_nondeterministic_functions, false, "Store results of queries with non-deterministic functions (e.g. rand(), now()) in the query cache", 0) \
M(QueryCacheNondeterministicFunctionHandling, query_cache_nondeterministic_function_handling, QueryCacheNondeterministicFunctionHandling::Throw, "How the query cache handles queries with non-deterministic functions, e.g. now()", 0) \
M(UInt64, query_cache_max_size_in_bytes, 0, "The maximum amount of memory (in bytes) the current user may allocate in the query cache. 0 means unlimited. ", 0) \
M(UInt64, query_cache_max_entries, 0, "The maximum number of query results the current user may store in the query cache. 0 means unlimited.", 0) \
M(UInt64, query_cache_min_query_runs, 0, "Minimum number a SELECT query must run before its result is stored in the query cache", 0) \
@ -879,6 +877,7 @@ class IColumn;
MAKE_OBSOLETE(M, Bool, optimize_duplicate_order_by_and_distinct, false) \
MAKE_OBSOLETE(M, UInt64, parallel_replicas_min_number_of_granules_to_enable, 0) \
MAKE_OBSOLETE(M, Bool, query_plan_optimize_projection, true) \
MAKE_OBSOLETE(M, Bool, query_cache_store_results_of_queries_with_nondeterministic_functions, false) \
/** The section above is for obsolete settings. Do not add anything there. */

6
src/Core/SettingsEnums.cpp
View File

@ -69,6 +69,12 @@ IMPLEMENT_SETTING_ENUM(DistributedProductMode, ErrorCodes::UNKNOWN_DISTRIBUTED_P
{"allow", DistributedProductMode::ALLOW}})
IMPLEMENT_SETTING_ENUM(QueryCacheNondeterministicFunctionHandling, ErrorCodes::BAD_ARGUMENTS,
{{"throw", QueryCacheNondeterministicFunctionHandling::Throw},
{"save", QueryCacheNondeterministicFunctionHandling::Save},
{"ignore", QueryCacheNondeterministicFunctionHandling::Ignore}})
IMPLEMENT_SETTING_ENUM(DateTimeInputFormat, ErrorCodes::BAD_ARGUMENTS,
{{"basic", FormatSettings::DateTimeInputFormat::Basic},
{"best_effort", FormatSettings::DateTimeInputFormat::BestEffort},

10
src/Core/SettingsEnums.h
View File

@ -70,6 +70,16 @@ enum class DistributedProductMode
DECLARE_SETTING_ENUM(DistributedProductMode)
/// How the query cache handles queries with non-deterministic functions, e.g. now()
enum class QueryCacheNondeterministicFunctionHandling
{
Throw,
Save,
Ignore
};
DECLARE_SETTING_ENUM(QueryCacheNondeterministicFunctionHandling)
DECLARE_SETTING_ENUM_WITH_RENAME(DateTimeInputFormat, FormatSettings::DateTimeInputFormat)

12
src/Daemon/BaseDaemon.cpp
View File

@ -60,7 +60,7 @@
#include <Loggers/OwnFormattingChannel.h>
#include <Loggers/OwnPatternFormatter.h>
#include "config_version.h"
#include <Common/config_version.h>
#if defined(OS_DARWIN)
# pragma clang diagnostic ignored "-Wunused-macros"
@ -485,10 +485,8 @@ private:
{
SentryWriter::onFault(sig, error_message, stack_trace);
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wunreachable-code"
/// Advice the user to send it manually.
if constexpr (std::string_view(VERSION_OFFICIAL).contains("official build"))
if (std::string_view(VERSION_OFFICIAL).contains("official build"))
{
const auto & date_lut = DateLUT::instance();
@ -506,8 +504,6 @@ private:
{
LOG_FATAL(log, "This ClickHouse version is not official and should be upgraded to the official build.");
}
#pragma clang diagnostic pop
}
/// ClickHouse Keeper does not link to some part of Settings.
@ -971,7 +967,7 @@ static void blockSignals(const std::vector<int> & signals)
throw Poco::Exception("Cannot block signal.");
}
extern String getGitHash();
extern const char * GIT_HASH;
void BaseDaemon::initializeTerminationAndSignalProcessing()
{
@ -1011,7 +1007,7 @@ void BaseDaemon::initializeTerminationAndSignalProcessing()
build_id = "";
#endif
git_hash = getGitHash();
git_hash = GIT_HASH;
#if defined(OS_LINUX)
std::string executable_path = getExecutablePath();

7
src/Daemon/GitHash.cpp.in
View File

@ -1,8 +1,3 @@
/// This file was autogenerated by CMake
#include <base/types.h>
String getGitHash()
{
return "@GIT_HASH@";
}
const char * GIT_HASH = "@GIT_HASH@";

2
src/Daemon/SentryWriter.cpp
View File

@ -14,7 +14,7 @@
#include <Core/ServerUUID.h>
#include "config.h"
#include "config_version.h"
#include <Common/config_version.h>
#if USE_SENTRY && !defined(CLICKHOUSE_KEEPER_STANDALONE_BUILD)

9
src/Databases/DDLLoadingDependencyVisitor.cpp
View File

@ -1,4 +1,5 @@
#include <Databases/DDLLoadingDependencyVisitor.h>
#include <Databases/DDLDependencyVisitor.h>
#include <Dictionaries/getDictionaryConfigurationFromAST.h>
#include <Interpreters/Context.h>
#include <Interpreters/misc.h>
@ -7,6 +8,7 @@
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTSelectWithUnionQuery.h>
#include <Parsers/ASTTTLElement.h>
#include <Poco/String.h>
@ -22,6 +24,7 @@ TableNamesSet getLoadingDependenciesFromCreateQuery(ContextPtr global_context, c
data.default_database = global_context->getCurrentDatabase();
data.create_query = ast;
data.global_context = global_context;
data.table_name = table;
TableLoadingDependenciesVisitor visitor{data};
visitor.visit(ast);
data.dependencies.erase(table);
@ -113,6 +116,12 @@ void DDLLoadingDependencyVisitor::visit(const ASTFunctionWithKeyValueArguments &
void DDLLoadingDependencyVisitor::visit(const ASTStorage & storage, Data & data)
{
if (storage.ttl_table)
{
auto ttl_dependensies = getDependenciesFromCreateQuery(data.global_context, data.table_name, storage.ttl_table->ptr());
data.dependencies.merge(ttl_dependensies);
}
if (!storage.engine)
return;

1
src/Databases/DDLLoadingDependencyVisitor.h
View File

@ -38,6 +38,7 @@ public:
TableNamesSet dependencies;
ContextPtr global_context;
ASTPtr create_query;
QualifiedTableName table_name;
};
using Visitor = ConstInDepthNodeVisitor<DDLLoadingDependencyVisitor, true>;

2
src/Functions/serverConstants.cpp
View File

@ -12,7 +12,7 @@
#include <Poco/Environment.h>
#include "config_version.h"
#include <Common/config_version.h>
namespace DB

2
src/IO/ReadWriteBufferFromHTTP.h
View File

@ -23,7 +23,7 @@
#include <Common/DNSResolver.h>
#include <Common/RemoteHostFilter.h>
#include "config.h"
#include "config_version.h"
#include <Common/config_version.h>
#include <filesystem>

5
src/Interpreters/ClientInfo.cpp
View File

@ -7,10 +7,11 @@
#include <base/getFQDNOrHostName.h>
#include <unistd.h>
#include "config_version.h"
#include <Common/config_version.h>
#include <format>
namespace DB
{
@ -196,7 +197,7 @@ void ClientInfo::setInitialQuery()
if (client_name.empty())
client_name = VERSION_NAME;
else
client_name = (VERSION_NAME " ") + client_name;
client_name = std::string(VERSION_NAME) + " " + client_name;
}
bool ClientInfo::clientVersionEquals(const ClientInfo & other, bool compare_patch) const

13
src/Interpreters/ClusterProxy/SelectStreamFactory.cpp
View File

@ -1,21 +1,20 @@
#include <Interpreters/ClusterProxy/SelectStreamFactory.h>
#include <Interpreters/Cluster.h>
#include <Storages/StorageReplicatedMergeTree.h>
#include <Storages/VirtualColumnUtils.h>
#include <Common/Exception.h>
#include <Common/ProfileEvents.h>
#include <Common/checkStackSize.h>
#include <Common/logger_useful.h>
#include <Common/FailPoint.h>
#include <TableFunctions/TableFunctionFactory.h>
#include <IO/ConnectionTimeouts.h>
#include <Interpreters/ClusterProxy/SelectStreamFactory.h>
#include <Interpreters/Cluster.h>
#include <Interpreters/AddDefaultDatabaseVisitor.h>
#include <Interpreters/RequiredSourceColumnsVisitor.h>
#include <Interpreters/TranslateQualifiedNamesVisitor.h>
#include <DataTypes/ObjectUtils.h>
#include <Client/IConnections.h>
#include <Parsers/ASTSelectQuery.h>
#include <Parsers/ASTSetQuery.h>
#include <Common/logger_useful.h>
#include <Common/FailPoint.h>
#include <Processors/QueryPlan/QueryPlan.h>
#include <Processors/QueryPlan/ReadFromRemote.h>
#include <Processors/QueryPlan/ExpressionStep.h>
@ -23,7 +22,6 @@
#include <Processors/QueryPlan/DistributedCreateLocalPlan.h>
#include <Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h>
namespace ProfileEvents
{
extern const Event DistributedConnectionMissingTable;
@ -123,7 +121,6 @@ void SelectStreamFactory::createForShard(
if (it != objects_by_shard.end())
replaceMissedSubcolumnsByConstants(storage_snapshot->object_columns, it->second, query_ast);
auto emplace_local_stream = [&]()
{
local_plans.emplace_back(createLocalPlan(

8
src/Interpreters/ClusterProxy/executeQuery.cpp
View File

@ -141,14 +141,6 @@ ContextMutablePtr updateSettingsForCluster(const Cluster & cluster,
new_settings.allow_experimental_parallel_reading_from_replicas = false;
}
if (settings.max_execution_time_leaf.value > 0)
{
/// Replace 'max_execution_time' of this sub-query with 'max_execution_time_leaf' and 'timeout_overflow_mode'
/// with 'timeout_overflow_mode_leaf'
new_settings.max_execution_time = settings.max_execution_time_leaf;
new_settings.timeout_overflow_mode = settings.timeout_overflow_mode_leaf;
}
auto new_context = Context::createCopy(context);
new_context->setSettings(new_settings);
return new_context;

2
src/Interpreters/CrashLog.cpp
View File

@ -8,7 +8,7 @@
#include <Common/SymbolIndex.h>
#include <Common/Stopwatch.h>
#include "config_version.h"
#include <Common/config_version.h>
namespace DB

19
src/Interpreters/executeQuery.cpp
View File

@ -96,7 +96,7 @@ namespace DB
namespace ErrorCodes
{
extern const int CANNOT_USE_QUERY_CACHE_WITH_NONDETERMINISTIC_FUNCTIONS;
extern const int QUERY_CACHE_USED_WITH_NONDETERMINISTIC_FUNCTIONS;
extern const int INTO_OUTFILE_NOT_ALLOWED;
extern const int INVALID_TRANSACTION;
extern const int LOGICAL_ERROR;
@ -1106,10 +1106,16 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
/// top of the pipeline which stores the result in the query cache.
if (can_use_query_cache && settings.enable_writes_to_query_cache)
{
if (astContainsNonDeterministicFunctions(ast, context) && !settings.query_cache_store_results_of_queries_with_nondeterministic_functions)
throw Exception(ErrorCodes::CANNOT_USE_QUERY_CACHE_WITH_NONDETERMINISTIC_FUNCTIONS,
"Unable to cache the query result because the query contains a non-deterministic function. Use setting `query_cache_store_results_of_queries_with_nondeterministic_functions = 1` to cache the query result regardless");
const bool ast_contains_nondeterministic_functions = astContainsNonDeterministicFunctions(ast, context);
const QueryCacheNondeterministicFunctionHandling nondeterministic_function_handling = settings.query_cache_nondeterministic_function_handling;
if (ast_contains_nondeterministic_functions && nondeterministic_function_handling == QueryCacheNondeterministicFunctionHandling::Throw)
throw Exception(ErrorCodes::QUERY_CACHE_USED_WITH_NONDETERMINISTIC_FUNCTIONS,
"The query result was not cached because the query contains a non-deterministic function."
" Use setting `query_cache_nondeterministic_function_handling = 'save'` or `= 'ignore'` to cache the query result regardless or to omit caching");
if (!ast_contains_nondeterministic_functions || nondeterministic_function_handling == QueryCacheNondeterministicFunctionHandling::Save)
{
QueryCache::Key key(
ast, res.pipeline.getHeader(),
context->getUserName(), settings.query_cache_share_between_users,
@ -1119,7 +1125,9 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
const size_t num_query_runs = query_cache->recordQueryRun(key);
if (num_query_runs <= settings.query_cache_min_query_runs)
{
LOG_TRACE(&Poco::Logger::get("QueryCache"), "Skipped insert because the query ran {} times but the minimum required number of query runs to cache the query result is {}", num_query_runs, settings.query_cache_min_query_runs);
LOG_TRACE(&Poco::Logger::get("QueryCache"),
"Skipped insert because the query ran {} times but the minimum required number of query runs to cache the query result is {}",
num_query_runs, settings.query_cache_min_query_runs);
}
else
{
@ -1134,6 +1142,7 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
query_cache_usage = QueryCache::Usage::Write;
}
}
}
}
}

2
src/Parsers/ASTAlterNamedCollectionQuery.cpp
View File

@ -14,7 +14,7 @@ ASTPtr ASTAlterNamedCollectionQuery::clone() const
void ASTAlterNamedCollectionQuery::formatImpl(const IAST::FormatSettings & settings, IAST::FormatState &, IAST::FormatStateStacked) const
{
settings.ostr << (settings.hilite ? hilite_keyword : "") << "Alter NAMED COLLECTION ";
settings.ostr << (settings.hilite ? hilite_keyword : "") << "ALTER NAMED COLLECTION ";
if (if_exists)
settings.ostr << "IF EXISTS ";
settings.ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(collection_name) << (settings.hilite ? hilite_none : "");

205
src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp
View File

@ -52,19 +52,19 @@ ORCInputStream::ORCInputStream(SeekableReadBuffer & in_, size_t file_size_) : in
{
}
uint64_t ORCInputStream::getLength() const
UInt64 ORCInputStream::getLength() const
{
return file_size;
}
uint64_t ORCInputStream::getNaturalReadSize() const
UInt64 ORCInputStream::getNaturalReadSize() const
{
return 128 * 1024;
}
void ORCInputStream::read(void * buf, uint64_t length, uint64_t offset)
void ORCInputStream::read(void * buf, UInt64 length, UInt64 offset)
{
if (offset != static_cast<uint64_t>(in.getPosition()))
if (offset != static_cast<UInt64>(in.getPosition()))
in.seek(offset, SEEK_SET);
in.readStrict(reinterpret_cast<char *>(buf), length);
@ -100,11 +100,11 @@ std::unique_ptr<orc::InputStream> asORCInputStreamLoadIntoMemory(ReadBuffer & in
return std::make_unique<ORCInputStreamFromString>(std::move(file_data), file_size);
}
static const orc::Type * getORCTypeByName(const orc::Type & schema, const String & name, bool case_insensitive_column_matching)
static const orc::Type * getORCTypeByName(const orc::Type & schema, const String & name, bool ignore_case)
{
for (uint64_t i = 0; i != schema.getSubtypeCount(); ++i)
for (UInt64 i = 0; i != schema.getSubtypeCount(); ++i)
if (boost::equals(schema.getFieldName(i), name)
|| (case_insensitive_column_matching && boost::iequals(schema.getFieldName(i), name)))
|| (ignore_case && boost::iequals(schema.getFieldName(i), name)))
return schema.getSubtype(i);
return nullptr;
}
@ -687,13 +687,8 @@ buildORCSearchArgument(const KeyCondition & key_condition, const Block & header,
return builder->build();
}
static void getFileReaderAndSchema(
ReadBuffer & in,
std::unique_ptr<orc::Reader> & file_reader,
Block & header,
const FormatSettings & format_settings,
std::atomic<int> & is_stopped)
static void getFileReader(
ReadBuffer & in, std::unique_ptr<orc::Reader> & file_reader, const FormatSettings & format_settings, std::atomic<int> & is_stopped)
{
if (is_stopped)
return;
@ -701,17 +696,144 @@ static void getFileReaderAndSchema(
orc::ReaderOptions options;
auto input_stream = asORCInputStream(in, format_settings, is_stopped);
file_reader = orc::createReader(std::move(input_stream), options);
const auto & schema = file_reader->getType();
}
for (size_t i = 0; i < schema.getSubtypeCount(); ++i)
static const orc::Type * traverseDownORCTypeByName(
const std::string & target,
const orc::Type * orc_type,
DataTypePtr & type,
bool ignore_case)
{
const std::string & name = schema.getFieldName(i);
const orc::Type * orc_type = schema.getSubtype(i);
if (target.empty())
return orc_type;
bool skipped = false;
DataTypePtr type = parseORCType(orc_type, format_settings.orc.skip_columns_with_unsupported_types_in_schema_inference, skipped);
if (!skipped)
header.insert(ColumnWithTypeAndName{type, name});
auto search_struct_field = [&](const std::string & target_, const orc::Type * type_) -> std::pair<std::string, const orc::Type *>
{
auto target_copy = target_;
if (ignore_case)
boost::to_lower(target_copy);
for (size_t i = 0; i < type_->getSubtypeCount(); ++i)
{
auto field_name = type_->getFieldName(i);
if (ignore_case)
boost::to_lower(field_name);
if (startsWith(target_copy, field_name) && (target_copy.size() == field_name.size() || target_copy[field_name.size()] == '.'))
{
return {target_copy.size() == field_name.size() ? "" : target_.substr(field_name.size() + 1), type_->getSubtype(i)};
}
}
return {"", nullptr};
};
if (orc::STRUCT == orc_type->getKind())
{
const auto [next_target, next_orc_type]= search_struct_field(target, orc_type);
return next_orc_type ? traverseDownORCTypeByName(next_target, next_orc_type, type, ignore_case) : nullptr;
}
else if (orc::LIST == orc_type->getKind())
{
/// For cases in which header contains subcolumns flattened from nested columns.
/// For example, "a Nested(x String, y Int64)" is flattened to "a.x Array(String), a.y Array(Int64)", and orc file schema is still "a array<struct<x string, y long>>".
/// In this case, we should skip possible array type and traverse down to its nested struct type.
const auto * array_type = typeid_cast<const DataTypeArray *>(removeNullable(type).get());
const auto * orc_nested_type = orc_type->getSubtype(0);
if (array_type && orc::STRUCT == orc_nested_type->getKind())
{
auto next_type_and_target = search_struct_field(target, orc_nested_type);
const auto & next_target = next_type_and_target.first;
const auto * next_orc_type = next_type_and_target.second;
if (next_orc_type)
{
/// Adjust CH type to avoid inconsistency between CH and ORC type brought by flattened Nested type.
type = array_type->getNestedType();
return traverseDownORCTypeByName(next_target, next_orc_type, type, ignore_case);
}
}
}
return nullptr;
}
static void updateIncludeTypeIds(
DataTypePtr type, const orc::Type * orc_type, bool ignore_case, std::unordered_set<UInt64> & include_typeids)
{
/// For primitive types, directly append column id into result
if (orc_type->getSubtypeCount() == 0)
{
include_typeids.insert(orc_type->getColumnId());
return;
}
auto non_nullable_type = removeNullable(type);
switch (orc_type->getKind())
{
case orc::LIST: {
const auto * array_type = typeid_cast<const DataTypeArray *>(non_nullable_type.get());
if (array_type)
{
updateIncludeTypeIds(
array_type->getNestedType(), orc_type->getSubtype(0), ignore_case, include_typeids);
}
return;
}
case orc::MAP: {
const auto * map_type = typeid_cast<const DataTypeMap *>(non_nullable_type.get());
if (map_type)
{
updateIncludeTypeIds(map_type->getKeyType(), orc_type->getSubtype(0), ignore_case, include_typeids);
updateIncludeTypeIds(map_type->getValueType(), orc_type->getSubtype(1), ignore_case, include_typeids);
}
return;
}
case orc::STRUCT: {
/// To make sure tuple field pruning work fine, we should include only the fields of orc struct type which are also contained in CH tuple types, instead of all fields of orc struct type.
/// For example, CH tupe type in header is "x Tuple(a String)", ORC struct type is "x struct<a:string, b:long>", then only type id of field "x.a" should be included.
/// For tuple field pruning purpose, we should never include "x.b" for it is not required in format header.
const auto * tuple_type = typeid_cast<const DataTypeTuple *>(non_nullable_type.get());
if (tuple_type)
{
if (tuple_type->haveExplicitNames())
{
std::unordered_map<String, size_t> orc_field_name_to_index;
orc_field_name_to_index.reserve(orc_type->getSubtypeCount());
for (size_t struct_i = 0; struct_i < orc_type->getSubtypeCount(); ++struct_i)
{
String field_name = orc_type->getFieldName(struct_i);
if (ignore_case)
boost::to_lower(field_name);
orc_field_name_to_index[field_name] = struct_i;
}
const auto & element_names = tuple_type->getElementNames();
for (size_t tuple_i = 0; tuple_i < element_names.size(); ++tuple_i)
{
String element_name = element_names[tuple_i];
if (ignore_case)
boost::to_lower(element_name);
if (orc_field_name_to_index.contains(element_name))
{
updateIncludeTypeIds(
tuple_type->getElement(tuple_i),
orc_type->getSubtype(orc_field_name_to_index[element_name]),
ignore_case,
include_typeids);
}
}
}
else
{
for (size_t i = 0; i < tuple_type->getElements().size() && i < orc_type->getSubtypeCount(); ++i)
updateIncludeTypeIds(
tuple_type->getElement(i), orc_type->getSubtype(i), ignore_case, include_typeids);
}
}
return;
}
default:
return;
}
}
@ -722,8 +844,7 @@ NativeORCBlockInputFormat::NativeORCBlockInputFormat(ReadBuffer & in_, Block hea
void NativeORCBlockInputFormat::prepareFileReader()
{
Block schema;
getFileReaderAndSchema(*in, file_reader, schema, format_settings, is_stopped);
getFileReader(*in, file_reader, format_settings, is_stopped);
if (is_stopped)
return;
@ -737,14 +858,17 @@ void NativeORCBlockInputFormat::prepareFileReader()
format_settings.orc.case_insensitive_column_matching);
const bool ignore_case = format_settings.orc.case_insensitive_column_matching;
std::unordered_set<String> nested_table_names = Nested::getAllTableNames(getPort().getHeader(), ignore_case);
for (size_t i = 0; i < schema.columns(); ++i)
const auto & header = getPort().getHeader();
const auto & file_schema = file_reader->getType();
std::unordered_set<UInt64> include_typeids;
for (const auto & column : header)
{
const auto & name = schema.getByPosition(i).name;
if (getPort().getHeader().has(name, ignore_case) || nested_table_names.contains(ignore_case ? boost::to_lower_copy(name) : name))
include_indices.push_back(static_cast<int>(i));
auto adjusted_type = column.type;
const auto * orc_type = traverseDownORCTypeByName(column.name, &file_schema, adjusted_type, ignore_case);
if (orc_type)
updateIncludeTypeIds(adjusted_type, orc_type, ignore_case, include_typeids);
}
include_indices.assign(include_typeids.begin(), include_typeids.end());
if (format_settings.orc.filter_push_down && key_condition && !sarg)
{
@ -769,7 +893,7 @@ bool NativeORCBlockInputFormat::prepareStripeReader()
throw Exception(ErrorCodes::INCORRECT_DATA, "ORC stripe {} has no rows", current_stripe);
orc::RowReaderOptions row_reader_options;
row_reader_options.include(include_indices);
row_reader_options.includeTypes(include_indices);
row_reader_options.range(current_stripe_info->getOffset(), current_stripe_info->getLength());
if (format_settings.orc.filter_push_down && sarg)
{
@ -854,17 +978,28 @@ NativeORCSchemaReader::NativeORCSchemaReader(ReadBuffer & in_, const FormatSetti
NamesAndTypesList NativeORCSchemaReader::readSchema()
{
Block header;
std::unique_ptr<orc::Reader> file_reader;
std::atomic<int> is_stopped = 0;
getFileReaderAndSchema(in, file_reader, header, format_settings, is_stopped);
getFileReader(in, file_reader, format_settings, is_stopped);
const auto & schema = file_reader->getType();
Block header;
for (size_t i = 0; i < schema.getSubtypeCount(); ++i)
{
const std::string & name = schema.getFieldName(i);
const orc::Type * orc_type = schema.getSubtype(i);
bool skipped = false;
DataTypePtr type = parseORCType(orc_type, format_settings.orc.skip_columns_with_unsupported_types_in_schema_inference, skipped);
if (!skipped)
header.insert(ColumnWithTypeAndName{type, name});
}
if (format_settings.schema_inference_make_columns_nullable)
return getNamesAndRecursivelyNullableTypes(header);
return header.getNamesAndTypesList();
}
ORCColumnToCHColumn::ORCColumnToCHColumn(
const Block & header_, bool allow_missing_columns_, bool null_as_default_, bool case_insensitive_matching_)
: header(header_)
@ -1385,8 +1520,8 @@ static ColumnWithTypeAndName readColumnFromORCColumn(
Columns tuple_elements;
DataTypes tuple_types;
std::vector<String> tuple_names;
const auto * tuple_type_hint = type_hint ? typeid_cast<const DataTypeTuple *>(type_hint.get()) : nullptr;
const auto * tuple_type_hint = type_hint ? typeid_cast<const DataTypeTuple *>(type_hint.get()) : nullptr;
const auto * orc_struct_column = dynamic_cast<const orc::StructVectorBatch *>(orc_column);
for (size_t i = 0; i < orc_type->getSubtypeCount(); ++i)
{

5
src/Processors/Formats/Impl/Parquet/Write.cpp
View File

@ -6,14 +6,13 @@
#include <lz4.h>
#include <Columns/MaskOperations.h>
#include <Columns/ColumnFixedString.h>
#include <Columns/ColumnNullable.h>
#include <Columns/ColumnString.h>
#include <Columns/ColumnArray.h>
#include <Columns/ColumnDecimal.h>
#include <Columns/ColumnTuple.h>
#include <Columns/ColumnMap.h>
#include <IO/WriteHelpers.h>
#include "config_version.h"
#include <Common/config_version.h>
#if USE_SNAPPY
#include <snappy.h>
@ -916,7 +915,7 @@ void writeFileFooter(std::vector<parq::RowGroup> row_groups, SchemaElements sche
meta.row_groups = std::move(row_groups);
for (auto & r : meta.row_groups)
meta.num_rows += r.num_rows;
meta.__set_created_by(VERSION_NAME " " VERSION_DESCRIBE);
meta.__set_created_by(std::string(VERSION_NAME) + " " + VERSION_DESCRIBE);
if (options.write_page_statistics || options.write_column_chunk_statistics)
{

2
src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp
View File

@ -1,6 +1,6 @@
#include <Processors/QueryPlan/DistributedCreateLocalPlan.h>
#include "config_version.h"
#include <Common/config_version.h>
#include <Common/checkStackSize.h>
#include <Core/ProtocolDefines.h>
#include <Interpreters/ActionsDAG.h>

6
src/Processors/examples/CMakeLists.txt
View File

@ -2,9 +2,3 @@ if (TARGET ch_contrib::hivemetastore)
clickhouse_add_executable (comma_separated_streams comma_separated_streams.cpp)
target_link_libraries (comma_separated_streams PRIVATE dbms)
endif()
if (USE_ORC)
clickhouse_add_executable (native_orc native_orc.cpp)
target_link_libraries (native_orc PRIVATE dbms)
target_include_directories (native_orc PRIVATE ${ClickHouse_SOURCE_DIR}/contrib/orc/c++/include)
endif ()

36
src/Processors/examples/native_orc.cpp
View File

@ -1,36 +0,0 @@
#include <string>
#include <IO/ReadBufferFromFile.h>
#include <Processors/Formats/Impl/NativeORCBlockInputFormat.h>
#include <IO/copyData.h>
using namespace DB;
int main()
{
/// Read schema from orc file
String path = "/path/to/orc/file";
// String path = "/data1/clickhouse_official/data/user_files/bigolive_audience_stats_orc.orc";
{
ReadBufferFromFile in(path);
NativeORCSchemaReader schema_reader(in, {});
auto schema = schema_reader.readSchema();
std::cout << "schema:" << schema.toString() << std::endl;
}
/// Read schema from string with orc data
{
ReadBufferFromFile in(path);
String content;
WriteBufferFromString out(content);
copyData(in, out);
content.resize(out.count());
ReadBufferFromString in2(content);
NativeORCSchemaReader schema_reader(in2, {});
auto schema = schema_reader.readSchema();
std::cout << "schema:" << schema.toString() << std::endl;
}
return 0;
}

3
src/Server/MySQLHandler.cpp
View File

@ -25,8 +25,7 @@
#include <Common/OpenSSLHelpers.h>
#include <Common/logger_useful.h>
#include <Common/setThreadName.h>
#include "config_version.h"
#include <Common/config_version.h>
#if USE_SSL
# include <Poco/Crypto/RSAKey.h>

3
src/Server/PostgreSQLHandler.cpp
View File

@ -11,8 +11,7 @@
#include <Common/setThreadName.h>
#include <base/scope_guard.h>
#include <pcg_random.hpp>
#include "config_version.h"
#include <Common/config_version.h>
#if USE_SSL
# include <Poco/Net/SecureStreamSocket.h>

6
src/Server/TCPHandler.cpp
View File

@ -59,11 +59,11 @@
# include <Poco/Net/SecureStreamSocketImpl.h>
#endif
#include "Core/Protocol.h"
#include "Storages/MergeTree/RequestResponse.h"
#include <Core/Protocol.h>
#include <Storages/MergeTree/RequestResponse.h>
#include "TCPHandler.h"
#include "config_version.h"
#include <Common/config_version.h>
using namespace std::literals;
using namespace DB;

2
src/Storages/AlterCommands.cpp
View File

@ -1159,7 +1159,7 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const
/// So we don't allow to do it for now.
if (command.data_type)
{
const GetColumnsOptions options(GetColumnsOptions::AllPhysical);
const GetColumnsOptions options(GetColumnsOptions::All);
const auto old_data_type = all_columns.getColumn(options, column_name).type;
bool new_type_has_object = command.data_type->hasDynamicSubcolumns();

6
src/Storages/Kafka/StorageKafka.cpp
View File

@ -41,11 +41,11 @@
#include <Common/setThreadName.h>
#include <Formats/FormatFactory.h>
#include "Storages/ColumnDefault.h"
#include "config_version.h"
#include <Storages/ColumnDefault.h>
#include <Common/config_version.h>
#include <Common/CurrentMetrics.h>
#include <Common/ProfileEvents.h>
#if USE_KRB5
#include <Access/KerberosInit.h>
#endif // USE_KRB5

10
src/Storages/tests/gtest_transform_query_for_external_database.cpp
View File

@ -279,9 +279,13 @@ TEST(TransformQueryForExternalDatabase, MultipleAndSubqueries)
{
const State & state = State::instance();
check(state, 1, {"column"},
"SELECT column FROM test.table WHERE 1 = 1 AND toString(column) = '42' AND column = 42 AND left(toString(column), 10) = RIGHT(toString(column), 10) AND column IN (1, 42) AND SUBSTRING(toString(column) FROM 1 FOR 2) = 'Hello' AND column != 4",
R"(SELECT "column" FROM "test"."table" WHERE 1 AND ("column" = 42) AND ("column" IN (1, 42)) AND ("column" != 4))");
check(
state,
1,
{"column"},
"SELECT column FROM test.table WHERE 1 = 1 AND toString(column) = '42' AND column = 42 AND left(toString(column), 10) = "
"RIGHT(toString(column), 10) AND column IN (1, 42) AND SUBSTRING(toString(column) FROM 1 FOR 2) = 'Hello' AND column != 4",
R"(SELECT "column" FROM "test"."table" WHERE (1 = 1) AND ("column" = 42) AND ("column" IN (1, 42)) AND ("column" != 4))");
check(state, 1, {"column"},
"SELECT column FROM test.table WHERE toString(column) = '42' AND left(toString(column), 10) = RIGHT(toString(column), 10) AND column = 42",
R"(SELECT "column" FROM "test"."table" WHERE "column" = 42)");

31
src/Storages/transformQueryForExternalDatabase.cpp
View File

@ -75,6 +75,33 @@ public:
}
};
struct ReplaceLiteralToExprVisitorData
{
using TypeToVisit = ASTFunction;
void visit(ASTFunction & func, ASTPtr &) const
{
if (func.name == "and" || func.name == "or")
{
for (auto & argument : func.arguments->children)
{
auto * literal_expr = typeid_cast<ASTLiteral *>(argument.get());
UInt64 value;
if (literal_expr && literal_expr->value.tryGet<UInt64>(value) && (value == 0 || value == 1))
{
/// 1 -> 1=1, 0 -> 1=0.
if (value)
argument = makeASTFunction("equals", std::make_shared<ASTLiteral>(1), std::make_shared<ASTLiteral>(1));
else
argument = makeASTFunction("equals", std::make_shared<ASTLiteral>(1), std::make_shared<ASTLiteral>(0));
}
}
}
}
};
using ReplaceLiteralToExprVisitor = InDepthNodeVisitor<OneTypeMatcher<ReplaceLiteralToExprVisitorData>, true>;
class DropAliasesMatcher
{
public:
@ -288,6 +315,10 @@ String transformQueryForExternalDatabaseImpl(
{
replaceConstantExpressions(original_where, context, available_columns);
/// Replace like WHERE 1 AND 1 to WHERE 1 = 1 AND 1 = 1
ReplaceLiteralToExprVisitor::Data replace_literal_to_expr_data;
ReplaceLiteralToExprVisitor(replace_literal_to_expr_data).visit(original_where);
if (isCompatible(original_where))
{
select->setExpression(ASTSelectQuery::Expression::WHERE, std::move(original_where));

3
tests/analyzer_tech_debt.txt
View File

@ -11,6 +11,7 @@
01244_optimize_distributed_group_by_sharding_key
01268_mv_scalars
01268_shard_avgweighted
01287_max_execution_speed
01455_shard_leaf_max_rows_bytes_to_read
01495_subqueries_in_with_statement
01560_merge_distributed_join
@ -42,6 +43,8 @@
00917_multiple_joins_denny_crane
00636_partition_key_parts_pruning
01825_type_json_multiple_files
02003_WithMergeableStateAfterAggregationAndLimit_LIMIT_BY_LIMIT_OFFSET
02404_memory_bound_merging
02725_agg_projection_resprect_PK
02721_url_cluster
02534_s3_cluster_insert_select_schema_inference

7
tests/ci/build_report_check.py
View File

@ -37,7 +37,10 @@ from commit_status_helper import (
from ci_config import CI_CONFIG
# Old way to read the neads_data
NEEDS_DATA_PATH = os.getenv("NEEDS_DATA_PATH", "")
# Now it's set here. Two-steps migration for backward compatibility
NEEDS_DATA = os.getenv("NEEDS_DATA", "")
def main():
@ -58,6 +61,10 @@ def main():
if os.path.exists(NEEDS_DATA_PATH):
with open(NEEDS_DATA_PATH, "rb") as file_handler:
needs_data = json.load(file_handler)
if NEEDS_DATA:
needs_data = json.loads(NEEDS_DATA)
required_builds = len(needs_data)
if needs_data:

20
tests/ci/ci_config.py
View File

@ -50,21 +50,27 @@ class CiConfig:
def validate(self) -> None:
errors = []
# All build configs must belong to build_report_config
for build_name in self.build_config.keys():
for name, build_config in self.build_config.items():
build_in_reports = False
for report_config in self.builds_report_config.values():
if build_name in report_config:
if name in report_config:
build_in_reports = True
break
# All build configs must belong to build_report_config
if not build_in_reports:
logging.error("Build name %s does not belong to build reports", name)
errors.append(f"Build name {name} does not belong to build reports")
# The name should be the same as build_config.name
if not build_config.name == name:
logging.error(
"Build name %s does not belong to build reports", build_name
"Build name '%s' does not match the config 'name' value '%s'",
name,
build_config.name,
)
errors.append(
f"Build name {build_name} does not belong to build reports"
f"Build name {name} does not match 'name' value '{build_config.name}'"
)
# And otherwise
# All build_report_config values should be in build_config.keys()
for build_report_name, build_names in self.builds_report_config.items():
missed_names = [
name for name in build_names if name not in self.build_config.keys()
@ -216,7 +222,7 @@ CI_CONFIG = CiConfig(
),
"fuzzers": BuildConfig(
name="fuzzers",
compiler="clang-16",
compiler="clang-17",
package_type="fuzzers",
),
},

150
tests/ci/codebrowser_check.py
View File

@ -1,150 +0,0 @@
#!/usr/bin/env python3
import logging
import os
from pathlib import Path
from github import Github
from commit_status_helper import get_commit, post_commit_status
from docker_pull_helper import get_image_with_version, DockerImage
from env_helper import (
IMAGES_PATH,
REPO_COPY,
S3_DOWNLOAD,
S3_BUILDS_BUCKET,
S3_TEST_REPORTS_BUCKET,
TEMP_PATH,
)
from get_robot_token import get_best_robot_token
from pr_info import PRInfo
from report import TestResult
from s3_helper import S3Helper
from stopwatch import Stopwatch
from tee_popen import TeePopen
from upload_result_helper import upload_results
NAME = "Woboq Build"
def get_run_command(
repo_path: Path, output_path: Path, image: DockerImage, sha: str
) -> str:
user = f"{os.geteuid()}:{os.getegid()}"
cmd = (
f"docker run --rm --user={user} --volume={repo_path}:/build "
f"--volume={output_path}:/workdir/output --network=host "
# use sccache, https://github.com/KDAB/codebrowser/issues/111
f"-e SCCACHE_BUCKET='{S3_BUILDS_BUCKET}' "
"-e SCCACHE_S3_KEY_PREFIX=ccache/sccache "
'-e CMAKE_FLAGS="$CMAKE_FLAGS -DCOMPILER_CACHE=sccache" '
f"-e 'DATA={S3_DOWNLOAD}/{S3_TEST_REPORTS_BUCKET}/codebrowser/data' "
f"-e SHA={sha} {image}"
)
return cmd
def main():
logging.basicConfig(level=logging.INFO)
stopwatch = Stopwatch()
gh = Github(get_best_robot_token(), per_page=100)
pr_info = PRInfo()
commit = get_commit(gh, pr_info.sha)
temp_path = Path(TEMP_PATH)
if not temp_path.exists():
os.makedirs(temp_path)
docker_image = get_image_with_version(IMAGES_PATH, "clickhouse/codebrowser")
# FIXME: the codebrowser is broken with clang-16, workaround with clang-15
# See https://github.com/ClickHouse/ClickHouse/issues/50077
docker_image.version = "49701-4dcdcf4c11b5604f1c5d3121c9c6fea3e957b605"
s3_helper = S3Helper()
result_path = temp_path / "result_path"
if not result_path.exists():
os.makedirs(result_path)
run_command = get_run_command(
Path(REPO_COPY), result_path, docker_image, pr_info.sha[:12]
)
logging.info("Going to run codebrowser: %s", run_command)
run_log_path = result_path / "run.log"
state = "success"
with TeePopen(run_command, run_log_path) as process:
retcode = process.wait()
if retcode == 0:
logging.info("Run successfully")
else:
logging.info("Run failed")
state = "failure"
report_path = result_path / "html_report"
logging.info("Report path %s", report_path)
s3_path_prefix = "codebrowser"
index_template = (
f'<a href="{S3_DOWNLOAD}/{S3_TEST_REPORTS_BUCKET}/{s3_path_prefix}/index.html">'
"{}</a>"
)
additional_logs = [path.absolute() for path in result_path.glob("*.log")]
test_results = [
TestResult(
index_template.format("Generate codebrowser site"),
state,
stopwatch.duration_seconds,
additional_logs,
)
]
if state == "success":
stopwatch.reset()
_ = s3_helper.fast_parallel_upload_dir(
report_path, s3_path_prefix, S3_TEST_REPORTS_BUCKET
)
test_results.append(
TestResult(
index_template.format("Upload codebrowser site"),
state,
stopwatch.duration_seconds,
)
)
# Check if the run log contains `FATAL Error:`, that means the code problem
stopwatch.reset()
fatal_error = "FATAL Error:"
logging.info("Search for '%s' in %s", fatal_error, run_log_path)
with open(run_log_path, "r", encoding="utf-8") as rlfd:
for line in rlfd.readlines():
if "FATAL Error:" in line:
logging.warning(
"The line '%s' found, mark the run as failure", fatal_error
)
state = "failure"
test_results.append(
TestResult(
"Indexing error",
state,
stopwatch.duration_seconds,
additional_logs,
)
)
break
report_url = upload_results(
s3_helper, pr_info.number, pr_info.sha, test_results, [], NAME
)
print(f"::notice ::Report url: {report_url}")
post_commit_status(commit, state, report_url, "Report built", NAME, pr_info)
if __name__ == "__main__":
main()

6
tests/ci/docs_check.py
View File

@ -17,7 +17,7 @@ from commit_status_helper import (
update_mergeable_check,
)
from docker_pull_helper import get_image_with_version
from env_helper import TEMP_PATH, REPO_COPY
from env_helper import TEMP_PATH, REPO_COPY, REPORTS_PATH
from get_robot_token import get_best_robot_token
from pr_info import PRInfo
from report import TestResults, TestResult
@ -57,6 +57,8 @@ def main():
temp_path = Path(TEMP_PATH)
temp_path.mkdir(parents=True, exist_ok=True)
reports_path = Path(REPORTS_PATH)
reports_path.mkdir(parents=True, exist_ok=True)
repo_path = Path(REPO_COPY)
pr_info = PRInfo(need_changed_files=True)
@ -82,7 +84,7 @@ def main():
elif args.force:
logging.info("Check the docs because of force flag")
docker_image = get_image_with_version(temp_path, "clickhouse/docs-builder")
docker_image = get_image_with_version(reports_path, "clickhouse/docs-builder")
test_output = temp_path / "docs_check_log"
test_output.mkdir(parents=True, exist_ok=True)

2
tests/ci/env_helper.py
View File

@ -24,7 +24,7 @@ GITHUB_WORKSPACE = os.getenv("GITHUB_WORKSPACE", git_root)
GITHUB_RUN_URL = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/actions/runs/{GITHUB_RUN_ID}"
IMAGES_PATH = os.getenv("IMAGES_PATH", TEMP_PATH)
REPORTS_PATH = os.getenv("REPORTS_PATH", p.abspath(p.join(module_dir, "./reports")))
REPO_COPY = os.getenv("REPO_COPY", git_root)
REPO_COPY = os.getenv("REPO_COPY", GITHUB_WORKSPACE)
RUNNER_TEMP = os.getenv("RUNNER_TEMP", p.abspath(p.join(module_dir, "./tmp")))
S3_BUILDS_BUCKET = os.getenv("S3_BUILDS_BUCKET", "clickhouse-builds")
S3_TEST_REPORTS_BUCKET = os.getenv("S3_TEST_REPORTS_BUCKET", "clickhouse-test-reports")

7
tests/ci/fast_test_check.py
View File

@ -24,7 +24,7 @@ from commit_status_helper import (
format_description,
)
from docker_pull_helper import get_image_with_version, DockerImage
from env_helper import S3_BUILDS_BUCKET, TEMP_PATH, REPO_COPY
from env_helper import S3_BUILDS_BUCKET, TEMP_PATH, REPO_COPY, REPORTS_PATH
from get_robot_token import get_best_robot_token
from pr_info import FORCE_TESTS_LABEL, PRInfo
from report import TestResult, TestResults, read_test_results
@ -117,8 +117,9 @@ def main():
args = parse_args()
temp_path = Path(TEMP_PATH)
temp_path.mkdir(parents=True, exist_ok=True)
reports_path = Path(REPORTS_PATH)
reports_path.mkdir(parents=True, exist_ok=True)
pr_info = PRInfo()
@ -135,7 +136,7 @@ def main():
sys.exit(1)
sys.exit(0)
docker_image = get_image_with_version(temp_path, "clickhouse/fasttest")
docker_image = get_image_with_version(reports_path, "clickhouse/fasttest")
s3_helper = S3Helper()

8
tests/ci/style_check.py
View File

@ -21,7 +21,7 @@ from commit_status_helper import (
update_mergeable_check,
)
from docker_pull_helper import get_image_with_version
from env_helper import GITHUB_WORKSPACE, TEMP_PATH
from env_helper import REPO_COPY, REPORTS_PATH, TEMP_PATH
from get_robot_token import get_best_robot_token
from github_helper import GitHub
from git_helper import git_runner
@ -139,9 +139,11 @@ def main():
stopwatch = Stopwatch()
repo_path = Path(GITHUB_WORKSPACE)
repo_path = Path(REPO_COPY)
temp_path = Path(TEMP_PATH)
temp_path.mkdir(parents=True, exist_ok=True)
reports_path = Path(REPORTS_PATH)
reports_path.mkdir(parents=True, exist_ok=True)
pr_info = PRInfo()
if args.push:
@ -161,7 +163,7 @@ def main():
code = int(state != "success")
sys.exit(code)
docker_image = get_image_with_version(temp_path, "clickhouse/style-test")
docker_image = get_image_with_version(reports_path, "clickhouse/style-test")
s3_helper = S3Helper()
cmd = (

15
tests/ci/test_ci_config.py Normal file
View File

@ -0,0 +1,15 @@
#!/usr/bin/env python3
import unittest
class TestCiConfig(unittest.TestCase):
def test_no_errors_in_ci_config(self):
raised = None
try:
from ci_config import ( # pylint: disable=import-outside-toplevel
CI_CONFIG as _,
)
except Exception as exc:
raised = exc
self.assertIsNone(raised, f"CI_CONFIG import raised error {raised}")

2
tests/config/config.d/merge_tree_settings.xml
View File

@ -2,7 +2,5 @@
<merge_tree>
<!-- 10 seconds (default is 1 minute) -->
<zookeeper_session_expiration_check_period>10</zookeeper_session_expiration_check_period>
<!-- Setting randomized for stress test, it is disabled here and this line is used to randomize it in the script -->
<allow_experimental_block_number_column>0</allow_experimental_block_number_column>
</merge_tree>
</clickhouse>

16
tests/performance/orc_tuple_field_prune.xml Normal file
View File

@ -0,0 +1,16 @@
<test>
<settings>
<output_format_orc_string_as_string>1</output_format_orc_string_as_string>
<input_format_orc_row_batch_size>10000</input_format_orc_row_batch_size>
</settings>
<fill_query>
insert into function file('test_orc_tfp.orc', 'ORC') select * from generateRandom('tuple_column Tuple(a Nullable(String), b Nullable(Float64), c Nullable(Int64)), array_tuple_column Nested(a Nullable(String), b Nullable(Float64), c Nullable(Int64)), map_tuple_column Map(String, Tuple(a Nullable(String), b Nullable(Float64), c Nullable(Int64)))') limit 1000000
</fill_query>
<drop_query>DROP TABLE IF EXISTS test_orc_tfp</drop_query>
<query>select * from file('test_orc_tfp.orc', 'ORC', 'tuple_column Tuple(c Nullable(Int64))') format Null</query>
<query>select * from file('test_orc_tfp.orc', 'ORC', 'array_tuple_column Nested(c Nullable(Int64))') format Null</query>
<query>select * from file('test_orc_tfp.orc', 'ORC', 'map_tuple_column Map(String, Tuple(c Nullable(Int64)))') format Null</query>
</test>

Some files were not shown because too many files have changed in this diff Show More