mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 15:12:02 +00:00
merge master
This commit is contained in:
taiyang-li
commit
2f4e7e1d4e
379
.github/workflows/backport_branches.yml
vendored
Normal file
379
.github/workflows/backport_branches.yml
vendored
Normal file
@ -0,0 +1,379 @@
|
||||
name: BackportPR
|
||||
on: # yamllint disable-line rule:truthy
|
||||
push:
|
||||
branches:
|
||||
- 'backport/**'
|
||||
jobs:
|
||||
DockerHubPush:
|
||||
runs-on: [self-hosted, style-checker]
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
- name: Images check
|
||||
run: |
|
||||
cd $GITHUB_WORKSPACE/tests/ci
|
||||
python3 docker_images_check.py
|
||||
- name: Upload images files to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: changed_images
|
||||
path: ${{ runner.temp }}/docker_images_check/changed_images.json
|
||||
CompatibilityCheck:
|
||||
needs: [BuilderDebRelease]
|
||||
runs-on: [self-hosted, style-checker]
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
- name: Download json reports
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
path: ${{runner.temp}}/reports_dir
|
||||
- name: CompatibilityCheck
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/compatibility_check
|
||||
REPO_COPY: ${{runner.temp}}/compatibility_check/ClickHouse
|
||||
REPORTS_PATH: ${{runner.temp}}/reports_dir
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 compatibility_check.py
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
#########################################################################################
|
||||
#################################### ORDINARY BUILDS ####################################
|
||||
#########################################################################################
|
||||
BuilderDebRelease:
|
||||
needs: [DockerHubPush]
|
||||
runs-on: [self-hosted, builder]
|
||||
steps:
|
||||
- name: Download changed images
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: changed_images
|
||||
path: ${{ runner.temp }}/images_path
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: 'recursive'
|
||||
fetch-depth: 0 # otherwise we will have no info about contributors
|
||||
- name: Build
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/build_check
|
||||
IMAGES_PATH: ${{runner.temp}}/images_path
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NAME: 'package_release'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: ${{ env.BUILD_NAME }}
|
||||
path: ${{ runner.temp }}/build_check/${{ env.BUILD_NAME }}.json
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
BuilderDebAsan:
|
||||
needs: [DockerHubPush]
|
||||
runs-on: [self-hosted, builder]
|
||||
steps:
|
||||
- name: Download changed images
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: changed_images
|
||||
path: ${{ runner.temp }}/images_path
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: 'recursive'
|
||||
fetch-depth: 0 # otherwise we will have no info about contributors
|
||||
- name: Build
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/build_check
|
||||
IMAGES_PATH: ${{runner.temp}}/images_path
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NAME: 'package_asan'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: ${{ env.BUILD_NAME }}
|
||||
path: ${{ runner.temp }}/build_check/${{ env.BUILD_NAME }}.json
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
BuilderDebTsan:
|
||||
needs: [DockerHubPush]
|
||||
runs-on: [self-hosted, builder]
|
||||
steps:
|
||||
- name: Download changed images
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: changed_images
|
||||
path: ${{ runner.temp }}/images_path
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: 'recursive'
|
||||
fetch-depth: 0 # otherwise we will have no info about contributors
|
||||
- name: Build
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/build_check
|
||||
IMAGES_PATH: ${{runner.temp}}/images_path
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NAME: 'package_tsan'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: ${{ env.BUILD_NAME }}
|
||||
path: ${{ runner.temp }}/build_check/${{ env.BUILD_NAME }}.json
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
BuilderDebDebug:
|
||||
needs: [DockerHubPush]
|
||||
runs-on: [self-hosted, builder]
|
||||
steps:
|
||||
- name: Download changed images
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: changed_images
|
||||
path: ${{ runner.temp }}/images_path
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: 'recursive'
|
||||
fetch-depth: 0 # otherwise we will have no info about contributors
|
||||
- name: Build
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/build_check
|
||||
IMAGES_PATH: ${{runner.temp}}/images_path
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NAME: 'package_debug'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: ${{ env.BUILD_NAME }}
|
||||
path: ${{ runner.temp }}/build_check/${{ env.BUILD_NAME }}.json
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
############################################################################################
|
||||
##################################### BUILD REPORTER #######################################
|
||||
############################################################################################
|
||||
BuilderReport:
|
||||
needs:
|
||||
- BuilderDebRelease
|
||||
- BuilderDebAsan
|
||||
- BuilderDebTsan
|
||||
- BuilderDebUBsan
|
||||
- BuilderDebMsan
|
||||
- BuilderDebDebug
|
||||
runs-on: [self-hosted, style-checker]
|
||||
steps:
|
||||
- name: Download json reports
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
path: ${{runner.temp}}/reports_dir
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
- name: Report Builder
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/report_check
|
||||
REPORTS_PATH: ${{runner.temp}}/reports_dir
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cd $GITHUB_WORKSPACE/tests/ci
|
||||
python3 build_report_check.py "$CHECK_NAME"
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
##############################################################################################
|
||||
########################### FUNCTIONAl STATELESS TESTS #######################################
|
||||
##############################################################################################
|
||||
FunctionalStatelessTestAsan:
|
||||
needs: [BuilderDebAsan]
|
||||
runs-on: [self-hosted, func-tester]
|
||||
steps:
|
||||
- name: Download json reports
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
path: ${{runner.temp}}/reports_dir
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
- name: Functional test
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/stateless_debug
|
||||
REPORTS_PATH: ${{runner.temp}}/reports_dir
|
||||
CHECK_NAME: 'Stateless tests (address, actions)'
|
||||
REPO_COPY: ${{runner.temp}}/stateless_debug/ClickHouse
|
||||
KILL_TIMEOUT: 10800
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci
|
||||
python3 functional_test_check.py "$CHECK_NAME" $KILL_TIMEOUT
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
##############################################################################################
|
||||
############################ FUNCTIONAl STATEFUL TESTS #######################################
|
||||
##############################################################################################
|
||||
FunctionalStatefulTestDebug:
|
||||
needs: [BuilderDebDebug]
|
||||
runs-on: [self-hosted, func-tester]
|
||||
steps:
|
||||
- name: Download json reports
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
path: ${{runner.temp}}/reports_dir
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
- name: Functional test
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/stateful_debug
|
||||
REPORTS_PATH: ${{runner.temp}}/reports_dir
|
||||
CHECK_NAME: 'Stateful tests (debug, actions)'
|
||||
REPO_COPY: ${{runner.temp}}/stateful_debug/ClickHouse
|
||||
KILL_TIMEOUT: 3600
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci
|
||||
python3 functional_test_check.py "$CHECK_NAME" $KILL_TIMEOUT
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
##############################################################################################
|
||||
######################################### STRESS TESTS #######################################
|
||||
##############################################################################################
|
||||
StressTestTsan:
|
||||
needs: [BuilderDebTsan]
|
||||
runs-on: [self-hosted, stress-tester]
|
||||
steps:
|
||||
- name: Download json reports
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
path: ${{runner.temp}}/reports_dir
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
- name: Stress test
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/stress_thread
|
||||
REPORTS_PATH: ${{runner.temp}}/reports_dir
|
||||
CHECK_NAME: 'Stress test (thread, actions)'
|
||||
REPO_COPY: ${{runner.temp}}/stress_thread/ClickHouse
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci
|
||||
python3 stress_check.py "$CHECK_NAME"
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
#############################################################################################
|
||||
############################# INTEGRATION TESTS #############################################
|
||||
#############################################################################################
|
||||
IntegrationTestsRelease:
|
||||
needs: [BuilderDebRelease, FunctionalStatelessTestRelease]
|
||||
runs-on: [self-hosted, stress-tester]
|
||||
steps:
|
||||
- name: Download json reports
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
path: ${{runner.temp}}/reports_dir
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
- name: Integration test
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/integration_tests_release
|
||||
REPORTS_PATH: ${{runner.temp}}/reports_dir
|
||||
CHECK_NAME: 'Integration tests (release, actions)'
|
||||
REPO_COPY: ${{runner.temp}}/integration_tests_release/ClickHouse
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci
|
||||
python3 integration_test_check.py "$CHECK_NAME"
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
FinishCheck:
|
||||
needs:
|
||||
- DockerHubPush
|
||||
- BuilderReport
|
||||
- FunctionalStatelessTestAsan
|
||||
- FunctionalStatefulTestDebug
|
||||
- StressTestTsan
|
||||
- IntegrationTestsRelease
|
||||
- CompatibilityCheck
|
||||
runs-on: [self-hosted, style-checker]
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
- name: Finish label
|
||||
run: |
|
||||
cd $GITHUB_WORKSPACE/tests/ci
|
||||
python3 finish_check.py
|
||||
3
.github/workflows/cancel.yml
vendored
3
.github/workflows/cancel.yml
vendored
@ -1,7 +1,7 @@
|
||||
name: Cancel
|
||||
on: # yamllint disable-line rule:truthy
|
||||
workflow_run:
|
||||
workflows: ["CIGithubActions", "ReleaseCI"]
|
||||
workflows: ["CIGithubActions", "ReleaseCI", "DocsCheck", "BackportPR"]
|
||||
types:
|
||||
- requested
|
||||
jobs:
|
||||
@ -10,4 +10,5 @@ jobs:
|
||||
steps:
|
||||
- uses: styfle/cancel-workflow-action@0.9.1
|
||||
with:
|
||||
all_but_latest: true
|
||||
workflow_id: ${{ github.event.workflow.id }}
|
||||
|
||||
62
.github/workflows/docs_check.yml
vendored
Normal file
62
.github/workflows/docs_check.yml
vendored
Normal file
@ -0,0 +1,62 @@
|
||||
name: DocsCheck
|
||||
on: # yamllint disable-line rule:truthy
|
||||
pull_request:
|
||||
types:
|
||||
- synchronize
|
||||
- reopened
|
||||
- opened
|
||||
branches:
|
||||
- master
|
||||
paths:
|
||||
- 'docs/**'
|
||||
- 'website/**'
|
||||
jobs:
|
||||
CheckLabels:
|
||||
runs-on: [self-hosted, style-checker]
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
- name: Labels check
|
||||
run: |
|
||||
cd $GITHUB_WORKSPACE/tests/ci
|
||||
python3 run_check.py
|
||||
DockerHubPush:
|
||||
needs: CheckLabels
|
||||
runs-on: [self-hosted, style-checker]
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
- name: Images check
|
||||
run: |
|
||||
cd $GITHUB_WORKSPACE/tests/ci
|
||||
python3 docker_images_check.py
|
||||
- name: Upload images files to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: changed_images
|
||||
path: ${{ runner.temp }}/docker_images_check/changed_images.json
|
||||
DocsCheck:
|
||||
needs: DockerHubPush
|
||||
runs-on: [self-hosted, func-tester]
|
||||
steps:
|
||||
- name: Download changed images
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: changed_images
|
||||
path: ${{ runner.temp }}/docs_check
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
- name: Docs Check
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/docs_check
|
||||
REPO_COPY: ${{runner.temp}}/docs_check/ClickHouse
|
||||
run: |
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci
|
||||
python3 docs_check.py
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
414
.github/workflows/main.yml
vendored
414
.github/workflows/main.yml
vendored
@ -2,13 +2,14 @@ name: CIGithubActions
|
||||
on: # yamllint disable-line rule:truthy
|
||||
pull_request:
|
||||
types:
|
||||
- labeled
|
||||
- unlabeled
|
||||
- synchronize
|
||||
- reopened
|
||||
- opened
|
||||
branches:
|
||||
- master
|
||||
paths-ignore:
|
||||
- 'docs/**'
|
||||
- 'website/**'
|
||||
##########################################################################################
|
||||
##################################### SMALL CHECKS #######################################
|
||||
##########################################################################################
|
||||
@ -60,34 +61,8 @@ jobs:
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
DocsCheck:
|
||||
needs: DockerHubPush
|
||||
runs-on: [self-hosted, func-tester]
|
||||
steps:
|
||||
- name: Download changed images
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: changed_images
|
||||
path: ${{ runner.temp }}/docs_check
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
- name: Docs Check
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/docs_check
|
||||
REPO_COPY: ${{runner.temp}}/docs_check/ClickHouse
|
||||
run: |
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci
|
||||
python3 docs_check.py
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
FastTest:
|
||||
needs: DockerHubPush
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
|
||||
runs-on: [self-hosted, builder]
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
@ -109,8 +84,7 @@ jobs:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
PVSCheck:
|
||||
needs: DockerHubPush
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
|
||||
needs: [DockerHubPush, FastTest]
|
||||
runs-on: [self-hosted, func-tester]
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
@ -134,7 +108,6 @@ jobs:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
CompatibilityCheck:
|
||||
needs: [BuilderDebRelease]
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
|
||||
runs-on: [self-hosted, style-checker]
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
@ -161,7 +134,6 @@ jobs:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
SplitBuildSmokeTest:
|
||||
needs: [BuilderDebSplitted]
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
|
||||
runs-on: [self-hosted, style-checker]
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
@ -191,7 +163,6 @@ jobs:
|
||||
#########################################################################################
|
||||
BuilderDebRelease:
|
||||
needs: [DockerHubPush, FastTest]
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
|
||||
runs-on: [self-hosted, builder]
|
||||
steps:
|
||||
- name: Download changed images
|
||||
@ -211,12 +182,12 @@ jobs:
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NUMBER: 0
|
||||
BUILD_NAME: 'package_release'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
@ -230,7 +201,6 @@ jobs:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
BuilderBinRelease:
|
||||
needs: [DockerHubPush, FastTest]
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
|
||||
runs-on: [self-hosted, builder]
|
||||
steps:
|
||||
- name: Download changed images
|
||||
@ -250,12 +220,12 @@ jobs:
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NUMBER: 8
|
||||
BUILD_NAME: 'binary_release'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
@ -269,7 +239,6 @@ jobs:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
BuilderDebAsan:
|
||||
needs: [DockerHubPush, FastTest]
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
|
||||
runs-on: [self-hosted, builder]
|
||||
steps:
|
||||
- name: Download changed images
|
||||
@ -289,12 +258,12 @@ jobs:
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NUMBER: 3
|
||||
BUILD_NAME: 'package_asan'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
@ -308,7 +277,6 @@ jobs:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
BuilderDebUBsan:
|
||||
needs: [DockerHubPush, FastTest]
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
|
||||
runs-on: [self-hosted, builder]
|
||||
steps:
|
||||
- name: Download changed images
|
||||
@ -328,12 +296,12 @@ jobs:
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NUMBER: 4
|
||||
BUILD_NAME: 'package_ubsan'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
@ -347,7 +315,6 @@ jobs:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
BuilderDebTsan:
|
||||
needs: [DockerHubPush, FastTest]
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
|
||||
runs-on: [self-hosted, builder]
|
||||
steps:
|
||||
- name: Download changed images
|
||||
@ -367,12 +334,12 @@ jobs:
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NUMBER: 5
|
||||
BUILD_NAME: 'package_tsan'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
@ -386,7 +353,6 @@ jobs:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
BuilderDebMsan:
|
||||
needs: [DockerHubPush, FastTest]
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
|
||||
runs-on: [self-hosted, builder]
|
||||
steps:
|
||||
- name: Download changed images
|
||||
@ -406,12 +372,12 @@ jobs:
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NUMBER: 6
|
||||
BUILD_NAME: 'package_msan'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
@ -425,7 +391,6 @@ jobs:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
BuilderDebDebug:
|
||||
needs: [DockerHubPush, FastTest]
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
|
||||
runs-on: [self-hosted, builder]
|
||||
steps:
|
||||
- name: Download changed images
|
||||
@ -445,12 +410,12 @@ jobs:
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NUMBER: 7
|
||||
BUILD_NAME: 'package_debug'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
@ -467,7 +432,6 @@ jobs:
|
||||
##########################################################################################
|
||||
BuilderDebSplitted:
|
||||
needs: [DockerHubPush, FastTest]
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
|
||||
runs-on: [self-hosted, builder]
|
||||
steps:
|
||||
- name: Download changed images
|
||||
@ -486,13 +450,241 @@ jobs:
|
||||
IMAGES_PATH: ${{runner.temp}}/images_path
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse special build check (actions)'
|
||||
BUILD_NUMBER: 1
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NAME: 'binary_splitted'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: ${{ env.BUILD_NAME }}
|
||||
path: ${{ runner.temp }}/build_check/${{ env.BUILD_NAME }}.json
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
BuilderBinTidy:
|
||||
needs: [DockerHubPush, FastTest]
|
||||
runs-on: [self-hosted, builder]
|
||||
steps:
|
||||
- name: Download changed images
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: changed_images
|
||||
path: ${{ runner.temp }}/images_path
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: 'recursive'
|
||||
fetch-depth: 0 # otherwise we will have no info about contributors
|
||||
- name: Build
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/build_check
|
||||
IMAGES_PATH: ${{runner.temp}}/images_path
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NAME: 'binary_tidy'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: ${{ env.BUILD_NAME }}
|
||||
path: ${{ runner.temp }}/build_check/${{ env.BUILD_NAME }}.json
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
BuilderBinDarwin:
|
||||
needs: [DockerHubPush, FastTest]
|
||||
runs-on: [self-hosted, builder]
|
||||
steps:
|
||||
- name: Download changed images
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: changed_images
|
||||
path: ${{ runner.temp }}/images_path
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: 'recursive'
|
||||
fetch-depth: 0 # otherwise we will have no info about contributors
|
||||
- name: Build
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/build_check
|
||||
IMAGES_PATH: ${{runner.temp}}/images_path
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NAME: 'binary_darwin'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: ${{ env.BUILD_NAME }}
|
||||
path: ${{ runner.temp }}/build_check/${{ env.BUILD_NAME }}.json
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
BuilderBinAarch64:
|
||||
needs: [DockerHubPush, FastTest]
|
||||
runs-on: [self-hosted, builder]
|
||||
steps:
|
||||
- name: Download changed images
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: changed_images
|
||||
path: ${{ runner.temp }}/images_path
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: 'recursive'
|
||||
fetch-depth: 0 # otherwise we will have no info about contributors
|
||||
- name: Build
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/build_check
|
||||
IMAGES_PATH: ${{runner.temp}}/images_path
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NAME: 'binary_aarch64'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: ${{ env.BUILD_NAME }}
|
||||
path: ${{ runner.temp }}/build_check/${{ env.BUILD_NAME }}.json
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
BuilderBinFreeBSD:
|
||||
needs: [DockerHubPush, FastTest]
|
||||
runs-on: [self-hosted, builder]
|
||||
steps:
|
||||
- name: Download changed images
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: changed_images
|
||||
path: ${{ runner.temp }}/images_path
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: 'recursive'
|
||||
fetch-depth: 0 # otherwise we will have no info about contributors
|
||||
- name: Build
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/build_check
|
||||
IMAGES_PATH: ${{runner.temp}}/images_path
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NAME: 'binary_freebsd'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: ${{ env.BUILD_NAME }}
|
||||
path: ${{ runner.temp }}/build_check/${{ env.BUILD_NAME }}.json
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
BuilderBinDarwinAarch64:
|
||||
needs: [DockerHubPush, FastTest]
|
||||
runs-on: [self-hosted, builder]
|
||||
steps:
|
||||
- name: Download changed images
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: changed_images
|
||||
path: ${{ runner.temp }}/images_path
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: 'recursive'
|
||||
fetch-depth: 0 # otherwise we will have no info about contributors
|
||||
- name: Build
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/build_check
|
||||
IMAGES_PATH: ${{runner.temp}}/images_path
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NAME: 'binary_darwin_aarch64'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: ${{ env.BUILD_NAME }}
|
||||
path: ${{ runner.temp }}/build_check/${{ env.BUILD_NAME }}.json
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
BuilderBinPPC64:
|
||||
needs: [DockerHubPush, FastTest]
|
||||
runs-on: [self-hosted, builder]
|
||||
steps:
|
||||
- name: Download changed images
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: changed_images
|
||||
path: ${{ runner.temp }}/images_path
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: 'recursive'
|
||||
fetch-depth: 0 # otherwise we will have no info about contributors
|
||||
- name: Build
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/build_check
|
||||
IMAGES_PATH: ${{runner.temp}}/images_path
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NAME: 'binary_ppc64le'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
@ -516,7 +708,6 @@ jobs:
|
||||
- BuilderDebUBsan
|
||||
- BuilderDebMsan
|
||||
- BuilderDebDebug
|
||||
- BuilderDebSplitted
|
||||
runs-on: [self-hosted, style-checker]
|
||||
if: always()
|
||||
steps:
|
||||
@ -542,6 +733,40 @@ jobs:
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
BuilderSpecialReport:
|
||||
needs:
|
||||
- BuilderDebSplitted
|
||||
- BuilderBinTidy
|
||||
- BuilderBinDarwin
|
||||
- BuilderBinAarch64
|
||||
- BuilderBinFreeBSD
|
||||
- BuilderBinDarwinAarch64
|
||||
- BuilderBinPPC64
|
||||
runs-on: [self-hosted, style-checker]
|
||||
if: always()
|
||||
steps:
|
||||
- name: Download json reports
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
path: ${{runner.temp}}/reports_dir
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
- name: Report Builder
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/report_check
|
||||
REPORTS_PATH: ${{runner.temp}}/reports_dir
|
||||
CHECK_NAME: 'ClickHouse special build check (actions)'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cd $GITHUB_WORKSPACE/tests/ci
|
||||
python3 build_report_check.py "$CHECK_NAME"
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
##############################################################################################
|
||||
########################### FUNCTIONAl STATELESS TESTS #######################################
|
||||
##############################################################################################
|
||||
@ -557,10 +782,68 @@ jobs:
|
||||
uses: actions/checkout@v2
|
||||
- name: Functional test
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/stateless_debug
|
||||
TEMP_PATH: ${{runner.temp}}/stateless_release
|
||||
REPORTS_PATH: ${{runner.temp}}/reports_dir
|
||||
CHECK_NAME: 'Stateless tests (release, actions)'
|
||||
REPO_COPY: ${{runner.temp}}/stateless_debug/ClickHouse
|
||||
REPO_COPY: ${{runner.temp}}/stateless_release/ClickHouse
|
||||
KILL_TIMEOUT: 10800
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci
|
||||
python3 functional_test_check.py "$CHECK_NAME" $KILL_TIMEOUT
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
FunctionalStatelessTestReleaseDatabaseReplicated:
|
||||
needs: [BuilderDebRelease]
|
||||
runs-on: [self-hosted, func-tester]
|
||||
steps:
|
||||
- name: Download json reports
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
path: ${{runner.temp}}/reports_dir
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
- name: Functional test
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/stateless_database_replicated
|
||||
REPORTS_PATH: ${{runner.temp}}/reports_dir
|
||||
CHECK_NAME: 'Stateless tests (release, DatabaseReplicated, actions)'
|
||||
REPO_COPY: ${{runner.temp}}/stateless_database_replicated/ClickHouse
|
||||
KILL_TIMEOUT: 10800
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci
|
||||
python3 functional_test_check.py "$CHECK_NAME" $KILL_TIMEOUT
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
FunctionalStatelessTestReleaseWideParts:
|
||||
needs: [BuilderDebRelease]
|
||||
runs-on: [self-hosted, func-tester]
|
||||
steps:
|
||||
- name: Download json reports
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
path: ${{runner.temp}}/reports_dir
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
- name: Functional test
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/stateless_wide_parts
|
||||
REPORTS_PATH: ${{runner.temp}}/reports_dir
|
||||
CHECK_NAME: 'Stateless tests (release, wide parts enabled, actions)'
|
||||
REPO_COPY: ${{runner.temp}}/stateless_wide_parts/ClickHouse
|
||||
KILL_TIMEOUT: 10800
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
@ -605,9 +888,7 @@ jobs:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
FunctionalStatelessTestTsan:
|
||||
needs: [BuilderDebTsan]
|
||||
# tests can consume more than 60GB of memory,
|
||||
# so use bigger server
|
||||
runs-on: [self-hosted, stress-tester]
|
||||
runs-on: [self-hosted, func-tester]
|
||||
steps:
|
||||
- name: Download json reports
|
||||
uses: actions/download-artifact@v2
|
||||
@ -765,10 +1046,10 @@ jobs:
|
||||
uses: actions/checkout@v2
|
||||
- name: Functional test
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/stateful_debug
|
||||
TEMP_PATH: ${{runner.temp}}/stateful_release
|
||||
REPORTS_PATH: ${{runner.temp}}/reports_dir
|
||||
CHECK_NAME: 'Stateful tests (release, actions)'
|
||||
REPO_COPY: ${{runner.temp}}/stateful_debug/ClickHouse
|
||||
REPO_COPY: ${{runner.temp}}/stateful_release/ClickHouse
|
||||
KILL_TIMEOUT: 3600
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
@ -1480,6 +1761,8 @@ jobs:
|
||||
- FastTest
|
||||
- FunctionalStatelessTestDebug
|
||||
- FunctionalStatelessTestRelease
|
||||
- FunctionalStatelessTestReleaseDatabaseReplicated
|
||||
- FunctionalStatelessTestReleaseWideParts
|
||||
- FunctionalStatelessTestAsan
|
||||
- FunctionalStatelessTestTsan
|
||||
- FunctionalStatelessTestMsan
|
||||
@ -1490,7 +1773,6 @@ jobs:
|
||||
- FunctionalStatefulTestTsan
|
||||
- FunctionalStatefulTestMsan
|
||||
- FunctionalStatefulTestUBsan
|
||||
- DocsCheck
|
||||
- StressTestDebug
|
||||
- StressTestAsan
|
||||
- StressTestTsan
|
||||
|
||||
397
.github/workflows/master.yml
vendored
397
.github/workflows/master.yml
vendored
@ -41,30 +41,6 @@ jobs:
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
PVSCheck:
|
||||
needs: DockerHubPush
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
|
||||
runs-on: [self-hosted, func-tester]
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: 'recursive'
|
||||
- name: PVS Check
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/pvs_check
|
||||
REPO_COPY: ${{runner.temp}}/pvs_check/ClickHouse
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 pvs_check.py
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
CompatibilityCheck:
|
||||
needs: [BuilderDebRelease]
|
||||
runs-on: [self-hosted, style-checker]
|
||||
@ -84,7 +60,7 @@ jobs:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 compatibility_check.py 0
|
||||
cd $REPO_COPY/tests/ci && python3 compatibility_check.py
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
@ -142,12 +118,12 @@ jobs:
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NUMBER: 0
|
||||
BUILD_NAME: 'package_release'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
@ -181,12 +157,12 @@ jobs:
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NUMBER: 8
|
||||
BUILD_NAME: 'binary_release'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
@ -219,12 +195,12 @@ jobs:
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NUMBER: 3
|
||||
BUILD_NAME: 'package_asan'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
@ -257,12 +233,12 @@ jobs:
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NUMBER: 4
|
||||
BUILD_NAME: 'package_ubsan'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
@ -295,12 +271,12 @@ jobs:
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NUMBER: 5
|
||||
BUILD_NAME: 'package_tsan'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
@ -333,12 +309,12 @@ jobs:
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NUMBER: 6
|
||||
BUILD_NAME: 'package_msan'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
@ -371,12 +347,12 @@ jobs:
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NUMBER: 7
|
||||
BUILD_NAME: 'package_debug'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
@ -412,13 +388,247 @@ jobs:
|
||||
IMAGES_PATH: ${{runner.temp}}/images_path
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse special build check (actions)'
|
||||
BUILD_NUMBER: 1
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NAME: 'binary_splitted'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: ${{ env.BUILD_NAME }}
|
||||
path: ${{ runner.temp }}/build_check/${{ env.BUILD_NAME }}.json
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
BuilderBinTidy:
|
||||
needs: [DockerHubPush]
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
|
||||
runs-on: [self-hosted, builder]
|
||||
steps:
|
||||
- name: Download changed images
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: changed_images
|
||||
path: ${{ runner.temp }}/images_path
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: 'recursive'
|
||||
fetch-depth: 0 # otherwise we will have no info about contributors
|
||||
- name: Build
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/build_check
|
||||
IMAGES_PATH: ${{runner.temp}}/images_path
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NAME: 'binary_tidy'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: ${{ env.BUILD_NAME }}
|
||||
path: ${{ runner.temp }}/build_check/${{ env.BUILD_NAME }}.json
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
BuilderBinDarwin:
|
||||
needs: [DockerHubPush]
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
|
||||
runs-on: [self-hosted, builder]
|
||||
steps:
|
||||
- name: Download changed images
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: changed_images
|
||||
path: ${{ runner.temp }}/images_path
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: 'recursive'
|
||||
fetch-depth: 0 # otherwise we will have no info about contributors
|
||||
- name: Build
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/build_check
|
||||
IMAGES_PATH: ${{runner.temp}}/images_path
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NAME: 'binary_darwin'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: ${{ env.BUILD_NAME }}
|
||||
path: ${{ runner.temp }}/build_check/${{ env.BUILD_NAME }}.json
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
BuilderBinAarch64:
|
||||
needs: [DockerHubPush]
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
|
||||
runs-on: [self-hosted, builder]
|
||||
steps:
|
||||
- name: Download changed images
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: changed_images
|
||||
path: ${{ runner.temp }}/images_path
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: 'recursive'
|
||||
fetch-depth: 0 # otherwise we will have no info about contributors
|
||||
- name: Build
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/build_check
|
||||
IMAGES_PATH: ${{runner.temp}}/images_path
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NAME: 'binary_aarch64'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: ${{ env.BUILD_NAME }}
|
||||
path: ${{ runner.temp }}/build_check/${{ env.BUILD_NAME }}.json
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
BuilderBinFreeBSD:
|
||||
needs: [DockerHubPush]
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
|
||||
runs-on: [self-hosted, builder]
|
||||
steps:
|
||||
- name: Download changed images
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: changed_images
|
||||
path: ${{ runner.temp }}/images_path
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: 'recursive'
|
||||
fetch-depth: 0 # otherwise we will have no info about contributors
|
||||
- name: Build
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/build_check
|
||||
IMAGES_PATH: ${{runner.temp}}/images_path
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NAME: 'binary_freebsd'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: ${{ env.BUILD_NAME }}
|
||||
path: ${{ runner.temp }}/build_check/${{ env.BUILD_NAME }}.json
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
BuilderBinDarwinAarch64:
|
||||
needs: [DockerHubPush]
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
|
||||
runs-on: [self-hosted, builder]
|
||||
steps:
|
||||
- name: Download changed images
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: changed_images
|
||||
path: ${{ runner.temp }}/images_path
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: 'recursive'
|
||||
fetch-depth: 0 # otherwise we will have no info about contributors
|
||||
- name: Build
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/build_check
|
||||
IMAGES_PATH: ${{runner.temp}}/images_path
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NAME: 'binary_darwin_aarch64'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: ${{ env.BUILD_NAME }}
|
||||
path: ${{ runner.temp }}/build_check/${{ env.BUILD_NAME }}.json
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
BuilderBinPPC64:
|
||||
needs: [DockerHubPush]
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
|
||||
runs-on: [self-hosted, builder]
|
||||
steps:
|
||||
- name: Download changed images
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: changed_images
|
||||
path: ${{ runner.temp }}/images_path
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: 'recursive'
|
||||
fetch-depth: 0 # otherwise we will have no info about contributors
|
||||
- name: Build
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/build_check
|
||||
IMAGES_PATH: ${{runner.temp}}/images_path
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NAME: 'binary_ppc64le'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
@ -442,7 +652,6 @@ jobs:
|
||||
- BuilderDebUBsan
|
||||
- BuilderDebMsan
|
||||
- BuilderDebDebug
|
||||
- BuilderDebSplitted
|
||||
runs-on: [self-hosted, style-checker]
|
||||
steps:
|
||||
- name: Download json reports
|
||||
@ -467,6 +676,39 @@ jobs:
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
BuilderSpecialReport:
|
||||
needs:
|
||||
- BuilderBinTidy
|
||||
- BuilderDebSplitted
|
||||
- BuilderBinDarwin
|
||||
- BuilderBinAarch64
|
||||
- BuilderBinFreeBSD
|
||||
- BuilderBinDarwinAarch64
|
||||
- BuilderBinPPC64
|
||||
runs-on: [self-hosted, style-checker]
|
||||
steps:
|
||||
- name: Download json reports
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
path: ${{runner.temp}}/reports_dir
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
- name: Report Builder
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/report_check
|
||||
REPORTS_PATH: ${{runner.temp}}/reports_dir
|
||||
CHECK_NAME: 'ClickHouse special build check (actions)'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cd $GITHUB_WORKSPACE/tests/ci
|
||||
python3 build_report_check.py "$CHECK_NAME"
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
##############################################################################################
|
||||
########################### FUNCTIONAl STATELESS TESTS #######################################
|
||||
##############################################################################################
|
||||
@ -482,10 +724,39 @@ jobs:
|
||||
uses: actions/checkout@v2
|
||||
- name: Functional test
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/stateless_debug
|
||||
TEMP_PATH: ${{runner.temp}}/stateless_release
|
||||
REPORTS_PATH: ${{runner.temp}}/reports_dir
|
||||
CHECK_NAME: 'Stateless tests (release, actions)'
|
||||
REPO_COPY: ${{runner.temp}}/stateless_debug/ClickHouse
|
||||
REPO_COPY: ${{runner.temp}}/stateless_release/ClickHouse
|
||||
KILL_TIMEOUT: 10800
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci
|
||||
python3 functional_test_check.py "$CHECK_NAME" $KILL_TIMEOUT
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
FunctionalStatelessTestReleaseDatabaseOrdinary:
|
||||
needs: [BuilderDebRelease]
|
||||
runs-on: [self-hosted, func-tester]
|
||||
steps:
|
||||
- name: Download json reports
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
path: ${{runner.temp}}/reports_dir
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
- name: Functional test
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/stateless_release_database_ordinary
|
||||
REPORTS_PATH: ${{runner.temp}}/reports_dir
|
||||
CHECK_NAME: 'Stateless tests (release, DatabaseOrdinary, actions)'
|
||||
REPO_COPY: ${{runner.temp}}/stateless_release_database_ordinary/ClickHouse
|
||||
KILL_TIMEOUT: 10800
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
@ -659,10 +930,39 @@ jobs:
|
||||
uses: actions/checkout@v2
|
||||
- name: Functional test
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/stateful_debug
|
||||
TEMP_PATH: ${{runner.temp}}/stateful_release
|
||||
REPORTS_PATH: ${{runner.temp}}/reports_dir
|
||||
CHECK_NAME: 'Stateful tests (release, actions)'
|
||||
REPO_COPY: ${{runner.temp}}/stateful_debug/ClickHouse
|
||||
REPO_COPY: ${{runner.temp}}/stateful_release/ClickHouse
|
||||
KILL_TIMEOUT: 3600
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci
|
||||
python3 functional_test_check.py "$CHECK_NAME" $KILL_TIMEOUT
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill $(docker ps -q) ||:
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
FunctionalStatefulTestReleaseDatabaseOrdinary:
|
||||
needs: [BuilderDebRelease]
|
||||
runs-on: [self-hosted, func-tester]
|
||||
steps:
|
||||
- name: Download json reports
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
path: ${{runner.temp}}/reports_dir
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
- name: Functional test
|
||||
env:
|
||||
TEMP_PATH: ${{runner.temp}}/stateful_release_database_ordinary
|
||||
REPORTS_PATH: ${{runner.temp}}/reports_dir
|
||||
CHECK_NAME: 'Stateful tests (release, DatabaseOrdinary, actions)'
|
||||
REPO_COPY: ${{runner.temp}}/stateful_release_database_ordinary/ClickHouse
|
||||
KILL_TIMEOUT: 3600
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
@ -1343,12 +1643,14 @@ jobs:
|
||||
- BuilderReport
|
||||
- FunctionalStatelessTestDebug
|
||||
- FunctionalStatelessTestRelease
|
||||
- FunctionalStatelessTestReleaseDatabaseOrdinary
|
||||
- FunctionalStatelessTestAsan
|
||||
- FunctionalStatelessTestTsan
|
||||
- FunctionalStatelessTestMsan
|
||||
- FunctionalStatelessTestUBsan
|
||||
- FunctionalStatefulTestDebug
|
||||
- FunctionalStatefulTestRelease
|
||||
- FunctionalStatefulTestReleaseDatabaseOrdinary
|
||||
- FunctionalStatefulTestAsan
|
||||
- FunctionalStatefulTestTsan
|
||||
- FunctionalStatefulTestMsan
|
||||
@ -1373,7 +1675,6 @@ jobs:
|
||||
- UnitTestsUBsan
|
||||
- UnitTestsReleaseClang
|
||||
- SplitBuildSmokeTest
|
||||
- PVSCheck
|
||||
runs-on: [self-hosted, style-checker]
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
|
||||
27
.github/workflows/release_branches.yml
vendored
27
.github/workflows/release_branches.yml
vendored
@ -6,7 +6,6 @@ on: # yamllint disable-line rule:truthy
|
||||
- '22.**'
|
||||
- '23.**'
|
||||
- '24.**'
|
||||
- 'backport/**'
|
||||
jobs:
|
||||
DockerHubPush:
|
||||
runs-on: [self-hosted, style-checker]
|
||||
@ -41,7 +40,7 @@ jobs:
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 compatibility_check.py 0
|
||||
cd $REPO_COPY/tests/ci && python3 compatibility_check.py
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
@ -72,12 +71,12 @@ jobs:
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NUMBER: 0
|
||||
BUILD_NAME: 'package_release'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
@ -110,12 +109,12 @@ jobs:
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NUMBER: 3
|
||||
BUILD_NAME: 'package_asan'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
@ -148,12 +147,12 @@ jobs:
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NUMBER: 4
|
||||
BUILD_NAME: 'package_ubsan'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
@ -186,12 +185,12 @@ jobs:
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NUMBER: 5
|
||||
BUILD_NAME: 'package_tsan'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
@ -224,12 +223,12 @@ jobs:
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NUMBER: 6
|
||||
BUILD_NAME: 'package_msan'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
@ -262,12 +261,12 @@ jobs:
|
||||
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH: ${{runner.temp}}/../ccaches
|
||||
CHECK_NAME: 'ClickHouse build check (actions)'
|
||||
BUILD_NUMBER: 7
|
||||
BUILD_NAME: 'package_debug'
|
||||
run: |
|
||||
sudo rm -fr $TEMP_PATH
|
||||
mkdir -p $TEMP_PATH
|
||||
cp -r $GITHUB_WORKSPACE $TEMP_PATH
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
|
||||
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
|
||||
- name: Upload build URLs to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
|
||||
5
.gitmodules
vendored
5
.gitmodules
vendored
@ -17,7 +17,7 @@
|
||||
[submodule "contrib/zlib-ng"]
|
||||
path = contrib/zlib-ng
|
||||
url = https://github.com/ClickHouse-Extras/zlib-ng.git
|
||||
branch = clickhouse-new
|
||||
branch = clickhouse-2.0.x
|
||||
[submodule "contrib/googletest"]
|
||||
path = contrib/googletest
|
||||
url = https://github.com/google/googletest.git
|
||||
@ -135,9 +135,6 @@
|
||||
[submodule "contrib/flatbuffers"]
|
||||
path = contrib/flatbuffers
|
||||
url = https://github.com/ClickHouse-Extras/flatbuffers.git
|
||||
[submodule "contrib/libc-headers"]
|
||||
path = contrib/libc-headers
|
||||
url = https://github.com/ClickHouse-Extras/libc-headers.git
|
||||
[submodule "contrib/replxx"]
|
||||
path = contrib/replxx
|
||||
url = https://github.com/ClickHouse-Extras/replxx.git
|
||||
|
||||
@ -169,9 +169,7 @@ endif ()
|
||||
include (cmake/check_flags.cmake)
|
||||
include (cmake/add_warning.cmake)
|
||||
|
||||
if (NOT MSVC)
|
||||
set (COMMON_WARNING_FLAGS "${COMMON_WARNING_FLAGS} -Wall") # -Werror and many more is also added inside cmake/warnings.cmake
|
||||
endif ()
|
||||
set (COMMON_WARNING_FLAGS "${COMMON_WARNING_FLAGS} -Wall") # -Werror and many more is also added inside cmake/warnings.cmake
|
||||
|
||||
if (COMPILER_CLANG)
|
||||
# clang: warning: argument unused during compilation: '-specs=/usr/share/dpkg/no-pie-compile.specs' [-Wunused-command-line-argument]
|
||||
@ -219,37 +217,13 @@ endif()
|
||||
# Make sure the final executable has symbols exported
|
||||
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -rdynamic")
|
||||
|
||||
find_program (OBJCOPY_PATH NAMES "llvm-objcopy" "llvm-objcopy-13" "llvm-objcopy-12" "llvm-objcopy-11" "llvm-objcopy-10" "llvm-objcopy-9" "llvm-objcopy-8" "objcopy")
|
||||
|
||||
if (NOT OBJCOPY_PATH AND OS_DARWIN)
|
||||
find_program (BREW_PATH NAMES "brew")
|
||||
if (BREW_PATH)
|
||||
execute_process (COMMAND ${BREW_PATH} --prefix llvm ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE LLVM_PREFIX)
|
||||
if (LLVM_PREFIX)
|
||||
find_program (OBJCOPY_PATH NAMES "llvm-objcopy" PATHS "${LLVM_PREFIX}/bin" NO_DEFAULT_PATH)
|
||||
endif ()
|
||||
if (NOT OBJCOPY_PATH)
|
||||
execute_process (COMMAND ${BREW_PATH} --prefix binutils ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE BINUTILS_PREFIX)
|
||||
if (BINUTILS_PREFIX)
|
||||
find_program (OBJCOPY_PATH NAMES "objcopy" PATHS "${BINUTILS_PREFIX}/bin" NO_DEFAULT_PATH)
|
||||
endif ()
|
||||
endif ()
|
||||
endif ()
|
||||
endif ()
|
||||
|
||||
if (OBJCOPY_PATH)
|
||||
message (STATUS "Using objcopy: ${OBJCOPY_PATH}")
|
||||
else ()
|
||||
message (FATAL_ERROR "Cannot find objcopy.")
|
||||
endif ()
|
||||
|
||||
if (OS_DARWIN)
|
||||
# The `-all_load` flag forces loading of all symbols from all libraries,
|
||||
# and leads to multiply-defined symbols. This flag allows force loading
|
||||
# from a _specific_ library, which is what we need.
|
||||
set(WHOLE_ARCHIVE -force_load)
|
||||
# The `-noall_load` flag is the default and now obsolete.
|
||||
set(NO_WHOLE_ARCHIVE "")
|
||||
set(NO_WHOLE_ARCHIVE "-undefined,error") # Effectively, a no-op. Here to avoid empty "-Wl, " sequence to be generated in the command line.
|
||||
else ()
|
||||
set(WHOLE_ARCHIVE --whole-archive)
|
||||
set(NO_WHOLE_ARCHIVE --no-whole-archive)
|
||||
@ -278,6 +252,13 @@ if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE")
|
||||
endif ()
|
||||
endif()
|
||||
|
||||
if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
|
||||
set(USE_DEBUG_HELPERS ON)
|
||||
else ()
|
||||
set(USE_DEBUG_HELPERS ON)
|
||||
endif()
|
||||
option(USE_DEBUG_HELPERS "Enable debug helpers" ${USE_DEBUG_HELPERS})
|
||||
|
||||
# Create BuildID when using lld. For other linkers it is created by default.
|
||||
if (LINKER_NAME MATCHES "lld$")
|
||||
# SHA1 is not cryptographically secure but it is the best what lld is offering.
|
||||
@ -312,6 +293,10 @@ include(cmake/cpu_features.cmake)
|
||||
# Enable it explicitly.
|
||||
set (COMPILER_FLAGS "${COMPILER_FLAGS} -fasynchronous-unwind-tables")
|
||||
|
||||
# Reproducible builds
|
||||
set (COMPILER_FLAGS "${COMPILER_FLAGS} -ffile-prefix-map=${CMAKE_SOURCE_DIR}=.")
|
||||
set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} -ffile-prefix-map=${CMAKE_SOURCE_DIR}=.")
|
||||
|
||||
if (${CMAKE_VERSION} VERSION_LESS "3.12.4")
|
||||
# CMake < 3.12 doesn't support setting 20 as a C++ standard version.
|
||||
# We will add C++ standard controlling flag in CMAKE_CXX_FLAGS manually for now.
|
||||
@ -411,25 +396,6 @@ if (COMPILER_CLANG)
|
||||
message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot enable ThinLTO")
|
||||
endif ()
|
||||
|
||||
# Always prefer llvm tools when using clang. For instance, we cannot use GNU ar when llvm LTO is enabled
|
||||
find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-13" "llvm-ar-12" "llvm-ar-11" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8")
|
||||
|
||||
if (LLVM_AR_PATH)
|
||||
message(STATUS "Using llvm-ar: ${LLVM_AR_PATH}.")
|
||||
set (CMAKE_AR ${LLVM_AR_PATH})
|
||||
else ()
|
||||
message(WARNING "Cannot find llvm-ar. System ar will be used instead. It does not work with ThinLTO.")
|
||||
endif ()
|
||||
|
||||
find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-13" "llvm-ranlib-12" "llvm-ranlib-11" "llvm-ranlib-10" "llvm-ranlib-9" "llvm-ranlib-8")
|
||||
|
||||
if (LLVM_RANLIB_PATH)
|
||||
message(STATUS "Using llvm-ranlib: ${LLVM_RANLIB_PATH}.")
|
||||
set (CMAKE_RANLIB ${LLVM_RANLIB_PATH})
|
||||
else ()
|
||||
message(WARNING "Cannot find llvm-ranlib. System ranlib will be used instead. It does not work with ThinLTO.")
|
||||
endif ()
|
||||
|
||||
elseif (ENABLE_THINLTO)
|
||||
message (${RECONFIGURE_MESSAGE_LEVEL} "ThinLTO is only available with CLang")
|
||||
endif ()
|
||||
@ -483,19 +449,6 @@ else ()
|
||||
set (CMAKE_POSITION_INDEPENDENT_CODE ON)
|
||||
endif ()
|
||||
|
||||
# https://github.com/include-what-you-use/include-what-you-use
|
||||
option (USE_INCLUDE_WHAT_YOU_USE "Automatically reduce unneeded includes in source code (external tool)" OFF)
|
||||
|
||||
if (USE_INCLUDE_WHAT_YOU_USE)
|
||||
find_program(IWYU_PATH NAMES include-what-you-use iwyu)
|
||||
if (NOT IWYU_PATH)
|
||||
message(FATAL_ERROR "Could not find the program include-what-you-use")
|
||||
endif()
|
||||
if (${CMAKE_VERSION} VERSION_LESS "3.3.0")
|
||||
message(FATAL_ERROR "include-what-you-use requires CMake version at least 3.3.")
|
||||
endif()
|
||||
endif ()
|
||||
|
||||
if (ENABLE_TESTS)
|
||||
message (STATUS "Unit tests are enabled")
|
||||
else()
|
||||
@ -597,8 +550,6 @@ include (cmake/find/mysqlclient.cmake)
|
||||
|
||||
# When testing for memory leaks with Valgrind, don't link tcmalloc or jemalloc.
|
||||
|
||||
include (cmake/print_flags.cmake)
|
||||
|
||||
if (TARGET global-group)
|
||||
install (EXPORT global DESTINATION cmake)
|
||||
endif ()
|
||||
@ -650,6 +601,7 @@ include_directories(${ConfigIncludePath})
|
||||
|
||||
# Add as many warnings as possible for our own code.
|
||||
include (cmake/warnings.cmake)
|
||||
include (cmake/print_flags.cmake)
|
||||
|
||||
add_subdirectory (base)
|
||||
add_subdirectory (src)
|
||||
|
||||
@ -29,7 +29,8 @@ elseif (ENABLE_READLINE)
|
||||
endif ()
|
||||
|
||||
if (USE_DEBUG_HELPERS)
|
||||
set (INCLUDE_DEBUG_HELPERS "-include \"${ClickHouse_SOURCE_DIR}/base/base/iostream_debug_helpers.h\"")
|
||||
get_target_property(MAGIC_ENUM_INCLUDE_DIR magic_enum INTERFACE_INCLUDE_DIRECTORIES)
|
||||
set (INCLUDE_DEBUG_HELPERS "-I\"${MAGIC_ENUM_INCLUDE_DIR}\" -include \"${ClickHouse_SOURCE_DIR}/base/base/iostream_debug_helpers.h\"")
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${INCLUDE_DEBUG_HELPERS}")
|
||||
endif ()
|
||||
|
||||
|
||||
@ -18,8 +18,8 @@ struct CachedFn
|
||||
{
|
||||
private:
|
||||
using Traits = FnTraits<decltype(Func)>;
|
||||
using DecayedArgs = TLMap<std::decay_t, typename Traits::Args>;
|
||||
using Key = TLChangeRoot<std::tuple, DecayedArgs>;
|
||||
using DecayedArgs = TypeListMap<std::decay_t, typename Traits::Args>;
|
||||
using Key = TypeListChangeRoot<std::tuple, DecayedArgs>;
|
||||
using Result = typename Traits::Ret;
|
||||
|
||||
std::map<Key, Result> cache; // Can't use hashmap as tuples are unhashable by default
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include "Typelist.h"
|
||||
#include "TypeList.h"
|
||||
|
||||
namespace detail
|
||||
{
|
||||
@ -14,7 +14,7 @@ struct FnTraits<R(A...)>
|
||||
static constexpr bool value = std::is_invocable_r_v<R, F, A...>;
|
||||
|
||||
using Ret = R;
|
||||
using Args = Typelist<A...>;
|
||||
using Args = TypeList<A...>;
|
||||
};
|
||||
|
||||
template <class R, class ...A>
|
||||
|
||||
44
base/base/TypeList.h
Normal file
44
base/base/TypeList.h
Normal file
@ -0,0 +1,44 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <type_traits>
|
||||
#include <utility>
|
||||
#include "defines.h"
|
||||
#include "TypePair.h"
|
||||
|
||||
/// General-purpose typelist. Easy on compilation times as it does not use recursion.
|
||||
template <typename ...Args>
|
||||
struct TypeList { static constexpr size_t size = sizeof...(Args); };
|
||||
|
||||
namespace TypeListUtils /// In some contexts it's more handy to use functions instead of aliases
|
||||
{
|
||||
template <typename ...LArgs, typename ...RArgs>
|
||||
constexpr TypeList<LArgs..., RArgs...> concat(TypeList<LArgs...>, TypeList<RArgs...>) { return {}; }
|
||||
|
||||
template <typename T, typename ...Args>
|
||||
constexpr TypeList<T, Args...> prepend(TypeList<Args...>) { return {}; }
|
||||
|
||||
template <typename T, typename ...Args>
|
||||
constexpr TypeList<Args..., T> append(TypeList<Args...>) { return {}; }
|
||||
|
||||
template <template <typename> typename F, typename ...Args>
|
||||
constexpr TypeList<F<Args>...> map(TypeList<Args...>) { return {}; }
|
||||
|
||||
template <template <typename...> typename Root, typename ...Args>
|
||||
constexpr Root<Args...> changeRoot(TypeList<Args...>) { return {}; }
|
||||
|
||||
template <typename F, typename ...Args>
|
||||
constexpr void forEach(TypeList<Args...>, F && f) { (std::forward<F>(f)(Id<Args>{}), ...); }
|
||||
}
|
||||
|
||||
template <typename TypeListLeft, typename TypeListRight>
|
||||
using TypeListConcat = decltype(TypeListUtils::concat(TypeListLeft{}, TypeListRight{}));
|
||||
|
||||
template <typename T, typename List> using TypeListPrepend = decltype(TypeListUtils::prepend<T>(List{}));
|
||||
template <typename T, typename List> using TypeListAppend = decltype(TypeListUtils::append<T>(List{}));
|
||||
|
||||
template <template <typename> typename F, typename List>
|
||||
using TypeListMap = decltype(TypeListUtils::map<F>(List{}));
|
||||
|
||||
template <template <typename...> typename Root, typename List>
|
||||
using TypeListChangeRoot = decltype(TypeListUtils::changeRoot<Root>(List{}));
|
||||
21
base/base/TypeLists.h
Normal file
21
base/base/TypeLists.h
Normal file
@ -0,0 +1,21 @@
|
||||
#pragma once
|
||||
|
||||
#include "TypeList.h"
|
||||
#include "extended_types.h"
|
||||
#include "Decimal.h"
|
||||
#include "UUID.h"
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
using TypeListNativeInt = TypeList<UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64>;
|
||||
using TypeListFloat = TypeList<Float32, Float64>;
|
||||
using TypeListNativeNumber = TypeListConcat<TypeListNativeInt, TypeListFloat>;
|
||||
using TypeListWideInt = TypeList<UInt128, Int128, UInt256, Int256>;
|
||||
using TypeListInt = TypeListConcat<TypeListNativeInt, TypeListWideInt>;
|
||||
using TypeListIntAndFloat = TypeListConcat<TypeListInt, TypeListFloat>;
|
||||
using TypeListDecimal = TypeList<Decimal32, Decimal64, Decimal128, Decimal256>;
|
||||
using TypeListNumber = TypeListConcat<TypeListIntAndFloat, TypeListDecimal>;
|
||||
using TypeListNumberWithUUID = TypeListAppend<UUID, TypeListNumber>;
|
||||
|
||||
}
|
||||
@ -1,4 +1,4 @@
|
||||
#pragma once
|
||||
|
||||
template <class T, class V> struct TypePair { };
|
||||
template <class T> struct Id { };
|
||||
template <typename T, typename V> struct TypePair {};
|
||||
template <typename T> struct Id {};
|
||||
|
||||
@ -1,44 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <type_traits>
|
||||
#include <utility>
|
||||
#include "defines.h"
|
||||
#include "TypePair.h"
|
||||
|
||||
/// General-purpose typelist. Easy on compilation times as it does not use recursion.
|
||||
template <class ...Args>
|
||||
struct Typelist { static constexpr size_t size = sizeof...(Args); };
|
||||
|
||||
namespace TLUtils /// In some contexts it's more handy to use functions instead of aliases
|
||||
{
|
||||
template <class ...LArgs, class ...RArgs>
|
||||
constexpr Typelist<LArgs..., RArgs...> concat(Typelist<LArgs...>, Typelist<RArgs...>) { return {}; }
|
||||
|
||||
template <class T, class ...Args>
|
||||
constexpr Typelist<T, Args...> prepend(Typelist<Args...>) { return {}; }
|
||||
|
||||
template <class T, class ...Args>
|
||||
constexpr Typelist<Args..., T> append(Typelist<Args...>) { return {}; }
|
||||
|
||||
template <template<class> class F, class ...Args>
|
||||
constexpr Typelist<F<Args>...> map(Typelist<Args...>) { return {}; }
|
||||
|
||||
template <template<class...> class Root, class ...Args>
|
||||
constexpr Root<Args...> changeRoot(Typelist<Args...>) { return {}; }
|
||||
|
||||
template <class F, class ...Args>
|
||||
constexpr void forEach(Typelist<Args...>, F && f) { (std::forward<F>(f)(Id<Args>{}), ...); }
|
||||
}
|
||||
|
||||
template <class TLLeft, class TLRight>
|
||||
using TLConcat = decltype(TLUtils::concat(TLLeft{}, TLRight{}));
|
||||
|
||||
template <class T, class Typelist> using TLPrepend = decltype(TLUtils::prepend<T>(Typelist{}));
|
||||
template <class T, class Typelist> using TLAppend = decltype(TLUtils::append<T>(Typelist{}));
|
||||
|
||||
template <template<class> class F, class Typelist>
|
||||
using TLMap = decltype(TLUtils::map<F>(Typelist{}));
|
||||
|
||||
template <template<class...> class Root, class Typelist>
|
||||
using TLChangeRoot = decltype(TLUtils::changeRoot<Root>(Typelist{}));
|
||||
@ -1,18 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include "Typelist.h"
|
||||
#include "extended_types.h"
|
||||
#include "Decimal.h"
|
||||
#include "UUID.h"
|
||||
|
||||
namespace DB
|
||||
{
|
||||
using TLIntegral = Typelist<UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64>;
|
||||
using TLExtendedIntegral = Typelist<UInt128, Int128, UInt256, Int256>;
|
||||
using TLDecimals = Typelist<Decimal32, Decimal64, Decimal128, Decimal256>;
|
||||
|
||||
using TLIntegralWithExtended = TLConcat<TLIntegral, TLExtendedIntegral>;
|
||||
|
||||
using TLNumbers = TLConcat<TLIntegralWithExtended, TLDecimals>;
|
||||
using TLNumbersWithUUID = TLAppend<UUID, TLNumbers>;
|
||||
}
|
||||
@ -5,5 +5,5 @@
|
||||
|
||||
namespace DB
|
||||
{
|
||||
using UUID = StrongTypedef<UInt128, struct UUIDTag>;
|
||||
using UUID = StrongTypedef<UInt128, struct UUIDTag>;
|
||||
}
|
||||
|
||||
@ -6,6 +6,7 @@
|
||||
#include <tuple>
|
||||
#include <iomanip>
|
||||
#include <iostream>
|
||||
#include <magic_enum.hpp>
|
||||
|
||||
/** Usage:
|
||||
*
|
||||
@ -61,6 +62,13 @@ std::enable_if_t<priority == 2, Out> & dumpImpl(Out & out, T && x, std::decay_t<
|
||||
}
|
||||
|
||||
|
||||
template <int priority, typename Out, typename T>
|
||||
std::enable_if_t<priority == 3 && std::is_enum_v<std::decay_t<T>>, Out> &
|
||||
dumpImpl(Out & out, T && x)
|
||||
{
|
||||
return out << magic_enum::enum_name(x);
|
||||
}
|
||||
|
||||
/// string and const char * - output not as container or pointer.
|
||||
|
||||
template <int priority, typename Out, typename T>
|
||||
@ -131,15 +139,26 @@ Out & dumpValue(Out & out, T && x)
|
||||
template <typename Out, typename T>
|
||||
Out & dump(Out & out, const char * name, T && x)
|
||||
{
|
||||
// Dumping string literal, printing name and demangled type is irrelevant.
|
||||
if constexpr (std::is_same_v<const char *, std::decay_t<std::remove_reference_t<T>>>)
|
||||
{
|
||||
const auto name_len = strlen(name);
|
||||
const auto value_len = strlen(x);
|
||||
// `name` is the same as quoted `x`
|
||||
if (name_len > 2 && value_len > 0 && name[0] == '"' && name[name_len - 1] == '"'
|
||||
&& strncmp(name + 1, x, std::min(value_len, name_len) - 1) == 0)
|
||||
return out << x;
|
||||
}
|
||||
|
||||
out << demangle(typeid(x).name()) << " " << name << " = ";
|
||||
return dumpValue(out, x);
|
||||
return dumpValue(out, x) << "; ";
|
||||
}
|
||||
|
||||
#ifdef __clang__
|
||||
#pragma clang diagnostic ignored "-Wgnu-zero-variadic-macro-arguments"
|
||||
#endif
|
||||
|
||||
#define DUMPVAR(VAR) ::dump(std::cerr, #VAR, (VAR)); std::cerr << "; ";
|
||||
#define DUMPVAR(VAR) ::dump(std::cerr, #VAR, (VAR));
|
||||
#define DUMPHEAD std::cerr << __FILE__ << ':' << __LINE__ << " [ " << getThreadId() << " ] ";
|
||||
#define DUMPTAIL std::cerr << '\n';
|
||||
|
||||
|
||||
@ -7,13 +7,9 @@ set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/freebsd-aarc
|
||||
|
||||
set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it doesn't work in CMake
|
||||
|
||||
set (CMAKE_AR "/usr/bin/ar" CACHE FILEPATH "" FORCE)
|
||||
set (CMAKE_RANLIB "/usr/bin/ranlib" CACHE FILEPATH "" FORCE)
|
||||
|
||||
set (LINKER_NAME "ld.lld" CACHE STRING "" FORCE)
|
||||
|
||||
set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld")
|
||||
set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld")
|
||||
# Will be changed later, but somehow needed to be set here.
|
||||
set (CMAKE_AR "ar")
|
||||
set (CMAKE_RANLIB "ranlib")
|
||||
|
||||
set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
|
||||
set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
|
||||
|
||||
@ -7,13 +7,9 @@ set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/freebsd-x86_
|
||||
|
||||
set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it doesn't work in CMake
|
||||
|
||||
set (CMAKE_AR "/usr/bin/ar" CACHE FILEPATH "" FORCE)
|
||||
set (CMAKE_RANLIB "/usr/bin/ranlib" CACHE FILEPATH "" FORCE)
|
||||
|
||||
set (LINKER_NAME "ld.lld" CACHE STRING "" FORCE)
|
||||
|
||||
set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld")
|
||||
set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld")
|
||||
# Will be changed later, but somehow needed to be set here.
|
||||
set (CMAKE_AR "ar")
|
||||
set (CMAKE_RANLIB "ranlib")
|
||||
|
||||
set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
|
||||
set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
|
||||
|
||||
@ -29,14 +29,6 @@ message(STATUS "Default libraries: ${DEFAULT_LIBS}")
|
||||
set(CMAKE_CXX_STANDARD_LIBRARIES ${DEFAULT_LIBS})
|
||||
set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS})
|
||||
|
||||
# glibc-compatibility library relies to constant version of libc headers
|
||||
# (because minor changes in function attributes between different glibc versions will introduce incompatibilities)
|
||||
# This is for x86_64. For other architectures we have separate toolchains.
|
||||
if (ARCH_AMD64 AND NOT CMAKE_CROSSCOMPILING)
|
||||
set(CMAKE_C_STANDARD_INCLUDE_DIRECTORIES ${ClickHouse_SOURCE_DIR}/contrib/libc-headers/x86_64-linux-gnu ${ClickHouse_SOURCE_DIR}/contrib/libc-headers)
|
||||
set(CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES ${ClickHouse_SOURCE_DIR}/contrib/libc-headers/x86_64-linux-gnu ${ClickHouse_SOURCE_DIR}/contrib/libc-headers)
|
||||
endif ()
|
||||
|
||||
# Unfortunately '-pthread' doesn't work with '-nodefaultlibs'.
|
||||
# Just make sure we have pthreads at all.
|
||||
set(THREADS_PREFER_PTHREAD_FLAG ON)
|
||||
|
||||
@ -6,25 +6,18 @@ set (CMAKE_C_COMPILER_TARGET "aarch64-linux-gnu")
|
||||
set (CMAKE_CXX_COMPILER_TARGET "aarch64-linux-gnu")
|
||||
set (CMAKE_ASM_COMPILER_TARGET "aarch64-linux-gnu")
|
||||
|
||||
# Will be changed later, but somehow needed to be set here.
|
||||
set (CMAKE_AR "ar")
|
||||
set (CMAKE_RANLIB "ranlib")
|
||||
|
||||
set (TOOLCHAIN_PATH "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-aarch64")
|
||||
|
||||
set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}/aarch64-linux-gnu/libc")
|
||||
|
||||
find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-13" "llvm-ar-12" "llvm-ar-11" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8")
|
||||
find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-13" "llvm-ranlib-12" "llvm-ranlib-11" "llvm-ranlib-10" "llvm-ranlib-9")
|
||||
|
||||
set (CMAKE_AR "${LLVM_AR_PATH}" CACHE FILEPATH "" FORCE)
|
||||
set (CMAKE_RANLIB "${LLVM_RANLIB_PATH}" CACHE FILEPATH "" FORCE)
|
||||
|
||||
set (CMAKE_C_FLAGS_INIT "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
|
||||
set (CMAKE_CXX_FLAGS_INIT "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
|
||||
set (CMAKE_ASM_FLAGS_INIT "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
|
||||
|
||||
set (LINKER_NAME "ld.lld" CACHE STRING "" FORCE)
|
||||
|
||||
set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld")
|
||||
set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld")
|
||||
|
||||
set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
|
||||
set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
|
||||
|
||||
|
||||
@ -6,25 +6,18 @@ set (CMAKE_C_COMPILER_TARGET "ppc64le-linux-gnu")
|
||||
set (CMAKE_CXX_COMPILER_TARGET "ppc64le-linux-gnu")
|
||||
set (CMAKE_ASM_COMPILER_TARGET "ppc64le-linux-gnu")
|
||||
|
||||
# Will be changed later, but somehow needed to be set here.
|
||||
set (CMAKE_AR "ar")
|
||||
set (CMAKE_RANLIB "ranlib")
|
||||
|
||||
set (TOOLCHAIN_PATH "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-powerpc64le")
|
||||
|
||||
set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}/powerpc64le-linux-gnu/libc")
|
||||
|
||||
find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-13" "llvm-ar-12" "llvm-ar-11" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8")
|
||||
find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-13" "llvm-ranlib-12" "llvm-ranlib-11" "llvm-ranlib-10" "llvm-ranlib-9")
|
||||
|
||||
set (CMAKE_AR "${LLVM_AR_PATH}" CACHE FILEPATH "" FORCE)
|
||||
set (CMAKE_RANLIB "${LLVM_RANLIB_PATH}" CACHE FILEPATH "" FORCE)
|
||||
|
||||
set (CMAKE_C_FLAGS_INIT "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
|
||||
set (CMAKE_CXX_FLAGS_INIT "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
|
||||
set (CMAKE_ASM_FLAGS_INIT "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
|
||||
|
||||
set (LINKER_NAME "ld.lld" CACHE STRING "" FORCE)
|
||||
|
||||
set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld")
|
||||
set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld")
|
||||
|
||||
set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
|
||||
set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
|
||||
|
||||
|
||||
@ -6,22 +6,18 @@ set (CMAKE_C_COMPILER_TARGET "riscv64-linux-gnu")
|
||||
set (CMAKE_CXX_COMPILER_TARGET "riscv64-linux-gnu")
|
||||
set (CMAKE_ASM_COMPILER_TARGET "riscv64-linux-gnu")
|
||||
|
||||
# Will be changed later, but somehow needed to be set here.
|
||||
set (CMAKE_AR "ar")
|
||||
set (CMAKE_RANLIB "ranlib")
|
||||
|
||||
set (TOOLCHAIN_PATH "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-riscv64")
|
||||
|
||||
set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}")
|
||||
|
||||
find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-13" "llvm-ar-12" "llvm-ar-11" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8")
|
||||
find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-13" "llvm-ranlib-12" "llvm-ranlib-11" "llvm-ranlib-10" "llvm-ranlib-9")
|
||||
|
||||
set (CMAKE_AR "${LLVM_AR_PATH}" CACHE FILEPATH "" FORCE)
|
||||
set (CMAKE_RANLIB "${LLVM_RANLIB_PATH}" CACHE FILEPATH "" FORCE)
|
||||
|
||||
set (CMAKE_C_FLAGS_INIT "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
|
||||
set (CMAKE_CXX_FLAGS_INIT "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
|
||||
set (CMAKE_ASM_FLAGS_INIT "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
|
||||
|
||||
set (LINKER_NAME "riscv64-linux-gnu-ld.bfd" CACHE STRING "" FORCE)
|
||||
|
||||
set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=bfd")
|
||||
set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=bfd")
|
||||
|
||||
|
||||
@ -6,25 +6,18 @@ set (CMAKE_C_COMPILER_TARGET "x86_64-linux-musl")
|
||||
set (CMAKE_CXX_COMPILER_TARGET "x86_64-linux-musl")
|
||||
set (CMAKE_ASM_COMPILER_TARGET "x86_64-linux-musl")
|
||||
|
||||
# Will be changed later, but somehow needed to be set here.
|
||||
set (CMAKE_AR "ar")
|
||||
set (CMAKE_RANLIB "ranlib")
|
||||
|
||||
set (TOOLCHAIN_PATH "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-x86_64-musl")
|
||||
|
||||
set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}")
|
||||
|
||||
find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-13" "llvm-ar-12" "llvm-ar-11" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8")
|
||||
find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-13" "llvm-ranlib-12" "llvm-ranlib-11" "llvm-ranlib-10" "llvm-ranlib-9")
|
||||
|
||||
set (CMAKE_AR "${LLVM_AR_PATH}" CACHE FILEPATH "" FORCE)
|
||||
set (CMAKE_RANLIB "${LLVM_RANLIB_PATH}" CACHE FILEPATH "" FORCE)
|
||||
|
||||
set (CMAKE_C_FLAGS_INIT "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
|
||||
set (CMAKE_CXX_FLAGS_INIT "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
|
||||
set (CMAKE_ASM_FLAGS_INIT "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
|
||||
|
||||
set (LINKER_NAME "ld.lld" CACHE STRING "" FORCE)
|
||||
|
||||
set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld")
|
||||
set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld")
|
||||
|
||||
set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
|
||||
set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
|
||||
|
||||
|
||||
@ -6,25 +6,18 @@ set (CMAKE_C_COMPILER_TARGET "x86_64-linux-gnu")
|
||||
set (CMAKE_CXX_COMPILER_TARGET "x86_64-linux-gnu")
|
||||
set (CMAKE_ASM_COMPILER_TARGET "x86_64-linux-gnu")
|
||||
|
||||
# Will be changed later, but somehow needed to be set here.
|
||||
set (CMAKE_AR "ar")
|
||||
set (CMAKE_RANLIB "ranlib")
|
||||
|
||||
set (TOOLCHAIN_PATH "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-x86_64")
|
||||
|
||||
set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}/x86_64-linux-gnu/libc")
|
||||
|
||||
find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-13" "llvm-ar-12" "llvm-ar-11" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8")
|
||||
find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-13" "llvm-ranlib-12" "llvm-ranlib-11" "llvm-ranlib-10" "llvm-ranlib-9")
|
||||
|
||||
set (CMAKE_AR "${LLVM_AR_PATH}" CACHE FILEPATH "" FORCE)
|
||||
set (CMAKE_RANLIB "${LLVM_RANLIB_PATH}" CACHE FILEPATH "" FORCE)
|
||||
|
||||
set (CMAKE_C_FLAGS_INIT "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
|
||||
set (CMAKE_CXX_FLAGS_INIT "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
|
||||
set (CMAKE_ASM_FLAGS_INIT "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
|
||||
|
||||
set (LINKER_NAME "ld.lld" CACHE STRING "" FORCE)
|
||||
|
||||
set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld")
|
||||
set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld")
|
||||
|
||||
set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
|
||||
set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
|
||||
|
||||
|
||||
@ -1,6 +1,12 @@
|
||||
set (FULL_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE_UC}}")
|
||||
set (FULL_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UC}}")
|
||||
set (FULL_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${CMAKE_EXE_LINKER_FLAGS_${CMAKE_BUILD_TYPE_UC}}")
|
||||
|
||||
message (STATUS "compiler C = ${CMAKE_C_COMPILER} ${FULL_C_FLAGS}")
|
||||
message (STATUS "compiler CXX = ${CMAKE_CXX_COMPILER} ${FULL_CXX_FLAGS}")
|
||||
message (STATUS "LINKER_FLAGS = ${FULL_EXE_LINKER_FLAGS}")
|
||||
|
||||
# Reproducible builds
|
||||
string (REPLACE "${CMAKE_SOURCE_DIR}" "." FULL_C_FLAGS_NORMALIZED "${FULL_C_FLAGS}")
|
||||
string (REPLACE "${CMAKE_SOURCE_DIR}" "." FULL_CXX_FLAGS_NORMALIZED "${FULL_CXX_FLAGS}")
|
||||
string (REPLACE "${CMAKE_SOURCE_DIR}" "." FULL_EXE_LINKER_FLAGS_NORMALIZED "${FULL_EXE_LINKER_FLAGS}")
|
||||
|
||||
@ -78,6 +78,15 @@ if (SANITIZE)
|
||||
|
||||
elseif (SANITIZE STREQUAL "undefined")
|
||||
set (UBSAN_FLAGS "-fsanitize=undefined -fno-sanitize-recover=all -fno-sanitize=float-divide-by-zero")
|
||||
if (ENABLE_FUZZING)
|
||||
# Unsigned integer overflow is well defined behaviour from a perspective of C++ standard,
|
||||
# compilers or CPU. We use in hash functions like SipHash and many other places in our codebase.
|
||||
# This flag is needed only because fuzzers are run inside oss-fuzz infrastructure
|
||||
# and they have a bunch of flags not halt the program if UIO happend and even to silence that warnings.
|
||||
# But for unknown reason that flags don't work with ClickHouse or we don't understand how to properly use them,
|
||||
# that's why we often receive reports about UIO. The simplest way to avoid this is just set this flag here.
|
||||
set(UBSAN_FLAGS "${SAN_FLAGS} -fno-sanitize=unsigned-integer-overflow")
|
||||
endif()
|
||||
if (COMPILER_CLANG)
|
||||
set (UBSAN_FLAGS "${UBSAN_FLAGS} -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/tests/ubsan_suppressions.txt")
|
||||
else()
|
||||
|
||||
@ -1,3 +1,5 @@
|
||||
# Compiler
|
||||
|
||||
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
|
||||
set (COMPILER_GCC 1)
|
||||
elseif (CMAKE_CXX_COMPILER_ID MATCHES "AppleClang")
|
||||
@ -6,6 +8,8 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
|
||||
set (COMPILER_CLANG 1)
|
||||
endif ()
|
||||
|
||||
execute_process(COMMAND ${CMAKE_CXX_COMPILER} --version)
|
||||
|
||||
if (COMPILER_GCC)
|
||||
# Require minimum version of gcc
|
||||
set (GCC_MINIMUM_VERSION 11)
|
||||
@ -18,9 +22,10 @@ if (COMPILER_GCC)
|
||||
elseif (COMPILER_CLANG)
|
||||
# Require minimum version of clang/apple-clang
|
||||
if (CMAKE_CXX_COMPILER_ID MATCHES "AppleClang")
|
||||
# If you are developer you can figure out what exact versions of AppleClang are Ok,
|
||||
# simply remove the following line.
|
||||
message (FATAL_ERROR "AppleClang is not supported, you should install clang from brew. See the instruction: https://clickhouse.com/docs/en/development/build-osx/")
|
||||
# (Experimental!) Specify "-DALLOW_APPLECLANG=ON" when running CMake configuration step, if you want to experiment with using it.
|
||||
if (NOT ALLOW_APPLECLANG AND NOT DEFINED ENV{ALLOW_APPLECLANG})
|
||||
message (FATAL_ERROR "AppleClang is not supported, you should install clang from brew. See the instruction: https://clickhouse.com/docs/en/development/build-osx/")
|
||||
endif ()
|
||||
|
||||
# AppleClang 10.0.1 (Xcode 10.2) corresponds to LLVM/Clang upstream version 7.0.0
|
||||
# AppleClang 11.0.0 (Xcode 11.0) corresponds to LLVM/Clang upstream version 8.0.0
|
||||
@ -44,8 +49,10 @@ else ()
|
||||
message (WARNING "You are using an unsupported compiler. Compilation has only been tested with Clang and GCC.")
|
||||
endif ()
|
||||
|
||||
STRING(REGEX MATCHALL "[0-9]+" COMPILER_VERSION_LIST ${CMAKE_CXX_COMPILER_VERSION})
|
||||
LIST(GET COMPILER_VERSION_LIST 0 COMPILER_VERSION_MAJOR)
|
||||
string (REGEX MATCHALL "[0-9]+" COMPILER_VERSION_LIST ${CMAKE_CXX_COMPILER_VERSION})
|
||||
list (GET COMPILER_VERSION_LIST 0 COMPILER_VERSION_MAJOR)
|
||||
|
||||
# Linker
|
||||
|
||||
# Example values: `lld-10`, `gold`.
|
||||
option (LINKER_NAME "Linker name or full path")
|
||||
@ -93,3 +100,69 @@ if (LINKER_NAME)
|
||||
|
||||
message(STATUS "Using custom linker by name: ${LINKER_NAME}")
|
||||
endif ()
|
||||
|
||||
# Archiver
|
||||
|
||||
if (COMPILER_GCC)
|
||||
find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-13" "llvm-ar-12" "llvm-ar-11")
|
||||
else ()
|
||||
find_program (LLVM_AR_PATH NAMES "llvm-ar-${COMPILER_VERSION_MAJOR}" "llvm-ar")
|
||||
endif ()
|
||||
|
||||
if (LLVM_AR_PATH)
|
||||
set (CMAKE_AR "${LLVM_AR_PATH}")
|
||||
endif ()
|
||||
|
||||
# Ranlib
|
||||
|
||||
if (COMPILER_GCC)
|
||||
find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-13" "llvm-ranlib-12" "llvm-ranlib-11")
|
||||
else ()
|
||||
find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib-${COMPILER_VERSION_MAJOR}" "llvm-ranlib")
|
||||
endif ()
|
||||
|
||||
if (LLVM_RANLIB_PATH)
|
||||
set (CMAKE_RANLIB "${LLVM_RANLIB_PATH}")
|
||||
endif ()
|
||||
|
||||
# Install Name Tool
|
||||
|
||||
if (COMPILER_GCC)
|
||||
find_program (LLVM_INSTALL_NAME_TOOL_PATH NAMES "llvm-install-name-tool" "llvm-install-name-tool-13" "llvm-install-name-tool-12" "llvm-install-name-tool-11")
|
||||
else ()
|
||||
find_program (LLVM_INSTALL_NAME_TOOL_PATH NAMES "llvm-install-name-tool-${COMPILER_VERSION_MAJOR}" "llvm-install-name-tool")
|
||||
endif ()
|
||||
|
||||
if (LLVM_INSTALL_NAME_TOOL_PATH)
|
||||
set (CMAKE_INSTALL_NAME_TOOL "${LLVM_INSTALL_NAME_TOOL_PATH}")
|
||||
endif ()
|
||||
|
||||
# Objcopy
|
||||
|
||||
if (COMPILER_GCC)
|
||||
find_program (OBJCOPY_PATH NAMES "llvm-objcopy" "llvm-objcopy-13" "llvm-objcopy-12" "llvm-objcopy-11" "objcopy")
|
||||
else ()
|
||||
find_program (OBJCOPY_PATH NAMES "llvm-objcopy-${COMPILER_VERSION_MAJOR}" "llvm-objcopy" "objcopy")
|
||||
endif ()
|
||||
|
||||
if (NOT OBJCOPY_PATH AND OS_DARWIN)
|
||||
find_program (BREW_PATH NAMES "brew")
|
||||
if (BREW_PATH)
|
||||
execute_process (COMMAND ${BREW_PATH} --prefix llvm ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE LLVM_PREFIX)
|
||||
if (LLVM_PREFIX)
|
||||
find_program (OBJCOPY_PATH NAMES "llvm-objcopy" PATHS "${LLVM_PREFIX}/bin" NO_DEFAULT_PATH)
|
||||
endif ()
|
||||
if (NOT OBJCOPY_PATH)
|
||||
execute_process (COMMAND ${BREW_PATH} --prefix binutils ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE BINUTILS_PREFIX)
|
||||
if (BINUTILS_PREFIX)
|
||||
find_program (OBJCOPY_PATH NAMES "objcopy" PATHS "${BINUTILS_PREFIX}/bin" NO_DEFAULT_PATH)
|
||||
endif ()
|
||||
endif ()
|
||||
endif ()
|
||||
endif ()
|
||||
|
||||
if (OBJCOPY_PATH)
|
||||
message (STATUS "Using objcopy: ${OBJCOPY_PATH}")
|
||||
else ()
|
||||
message (FATAL_ERROR "Cannot find objcopy.")
|
||||
endif ()
|
||||
|
||||
@ -26,46 +26,6 @@ if (COMPILER_CLANG)
|
||||
no_warning(zero-length-array)
|
||||
no_warning(c11-extensions)
|
||||
|
||||
add_warning(comma)
|
||||
add_warning(conditional-uninitialized)
|
||||
add_warning(covered-switch-default)
|
||||
add_warning(deprecated)
|
||||
add_warning(embedded-directive)
|
||||
add_warning(empty-init-stmt) # linux-only
|
||||
add_warning(extra-semi-stmt) # linux-only
|
||||
add_warning(extra-semi)
|
||||
add_warning(gnu-case-range)
|
||||
add_warning(inconsistent-missing-destructor-override)
|
||||
add_warning(newline-eof)
|
||||
add_warning(old-style-cast)
|
||||
add_warning(range-loop-analysis)
|
||||
add_warning(redundant-parens)
|
||||
add_warning(reserved-id-macro)
|
||||
add_warning(shadow-field)
|
||||
add_warning(shadow-uncaptured-local)
|
||||
add_warning(shadow)
|
||||
add_warning(string-plus-int)
|
||||
add_warning(undef)
|
||||
add_warning(unreachable-code-return)
|
||||
add_warning(unreachable-code)
|
||||
add_warning(unused-exception-parameter)
|
||||
add_warning(unused-macros)
|
||||
add_warning(unused-member-function)
|
||||
add_warning(unneeded-internal-declaration)
|
||||
add_warning(implicit-int-float-conversion)
|
||||
add_warning(no-delete-null-pointer-checks)
|
||||
add_warning(anon-enum-enum-conversion)
|
||||
add_warning(assign-enum)
|
||||
add_warning(bitwise-op-parentheses)
|
||||
add_warning(int-in-bool-context)
|
||||
add_warning(sometimes-uninitialized)
|
||||
add_warning(tautological-bitwise-compare)
|
||||
|
||||
# XXX: libstdc++ has some of these for 3way compare
|
||||
if (USE_LIBCXX)
|
||||
add_warning(zero-as-null-pointer-constant)
|
||||
endif()
|
||||
|
||||
if (WEVERYTHING)
|
||||
add_warning(everything)
|
||||
no_warning(c++98-compat-pedantic)
|
||||
@ -101,6 +61,47 @@ if (COMPILER_CLANG)
|
||||
endif()
|
||||
|
||||
# TODO Enable conversion, sign-conversion, double-promotion warnings.
|
||||
else ()
|
||||
add_warning(comma)
|
||||
add_warning(conditional-uninitialized)
|
||||
add_warning(covered-switch-default)
|
||||
add_warning(deprecated)
|
||||
add_warning(embedded-directive)
|
||||
add_warning(empty-init-stmt) # linux-only
|
||||
add_warning(extra-semi-stmt) # linux-only
|
||||
add_warning(extra-semi)
|
||||
add_warning(gnu-case-range)
|
||||
add_warning(inconsistent-missing-destructor-override)
|
||||
add_warning(newline-eof)
|
||||
add_warning(old-style-cast)
|
||||
add_warning(range-loop-analysis)
|
||||
add_warning(redundant-parens)
|
||||
add_warning(reserved-id-macro)
|
||||
add_warning(shadow-field)
|
||||
add_warning(shadow-uncaptured-local)
|
||||
add_warning(shadow)
|
||||
add_warning(string-plus-int)
|
||||
add_warning(undef)
|
||||
add_warning(unreachable-code-return)
|
||||
add_warning(unreachable-code)
|
||||
add_warning(unused-exception-parameter)
|
||||
add_warning(unused-macros)
|
||||
add_warning(unused-member-function)
|
||||
add_warning(unneeded-internal-declaration)
|
||||
add_warning(implicit-int-float-conversion)
|
||||
add_warning(no-delete-null-pointer-checks)
|
||||
add_warning(anon-enum-enum-conversion)
|
||||
add_warning(assign-enum)
|
||||
add_warning(bitwise-op-parentheses)
|
||||
add_warning(int-in-bool-context)
|
||||
add_warning(sometimes-uninitialized)
|
||||
add_warning(tautological-bitwise-compare)
|
||||
|
||||
# XXX: libstdc++ has some of these for 3way compare
|
||||
if (USE_LIBCXX)
|
||||
add_warning(zero-as-null-pointer-constant)
|
||||
endif()
|
||||
|
||||
endif ()
|
||||
elseif (COMPILER_GCC)
|
||||
# Add compiler options only to c++ compiler
|
||||
|
||||
5
contrib/CMakeLists.txt
vendored
5
contrib/CMakeLists.txt
vendored
@ -308,9 +308,10 @@ endif()
|
||||
# instead of controlling it via CMAKE_FOLDER.
|
||||
|
||||
function (ensure_target_rooted_in _target _folder)
|
||||
# Skip INTERFACE library targets, since FOLDER property is not available for them.
|
||||
# Skip aliases and INTERFACE library targets, since FOLDER property is not available/writable for them.
|
||||
get_target_property (_target_aliased "${_target}" ALIASED_TARGET)
|
||||
get_target_property (_target_type "${_target}" TYPE)
|
||||
if (_target_type STREQUAL "INTERFACE_LIBRARY")
|
||||
if (_target_aliased OR _target_type STREQUAL "INTERFACE_LIBRARY")
|
||||
return ()
|
||||
endif ()
|
||||
|
||||
|
||||
@ -82,7 +82,7 @@ if (NOT EXTERNAL_CCTZ_LIBRARY_FOUND OR NOT EXTERNAL_CCTZ_LIBRARY_WORKS)
|
||||
|
||||
# each file in that dir (except of tab and localtime) store the info about timezone
|
||||
execute_process(COMMAND
|
||||
bash -c "cd ${TZDIR} && find * -type f -and ! -name '*.tab' -and ! -name 'localtime' | sort | paste -sd ';' -"
|
||||
bash -c "cd ${TZDIR} && find * -type f -and ! -name '*.tab' -and ! -name 'localtime' | LC_ALL=C sort | paste -sd ';' -"
|
||||
OUTPUT_STRIP_TRAILING_WHITESPACE
|
||||
OUTPUT_VARIABLE TIMEZONES)
|
||||
|
||||
|
||||
1
contrib/libc-headers
vendored
1
contrib/libc-headers
vendored
@ -1 +0,0 @@
|
||||
Subproject commit aa5429bf67a346e48ad60efd88bcefc286644bf3
|
||||
@ -73,6 +73,11 @@ target_compile_options(cxx PRIVATE -w)
|
||||
|
||||
target_link_libraries(cxx PUBLIC cxxabi)
|
||||
|
||||
# For __udivmodti4, __divmodti4.
|
||||
if (OS_DARWIN AND COMPILER_GCC)
|
||||
target_link_libraries(cxx PRIVATE gcc)
|
||||
endif ()
|
||||
|
||||
install(
|
||||
TARGETS cxx
|
||||
EXPORT global
|
||||
|
||||
@ -28,11 +28,16 @@ set (SRCS
|
||||
${SRC_DIR}/src/sentry_unix_pageallocator.c
|
||||
${SRC_DIR}/src/path/sentry_path_unix.c
|
||||
${SRC_DIR}/src/symbolizer/sentry_symbolizer_unix.c
|
||||
${SRC_DIR}/src/modulefinder/sentry_modulefinder_linux.c
|
||||
${SRC_DIR}/src/transports/sentry_transport_curl.c
|
||||
${SRC_DIR}/src/backends/sentry_backend_none.c
|
||||
)
|
||||
|
||||
if(APPLE)
|
||||
list(APPEND SRCS ${SRC_DIR}/src/modulefinder/sentry_modulefinder_apple.c)
|
||||
else()
|
||||
list(APPEND SRCS ${SRC_DIR}/src/modulefinder/sentry_modulefinder_linux.c)
|
||||
endif()
|
||||
|
||||
add_library(sentry ${SRCS})
|
||||
add_library(sentry::sentry ALIAS sentry)
|
||||
|
||||
|
||||
@ -6,4 +6,6 @@ add_library(simdjson ${SIMDJSON_SRC})
|
||||
target_include_directories(simdjson SYSTEM PUBLIC "${SIMDJSON_INCLUDE_DIR}" PRIVATE "${SIMDJSON_SRC_DIR}")
|
||||
|
||||
# simdjson is using its own CPU dispatching and get confused if we enable AVX/AVX2 flags.
|
||||
target_compile_options(simdjson PRIVATE -mno-avx -mno-avx2)
|
||||
if(ARCH_AMD64)
|
||||
target_compile_options(simdjson PRIVATE -mno-avx -mno-avx2)
|
||||
endif()
|
||||
|
||||
2
contrib/zlib-ng
vendored
2
contrib/zlib-ng
vendored
@ -1 +1 @@
|
||||
Subproject commit 6a5e93b9007782115f7f7e5235dedc81c4f1facb
|
||||
Subproject commit bffad6f6fe74d6a2f92e2668390664a926c68733
|
||||
@ -1,8 +1,10 @@
|
||||
FROM ubuntu:20.04
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13
|
||||
# ARG for quick switch to a given ubuntu mirror
|
||||
ARG apt_archive="http://archive.ubuntu.com"
|
||||
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
|
||||
|
||||
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
|
||||
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install ca-certificates lsb-release wget gnupg apt-transport-https \
|
||||
|
||||
@ -1,10 +1,12 @@
|
||||
FROM ubuntu:18.04
|
||||
|
||||
# ARG for quick switch to a given ubuntu mirror
|
||||
ARG apt_archive="http://archive.ubuntu.com"
|
||||
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
|
||||
|
||||
ARG repository="deb https://repo.clickhouse.com/deb/stable/ main/"
|
||||
ARG version=21.12.1.*
|
||||
|
||||
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install --yes --no-install-recommends \
|
||||
apt-transport-https \
|
||||
|
||||
@ -1,9 +1,11 @@
|
||||
# docker build -t clickhouse/docs-build .
|
||||
FROM ubuntu:20.04
|
||||
|
||||
ENV LANG=C.UTF-8
|
||||
# ARG for quick switch to a given ubuntu mirror
|
||||
ARG apt_archive="http://archive.ubuntu.com"
|
||||
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
|
||||
|
||||
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
|
||||
ENV LANG=C.UTF-8
|
||||
|
||||
RUN apt-get update \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends \
|
||||
|
||||
@ -1,9 +1,11 @@
|
||||
# docker build -t clickhouse/binary-builder .
|
||||
FROM ubuntu:20.04
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13
|
||||
# ARG for quick switch to a given ubuntu mirror
|
||||
ARG apt_archive="http://archive.ubuntu.com"
|
||||
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
|
||||
|
||||
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
|
||||
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install \
|
||||
|
||||
@ -1,9 +1,11 @@
|
||||
# docker build -t clickhouse/deb-builder .
|
||||
FROM ubuntu:20.04
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13
|
||||
# ARG for quick switch to a given ubuntu mirror
|
||||
ARG apt_archive="http://archive.ubuntu.com"
|
||||
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
|
||||
|
||||
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
|
||||
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install ca-certificates lsb-release wget gnupg apt-transport-https \
|
||||
|
||||
@ -63,7 +63,7 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
|
||||
PPC_SUFFIX = '-ppc64le'
|
||||
|
||||
result = []
|
||||
cmake_flags = ['$CMAKE_FLAGS', '-DADD_GDB_INDEX_FOR_GOLD=1']
|
||||
cmake_flags = ['$CMAKE_FLAGS']
|
||||
|
||||
is_clang = compiler.startswith(CLANG_PREFIX)
|
||||
is_cross_darwin = compiler.endswith(DARWIN_SUFFIX)
|
||||
@ -73,11 +73,6 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
|
||||
is_cross_freebsd = compiler.endswith(FREEBSD_SUFFIX)
|
||||
is_cross_compile = is_cross_darwin or is_cross_darwin_arm or is_cross_arm or is_cross_freebsd or is_cross_ppc
|
||||
|
||||
# Explicitly use LLD with Clang by default.
|
||||
# Don't force linker for cross-compilation.
|
||||
if is_clang and not is_cross_compile:
|
||||
cmake_flags.append("-DLINKER_NAME=ld.lld")
|
||||
|
||||
if is_cross_darwin:
|
||||
cc = compiler[:-len(DARWIN_SUFFIX)]
|
||||
cmake_flags.append("-DCMAKE_AR:FILEPATH=/cctools/bin/x86_64-apple-darwin-ar")
|
||||
|
||||
@ -1,5 +1,9 @@
|
||||
FROM ubuntu:20.04
|
||||
|
||||
# ARG for quick switch to a given ubuntu mirror
|
||||
ARG apt_archive="http://archive.ubuntu.com"
|
||||
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
|
||||
|
||||
ARG repository="deb https://repo.clickhouse.com/deb/stable/ main/"
|
||||
ARG version=21.12.1.*
|
||||
ARG gosu_ver=1.10
|
||||
@ -26,8 +30,6 @@ ARG DEBIAN_FRONTEND=noninteractive
|
||||
# installed to prevent picking those uid / gid by some unrelated software.
|
||||
# The same uid / gid (101) is used both for alpine and ubuntu.
|
||||
|
||||
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
|
||||
|
||||
RUN groupadd -r clickhouse --gid=101 \
|
||||
&& useradd -r -g clickhouse --uid=101 --home-dir=/var/lib/clickhouse --shell=/bin/bash clickhouse \
|
||||
&& apt-get update \
|
||||
|
||||
@ -1,9 +1,11 @@
|
||||
# docker build -t clickhouse/test-base .
|
||||
FROM clickhouse/test-util
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13
|
||||
# ARG for quick switch to a given ubuntu mirror
|
||||
ARG apt_archive="http://archive.ubuntu.com"
|
||||
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
|
||||
|
||||
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
|
||||
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install ca-certificates lsb-release wget gnupg apt-transport-https \
|
||||
|
||||
@ -2,7 +2,9 @@
|
||||
# docker run --volume=path_to_repo:/repo_folder --volume=path_to_result:/test_output clickhouse/codebrowser
|
||||
FROM clickhouse/binary-builder
|
||||
|
||||
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
|
||||
# ARG for quick switch to a given ubuntu mirror
|
||||
ARG apt_archive="http://archive.ubuntu.com"
|
||||
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
|
||||
|
||||
RUN apt-get update && apt-get --yes --allow-unauthenticated install clang-9 libllvm9 libclang-9-dev
|
||||
|
||||
|
||||
@ -1,9 +1,11 @@
|
||||
# docker build -t clickhouse/fasttest .
|
||||
FROM clickhouse/test-util
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13
|
||||
# ARG for quick switch to a given ubuntu mirror
|
||||
ARG apt_archive="http://archive.ubuntu.com"
|
||||
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
|
||||
|
||||
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
|
||||
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install ca-certificates lsb-release wget gnupg apt-transport-https \
|
||||
|
||||
@ -174,7 +174,6 @@ function clone_submodules
|
||||
contrib/double-conversion
|
||||
contrib/libcxx
|
||||
contrib/libcxxabi
|
||||
contrib/libc-headers
|
||||
contrib/lz4
|
||||
contrib/zstd
|
||||
contrib/fastops
|
||||
@ -235,6 +234,9 @@ function build
|
||||
time ninja clickhouse-bundle 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/build_log.txt"
|
||||
if [ "$COPY_CLICKHOUSE_BINARY_TO_OUTPUT" -eq "1" ]; then
|
||||
cp programs/clickhouse "$FASTTEST_OUTPUT/clickhouse"
|
||||
|
||||
strip programs/clickhouse -o "$FASTTEST_OUTPUT/clickhouse-stripped"
|
||||
gzip "$FASTTEST_OUTPUT/clickhouse-stripped"
|
||||
fi
|
||||
ccache --show-stats ||:
|
||||
)
|
||||
|
||||
@ -1,12 +1,14 @@
|
||||
# docker build -t clickhouse/fuzzer .
|
||||
FROM clickhouse/test-base
|
||||
|
||||
# ARG for quick switch to a given ubuntu mirror
|
||||
ARG apt_archive="http://archive.ubuntu.com"
|
||||
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
|
||||
|
||||
ENV LANG=C.UTF-8
|
||||
ENV TZ=Europe/Moscow
|
||||
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
|
||||
|
||||
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
|
||||
|
||||
RUN apt-get update \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends \
|
||||
ca-certificates \
|
||||
|
||||
@ -77,7 +77,7 @@ function configure
|
||||
|
||||
function watchdog
|
||||
{
|
||||
sleep 3600
|
||||
sleep 1800
|
||||
|
||||
echo "Fuzzing run has timed out"
|
||||
for _ in {1..10}
|
||||
|
||||
@ -1,7 +1,9 @@
|
||||
# docker build -t clickhouse/integration-tests-runner .
|
||||
FROM ubuntu:20.04
|
||||
|
||||
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
|
||||
# ARG for quick switch to a given ubuntu mirror
|
||||
ARG apt_archive="http://archive.ubuntu.com"
|
||||
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
|
||||
|
||||
RUN apt-get update \
|
||||
&& env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
|
||||
|
||||
@ -1,12 +1,14 @@
|
||||
# docker build -t clickhouse/performance-comparison .
|
||||
FROM ubuntu:18.04
|
||||
|
||||
# ARG for quick switch to a given ubuntu mirror
|
||||
ARG apt_archive="http://archive.ubuntu.com"
|
||||
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
|
||||
|
||||
ENV LANG=C.UTF-8
|
||||
ENV TZ=Europe/Moscow
|
||||
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
|
||||
|
||||
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
|
||||
|
||||
RUN apt-get update \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends \
|
||||
bash \
|
||||
|
||||
@ -1,7 +1,9 @@
|
||||
# docker build -t clickhouse/sqlancer-test .
|
||||
FROM ubuntu:20.04
|
||||
|
||||
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
|
||||
# ARG for quick switch to a given ubuntu mirror
|
||||
ARG apt_archive="http://archive.ubuntu.com"
|
||||
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
|
||||
|
||||
RUN apt-get update --yes && env DEBIAN_FRONTEND=noninteractive apt-get install wget unzip git default-jdk maven python3 --yes --no-install-recommends
|
||||
RUN wget https://github.com/sqlancer/sqlancer/archive/master.zip -O /sqlancer.zip
|
||||
|
||||
@ -151,8 +151,8 @@ zgrep -Fa " <Fatal> " /var/log/clickhouse-server/clickhouse-server.log*
|
||||
# Grep logs for sanitizer asserts, crashes and other critical errors
|
||||
|
||||
# Sanitizer asserts
|
||||
zgrep -Fa "==================" /var/log/clickhouse-server/stderr.log >> /test_output/tmp
|
||||
zgrep -Fa "WARNING" /var/log/clickhouse-server/stderr.log >> /test_output/tmp
|
||||
grep -Fa "==================" /var/log/clickhouse-server/stderr.log | grep -v "in query:" >> /test_output/tmp
|
||||
grep -Fa "WARNING" /var/log/clickhouse-server/stderr.log >> /test_output/tmp
|
||||
zgrep -Fav "ASan doesn't fully support makecontext/swapcontext functions" /test_output/tmp > /dev/null \
|
||||
&& echo -e 'Sanitizer assert (in stderr.log)\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'No sanitizer asserts\tOK' >> /test_output/test_results.tsv
|
||||
|
||||
@ -67,52 +67,70 @@ def compress_stress_logs(output_path, files_prefix):
|
||||
cmd = f"cd {output_path} && tar -zcf stress_run_logs.tar.gz {files_prefix}* && rm {files_prefix}*"
|
||||
check_output(cmd, shell=True)
|
||||
|
||||
def call_with_retry(query, timeout=30, retry_count=5):
|
||||
for i in range(retry_count):
|
||||
code = call(query, shell=True, stderr=STDOUT, timeout=timeout)
|
||||
if code != 0:
|
||||
time.sleep(i)
|
||||
else:
|
||||
break
|
||||
|
||||
|
||||
def prepare_for_hung_check(drop_databases):
|
||||
# FIXME this function should not exist, but...
|
||||
|
||||
# ThreadFuzzer significantly slows down server and causes false-positive hung check failures
|
||||
call("clickhouse client -q 'SYSTEM STOP THREAD FUZZER'", shell=True, stderr=STDOUT, timeout=30)
|
||||
call_with_retry("clickhouse client -q 'SYSTEM STOP THREAD FUZZER'")
|
||||
|
||||
# We attach gdb to clickhouse-server before running tests
|
||||
# to print stacktraces of all crashes even if clickhouse cannot print it for some reason.
|
||||
# However, it obstruct checking for hung queries.
|
||||
logging.info("Will terminate gdb (if any)")
|
||||
call("kill -TERM $(pidof gdb)", shell=True, stderr=STDOUT, timeout=30)
|
||||
call_with_retry("kill -TERM $(pidof gdb)")
|
||||
|
||||
# Some tests set too low memory limit for default user and forget to reset in back.
|
||||
# It may cause SYSTEM queries to fail, let's disable memory limit.
|
||||
call("clickhouse client --max_memory_usage_for_user=0 -q 'SELECT 1 FORMAT Null'", shell=True, stderr=STDOUT, timeout=30)
|
||||
call_with_retry("clickhouse client --max_memory_usage_for_user=0 -q 'SELECT 1 FORMAT Null'")
|
||||
|
||||
# Some tests execute SYSTEM STOP MERGES or similar queries.
|
||||
# It may cause some ALTERs to hang.
|
||||
# Possibly we should fix tests and forbid to use such queries without specifying table.
|
||||
call("clickhouse client -q 'SYSTEM START MERGES'", shell=True, stderr=STDOUT, timeout=30)
|
||||
call("clickhouse client -q 'SYSTEM START DISTRIBUTED SENDS'", shell=True, stderr=STDOUT, timeout=30)
|
||||
call("clickhouse client -q 'SYSTEM START TTL MERGES'", shell=True, stderr=STDOUT, timeout=30)
|
||||
call("clickhouse client -q 'SYSTEM START MOVES'", shell=True, stderr=STDOUT, timeout=30)
|
||||
call("clickhouse client -q 'SYSTEM START FETCHES'", shell=True, stderr=STDOUT, timeout=30)
|
||||
call("clickhouse client -q 'SYSTEM START REPLICATED SENDS'", shell=True, stderr=STDOUT, timeout=30)
|
||||
call("clickhouse client -q 'SYSTEM START REPLICATION QUEUES'", shell=True, stderr=STDOUT, timeout=30)
|
||||
call_with_retry("clickhouse client -q 'SYSTEM START MERGES'")
|
||||
call_with_retry("clickhouse client -q 'SYSTEM START DISTRIBUTED SENDS'")
|
||||
call_with_retry("clickhouse client -q 'SYSTEM START TTL MERGES'")
|
||||
call_with_retry("clickhouse client -q 'SYSTEM START MOVES'")
|
||||
call_with_retry("clickhouse client -q 'SYSTEM START FETCHES'")
|
||||
call_with_retry("clickhouse client -q 'SYSTEM START REPLICATED SENDS'")
|
||||
call_with_retry("clickhouse client -q 'SYSTEM START REPLICATION QUEUES'")
|
||||
|
||||
# Issue #21004, live views are experimental, so let's just suppress it
|
||||
call("""clickhouse client -q "KILL QUERY WHERE upper(query) LIKE 'WATCH %'" """, shell=True, stderr=STDOUT, timeout=30)
|
||||
call_with_retry("""clickhouse client -q "KILL QUERY WHERE upper(query) LIKE 'WATCH %'" """)
|
||||
|
||||
# Kill other queries which known to be slow
|
||||
# It's query from 01232_preparing_sets_race_condition_long, it may take up to 1000 seconds in slow builds
|
||||
call("""clickhouse client -q "KILL QUERY WHERE query LIKE 'insert into tableB select %'" """, shell=True, stderr=STDOUT, timeout=30)
|
||||
call_with_retry("""clickhouse client -q "KILL QUERY WHERE query LIKE 'insert into tableB select %'" """)
|
||||
# Long query from 00084_external_agregation
|
||||
call("""clickhouse client -q "KILL QUERY WHERE query LIKE 'SELECT URL, uniq(SearchPhrase) AS u FROM test.hits GROUP BY URL ORDER BY u %'" """, shell=True, stderr=STDOUT, timeout=30)
|
||||
call_with_retry("""clickhouse client -q "KILL QUERY WHERE query LIKE 'SELECT URL, uniq(SearchPhrase) AS u FROM test.hits GROUP BY URL ORDER BY u %'" """)
|
||||
|
||||
if drop_databases:
|
||||
# Here we try to drop all databases in async mode. If some queries really hung, than drop will hung too.
|
||||
# Otherwise we will get rid of queries which wait for background pool. It can take a long time on slow builds (more than 900 seconds).
|
||||
databases = check_output('clickhouse client -q "SHOW DATABASES"', shell=True, timeout=30).decode('utf-8').strip().split()
|
||||
for db in databases:
|
||||
if db == "system":
|
||||
continue
|
||||
command = f'clickhouse client -q "DROP DATABASE {db}"'
|
||||
# we don't wait for drop
|
||||
Popen(command, shell=True)
|
||||
for i in range(5):
|
||||
try:
|
||||
# Here we try to drop all databases in async mode. If some queries really hung, than drop will hung too.
|
||||
# Otherwise we will get rid of queries which wait for background pool. It can take a long time on slow builds (more than 900 seconds).
|
||||
databases = check_output('clickhouse client -q "SHOW DATABASES"', shell=True, timeout=30).decode('utf-8').strip().split()
|
||||
for db in databases:
|
||||
if db == "system":
|
||||
continue
|
||||
command = f'clickhouse client -q "DROP DATABASE {db}"'
|
||||
# we don't wait for drop
|
||||
Popen(command, shell=True)
|
||||
break
|
||||
except Exception as ex:
|
||||
print("Failed to SHOW or DROP databasese, will retry", ex)
|
||||
time.sleep(i)
|
||||
else:
|
||||
raise Exception("Cannot drop databases after stress tests. Probably server consumed too much memory and cannot execute simple queries")
|
||||
|
||||
|
||||
# Wait for last queries to finish if any, not longer than 300 seconds
|
||||
call("""clickhouse client -q "select sleepEachRow((
|
||||
@ -138,7 +156,7 @@ if __name__ == "__main__":
|
||||
parser.add_argument("--client-cmd", default='clickhouse-client')
|
||||
parser.add_argument("--server-log-folder", default='/var/log/clickhouse-server')
|
||||
parser.add_argument("--output-folder")
|
||||
parser.add_argument("--global-time-limit", type=int, default=3600)
|
||||
parser.add_argument("--global-time-limit", type=int, default=1800)
|
||||
parser.add_argument("--num-parallel", type=int, default=cpu_count())
|
||||
parser.add_argument('--hung-check', action='store_true', default=False)
|
||||
# make sense only for hung check
|
||||
|
||||
@ -1,7 +1,9 @@
|
||||
# docker build -t clickhouse/style-test .
|
||||
FROM ubuntu:20.04
|
||||
|
||||
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
|
||||
# ARG for quick switch to a given ubuntu mirror
|
||||
ARG apt_archive="http://archive.ubuntu.com"
|
||||
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
|
||||
|
||||
RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
|
||||
shellcheck \
|
||||
|
||||
@ -1,7 +1,9 @@
|
||||
# docker build -t clickhouse/testflows-runner .
|
||||
FROM ubuntu:20.04
|
||||
|
||||
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
|
||||
# ARG for quick switch to a given ubuntu mirror
|
||||
ARG apt_archive="http://archive.ubuntu.com"
|
||||
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
|
||||
|
||||
RUN apt-get update \
|
||||
&& env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
|
||||
|
||||
@ -3,15 +3,14 @@ toc_priority: 65
|
||||
toc_title: Build on Mac OS X
|
||||
---
|
||||
|
||||
# You don't have to build ClickHouse
|
||||
|
||||
You can install ClickHouse as follows: https://clickhouse.com/#quick-start
|
||||
Choose Mac x86 or M1.
|
||||
|
||||
# How to Build ClickHouse on Mac OS X {#how-to-build-clickhouse-on-mac-os-x}
|
||||
|
||||
Build should work on x86_64 (Intel) and arm64 (Apple Silicon) based macOS 10.15 (Catalina) and higher with Homebrew's vanilla Clang.
|
||||
It is always recommended to use `clang` compiler. It is possible to use XCode's `AppleClang` or `gcc` but it's strongly discouraged.
|
||||
!!! info "You don't have to build ClickHouse yourself!"
|
||||
You can install pre-built ClickHouse as described in [Quick Start](https://clickhouse.com/#quick-start).
|
||||
Follow `macOS (Intel)` or `macOS (Apple silicon)` installation instructions.
|
||||
|
||||
Build should work on x86_64 (Intel) and arm64 (Apple silicon) based macOS 10.15 (Catalina) and higher with Homebrew's vanilla Clang.
|
||||
It is always recommended to use vanilla `clang` compiler. It is possible to use XCode's `apple-clang` or `gcc` but it's strongly discouraged.
|
||||
|
||||
## Install Homebrew {#install-homebrew}
|
||||
|
||||
@ -33,8 +32,6 @@ sudo rm -rf /Library/Developer/CommandLineTools
|
||||
sudo xcode-select --install
|
||||
```
|
||||
|
||||
Reboot.
|
||||
|
||||
## Install Required Compilers, Tools, and Libraries {#install-required-compilers-tools-and-libraries}
|
||||
|
||||
``` bash
|
||||
@ -51,40 +48,41 @@ git clone --recursive git@github.com:ClickHouse/ClickHouse.git
|
||||
|
||||
## Build ClickHouse {#build-clickhouse}
|
||||
|
||||
To build using Homebrew's vanilla Clang compiler:
|
||||
To build using Homebrew's vanilla Clang compiler (the only **recommended** way):
|
||||
|
||||
``` bash
|
||||
cd ClickHouse
|
||||
rm -rf build
|
||||
mkdir build
|
||||
cd build
|
||||
cmake -DCMAKE_C_COMPILER=$(brew --prefix llvm)/bin/clang -DCMAKE_CXX_COMPILER=$(brew --prefix llvm)/bin/clang++ -DCMAKE_BUILD_TYPE=RelWithDebInfo ..
|
||||
cmake -DCMAKE_C_COMPILER=$(brew --prefix llvm)/bin/clang -DCMAKE_CXX_COMPILER=$(brew --prefix llvm)/bin/clang++ -DCMAKE_AR=$(brew --prefix llvm)/bin/llvm-ar -DCMAKE_RANLIB=$(brew --prefix llvm)/bin/llvm-ranlib -DOBJCOPY_PATH=$(brew --prefix llvm)/bin/llvm-objcopy -DCMAKE_BUILD_TYPE=RelWithDebInfo ..
|
||||
cmake --build . --config RelWithDebInfo
|
||||
cd ..
|
||||
# The resulting binary will be created at: ./programs/clickhouse
|
||||
```
|
||||
|
||||
To build using Xcode's native AppleClang compiler (this option is strongly not recommended; use the option above):
|
||||
To build using Xcode's native AppleClang compiler in Xcode IDE (this option is only for development builds and workflows, and is **not recommended** unless you know what you are doing):
|
||||
|
||||
``` bash
|
||||
cd ClickHouse
|
||||
rm -rf build
|
||||
mkdir build
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo ..
|
||||
cmake --build . --config RelWithDebInfo
|
||||
cd ..
|
||||
XCODE_IDE=1 ALLOW_APPLECLANG=1 cmake -G Xcode -DCMAKE_BUILD_TYPE=Debug -DENABLE_JEMALLOC=OFF ..
|
||||
cmake --open .
|
||||
# ...then, in Xcode IDE select ALL_BUILD scheme and start the building process.
|
||||
# The resulting binary will be created at: ./programs/Debug/clickhouse
|
||||
```
|
||||
|
||||
To build using Homebrew's vanilla GCC compiler (this option is absolutely not recommended, I'm wondering why do we ever have it):
|
||||
To build using Homebrew's vanilla GCC compiler (this option is only for development experiments, and is **absolutely not recommended** unless you really know what you are doing):
|
||||
|
||||
``` bash
|
||||
cd ClickHouse
|
||||
rm -rf build
|
||||
mkdir build
|
||||
cd build
|
||||
cmake -DCMAKE_C_COMPILER=$(brew --prefix gcc)/bin/gcc-11 -DCMAKE_CXX_COMPILER=$(brew --prefix gcc)/bin/g++-11 -DCMAKE_BUILD_TYPE=RelWithDebInfo ..
|
||||
cmake -DCMAKE_C_COMPILER=$(brew --prefix gcc)/bin/gcc-11 -DCMAKE_CXX_COMPILER=$(brew --prefix gcc)/bin/g++-11 -DCMAKE_AR=$(brew --prefix gcc)/bin/gcc-ar-11 -DCMAKE_RANLIB=$(brew --prefix gcc)/bin/gcc-ranlib-11 -DOBJCOPY_PATH=$(brew --prefix binutils)/bin/objcopy -DCMAKE_BUILD_TYPE=RelWithDebInfo ..
|
||||
cmake --build . --config RelWithDebInfo
|
||||
cd ..
|
||||
# The resulting binary will be created at: ./programs/clickhouse
|
||||
```
|
||||
|
||||
## Caveats {#caveats}
|
||||
@ -140,9 +138,9 @@ sudo launchctl load -w /Library/LaunchDaemons/limit.maxfiles.plist
|
||||
|
||||
To check if it’s working, use the `ulimit -n` or `launchctl limit maxfiles` commands.
|
||||
|
||||
## Run ClickHouse server:
|
||||
## Running ClickHouse server
|
||||
|
||||
```
|
||||
``` bash
|
||||
cd ClickHouse
|
||||
./build/programs/clickhouse-server --config-file ./programs/server/config.xml
|
||||
```
|
||||
|
||||
@ -131,18 +131,18 @@ Builds ClickHouse in various configurations for use in further steps. You have t
|
||||
- **Build log**: link to the building and files copying log, useful when build failed.
|
||||
- **Build time**.
|
||||
- **Artifacts**: build result files (with `XXX` being the server version e.g. `20.8.1.4344`).
|
||||
- `clickhouse-client_XXX_all.deb`
|
||||
- `clickhouse-common-static-dbg_XXX[+asan, +msan, +ubsan, +tsan]_amd64.deb`
|
||||
- `clickhouse-common-staticXXX_amd64.deb`
|
||||
- `clickhouse-server_XXX_all.deb`
|
||||
- `clickhouse-test_XXX_all.deb`
|
||||
- `clickhouse_XXX_amd64.buildinfo`
|
||||
- `clickhouse_XXX_amd64.changes`
|
||||
- `clickhouse`: Main built binary.
|
||||
- `clickhouse-odbc-bridge`
|
||||
- `unit_tests_dbms`: GoogleTest binary with ClickHouse unit tests.
|
||||
- `shared_build.tgz`: build with shared libraries.
|
||||
- `performance.tgz`: Special package for performance tests.
|
||||
- `clickhouse-client_XXX_all.deb`
|
||||
- `clickhouse-common-static-dbg_XXX[+asan, +msan, +ubsan, +tsan]_amd64.deb`
|
||||
- `clickhouse-common-staticXXX_amd64.deb`
|
||||
- `clickhouse-server_XXX_all.deb`
|
||||
- `clickhouse-test_XXX_all.deb`
|
||||
- `clickhouse_XXX_amd64.buildinfo`
|
||||
- `clickhouse_XXX_amd64.changes`
|
||||
- `clickhouse`: Main built binary.
|
||||
- `clickhouse-odbc-bridge`
|
||||
- `unit_tests_dbms`: GoogleTest binary with ClickHouse unit tests.
|
||||
- `shared_build.tgz`: build with shared libraries.
|
||||
- `performance.tgz`: Special package for performance tests.
|
||||
|
||||
|
||||
## Special Build Check
|
||||
|
||||
@ -39,12 +39,16 @@ DETACH TABLE postgres_database.table_to_remove;
|
||||
|
||||
## Settings {#settings}
|
||||
|
||||
- [materialized_postgresql_max_block_size](../../operations/settings/settings.md#materialized-postgresql-max-block-size)
|
||||
|
||||
- [materialized_postgresql_tables_list](../../operations/settings/settings.md#materialized-postgresql-tables-list)
|
||||
|
||||
- [materialized_postgresql_schema](../../operations/settings/settings.md#materialized-postgresql-schema)
|
||||
|
||||
- [materialized_postgresql_schema_list](../../operations/settings/settings.md#materialized-postgresql-schema-list)
|
||||
|
||||
- [materialized_postgresql_allow_automatic_update](../../operations/settings/settings.md#materialized-postgresql-allow-automatic-update)
|
||||
|
||||
- [materialized_postgresql_max_block_size](../../operations/settings/settings.md#materialized-postgresql-max-block-size)
|
||||
|
||||
- [materialized_postgresql_replication_slot](../../operations/settings/settings.md#materialized-postgresql-replication-slot)
|
||||
|
||||
- [materialized_postgresql_snapshot](../../operations/settings/settings.md#materialized-postgresql-snapshot)
|
||||
@ -52,8 +56,7 @@ DETACH TABLE postgres_database.table_to_remove;
|
||||
``` sql
|
||||
CREATE DATABASE database1
|
||||
ENGINE = MaterializedPostgreSQL('postgres1:5432', 'postgres_database', 'postgres_user', 'postgres_password')
|
||||
SETTINGS materialized_postgresql_max_block_size = 65536,
|
||||
materialized_postgresql_tables_list = 'table1,table2,table3';
|
||||
SETTINGS materialized_postgresql_tables_list = 'table1,table2,table3';
|
||||
|
||||
SELECT * FROM database1.table1;
|
||||
```
|
||||
@ -64,6 +67,55 @@ The settings can be changed, if necessary, using a DDL query. But it is impossib
|
||||
ALTER DATABASE postgres_database MODIFY SETTING materialized_postgresql_max_block_size = <new_size>;
|
||||
```
|
||||
|
||||
|
||||
## PostgreSQL schema {#schema}
|
||||
|
||||
PostgreSQL [schema](https://www.postgresql.org/docs/9.1/ddl-schemas.html) can be used in two ways.
|
||||
|
||||
1. One schema for one `MaterializedPostgreSQL` database engine. Requires to use setting `materialized_postgresql_schema`.
|
||||
Tables are accessed via table name only:
|
||||
|
||||
``` sql
|
||||
CREATE DATABASE postgres_database
|
||||
ENGINE = MaterializedPostgreSQL('postgres1:5432', 'postgres_database', 'postgres_user', 'postgres_password')
|
||||
SETTINGS materialized_postgresql_schema = 'postgres_schema';
|
||||
|
||||
SELECT * FROM postgres_database.table1;
|
||||
```
|
||||
|
||||
2. Any number of schemas with specified set of tables for one `MaterializedPostgreSQL` database engine. Requires to use setting `materialized_postgresql_tables_list`. Each table is written along with its schema.
|
||||
Tables are accessed via schema name and table name at the same time:
|
||||
|
||||
``` sql
|
||||
CREATE DATABASE database1
|
||||
ENGINE = MaterializedPostgreSQL('postgres1:5432', 'postgres_database', 'postgres_user', 'postgres_password')
|
||||
SETTINGS materialized_postgresql_tables_list = 'schema1.table1,schema2.table2,schema1.table3';
|
||||
materialized_postgresql_tables_list_with_schema = 1;
|
||||
|
||||
SELECT * FROM database1.`schema1.table1`;
|
||||
SELECT * FROM database1.`schema2.table2`;
|
||||
```
|
||||
|
||||
But in this case all tables in `materialized_postgresql_tables_list` must be written with its schema name.
|
||||
Requires `materialized_postgresql_tables_list_with_schema = 1`.
|
||||
|
||||
Warning: for this case dots in table name are not allowed.
|
||||
|
||||
3. Any number of schemas with full set of tables for one `MaterializedPostgreSQL` database engine. Requires to use setting `materialized_postgresql_schema_list`.
|
||||
|
||||
``` sql
|
||||
CREATE DATABASE database1
|
||||
ENGINE = MaterializedPostgreSQL('postgres1:5432', 'postgres_database', 'postgres_user', 'postgres_password')
|
||||
SETTINGS materialized_postgresql_schema_list = 'schema1,schema2,schema3';
|
||||
|
||||
SELECT * FROM database1.`schema1.table1`;
|
||||
SELECT * FROM database1.`schema1.table2`;
|
||||
SELECT * FROM database1.`schema2.table2`;
|
||||
```
|
||||
|
||||
Warning: for this case dots in table name are not allowed.
|
||||
|
||||
|
||||
## Requirements {#requirements}
|
||||
|
||||
1. The [wal_level](https://www.postgresql.org/docs/current/runtime-config-wal.html) setting must have a value `logical` and `max_replication_slots` parameter must have a value at least `2` in the PostgreSQL config file.
|
||||
|
||||
@ -33,7 +33,7 @@ CREATE TABLE test
|
||||
`key` String,
|
||||
`v1` UInt32,
|
||||
`v2` String,
|
||||
`v3` Float32,
|
||||
`v3` Float32
|
||||
)
|
||||
ENGINE = EmbeddedRocksDB
|
||||
PRIMARY KEY key
|
||||
|
||||
@ -133,8 +133,7 @@ Example:
|
||||
|
||||
SELECT level, sum(total) FROM daily GROUP BY level;
|
||||
```
|
||||
|
||||
To improve performance, received messages are grouped into blocks the size of [max_insert_block_size](../../../operations/server-configuration-parameters/settings.md#settings-max_insert_block_size). If the block wasn’t formed within [stream_flush_interval_ms](../../../operations/server-configuration-parameters/settings.md) milliseconds, the data will be flushed to the table regardless of the completeness of the block.
|
||||
To improve performance, received messages are grouped into blocks the size of [max_insert_block_size](../../../operations/settings/settings/#settings-max_insert_block_size). If the block wasn’t formed within [stream_flush_interval_ms](../../../operations/settings/settings/#stream-flush-interval-ms) milliseconds, the data will be flushed to the table regardless of the completeness of the block.
|
||||
|
||||
To stop receiving topic data or to change the conversion logic, detach the materialized view:
|
||||
|
||||
@ -192,6 +191,6 @@ Example:
|
||||
**See Also**
|
||||
|
||||
- [Virtual columns](../../../engines/table-engines/index.md#table_engines-virtual_columns)
|
||||
- [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size)
|
||||
- [background_message_broker_schedule_pool_size](../../../operations/settings/settings.md#background_message_broker_schedule_pool_size)
|
||||
|
||||
[Original article](https://clickhouse.com/docs/en/engines/table-engines/integrations/kafka/) <!--hide-->
|
||||
|
||||
@ -108,9 +108,43 @@ SELECT * FROM mysql_table
|
||||
└────────────────┴────────┘
|
||||
```
|
||||
|
||||
## Settings {#mysql-settings}
|
||||
|
||||
Default settings are not very efficient, since they do not even reuse connections. These settings allow you to increase the number of queries run by the server per second.
|
||||
|
||||
### connection_auto_close {#connection-auto-close}
|
||||
|
||||
Allows to automatically close the connection after query execution, i.e. disable connection reuse.
|
||||
|
||||
Possible values:
|
||||
|
||||
- 1 — Auto-close connection is allowed, so the connection reuse is disabled
|
||||
- 0 — Auto-close connection is not allowed, so the connection reuse is enabled
|
||||
|
||||
Default value: `1`.
|
||||
|
||||
### connection_max_tries {#connection-max-tries}
|
||||
|
||||
Sets the number of retries for pool with failover.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Positive integer.
|
||||
- 0 — There are no retries for pool with failover.
|
||||
|
||||
Default value: `3`.
|
||||
|
||||
### connection_pool_size {#connection-pool-size}
|
||||
|
||||
Size of connection pool (if all connections are in use, the query will wait until some connection will be freed).
|
||||
|
||||
Possible values:
|
||||
|
||||
- Positive integer.
|
||||
|
||||
Default value: `16`.
|
||||
|
||||
## See Also {#see-also}
|
||||
|
||||
- [The ‘mysql’ table function](../../../sql-reference/table-functions/mysql.md)
|
||||
- [The mysql table function](../../../sql-reference/table-functions/mysql.md)
|
||||
- [Using MySQL as a source of external dictionary](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md#dicts-external_dicts_dict_sources-mysql)
|
||||
|
||||
[Original article](https://clickhouse.com/docs/en/engines/table-engines/integrations/mysql/) <!--hide-->
|
||||
|
||||
@ -24,6 +24,8 @@ The supported formats are:
|
||||
| [CSVWithNames](#csvwithnames) | ✔ | ✔ |
|
||||
| [CSVWithNamesAndTypes](#csvwithnamesandtypes) | ✔ | ✔ |
|
||||
| [CustomSeparated](#format-customseparated) | ✔ | ✔ |
|
||||
| [CustomSeparatedWithNames](#customseparatedwithnames) | ✔ | ✔ |
|
||||
| [CustomSeparatedWithNamesAndTypes](#customseparatedwithnamesandtypes) | ✔ | ✔ |
|
||||
| [Values](#data-format-values) | ✔ | ✔ |
|
||||
| [Vertical](#vertical) | ✗ | ✔ |
|
||||
| [JSON](#json) | ✗ | ✔ |
|
||||
@ -429,8 +431,17 @@ Also prints two header rows with column names and types, similar to [TabSeparate
|
||||
|
||||
## CustomSeparated {#format-customseparated}
|
||||
|
||||
Similar to [Template](#format-template), but it prints or reads all columns and uses escaping rule from setting `format_custom_escaping_rule` and delimiters from settings `format_custom_field_delimiter`, `format_custom_row_before_delimiter`, `format_custom_row_after_delimiter`, `format_custom_row_between_delimiter`, `format_custom_result_before_delimiter` and `format_custom_result_after_delimiter`, not from format strings.
|
||||
There is also `CustomSeparatedIgnoreSpaces` format, which is similar to `TemplateIgnoreSpaces`.
|
||||
Similar to [Template](#format-template), but it prints or reads all names and types of columns and uses escaping rule from [format_custom_escaping_rule](../operations/settings/settings.md#format-custom-escaping-rule) setting and delimiters from [format_custom_field_delimiter](../operations/settings/settings.md#format-custom-field-delimiter), [format_custom_row_before_delimiter](../operations/settings/settings.md#format-custom-row-before-delimiter), [format_custom_row_after_delimiter](../operations/settings/settings.md#format-custom-row-after-delimiter), [format_custom_row_between_delimiter](../operations/settings/settings.md#format-custom-row-between-delimiter), [format_custom_result_before_delimiter](../operations/settings/settings.md#format-custom-result-before-delimiter) and [format_custom_result_after_delimiter](../operations/settings/settings.md#format-custom-result-after-delimiter) settings, not from format strings.
|
||||
|
||||
There is also `CustomSeparatedIgnoreSpaces` format, which is similar to [TemplateIgnoreSpaces](#templateignorespaces).
|
||||
|
||||
## CustomSeparatedWithNames {#customseparatedwithnames}
|
||||
|
||||
Also prints the header row with column names, similar to [TabSeparatedWithNames](#tabseparatedwithnames).
|
||||
|
||||
## CustomSeparatedWithNamesAndTypes {#customseparatedwithnamesandtypes}
|
||||
|
||||
Also prints two header rows with column names and types, similar to [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes).
|
||||
|
||||
## JSON {#json}
|
||||
|
||||
@ -1536,14 +1547,17 @@ Each line of imported data is parsed according to the regular expression.
|
||||
|
||||
When working with the `Regexp` format, you can use the following settings:
|
||||
|
||||
- `format_regexp` — [String](../sql-reference/data-types/string.md). Contains regular expression in the [re2](https://github.com/google/re2/wiki/Syntax) format.
|
||||
- `format_regexp_escaping_rule` — [String](../sql-reference/data-types/string.md). The following escaping rules are supported:
|
||||
- CSV (similarly to [CSV](#csv))
|
||||
- JSON (similarly to [JSONEachRow](#jsoneachrow))
|
||||
- Escaped (similarly to [TSV](#tabseparated))
|
||||
- Quoted (similarly to [Values](#data-format-values))
|
||||
- Raw (extracts subpatterns as a whole, no escaping rules)
|
||||
- `format_regexp_skip_unmatched` — [UInt8](../sql-reference/data-types/int-uint.md). Defines the need to throw an exeption in case the `format_regexp` expression does not match the imported data. Can be set to `0` or `1`.
|
||||
- `format_regexp` — [String](../sql-reference/data-types/string.md). Contains regular expression in the [re2](https://github.com/google/re2/wiki/Syntax) format.
|
||||
|
||||
- `format_regexp_escaping_rule` — [String](../sql-reference/data-types/string.md). The following escaping rules are supported:
|
||||
|
||||
- CSV (similarly to [CSV](#csv))
|
||||
- JSON (similarly to [JSONEachRow](#jsoneachrow))
|
||||
- Escaped (similarly to [TSV](#tabseparated))
|
||||
- Quoted (similarly to [Values](#data-format-values))
|
||||
- Raw (extracts subpatterns as a whole, no escaping rules, similarly to [TSVRaw](#tabseparatedraw))
|
||||
|
||||
- `format_regexp_skip_unmatched` — [UInt8](../sql-reference/data-types/int-uint.md). Defines the need to throw an exeption in case the `format_regexp` expression does not match the imported data. Can be set to `0` or `1`.
|
||||
|
||||
**Usage**
|
||||
|
||||
|
||||
@ -69,8 +69,6 @@ If no conditions met for a data part, ClickHouse uses the `lz4` compression.
|
||||
</compression>
|
||||
```
|
||||
|
||||
<!--
|
||||
|
||||
## encryption {#server-settings-encryption}
|
||||
|
||||
Configures a command to obtain a key to be used by [encryption codecs](../../sql-reference/statements/create/table.md#create-query-encryption-codecs). Key (or keys) should be written in environment variables or set in the configuration file.
|
||||
@ -150,7 +148,6 @@ Or it can be set in hex:
|
||||
|
||||
Everything mentioned above can be applied for `aes_256_gcm_siv` (but the key must be 32 bytes long).
|
||||
|
||||
-->
|
||||
|
||||
## custom_settings_prefixes {#custom_settings_prefixes}
|
||||
|
||||
|
||||
@ -3691,6 +3691,14 @@ Sets a comma-separated list of PostgreSQL database tables, which will be replica
|
||||
|
||||
Default value: empty list — means whole PostgreSQL database will be replicated.
|
||||
|
||||
## materialized_postgresql_schema {#materialized-postgresql-schema}
|
||||
|
||||
Default value: empty string. (Default schema is used)
|
||||
|
||||
## materialized_postgresql_schema_list {#materialized-postgresql-schema-list}
|
||||
|
||||
Default value: empty list. (Default schema is used)
|
||||
|
||||
## materialized_postgresql_allow_automatic_update {#materialized-postgresql-allow-automatic-update}
|
||||
|
||||
Allows reloading table in the background, when schema changes are detected. DDL queries on the PostgreSQL side are not replicated via ClickHouse [MaterializedPostgreSQL](../../engines/database-engines/materialized-postgresql.md) engine, because it is not allowed with PostgreSQL logical replication protocol, but the fact of DDL changes is detected transactionally. In this case, the default behaviour is to stop replicating those tables once DDL is detected. However, if this setting is enabled, then, instead of stopping the replication of those tables, they will be reloaded in the background via database snapshot without data losses and replication will continue for them.
|
||||
@ -4071,3 +4079,54 @@ Possible values:
|
||||
- 0 — Big files read with only copying data from kernel to userspace.
|
||||
|
||||
Default value: `0`.
|
||||
|
||||
## format_custom_escaping_rule {#format-custom-escaping-rule}
|
||||
|
||||
Sets the field escaping rule for [CustomSeparated](../../interfaces/formats.md#format-customseparated) data format.
|
||||
|
||||
Possible values:
|
||||
|
||||
- `'Escaped'` — Similarly to [TSV](../../interfaces/formats.md#tabseparated).
|
||||
- `'Quoted'` — Similarly to [Values](../../interfaces/formats.md#data-format-values).
|
||||
- `'CSV'` — Similarly to [CSV](../../interfaces/formats.md#csv).
|
||||
- `'JSON'` — Similarly to [JSONEachRow](../../interfaces/formats.md#jsoneachrow).
|
||||
- `'XML'` — Similarly to [XML](../../interfaces/formats.md#xml).
|
||||
- `'Raw'` — Extracts subpatterns as a whole, no escaping rules, similarly to [TSVRaw](../../interfaces/formats.md#tabseparatedraw).
|
||||
|
||||
Default value: `'Escaped'`.
|
||||
|
||||
## format_custom_field_delimiter {#format-custom-field-delimiter}
|
||||
|
||||
Sets the character that is interpreted as a delimiter between the fields for [CustomSeparated](../../interfaces/formats.md#format-customseparated) data format.
|
||||
|
||||
Default value: `'\t'`.
|
||||
|
||||
## format_custom_row_before_delimiter {#format-custom-row-before-delimiter}
|
||||
|
||||
Sets the character that is interpreted as a delimiter before the field of the first column for [CustomSeparated](../../interfaces/formats.md#format-customseparated) data format.
|
||||
|
||||
Default value: `''`.
|
||||
|
||||
## format_custom_row_after_delimiter {#format-custom-row-after-delimiter}
|
||||
|
||||
Sets the character that is interpreted as a delimiter after the field of the last column for [CustomSeparated](../../interfaces/formats.md#format-customseparated) data format.
|
||||
|
||||
Default value: `'\n'`.
|
||||
|
||||
## format_custom_row_between_delimiter {#format-custom-row-between-delimiter}
|
||||
|
||||
Sets the character that is interpreted as a delimiter between the rows for [CustomSeparated](../../interfaces/formats.md#format-customseparated) data format.
|
||||
|
||||
Default value: `''`.
|
||||
|
||||
## format_custom_result_before_delimiter {#format-custom-result-before-delimiter}
|
||||
|
||||
Sets the character that is interpreted as a prefix before the result set for [CustomSeparated](../../interfaces/formats.md#format-customseparated) data format.
|
||||
|
||||
Default value: `''`.
|
||||
|
||||
## format_custom_result_after_delimiter {#format-custom-result-after-delimiter}
|
||||
|
||||
Sets the character that is interpreted as a suffix after the result set for [CustomSeparated](../../interfaces/formats.md#format-customseparated) data format.
|
||||
|
||||
Default value: `''`.
|
||||
|
||||
@ -1,98 +1,114 @@
|
||||
---
|
||||
toc_priority: 65
|
||||
toc_title: clickhouse-format
|
||||
---
|
||||
|
||||
# clickhouse-format {#clickhouse-format}
|
||||
|
||||
Allows formatting input queries.
|
||||
|
||||
Keys:
|
||||
|
||||
- `--help` or`-h` — Produce help message.
|
||||
- `--hilite` — Add syntax highlight with ANSI terminal escape sequences.
|
||||
- `--oneline` — Format in single line.
|
||||
- `--quiet` or `-q` — Just check syntax, no output on success.
|
||||
- `--multiquery` or `-n` — Allow multiple queries in the same file.
|
||||
- `--obfuscate` — Obfuscate instead of formatting.
|
||||
- `--seed <string>` — Seed arbitrary string that determines the result of obfuscation.
|
||||
- `--backslash` — Add a backslash at the end of each line of the formatted query. Can be useful when you copy a query from web or somewhere else with multiple lines, and want to execute it in command line.
|
||||
|
||||
## Examples {#examples}
|
||||
|
||||
1. Highlighting and single line:
|
||||
|
||||
```bash
|
||||
$ clickhouse-format --oneline --hilite <<< "SELECT sum(number) FROM numbers(5);"
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```sql
|
||||
SELECT sum(number) FROM numbers(5)
|
||||
```
|
||||
|
||||
2. Multiqueries:
|
||||
|
||||
```bash
|
||||
$ clickhouse-format -n <<< "SELECT * FROM (SELECT 1 AS x UNION ALL SELECT 1 UNION DISTINCT SELECT 3);"
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
SELECT *
|
||||
FROM
|
||||
(
|
||||
SELECT 1 AS x
|
||||
UNION ALL
|
||||
SELECT 1
|
||||
UNION DISTINCT
|
||||
SELECT 3
|
||||
)
|
||||
;
|
||||
```
|
||||
|
||||
3. Obfuscating:
|
||||
|
||||
```bash
|
||||
$ clickhouse-format --seed Hello --obfuscate <<< "SELECT cost_first_screen BETWEEN a AND b, CASE WHEN x >= 123 THEN y ELSE NULL END;"
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
SELECT treasury_mammoth_hazelnut BETWEEN nutmeg AND span, CASE WHEN chive >= 116 THEN switching ELSE ANYTHING END;
|
||||
```
|
||||
|
||||
Same query and another seed string:
|
||||
|
||||
```bash
|
||||
$ clickhouse-format --seed World --obfuscate <<< "SELECT cost_first_screen BETWEEN a AND b, CASE WHEN x >= 123 THEN y ELSE NULL END;"
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
SELECT horse_tape_summer BETWEEN folklore AND moccasins, CASE WHEN intestine >= 116 THEN nonconformist ELSE FORESTRY END;
|
||||
```
|
||||
|
||||
4. Adding backslash:
|
||||
|
||||
```bash
|
||||
$ clickhouse-format --backslash <<< "SELECT * FROM (SELECT 1 AS x UNION ALL SELECT 1 UNION DISTINCT SELECT 3);"
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
SELECT * \
|
||||
FROM \
|
||||
( \
|
||||
SELECT 1 AS x \
|
||||
UNION ALL \
|
||||
SELECT 1 \
|
||||
UNION DISTINCT \
|
||||
SELECT 3 \
|
||||
)
|
||||
```
|
||||
---
|
||||
toc_priority: 65
|
||||
toc_title: clickhouse-format
|
||||
---
|
||||
|
||||
# clickhouse-format {#clickhouse-format}
|
||||
|
||||
Allows formatting input queries.
|
||||
|
||||
Keys:
|
||||
|
||||
- `--help` or`-h` — Produce help message.
|
||||
- `--query` — Format queries of any length and complexity.
|
||||
- `--hilite` — Add syntax highlight with ANSI terminal escape sequences.
|
||||
- `--oneline` — Format in single line.
|
||||
- `--quiet` or `-q` — Just check syntax, no output on success.
|
||||
- `--multiquery` or `-n` — Allow multiple queries in the same file.
|
||||
- `--obfuscate` — Obfuscate instead of formatting.
|
||||
- `--seed <string>` — Seed arbitrary string that determines the result of obfuscation.
|
||||
- `--backslash` — Add a backslash at the end of each line of the formatted query. Can be useful when you copy a query from web or somewhere else with multiple lines, and want to execute it in command line.
|
||||
|
||||
## Examples {#examples}
|
||||
|
||||
1. Formatting a query:
|
||||
|
||||
```bash
|
||||
$ clickhouse-format --query "select number from numbers(10) where number%2 order by number desc;"
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
SELECT number
|
||||
FROM numbers(10)
|
||||
WHERE number % 2
|
||||
ORDER BY number DESC
|
||||
```
|
||||
|
||||
2. Highlighting and single line:
|
||||
|
||||
```bash
|
||||
$ clickhouse-format --oneline --hilite <<< "SELECT sum(number) FROM numbers(5);"
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```sql
|
||||
SELECT sum(number) FROM numbers(5)
|
||||
```
|
||||
|
||||
3. Multiqueries:
|
||||
|
||||
```bash
|
||||
$ clickhouse-format -n <<< "SELECT * FROM (SELECT 1 AS x UNION ALL SELECT 1 UNION DISTINCT SELECT 3);"
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
SELECT *
|
||||
FROM
|
||||
(
|
||||
SELECT 1 AS x
|
||||
UNION ALL
|
||||
SELECT 1
|
||||
UNION DISTINCT
|
||||
SELECT 3
|
||||
)
|
||||
;
|
||||
```
|
||||
|
||||
4. Obfuscating:
|
||||
|
||||
```bash
|
||||
$ clickhouse-format --seed Hello --obfuscate <<< "SELECT cost_first_screen BETWEEN a AND b, CASE WHEN x >= 123 THEN y ELSE NULL END;"
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
SELECT treasury_mammoth_hazelnut BETWEEN nutmeg AND span, CASE WHEN chive >= 116 THEN switching ELSE ANYTHING END;
|
||||
```
|
||||
|
||||
Same query and another seed string:
|
||||
|
||||
```bash
|
||||
$ clickhouse-format --seed World --obfuscate <<< "SELECT cost_first_screen BETWEEN a AND b, CASE WHEN x >= 123 THEN y ELSE NULL END;"
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
SELECT horse_tape_summer BETWEEN folklore AND moccasins, CASE WHEN intestine >= 116 THEN nonconformist ELSE FORESTRY END;
|
||||
```
|
||||
|
||||
5. Adding backslash:
|
||||
|
||||
```bash
|
||||
$ clickhouse-format --backslash <<< "SELECT * FROM (SELECT 1 AS x UNION ALL SELECT 1 UNION DISTINCT SELECT 3);"
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
SELECT * \
|
||||
FROM \
|
||||
( \
|
||||
SELECT 1 AS x \
|
||||
UNION ALL \
|
||||
SELECT 1 \
|
||||
UNION DISTINCT \
|
||||
SELECT 3 \
|
||||
)
|
||||
```
|
||||
|
||||
@ -0,0 +1,148 @@
|
||||
---
|
||||
toc_priority: 108
|
||||
---
|
||||
|
||||
## exponentialMovingAverage {#exponential-moving-average}
|
||||
|
||||
Сalculates the exponential moving average of values for the determined time.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
exponentialMovingAverage(x)(value, timestamp)
|
||||
```
|
||||
|
||||
Each `value` corresponds to the determinate `timestamp`. The half-life `x` is the time lag at which the exponential weights decay by one-half. The function returns a weighted average: the older the time point, the less weight the corresponding value is considered to be.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` — Value. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md).
|
||||
- `timestamp` — Timestamp. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `x` — Half-life period. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned values**
|
||||
|
||||
- Returnes an [exponentially smoothed moving average](https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average) of the values for the past `x` time at the latest point of time.
|
||||
|
||||
Type: [Float64](../../../sql-reference/data-types/float.md#float32-float64).
|
||||
|
||||
**Examples**
|
||||
|
||||
Input table:
|
||||
|
||||
``` text
|
||||
┌──temperature─┬─timestamp──┐
|
||||
│ 95 │ 1 │
|
||||
│ 95 │ 2 │
|
||||
│ 95 │ 3 │
|
||||
│ 96 │ 4 │
|
||||
│ 96 │ 5 │
|
||||
│ 96 │ 6 │
|
||||
│ 96 │ 7 │
|
||||
│ 97 │ 8 │
|
||||
│ 97 │ 9 │
|
||||
│ 97 │ 10 │
|
||||
│ 97 │ 11 │
|
||||
│ 98 │ 12 │
|
||||
│ 98 │ 13 │
|
||||
│ 98 │ 14 │
|
||||
│ 98 │ 15 │
|
||||
│ 99 │ 16 │
|
||||
│ 99 │ 17 │
|
||||
│ 99 │ 18 │
|
||||
│ 100 │ 19 │
|
||||
│ 100 │ 20 │
|
||||
└──────────────┴────────────┘
|
||||
```
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT exponentialMovingAverage(5)(temperature, timestamp);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌──exponentialMovingAverage(5)(temperature, timestamp)──┐
|
||||
│ 92.25779635374204 │
|
||||
└───────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
value,
|
||||
time,
|
||||
round(exp_smooth, 3),
|
||||
bar(exp_smooth, 0, 1, 50) AS bar
|
||||
FROM
|
||||
(
|
||||
SELECT
|
||||
(number = 0) OR (number >= 25) AS value,
|
||||
number AS time,
|
||||
exponentialMovingAverage(10)(value, time) OVER (Rows BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS exp_smooth
|
||||
FROM numbers(50)
|
||||
)
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─value─┬─time─┬─round(exp_smooth, 3)─┬─bar────────────────────────────────────────┐
|
||||
│ 1 │ 0 │ 0.067 │ ███▎ │
|
||||
│ 0 │ 1 │ 0.062 │ ███ │
|
||||
│ 0 │ 2 │ 0.058 │ ██▊ │
|
||||
│ 0 │ 3 │ 0.054 │ ██▋ │
|
||||
│ 0 │ 4 │ 0.051 │ ██▌ │
|
||||
│ 0 │ 5 │ 0.047 │ ██▎ │
|
||||
│ 0 │ 6 │ 0.044 │ ██▏ │
|
||||
│ 0 │ 7 │ 0.041 │ ██ │
|
||||
│ 0 │ 8 │ 0.038 │ █▊ │
|
||||
│ 0 │ 9 │ 0.036 │ █▋ │
|
||||
│ 0 │ 10 │ 0.033 │ █▋ │
|
||||
│ 0 │ 11 │ 0.031 │ █▌ │
|
||||
│ 0 │ 12 │ 0.029 │ █▍ │
|
||||
│ 0 │ 13 │ 0.027 │ █▎ │
|
||||
│ 0 │ 14 │ 0.025 │ █▎ │
|
||||
│ 0 │ 15 │ 0.024 │ █▏ │
|
||||
│ 0 │ 16 │ 0.022 │ █ │
|
||||
│ 0 │ 17 │ 0.021 │ █ │
|
||||
│ 0 │ 18 │ 0.019 │ ▊ │
|
||||
│ 0 │ 19 │ 0.018 │ ▊ │
|
||||
│ 0 │ 20 │ 0.017 │ ▋ │
|
||||
│ 0 │ 21 │ 0.016 │ ▋ │
|
||||
│ 0 │ 22 │ 0.015 │ ▋ │
|
||||
│ 0 │ 23 │ 0.014 │ ▋ │
|
||||
│ 0 │ 24 │ 0.013 │ ▋ │
|
||||
│ 1 │ 25 │ 0.079 │ ███▊ │
|
||||
│ 1 │ 26 │ 0.14 │ ███████ │
|
||||
│ 1 │ 27 │ 0.198 │ █████████▊ │
|
||||
│ 1 │ 28 │ 0.252 │ ████████████▌ │
|
||||
│ 1 │ 29 │ 0.302 │ ███████████████ │
|
||||
│ 1 │ 30 │ 0.349 │ █████████████████▍ │
|
||||
│ 1 │ 31 │ 0.392 │ ███████████████████▌ │
|
||||
│ 1 │ 32 │ 0.433 │ █████████████████████▋ │
|
||||
│ 1 │ 33 │ 0.471 │ ███████████████████████▌ │
|
||||
│ 1 │ 34 │ 0.506 │ █████████████████████████▎ │
|
||||
│ 1 │ 35 │ 0.539 │ ██████████████████████████▊ │
|
||||
│ 1 │ 36 │ 0.57 │ ████████████████████████████▌ │
|
||||
│ 1 │ 37 │ 0.599 │ █████████████████████████████▊ │
|
||||
│ 1 │ 38 │ 0.626 │ ███████████████████████████████▎ │
|
||||
│ 1 │ 39 │ 0.651 │ ████████████████████████████████▌ │
|
||||
│ 1 │ 40 │ 0.674 │ █████████████████████████████████▋ │
|
||||
│ 1 │ 41 │ 0.696 │ ██████████████████████████████████▋ │
|
||||
│ 1 │ 42 │ 0.716 │ ███████████████████████████████████▋ │
|
||||
│ 1 │ 43 │ 0.735 │ ████████████████████████████████████▋ │
|
||||
│ 1 │ 44 │ 0.753 │ █████████████████████████████████████▋ │
|
||||
│ 1 │ 45 │ 0.77 │ ██████████████████████████████████████▍ │
|
||||
│ 1 │ 46 │ 0.785 │ ███████████████████████████████████████▎ │
|
||||
│ 1 │ 47 │ 0.8 │ ███████████████████████████████████████▊ │
|
||||
│ 1 │ 48 │ 0.813 │ ████████████████████████████████████████▋ │
|
||||
│ 1 │ 49 │ 0.825 │ █████████████████████████████████████████▎│
|
||||
└───────┴──────┴──────────────────────┴────────────────────────────────────────────┘
|
||||
```
|
||||
@ -3,13 +3,12 @@ toc_priority: 46
|
||||
toc_title: Polygon Dictionaries With Grids
|
||||
---
|
||||
|
||||
|
||||
# Polygon dictionaries {#polygon-dictionaries}
|
||||
|
||||
Polygon dictionaries allow you to efficiently search for the polygon containing specified points.
|
||||
For example: defining a city area by geographical coordinates.
|
||||
|
||||
Example configuration:
|
||||
Example of a polygon dictionary configuration:
|
||||
|
||||
``` xml
|
||||
<dictionary>
|
||||
@ -30,13 +29,15 @@ Example configuration:
|
||||
<type>UInt64</type>
|
||||
<null_value>0</null_value>
|
||||
</attribute>
|
||||
|
||||
</structure>
|
||||
|
||||
<layout>
|
||||
<polygon />
|
||||
<polygon>
|
||||
<store_polygon_key_column>1</store_polygon_key_column>
|
||||
</polygon>
|
||||
</layout>
|
||||
|
||||
...
|
||||
</dictionary>
|
||||
```
|
||||
|
||||
@ -48,11 +49,12 @@ CREATE DICTIONARY polygon_dict_name (
|
||||
value UInt64
|
||||
)
|
||||
PRIMARY KEY key
|
||||
LAYOUT(POLYGON())
|
||||
LAYOUT(POLYGON(STORE_POLYGON_KEY_COLUMN 1))
|
||||
...
|
||||
```
|
||||
|
||||
When configuring the polygon dictionary, the key must have one of two types:
|
||||
|
||||
- A simple polygon. It is an array of points.
|
||||
- MultiPolygon. It is an array of polygons. Each polygon is a two-dimensional array of points. The first element of this array is the outer boundary of the polygon, and subsequent elements specify areas to be excluded from it.
|
||||
|
||||
@ -60,25 +62,27 @@ Points can be specified as an array or a tuple of their coordinates. In the curr
|
||||
|
||||
The user can [upload their own data](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md) in all formats supported by ClickHouse.
|
||||
|
||||
|
||||
There are 3 types of [in-memory storage](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md) available:
|
||||
|
||||
- POLYGON_SIMPLE. This is a naive implementation, where a linear pass through all polygons is made for each query, and membership is checked for each one without using additional indexes.
|
||||
- `POLYGON_SIMPLE`. This is a naive implementation, where a linear pass through all polygons is made for each query, and membership is checked for each one without using additional indexes.
|
||||
|
||||
- POLYGON_INDEX_EACH. A separate index is built for each polygon, which allows you to quickly check whether it belongs in most cases (optimized for geographical regions).
|
||||
- `POLYGON_INDEX_EACH`. A separate index is built for each polygon, which allows you to quickly check whether it belongs in most cases (optimized for geographical regions).
|
||||
Also, a grid is superimposed on the area under consideration, which significantly narrows the number of polygons under consideration.
|
||||
The grid is created by recursively dividing the cell into 16 equal parts and is configured with two parameters.
|
||||
The division stops when the recursion depth reaches MAX_DEPTH or when the cell crosses no more than MIN_INTERSECTIONS polygons.
|
||||
The division stops when the recursion depth reaches `MAX_DEPTH` or when the cell crosses no more than `MIN_INTERSECTIONS` polygons.
|
||||
To respond to the query, there is a corresponding cell, and the index for the polygons stored in it is accessed alternately.
|
||||
|
||||
- POLYGON_INDEX_CELL. This placement also creates the grid described above. The same options are available. For each sheet cell, an index is built on all pieces of polygons that fall into it, which allows you to quickly respond to a request.
|
||||
- `POLYGON_INDEX_CELL`. This placement also creates the grid described above. The same options are available. For each sheet cell, an index is built on all pieces of polygons that fall into it, which allows you to quickly respond to a request.
|
||||
|
||||
- POLYGON. Synonym to POLYGON_INDEX_CELL.
|
||||
- `POLYGON`. Synonym to `POLYGON_INDEX_CELL`.
|
||||
|
||||
Dictionary queries are carried out using standard [functions](../../../sql-reference/functions/ext-dict-functions.md) for working with external dictionaries.
|
||||
An important difference is that here the keys will be the points for which you want to find the polygon containing them.
|
||||
|
||||
**Example**
|
||||
|
||||
Example of working with the dictionary defined above:
|
||||
|
||||
``` sql
|
||||
CREATE TABLE points (
|
||||
x Float64,
|
||||
@ -89,3 +93,39 @@ SELECT tuple(x, y) AS key, dictGet(dict_name, 'name', key), dictGet(dict_name, '
|
||||
```
|
||||
|
||||
As a result of executing the last command for each point in the 'points' table, a minimum area polygon containing this point will be found, and the requested attributes will be output.
|
||||
|
||||
**Example**
|
||||
|
||||
You can read columns from polygon dictionaries via SELECT query, just turn on the `store_polygon_key_column = 1` in the dictionary configuration or corresponding DDL-query.
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
CREATE TABLE polygons_test_table
|
||||
(
|
||||
key Array(Array(Array(Tuple(Float64, Float64)))),
|
||||
name String
|
||||
) ENGINE = TinyLog;
|
||||
|
||||
INSERT INTO polygons_test_table VALUES ([[[(3, 1), (0, 1), (0, -1), (3, -1)]]], 'Value');
|
||||
|
||||
CREATE DICTIONARY polygons_test_dictionary
|
||||
(
|
||||
key Array(Array(Array(Tuple(Float64, Float64)))),
|
||||
name String
|
||||
)
|
||||
PRIMARY KEY key
|
||||
SOURCE(CLICKHOUSE(TABLE 'polygons_test_table'))
|
||||
LAYOUT(POLYGON(STORE_POLYGON_KEY_COLUMN 1))
|
||||
LIFETIME(0);
|
||||
|
||||
SELECT * FROM polygons_test_dictionary;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─key─────────────────────────────┬─name──┐
|
||||
│ [[[(3,1),(0,1),(0,-1),(3,-1)]]] │ Value │
|
||||
└─────────────────────────────────┴───────┘
|
||||
```
|
||||
|
||||
@ -89,9 +89,39 @@ SELECT sipHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00
|
||||
|
||||
## sipHash128 {#hash_functions-siphash128}
|
||||
|
||||
Calculates SipHash from a string.
|
||||
Accepts a String-type argument. Returns FixedString(16).
|
||||
Differs from sipHash64 in that the final xor-folding state is only done up to 128 bits.
|
||||
Produces a 128-bit [SipHash](https://131002.net/siphash/) hash value. Differs from [sipHash64](#hash_functions-siphash64) in that the final xor-folding state is done up to 128 bits.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
sipHash128(par1,...)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
The function takes a variable number of input parameters. Arguments can be any of the [supported data types](../../sql-reference/data-types/index.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
A 128-bit `SipHash` hash value.
|
||||
|
||||
Type: [FixedString(16)](../../sql-reference/data-types/fixedstring.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT hex(sipHash128('foo', '\x01', 3));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─hex(sipHash128('foo', '', 3))────┐
|
||||
│ 9DE516A64A414D4B1B609415E4523F24 │
|
||||
└──────────────────────────────────┘
|
||||
```
|
||||
|
||||
## cityHash64 {#cityhash64}
|
||||
|
||||
@ -459,28 +489,36 @@ SELECT murmurHash3_32(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:
|
||||
|
||||
Produces a 128-bit [MurmurHash3](https://github.com/aappleby/smhasher) hash value.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
murmurHash3_128( expr )
|
||||
murmurHash3_128(expr)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `expr` — [Expressions](../../sql-reference/syntax.md#syntax-expressions) returning a [String](../../sql-reference/data-types/string.md)-type value.
|
||||
- `expr` — A list of [expressions](../../sql-reference/syntax.md#syntax-expressions). [String](../../sql-reference/data-types/string.md).
|
||||
|
||||
**Returned Value**
|
||||
**Returned value**
|
||||
|
||||
A [FixedString(16)](../../sql-reference/data-types/fixedstring.md) data type hash value.
|
||||
A 128-bit `MurmurHash3` hash value.
|
||||
|
||||
Type: [FixedString(16)](../../sql-reference/data-types/fixedstring.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT hex(murmurHash3_128('example_string')) AS MurmurHash3, toTypeName(MurmurHash3) AS type;
|
||||
SELECT hex(murmurHash3_128('foo', 'foo', 'foo'));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─MurmurHash3──────────────────────┬─type───┐
|
||||
│ 368A1A311CB7342253354B548E7E7E71 │ String │
|
||||
└──────────────────────────────────┴────────┘
|
||||
┌─hex(murmurHash3_128('foo', 'foo', 'foo'))─┐
|
||||
│ F8F7AD9B6CD4CF117A71E277E2EC2931 │
|
||||
└───────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## xxHash32, xxHash64 {#hash-functions-xxhash32}
|
||||
|
||||
@ -290,7 +290,7 @@ ngrams(string, ngramsize)
|
||||
|
||||
- Array with n-grams.
|
||||
|
||||
Type: [Array](../../sql-reference/data-types/array.md)([FixedString](../../sql-reference/data-types/fixedstring.md)).
|
||||
Type: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)).
|
||||
|
||||
**Example**
|
||||
|
||||
|
||||
@ -10,7 +10,7 @@ A set of queries that allow changing the table structure.
|
||||
Syntax:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE [db].name [ON CLUSTER cluster] ADD|DROP|RENAME|CLEAR|COMMENT|MODIFY|MATERIALIZE COLUMN ...
|
||||
ALTER TABLE [db].name [ON CLUSTER cluster] ADD|DROP|RENAME|CLEAR|COMMENT|{MODIFY|ALTER}|MATERIALIZE COLUMN ...
|
||||
```
|
||||
|
||||
In the query, specify a list of one or more comma-separated actions.
|
||||
@ -138,6 +138,7 @@ ALTER TABLE visits COMMENT COLUMN browser 'The table shows the browser used for
|
||||
|
||||
``` sql
|
||||
MODIFY COLUMN [IF EXISTS] name [type] [default_expr] [codec] [TTL] [AFTER name_after | FIRST]
|
||||
ALTER COLUMN [IF EXISTS] name TYPE [type] [default_expr] [codec] [TTL] [AFTER name_after | FIRST]
|
||||
```
|
||||
|
||||
This query changes the `name` column properties:
|
||||
|
||||
@ -5,9 +5,9 @@ toc_title: INSERT INTO
|
||||
|
||||
## INSERT INTO Statement {#insert}
|
||||
|
||||
Adding data.
|
||||
Inserts data into a table.
|
||||
|
||||
Basic query format:
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
INSERT INTO [db.]table [(c1, c2, c3)] VALUES (v11, v12, v13), (v21, v22, v23), ...
|
||||
@ -92,6 +92,8 @@ If table has [constraints](../../sql-reference/statements/create/table.md#constr
|
||||
|
||||
### Inserting the Results of `SELECT` {#insert_query_insert-select}
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
INSERT INTO [db.]table [(c1, c2, c3)] SELECT ...
|
||||
```
|
||||
@ -107,6 +109,68 @@ However, you can delete old data using `ALTER TABLE ... DROP PARTITION`.
|
||||
|
||||
To insert a default value instead of `NULL` into a column with not nullable data type, enable [insert_null_as_default](../../operations/settings/settings.md#insert_null_as_default) setting.
|
||||
|
||||
### Inserting Data from a File {#inserting-data-from-a-file}
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
INSERT INTO [db.]table [(c1, c2, c3)] FROM INFILE file_name [COMPRESSION type] FORMAT format_name
|
||||
```
|
||||
|
||||
Use the syntax above to insert data from a file stored on a **client** side. `file_name` and `type` are string literals. Input file [format](../../interfaces/formats.md) must be set in the `FORMAT` clause.
|
||||
|
||||
Compressed files are supported. Compression type is detected by the extension of the file name. Or it can be explicitly specified in a `COMPRESSION` clause. Supported types are: `'none'`, `'gzip'`, `'deflate'`, `'br'`, `'xz'`, `'zstd'`, `'lz4'`, `'bz2'`.
|
||||
|
||||
This functionality is available in the [command-line client](../../interfaces/cli.md) and [clickhouse-local](../../operations/utilities/clickhouse-local.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Execute the following queries using [command-line client](../../interfaces/cli.md):
|
||||
|
||||
```bash
|
||||
echo 1,A > input.csv ; echo 2,B >> input.csv
|
||||
clickhouse-client --query="CREATE TABLE table_from_file (id UInt32, text String) ENGINE=MergeTree() ORDER BY id;"
|
||||
clickhouse-client --query="INSERT INTO table_from_file FROM INFILE 'input.csv' FORMAT CSV;"
|
||||
clickhouse-client --query="SELECT * FROM table_from_file FORMAT PrettyCompact;"
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─id─┬─text─┐
|
||||
│ 1 │ A │
|
||||
│ 2 │ B │
|
||||
└────┴──────┘
|
||||
```
|
||||
|
||||
### Inserting into Table Function {#inserting-into-table-function}
|
||||
|
||||
Data can be inserted into tables referenced by [table functions](../../sql-reference/table-functions/index.md).
|
||||
|
||||
**Syntax**
|
||||
``` sql
|
||||
INSERT INTO [TABLE] FUNCTION table_func ...
|
||||
```
|
||||
|
||||
**Example**
|
||||
|
||||
[remote](../../sql-reference/table-functions/index.md#remote) table function is used in the following queries:
|
||||
|
||||
``` sql
|
||||
CREATE TABLE simple_table (id UInt32, text String) ENGINE=MergeTree() ORDER BY id;
|
||||
INSERT INTO TABLE FUNCTION remote('localhost', default.simple_table)
|
||||
VALUES (100, 'inserted via remote()');
|
||||
SELECT * FROM simple_table;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌──id─┬─text──────────────────┐
|
||||
│ 100 │ inserted via remote() │
|
||||
└─────┴───────────────────────┘
|
||||
```
|
||||
|
||||
### Performance Considerations {#performance-considerations}
|
||||
|
||||
`INSERT` sorts the input data by primary key and splits them into partitions by a partition key. If you insert data into several partitions at once, it can significantly reduce the performance of the `INSERT` query. To avoid this:
|
||||
|
||||
@ -27,7 +27,7 @@ SELECT [DISTINCT [ON (column1, column2, ...)]] expr_list
|
||||
[LIMIT [n, ]m] [WITH TIES]
|
||||
[SETTINGS ...]
|
||||
[UNION ...]
|
||||
[INTO OUTFILE filename]
|
||||
[INTO OUTFILE filename [COMPRESSION type] ]
|
||||
[FORMAT format]
|
||||
```
|
||||
|
||||
|
||||
@ -4,10 +4,35 @@ toc_title: INTO OUTFILE
|
||||
|
||||
# INTO OUTFILE Clause {#into-outfile-clause}
|
||||
|
||||
Add the `INTO OUTFILE filename` clause (where filename is a string literal) to `SELECT query` to redirect its output to the specified file on the client-side.
|
||||
`INTO OUTFILE` clause redirects the result of a `SELECT` query to a file on the **client** side.
|
||||
|
||||
Compressed files are supported. Compression type is detected by the extension of the file name (mode `'auto'` is used by default). Or it can be explicitly specified in a `COMPRESSION` clause.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
SELECT <expr_list> INTO OUTFILE file_name [COMPRESSION type]
|
||||
```
|
||||
|
||||
`file_name` and `type` are string literals. Supported compression types are: `'none'`, `'gzip'`, `'deflate'`, `'br'`, `'xz'`, `'zstd'`, `'lz4'`, `'bz2'`.
|
||||
|
||||
## Implementation Details {#implementation-details}
|
||||
|
||||
- This functionality is available in the [command-line client](../../../interfaces/cli.md) and [clickhouse-local](../../../operations/utilities/clickhouse-local.md). Thus a query sent via [HTTP interface](../../../interfaces/http.md) will fail.
|
||||
- The query will fail if a file with the same filename already exists.
|
||||
- The default [output format](../../../interfaces/formats.md) is `TabSeparated` (like in the command-line client batch mode).
|
||||
- The query will fail if a file with the same file name already exists.
|
||||
- The default [output format](../../../interfaces/formats.md) is `TabSeparated` (like in the command-line client batch mode). Use [FORMAT](format.md) clause to change it.
|
||||
|
||||
**Example**
|
||||
|
||||
Execute the following query using [command-line client](../../../interfaces/cli.md):
|
||||
|
||||
```bash
|
||||
clickhouse-client --query="SELECT 1,'ABC' INTO OUTFILE 'select.gz' FORMAT CSV;"
|
||||
zcat select.gz
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
1,"ABC"
|
||||
```
|
||||
|
||||
@ -18,6 +18,8 @@ You can use table functions in:
|
||||
|
||||
It's one of the methods of creating a table.
|
||||
|
||||
- [INSERT INTO TABLE FUNCTION](../../sql-reference/statements/insert-into.md#inserting-into-table-function) query.
|
||||
|
||||
!!! warning "Warning"
|
||||
You can’t use table functions if the [allow_ddl](../../operations/settings/permissions-for-queries.md#settings_allow_ddl) setting is disabled.
|
||||
|
||||
|
||||
@ -15,7 +15,13 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
|
||||
name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [TTL expr1],
|
||||
name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2] [TTL expr2],
|
||||
...
|
||||
) ENGINE = MySQL('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_duplicate_clause']);
|
||||
) ENGINE = MySQL('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_duplicate_clause'])
|
||||
SETTINGS
|
||||
[connection_pool_size=16, ]
|
||||
[connection_max_tries=3, ]
|
||||
[connection_wait_timeout=5, ] /* 0 -- не ждать */
|
||||
[connection_auto_close=true ]
|
||||
;
|
||||
```
|
||||
|
||||
Смотрите подробное описание запроса [CREATE TABLE](../../../sql-reference/statements/create/table.md#create-table-query).
|
||||
@ -102,8 +108,43 @@ SELECT * FROM mysql_table
|
||||
└────────────────┴────────┘
|
||||
```
|
||||
|
||||
## Смотрите также {#smotrite-takzhe}
|
||||
## Настройки {#mysql-settings}
|
||||
|
||||
- [Табличная функция ‘mysql’](../../../engines/table-engines/integrations/mysql.md)
|
||||
Настройки по умолчанию не очень эффективны, так как они не используют повторное соединение. Эти настройки позволяют увеличить количество запросов, выполняемых сервером в секунду.
|
||||
|
||||
### connection_auto_close {#connection-auto-close}
|
||||
|
||||
Позволяет автоматически закрыть соединение после выполнения запроса, то есть отключить повторное использование соединения.
|
||||
|
||||
Возможные значения:
|
||||
|
||||
- 1 — автоматическое закрытие соединения разрешено (повторное использование отключается).
|
||||
- 0 — автоматическое закрытие соединения запрещено (повторное использование включается).
|
||||
|
||||
Значение по умолчанию: `1`.
|
||||
|
||||
### connection_max_tries {#connection-max-tries}
|
||||
|
||||
Устанавливает количество повторных попыток для пула со сбоями соединения.
|
||||
|
||||
Возможные значения:
|
||||
|
||||
- Положительное целое число.
|
||||
- 0 — отсутствуют повторные попытки для пула со сбоями соединения.
|
||||
|
||||
Значение по умолчанию: `3`.
|
||||
|
||||
### connection_pool_size {#connection-pool-size}
|
||||
|
||||
Задает размер пула соединений (если используются все соединения, запрос будет ждать, пока какое-либо соединение не будет освобождено).
|
||||
|
||||
Возможные значения:
|
||||
|
||||
- Положительное целое число.
|
||||
|
||||
Значение по умолчанию: `16`.
|
||||
|
||||
## См. также {#see-also}
|
||||
|
||||
- [Табличная функция mysql](../../../engines/table-engines/integrations/mysql.md)
|
||||
- [Использование MySQL в качестве источника для внешнего словаря](../../../engines/table-engines/integrations/mysql.md#dicts-external_dicts_dict_sources-mysql)
|
||||
|
||||
|
||||
@ -20,6 +20,8 @@ ClickHouse может принимать (`INSERT`) и отдавать (`SELECT
|
||||
| [CSV](#csv) | ✔ | ✔ |
|
||||
| [CSVWithNames](#csvwithnames) | ✔ | ✔ |
|
||||
| [CustomSeparated](#format-customseparated) | ✔ | ✔ |
|
||||
| [CustomSeparatedWithNames](#customseparatedwithnames) | ✔ | ✔ |
|
||||
| [CustomSeparatedWithNamesAndTypes](#customseparatedwithnamesandtypes) | ✔ | ✔ |
|
||||
| [Values](#data-format-values) | ✔ | ✔ |
|
||||
| [Vertical](#vertical) | ✗ | ✔ |
|
||||
| [JSON](#json) | ✗ | ✔ |
|
||||
@ -368,8 +370,17 @@ $ clickhouse-client --format_csv_delimiter="|" --query="INSERT INTO test.csv FOR
|
||||
|
||||
## CustomSeparated {#format-customseparated}
|
||||
|
||||
Аналогичен [Template](#format-template), но выводит (или считывает) все столбцы, используя для них правило экранирования из настройки `format_custom_escaping_rule` и разделители из настроек `format_custom_field_delimiter`, `format_custom_row_before_delimiter`, `format_custom_row_after_delimiter`, `format_custom_row_between_delimiter`, `format_custom_result_before_delimiter` и `format_custom_result_after_delimiter`, а не из форматных строк.
|
||||
Также существует формат `CustomSeparatedIgnoreSpaces`, аналогичный `TemplateIgnoreSpaces`.
|
||||
Аналогичен [Template](#format-template), но выводит (или считывает) все имена и типы столбцов, используя для них правило экранирования из настройки [format_custom_escaping_rule](../operations/settings/settings.md#format-custom-escaping-rule) и разделители из настроек [format_custom_field_delimiter](../operations/settings/settings.md#format-custom-field-delimiter), [format_custom_row_before_delimiter](../operations/settings/settings.md#format-custom-row-before-delimiter), [format_custom_row_after_delimiter](../operations/settings/settings.md#format-custom-row-after-delimiter), [format_custom_row_between_delimiter](../operations/settings/settings.md#format-custom-row-between-delimiter), [format_custom_result_before_delimiter](../operations/settings/settings.md#format-custom-result-before-delimiter) и [format_custom_result_after_delimiter](../operations/settings/settings.md#format-custom-result-after-delimiter), а не из форматных строк.
|
||||
|
||||
Также существует формат `CustomSeparatedIgnoreSpaces`, аналогичный формату [TemplateIgnoreSpaces](#templateignorespaces).
|
||||
|
||||
## CustomSeparatedWithNames {#customseparatedwithnames}
|
||||
|
||||
Выводит также заголовок с именами столбцов, аналогичен формату [TabSeparatedWithNames](#tabseparatedwithnames).
|
||||
|
||||
## CustomSeparatedWithNamesAndTypes {#customseparatedwithnamesandtypes}
|
||||
|
||||
Выводит также два заголовка с именами и типами столбцов, аналогичен формату [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes).
|
||||
|
||||
## JSON {#json}
|
||||
|
||||
@ -1399,14 +1410,17 @@ SELECT * FROM line_as_string;
|
||||
|
||||
При работе с форматом `Regexp` можно использовать следующие параметры:
|
||||
|
||||
- `format_regexp` — [String](../sql-reference/data-types/string.md). Строка с регулярным выражением в формате [re2](https://github.com/google/re2/wiki/Syntax).
|
||||
- `format_regexp_escaping_rule` — [String](../sql-reference/data-types/string.md). Правило сериализации. Поддерживаются следующие правила:
|
||||
- CSV (как в [CSV](#csv))
|
||||
- JSON (как в [JSONEachRow](#jsoneachrow))
|
||||
- Escaped (как в [TSV](#tabseparated))
|
||||
- Quoted (как в [Values](#data-format-values))
|
||||
- Raw (данные импортируются как есть, без сериализации)
|
||||
- `format_regexp_skip_unmatched` — [UInt8](../sql-reference/data-types/int-uint.md). Признак, будет ли генерироваться исключение в случае, если импортируемые данные не соответствуют регулярному выражению `format_regexp`. Может принимать значение `0` или `1`.
|
||||
- `format_regexp` — [String](../sql-reference/data-types/string.md). Строка с регулярным выражением в формате [re2](https://github.com/google/re2/wiki/Syntax).
|
||||
|
||||
- `format_regexp_escaping_rule` — [String](../sql-reference/data-types/string.md). Правило экранирования. Поддерживаются следующие правила:
|
||||
|
||||
- CSV (как в формате [CSV](#csv))
|
||||
- JSON (как в формате [JSONEachRow](#jsoneachrow))
|
||||
- Escaped (как в формате [TSV](#tabseparated))
|
||||
- Quoted (как в формате [Values](#data-format-values))
|
||||
- Raw (данные импортируются как есть, без экранирования, как в формате [TSVRaw](#tabseparatedraw))
|
||||
|
||||
- `format_regexp_skip_unmatched` — [UInt8](../sql-reference/data-types/int-uint.md). Признак, будет ли генерироваться исключение в случае, если импортируемые данные не соответствуют регулярному выражению `format_regexp`. Может принимать значение `0` или `1`.
|
||||
|
||||
**Использование**
|
||||
|
||||
|
||||
@ -131,7 +131,7 @@ ClickHouse проверяет условия для `min_part_size` и `min_part
|
||||
```xml
|
||||
<encryption_codecs>
|
||||
<aes_128_gcm_siv>
|
||||
<nonce>0123456789101</nonce>
|
||||
<nonce>012345678910</nonce>
|
||||
</aes_128_gcm_siv>
|
||||
</encryption_codecs>
|
||||
```
|
||||
|
||||
@ -3830,3 +3830,54 @@ SELECT * FROM positional_arguments ORDER BY 2,3;
|
||||
- 0 — большие файлы считываются только с копированием данных из ядра в пространство пользователей.
|
||||
|
||||
Значение по умолчанию: `0`.
|
||||
|
||||
## format_custom_escaping_rule {#format-custom-escaping-rule}
|
||||
|
||||
Устанавливает правило экранирования данных формата [CustomSeparated](../../interfaces/formats.md#format-customseparated).
|
||||
|
||||
Возможные значения:
|
||||
|
||||
- `'Escaped'` — как в формате [TSV](../../interfaces/formats.md#tabseparated).
|
||||
- `'Quoted'` — как в формате [Values](../../interfaces/formats.md#data-format-values).
|
||||
- `'CSV'` — как в формате [CSV](../../interfaces/formats.md#csv).
|
||||
- `'JSON'` — как в формате [JSONEachRow](../../interfaces/formats.md#jsoneachrow).
|
||||
- `'XML'` — как в формате [XML](../../interfaces/formats.md#xml).
|
||||
- `'Raw'` — данные импортируются как есть, без экранирования, как в формате [TSVRaw](../../interfaces/formats.md#tabseparatedraw).
|
||||
|
||||
Значение по умолчанию: `'Escaped'`.
|
||||
|
||||
## format_custom_field_delimiter {#format-custom-field-delimiter}
|
||||
|
||||
Задает символ, который интерпретируется как разделитель между полями данных формата [CustomSeparated](../../interfaces/formats.md#format-customseparated).
|
||||
|
||||
Значение по умолчанию: `'\t'`.
|
||||
|
||||
## format_custom_row_before_delimiter {#format-custom-row-before-delimiter}
|
||||
|
||||
Задает символ, который интерпретируется как разделитель перед полем первого столбца данных формата [CustomSeparated](../../interfaces/formats.md#format-customseparated).
|
||||
|
||||
Значение по умолчанию: `''`.
|
||||
|
||||
## format_custom_row_after_delimiter {#format-custom-row-after-delimiter}
|
||||
|
||||
Задает символ, который интерпретируется как разделитель после поля последнего столбца данных формата [CustomSeparated](../../interfaces/formats.md#format-customseparated).
|
||||
|
||||
Значение по умолчанию: `'\n'`.
|
||||
|
||||
## format_custom_row_between_delimiter {#format-custom-row-between-delimiter}
|
||||
|
||||
Задает символ, который интерпретируется как разделитель между строками данных формата [CustomSeparated](../../interfaces/formats.md#format-customseparated).
|
||||
|
||||
Значение по умолчанию: `''`.
|
||||
|
||||
## format_custom_result_before_delimiter {#format-custom-result-before-delimiter}
|
||||
|
||||
Задает символ, который интерпретируется как префикс перед результирующим набором данных формата [CustomSeparated](../../interfaces/formats.md#format-customseparated).
|
||||
|
||||
Значение по умолчанию: `''`.
|
||||
|
||||
## format_custom_result_after_delimiter {#format-custom-result-after-delimiter}
|
||||
|
||||
Задает символ, который интерпретируется как суффикс после результирующего набора данных формата [CustomSeparated](../../interfaces/formats.md#format-customseparated).
|
||||
|
||||
Значение по умолчанию: `''`.
|
||||
|
||||
@ -1,98 +1,114 @@
|
||||
---
|
||||
toc_priority: 65
|
||||
toc_title: clickhouse-format
|
||||
---
|
||||
|
||||
# clickhouse-format {#clickhouse-format}
|
||||
|
||||
Позволяет форматировать входящие запросы.
|
||||
|
||||
Ключи:
|
||||
|
||||
- `--help` или`-h` — выводит описание ключей.
|
||||
- `--hilite` — добавляет подсветку синтаксиса с экранированием символов.
|
||||
- `--oneline` — форматирование в одну строку.
|
||||
- `--quiet` или `-q` — проверяет синтаксис без вывода результата.
|
||||
- `--multiquery` or `-n` — поддерживает несколько запросов в одной строке.
|
||||
- `--obfuscate` — обфусцирует вместо форматирования.
|
||||
- `--seed <строка>` — задает строку, которая определяет результат обфускации.
|
||||
- `--backslash` — добавляет обратный слеш в конце каждой строки отформатированного запроса. Удобно использовать если многострочный запрос скопирован из интернета или другого источника и его нужно выполнить из командной строки.
|
||||
|
||||
## Примеры {#examples}
|
||||
|
||||
1. Подсветка синтаксиса и форматирование в одну строку:
|
||||
|
||||
```bash
|
||||
$ clickhouse-format --oneline --hilite <<< "SELECT sum(number) FROM numbers(5);"
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
```sql
|
||||
SELECT sum(number) FROM numbers(5)
|
||||
```
|
||||
|
||||
2. Несколько запросов в одной строке:
|
||||
|
||||
```bash
|
||||
$ clickhouse-format -n <<< "SELECT * FROM (SELECT 1 AS x UNION ALL SELECT 1 UNION DISTINCT SELECT 3);"
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
```text
|
||||
SELECT *
|
||||
FROM
|
||||
(
|
||||
SELECT 1 AS x
|
||||
UNION ALL
|
||||
SELECT 1
|
||||
UNION DISTINCT
|
||||
SELECT 3
|
||||
)
|
||||
;
|
||||
```
|
||||
|
||||
3. Обфускация:
|
||||
|
||||
```bash
|
||||
$ clickhouse-format --seed Hello --obfuscate <<< "SELECT cost_first_screen BETWEEN a AND b, CASE WHEN x >= 123 THEN y ELSE NULL END;"
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
```text
|
||||
SELECT treasury_mammoth_hazelnut BETWEEN nutmeg AND span, CASE WHEN chive >= 116 THEN switching ELSE ANYTHING END;
|
||||
```
|
||||
|
||||
Тот же запрос с другой инициализацией обфускатора:
|
||||
|
||||
```bash
|
||||
$ clickhouse-format --seed World --obfuscate <<< "SELECT cost_first_screen BETWEEN a AND b, CASE WHEN x >= 123 THEN y ELSE NULL END;"
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
```text
|
||||
SELECT horse_tape_summer BETWEEN folklore AND moccasins, CASE WHEN intestine >= 116 THEN nonconformist ELSE FORESTRY END;
|
||||
```
|
||||
|
||||
4. Добавление обратного слеша:
|
||||
|
||||
```bash
|
||||
$ clickhouse-format --backslash <<< "SELECT * FROM (SELECT 1 AS x UNION ALL SELECT 1 UNION DISTINCT SELECT 3);"
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
```text
|
||||
SELECT * \
|
||||
FROM \
|
||||
( \
|
||||
SELECT 1 AS x \
|
||||
UNION ALL \
|
||||
SELECT 1 \
|
||||
UNION DISTINCT \
|
||||
SELECT 3 \
|
||||
)
|
||||
```
|
||||
---
|
||||
toc_priority: 65
|
||||
toc_title: clickhouse-format
|
||||
---
|
||||
|
||||
# clickhouse-format {#clickhouse-format}
|
||||
|
||||
Позволяет форматировать входящие запросы.
|
||||
|
||||
Ключи:
|
||||
|
||||
- `--help` или`-h` — выводит описание ключей.
|
||||
- `--query` — форматирует запрос любой длины и сложности.
|
||||
- `--hilite` — добавляет подсветку синтаксиса с экранированием символов.
|
||||
- `--oneline` — форматирование в одну строку.
|
||||
- `--quiet` или `-q` — проверяет синтаксис без вывода результата.
|
||||
- `--multiquery` or `-n` — поддерживает несколько запросов в одной строке.
|
||||
- `--obfuscate` — обфусцирует вместо форматирования.
|
||||
- `--seed <строка>` — задает строку, которая определяет результат обфускации.
|
||||
- `--backslash` — добавляет обратный слеш в конце каждой строки отформатированного запроса. Удобно использовать если многострочный запрос скопирован из интернета или другого источника и его нужно выполнить из командной строки.
|
||||
|
||||
## Примеры {#examples}
|
||||
|
||||
1. Форматирование запроса:
|
||||
|
||||
```bash
|
||||
$ clickhouse-format --query "select number from numbers(10) where number%2 order by number desc;"
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
```text
|
||||
SELECT number
|
||||
FROM numbers(10)
|
||||
WHERE number % 2
|
||||
ORDER BY number DESC
|
||||
```
|
||||
|
||||
2. Подсветка синтаксиса и форматирование в одну строку:
|
||||
|
||||
```bash
|
||||
$ clickhouse-format --oneline --hilite <<< "SELECT sum(number) FROM numbers(5);"
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
```sql
|
||||
SELECT sum(number) FROM numbers(5)
|
||||
```
|
||||
|
||||
3. Несколько запросов в одной строке:
|
||||
|
||||
```bash
|
||||
$ clickhouse-format -n <<< "SELECT * FROM (SELECT 1 AS x UNION ALL SELECT 1 UNION DISTINCT SELECT 3);"
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
```text
|
||||
SELECT *
|
||||
FROM
|
||||
(
|
||||
SELECT 1 AS x
|
||||
UNION ALL
|
||||
SELECT 1
|
||||
UNION DISTINCT
|
||||
SELECT 3
|
||||
)
|
||||
;
|
||||
```
|
||||
|
||||
4. Обфускация:
|
||||
|
||||
```bash
|
||||
$ clickhouse-format --seed Hello --obfuscate <<< "SELECT cost_first_screen BETWEEN a AND b, CASE WHEN x >= 123 THEN y ELSE NULL END;"
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
```text
|
||||
SELECT treasury_mammoth_hazelnut BETWEEN nutmeg AND span, CASE WHEN chive >= 116 THEN switching ELSE ANYTHING END;
|
||||
```
|
||||
|
||||
Тот же запрос с другой инициализацией обфускатора:
|
||||
|
||||
```bash
|
||||
$ clickhouse-format --seed World --obfuscate <<< "SELECT cost_first_screen BETWEEN a AND b, CASE WHEN x >= 123 THEN y ELSE NULL END;"
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
```text
|
||||
SELECT horse_tape_summer BETWEEN folklore AND moccasins, CASE WHEN intestine >= 116 THEN nonconformist ELSE FORESTRY END;
|
||||
```
|
||||
|
||||
5. Добавление обратного слеша:
|
||||
|
||||
```bash
|
||||
$ clickhouse-format --backslash <<< "SELECT * FROM (SELECT 1 AS x UNION ALL SELECT 1 UNION DISTINCT SELECT 3);"
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
```text
|
||||
SELECT * \
|
||||
FROM \
|
||||
( \
|
||||
SELECT 1 AS x \
|
||||
UNION ALL \
|
||||
SELECT 1 \
|
||||
UNION DISTINCT \
|
||||
SELECT 3 \
|
||||
)
|
||||
```
|
||||
|
||||
@ -0,0 +1,148 @@
|
||||
---
|
||||
toc_priority: 108
|
||||
---
|
||||
|
||||
## exponentialMovingAverage {#exponential-moving-average}
|
||||
|
||||
Вычисляет экспоненциальное скользящее среднее за определенный промежуток времени.
|
||||
|
||||
**Синтакис:**
|
||||
|
||||
```sql
|
||||
exponentialMovingAverage(x)(value, timestamp)
|
||||
```
|
||||
|
||||
Каждой точке `timestamp` на временном отрезке соответствует определенное значение `value`. Период полураспада — временной интервал `х`, через который вес значений уменьшается в 2 раза. Функция возвращает взвешенное среднее: чем старше временная точка, тем c меньшим весом считается соответствующее ей значение.
|
||||
|
||||
**Аргументы**
|
||||
|
||||
- `value` — входные значения. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) или [Decimal](../../../sql-reference/data-types/decimal.md).
|
||||
- `timestamp` — параметр для упорядочивания значений. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) или [Decimal](../../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Параметры**
|
||||
|
||||
- `x` — период полураспада. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) или [Decimal](../../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Возвращаемые значения**
|
||||
|
||||
- Возвращает [экспоненциальное скользящее среднее](https://ru.wikipedia.org/wiki/Скользящая_средняя#Экспоненциально_взвешенное_скользящее_среднее) за прошедшее время `x` в последний момент времени.
|
||||
|
||||
Тип: [Float64](../../../sql-reference/data-types/float.md#float32-float64).
|
||||
|
||||
**Пример**
|
||||
|
||||
Исходная таблица:
|
||||
|
||||
``` text
|
||||
┌──temperature─┬─timestamp──┐
|
||||
│ 95 │ 1 │
|
||||
│ 95 │ 2 │
|
||||
│ 95 │ 3 │
|
||||
│ 96 │ 4 │
|
||||
│ 96 │ 5 │
|
||||
│ 96 │ 6 │
|
||||
│ 96 │ 7 │
|
||||
│ 97 │ 8 │
|
||||
│ 97 │ 9 │
|
||||
│ 97 │ 10 │
|
||||
│ 97 │ 11 │
|
||||
│ 98 │ 12 │
|
||||
│ 98 │ 13 │
|
||||
│ 98 │ 14 │
|
||||
│ 98 │ 15 │
|
||||
│ 99 │ 16 │
|
||||
│ 99 │ 17 │
|
||||
│ 99 │ 18 │
|
||||
│ 100 │ 19 │
|
||||
│ 100 │ 20 │
|
||||
└──────────────┴────────────┘
|
||||
```
|
||||
|
||||
Запрос:
|
||||
|
||||
```sql
|
||||
SELECT exponentialMovingAverage(5)(temperature, timestamp);
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
``` text
|
||||
┌──exponentialMovingAverage(5)(temperature, timestamp)──┐
|
||||
│ 92.25779635374204 │
|
||||
└───────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Запрос:
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
value,
|
||||
time,
|
||||
round(exp_smooth, 3),
|
||||
bar(exp_smooth, 0, 1, 50) AS bar
|
||||
FROM
|
||||
(
|
||||
SELECT
|
||||
(number = 0) OR (number >= 25) AS value,
|
||||
number AS time,
|
||||
exponentialMovingAverage(10)(value, time) OVER (Rows BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS exp_smooth
|
||||
FROM numbers(50)
|
||||
)
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
``` text
|
||||
┌─value─┬─time─┬─round(exp_smooth, 3)─┬─bar────────────────────────────────────────┐
|
||||
│ 1 │ 0 │ 0.067 │ ███▎ │
|
||||
│ 0 │ 1 │ 0.062 │ ███ │
|
||||
│ 0 │ 2 │ 0.058 │ ██▊ │
|
||||
│ 0 │ 3 │ 0.054 │ ██▋ │
|
||||
│ 0 │ 4 │ 0.051 │ ██▌ │
|
||||
│ 0 │ 5 │ 0.047 │ ██▎ │
|
||||
│ 0 │ 6 │ 0.044 │ ██▏ │
|
||||
│ 0 │ 7 │ 0.041 │ ██ │
|
||||
│ 0 │ 8 │ 0.038 │ █▊ │
|
||||
│ 0 │ 9 │ 0.036 │ █▋ │
|
||||
│ 0 │ 10 │ 0.033 │ █▋ │
|
||||
│ 0 │ 11 │ 0.031 │ █▌ │
|
||||
│ 0 │ 12 │ 0.029 │ █▍ │
|
||||
│ 0 │ 13 │ 0.027 │ █▎ │
|
||||
│ 0 │ 14 │ 0.025 │ █▎ │
|
||||
│ 0 │ 15 │ 0.024 │ █▏ │
|
||||
│ 0 │ 16 │ 0.022 │ █ │
|
||||
│ 0 │ 17 │ 0.021 │ █ │
|
||||
│ 0 │ 18 │ 0.019 │ ▊ │
|
||||
│ 0 │ 19 │ 0.018 │ ▊ │
|
||||
│ 0 │ 20 │ 0.017 │ ▋ │
|
||||
│ 0 │ 21 │ 0.016 │ ▋ │
|
||||
│ 0 │ 22 │ 0.015 │ ▋ │
|
||||
│ 0 │ 23 │ 0.014 │ ▋ │
|
||||
│ 0 │ 24 │ 0.013 │ ▋ │
|
||||
│ 1 │ 25 │ 0.079 │ ███▊ │
|
||||
│ 1 │ 26 │ 0.14 │ ███████ │
|
||||
│ 1 │ 27 │ 0.198 │ █████████▊ │
|
||||
│ 1 │ 28 │ 0.252 │ ████████████▌ │
|
||||
│ 1 │ 29 │ 0.302 │ ███████████████ │
|
||||
│ 1 │ 30 │ 0.349 │ █████████████████▍ │
|
||||
│ 1 │ 31 │ 0.392 │ ███████████████████▌ │
|
||||
│ 1 │ 32 │ 0.433 │ █████████████████████▋ │
|
||||
│ 1 │ 33 │ 0.471 │ ███████████████████████▌ │
|
||||
│ 1 │ 34 │ 0.506 │ █████████████████████████▎ │
|
||||
│ 1 │ 35 │ 0.539 │ ██████████████████████████▊ │
|
||||
│ 1 │ 36 │ 0.57 │ ████████████████████████████▌ │
|
||||
│ 1 │ 37 │ 0.599 │ █████████████████████████████▊ │
|
||||
│ 1 │ 38 │ 0.626 │ ███████████████████████████████▎ │
|
||||
│ 1 │ 39 │ 0.651 │ ████████████████████████████████▌ │
|
||||
│ 1 │ 40 │ 0.674 │ █████████████████████████████████▋ │
|
||||
│ 1 │ 41 │ 0.696 │ ██████████████████████████████████▋ │
|
||||
│ 1 │ 42 │ 0.716 │ ███████████████████████████████████▋ │
|
||||
│ 1 │ 43 │ 0.735 │ ████████████████████████████████████▋ │
|
||||
│ 1 │ 44 │ 0.753 │ █████████████████████████████████████▋ │
|
||||
│ 1 │ 45 │ 0.77 │ ██████████████████████████████████████▍ │
|
||||
│ 1 │ 46 │ 0.785 │ ███████████████████████████████████████▎ │
|
||||
│ 1 │ 47 │ 0.8 │ ███████████████████████████████████████▊ │
|
||||
│ 1 │ 48 │ 0.813 │ ████████████████████████████████████████▋ │
|
||||
│ 1 │ 49 │ 0.825 │ █████████████████████████████████████████▎│
|
||||
└───────┴──────┴──────────────────────┴────────────────────────────────────────────┘
|
||||
```
|
||||
@ -1,9 +1,14 @@
|
||||
---
|
||||
toc_priority: 46
|
||||
toc_title: Cловари полигонов
|
||||
---
|
||||
|
||||
# Cловари полигонов {#polygon-dictionaries}
|
||||
|
||||
Словари полигонов позволяют эффективно искать полигон, в который попадают данные точки, среди множества полигонов.
|
||||
Для примера: определение района города по географическим координатам.
|
||||
|
||||
Пример конфигурации:
|
||||
Пример конфигурации словаря полигонов:
|
||||
|
||||
``` xml
|
||||
<dictionary>
|
||||
@ -28,9 +33,12 @@
|
||||
</structure>
|
||||
|
||||
<layout>
|
||||
<polygon />
|
||||
<polygon>
|
||||
<store_polygon_key_column>1</store_polygon_key_column>
|
||||
</polygon>
|
||||
</layout>
|
||||
|
||||
...
|
||||
</dictionary>
|
||||
```
|
||||
|
||||
@ -42,11 +50,12 @@ CREATE DICTIONARY polygon_dict_name (
|
||||
value UInt64
|
||||
)
|
||||
PRIMARY KEY key
|
||||
LAYOUT(POLYGON())
|
||||
LAYOUT(POLYGON(STORE_POLYGON_KEY_COLUMN 1))
|
||||
...
|
||||
```
|
||||
|
||||
При конфигурации словаря полигонов ключ должен иметь один из двух типов:
|
||||
|
||||
- Простой полигон. Представляет из себя массив точек.
|
||||
- Мультиполигон. Представляет из себя массив полигонов. Каждый полигон задается двумерным массивом точек — первый элемент этого массива задает внешнюю границу полигона,
|
||||
последующие элементы могут задавать дырки, вырезаемые из него.
|
||||
@ -55,24 +64,25 @@ LAYOUT(POLYGON())
|
||||
|
||||
Пользователь может [загружать свои собственные данные](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md) во всех поддерживаемых ClickHouse форматах.
|
||||
|
||||
|
||||
Доступно 3 типа [хранения данных в памяти](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md):
|
||||
|
||||
- POLYGON_SIMPLE. Это наивная реализация, в которой на каждый запрос делается линейный проход по всем полигонам, и для каждого проверяется принадлежность без использования дополнительных индексов.
|
||||
- `POLYGON_SIMPLE`. Это наивная реализация, в которой на каждый запрос делается линейный проход по всем полигонам, и для каждого проверяется принадлежность без использования дополнительных индексов.
|
||||
|
||||
- POLYGON_INDEX_EACH. Для каждого полигона строится отдельный индекс, который позволяет быстро проверять принадлежность в большинстве случаев (оптимизирован под географические регионы).
|
||||
- `POLYGON_INDEX_EACH`. Для каждого полигона строится отдельный индекс, который позволяет быстро проверять принадлежность в большинстве случаев (оптимизирован под географические регионы).
|
||||
Также на рассматриваемую область накладывается сетка, которая значительно сужает количество рассматриваемых полигонов.
|
||||
Сетка строится рекурсивным делением ячейки на 16 равных частей и конфигурируется двумя параметрами.
|
||||
Деление прекращается при достижении глубины рекурсии MAX_DEPTH или в тот момент, когда ячейку пересекают не более MIN_INTERSECTIONS полигонов.
|
||||
Деление прекращается при достижении глубины рекурсии `MAX_DEPTH` или в тот момент, когда ячейку пересекают не более `MIN_INTERSECTIONS` полигонов.
|
||||
Для ответа на запрос находится соответствующая ячейка, и происходит поочередное обращение к индексу для сохранных в ней полигонов.
|
||||
|
||||
- POLYGON_INDEX_CELL. В этом размещении также строится сетка, описанная выше. Доступны такие же параметры. Для каждой ячейки-листа строится индекс на всех попадающих в неё кусках полигонов, который позволяет быстро отвечать на запрос.
|
||||
- `POLYGON_INDEX_CELL`. В этом размещении также строится сетка, описанная выше. Доступны такие же параметры. Для каждой ячейки-листа строится индекс на всех попадающих в неё кусках полигонов, который позволяет быстро отвечать на запрос.
|
||||
|
||||
- POLYGON. Синоним к POLYGON_INDEX_CELL.
|
||||
- `POLYGON`. Синоним к `POLYGON_INDEX_CELL`.
|
||||
|
||||
Запросы к словарю осуществляются с помощью стандартных [функций](../../../sql-reference/functions/ext-dict-functions.md) для работы со внешними словарями.
|
||||
Важным отличием является то, что здесь ключами будут являются точки, для которых хочется найти содержащий их полигон.
|
||||
|
||||
**Пример**
|
||||
|
||||
Пример работы со словарем, определенным выше:
|
||||
``` sql
|
||||
CREATE TABLE points (
|
||||
@ -84,3 +94,40 @@ SELECT tuple(x, y) AS key, dictGet(dict_name, 'name', key), dictGet(dict_name, '
|
||||
```
|
||||
|
||||
В результате исполнения последней команды для каждой точки в таблице `points` будет найден полигон минимальной площади, содержащий данную точку, и выведены запрошенные аттрибуты.
|
||||
|
||||
**Пример**
|
||||
|
||||
Вы можете читать столбцы из полигональных словарей с помощью SELECT, для этого включите `store_polygon_key_column = 1` в конфигурации словаря или соответствующего DDL-запроса.
|
||||
|
||||
Запрос:
|
||||
|
||||
``` sql
|
||||
CREATE TABLE polygons_test_table
|
||||
(
|
||||
key Array(Array(Array(Tuple(Float64, Float64)))),
|
||||
name String
|
||||
) ENGINE = TinyLog;
|
||||
|
||||
INSERT INTO polygons_test_table VALUES ([[[(3, 1), (0, 1), (0, -1), (3, -1)]]], 'Value');
|
||||
|
||||
CREATE DICTIONARY polygons_test_dictionary
|
||||
(
|
||||
key Array(Array(Array(Tuple(Float64, Float64)))),
|
||||
name String
|
||||
)
|
||||
PRIMARY KEY key
|
||||
SOURCE(CLICKHOUSE(TABLE 'polygons_test_table'))
|
||||
LAYOUT(POLYGON(STORE_POLYGON_KEY_COLUMN 1))
|
||||
LIFETIME(0);
|
||||
|
||||
SELECT * FROM polygons_test_dictionary;
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
``` text
|
||||
┌─key─────────────────────────────┬─name──┐
|
||||
│ [[[(3,1),(0,1),(0,-1),(3,-1)]]] │ Value │
|
||||
└─────────────────────────────────┴───────┘
|
||||
```
|
||||
|
||||
|
||||
@ -748,7 +748,7 @@ SOURCE(REDIS(
|
||||
!!! info "Примечание"
|
||||
Поля `column_family` или `where` не могут быть использованы вместе с полем `query`. Также обязательно должен быть один из источников данных: `column_family` или `query`.
|
||||
|
||||
### PosgreSQL {#dicts-external_dicts_dict_sources-postgresql}
|
||||
### PostgreSQL {#dicts-external_dicts_dict_sources-postgresql}
|
||||
|
||||
Пример настройки:
|
||||
|
||||
|
||||
@ -89,9 +89,39 @@ SELECT sipHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00
|
||||
|
||||
## sipHash128 {#hash_functions-siphash128}
|
||||
|
||||
Вычисляет SipHash от строки.
|
||||
Принимает аргумент типа String. Возвращает FixedString(16).
|
||||
Отличается от sipHash64 тем, что финальный xor-folding состояния делается только до 128 бит.
|
||||
Генерирует 128-битное хеш-значение [SipHash](https://131002.net/siphash/). Отличается от [sipHash64](#hash_functions-siphash64) тем, что финальный xor-folding состояния делается до 128 бит.
|
||||
|
||||
**Синтаксис**
|
||||
|
||||
``` sql
|
||||
sipHash128(par1,...)
|
||||
```
|
||||
|
||||
**Аргументы**
|
||||
|
||||
Функция принимает переменное число входных параметров. Аргументы могут быть любого [поддерживаемого типа данных](../../sql-reference/functions/hash-functions.md).
|
||||
|
||||
**Возвращаемое значение**
|
||||
|
||||
128-битное хеш-значение `SipHash`.
|
||||
|
||||
Тип: [FixedString(16)](../../sql-reference/data-types/fixedstring.md).
|
||||
|
||||
**Пример**
|
||||
|
||||
Запрос:
|
||||
|
||||
``` sql
|
||||
SELECT hex(sipHash128('foo', '\x01', 3));
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
``` text
|
||||
┌─hex(sipHash128('foo', '', 3))────┐
|
||||
│ 9DE516A64A414D4B1B609415E4523F24 │
|
||||
└──────────────────────────────────┘
|
||||
```
|
||||
|
||||
## cityHash64 {#cityhash64}
|
||||
|
||||
@ -459,30 +489,38 @@ SELECT murmurHash3_32(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:
|
||||
|
||||
## murmurHash3_128 {#murmurhash3-128}
|
||||
|
||||
Генерирует значение [MurmurHash3](https://github.com/aappleby/smhasher).
|
||||
Генерирует 128-битное хеш-значение [MurmurHash3](https://github.com/aappleby/smhasher).
|
||||
|
||||
**Синтаксис**
|
||||
|
||||
``` sql
|
||||
murmurHash3_128( expr )
|
||||
murmurHash3_128(expr)
|
||||
```
|
||||
|
||||
**Аргументы**
|
||||
|
||||
- `expr` — [выражение](../syntax.md#syntax-expressions), возвращающее значение типа [String](../../sql-reference/functions/hash-functions.md).
|
||||
- `expr` — список [выражений](../../sql-reference/syntax.md#syntax-expressions). [String](../../sql-reference/data-types/string.md).
|
||||
|
||||
**Возвращаемое значение**
|
||||
|
||||
Хэш-значение типа [FixedString(16)](../../sql-reference/functions/hash-functions.md).
|
||||
128-битное значение хеш-значение `MurmurHash3`.
|
||||
|
||||
Тип: [FixedString(16)](../../sql-reference/data-types/fixedstring.md).
|
||||
|
||||
**Пример**
|
||||
|
||||
Запрос:
|
||||
|
||||
``` sql
|
||||
SELECT hex(murmurHash3_128('example_string')) AS MurmurHash3, toTypeName(MurmurHash3) AS type;
|
||||
SELECT hex(murmurHash3_128('foo', 'foo', 'foo'));
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
``` text
|
||||
┌─MurmurHash3──────────────────────┬─type───┐
|
||||
│ 368A1A311CB7342253354B548E7E7E71 │ String │
|
||||
└──────────────────────────────────┴────────┘
|
||||
┌─hex(murmurHash3_128('foo', 'foo', 'foo'))─┐
|
||||
│ F8F7AD9B6CD4CF117A71E277E2EC2931 │
|
||||
└───────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## xxHash32, xxHash64 {#hash-functions-xxhash32-xxhash64}
|
||||
|
||||
@ -252,7 +252,7 @@ ngrams(string, ngramsize)
|
||||
|
||||
- Массив с n-граммами.
|
||||
|
||||
Тип: [Array](../../sql-reference/data-types/array.md)([FixedString](../../sql-reference/data-types/fixedstring.md)).
|
||||
Тип: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)).
|
||||
|
||||
**Пример**
|
||||
|
||||
|
||||
@ -10,7 +10,7 @@ toc_title: "Манипуляции со столбцами"
|
||||
Синтаксис:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE [db].name [ON CLUSTER cluster] ADD|DROP|RENAME|CLEAR|COMMENT|MODIFY|MATERIALIZE COLUMN ...
|
||||
ALTER TABLE [db].name [ON CLUSTER cluster] ADD|DROP|RENAME|CLEAR|COMMENT|{MODIFY|ALTER}|MATERIALIZE COLUMN ...
|
||||
```
|
||||
|
||||
В запросе можно указать сразу несколько действий над одной таблицей через запятую.
|
||||
@ -138,6 +138,7 @@ ALTER TABLE visits COMMENT COLUMN browser 'Столбец показывает,
|
||||
|
||||
``` sql
|
||||
MODIFY COLUMN [IF EXISTS] name [type] [default_expr] [codec] [TTL] [AFTER name_after | FIRST]
|
||||
ALTER COLUMN [IF EXISTS] name TYPE [type] [default_expr] [codec] [TTL] [AFTER name_after | FIRST]
|
||||
```
|
||||
|
||||
Запрос изменяет следующие свойства столбца `name`:
|
||||
|
||||
@ -3,11 +3,11 @@ toc_priority: 33
|
||||
toc_title: INSERT INTO
|
||||
---
|
||||
|
||||
## INSERT {#insert}
|
||||
## INSERT INTO {#insert}
|
||||
|
||||
Добавление данных.
|
||||
Добавляет данные в таблицу.
|
||||
|
||||
Базовый формат запроса:
|
||||
**Синтаксис**
|
||||
|
||||
``` sql
|
||||
INSERT INTO [db.]table [(c1, c2, c3)] VALUES (v11, v12, v13), (v21, v22, v23), ...
|
||||
@ -21,17 +21,15 @@ INSERT INTO [db.]table [(c1, c2, c3)] VALUES (v11, v12, v13), (v21, v22, v23), .
|
||||
SHOW CREATE insert_select_testtable
|
||||
```
|
||||
|
||||
```
|
||||
┌─statement────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐
|
||||
│ CREATE TABLE insert_select_testtable
|
||||
```text
|
||||
CREATE TABLE insert_select_testtable
|
||||
(
|
||||
`a` Int8,
|
||||
`b` String,
|
||||
`c` Int8
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
ORDER BY a │
|
||||
└──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
|
||||
ORDER BY a
|
||||
```
|
||||
|
||||
``` sql
|
||||
@ -93,6 +91,8 @@ INSERT INTO t FORMAT TabSeparated
|
||||
|
||||
### Вставка результатов `SELECT` {#insert_query_insert-select}
|
||||
|
||||
**Синтаксис**
|
||||
|
||||
``` sql
|
||||
INSERT INTO [db.]table [(c1, c2, c3)] SELECT ...
|
||||
```
|
||||
@ -109,6 +109,68 @@ INSERT INTO [db.]table [(c1, c2, c3)] SELECT ...
|
||||
|
||||
Чтобы вставить значение по умолчанию вместо `NULL` в столбец, который не позволяет хранить `NULL`, включите настройку [insert_null_as_default](../../operations/settings/settings.md#insert_null_as_default).
|
||||
|
||||
### Вставка данных из файла {#inserting-data-from-a-file}
|
||||
|
||||
**Синтаксис**
|
||||
|
||||
``` sql
|
||||
INSERT INTO [db.]table [(c1, c2, c3)] FROM INFILE file_name [COMPRESSION type] FORMAT format_name
|
||||
```
|
||||
|
||||
Используйте этот синтаксис, чтобы вставить данные из файла, который хранится на стороне **клиента**. `file_name` и `type` задаются в виде строковых литералов. [Формат](../../interfaces/formats.md) входного файла должен быть задан в секции `FORMAT`.
|
||||
|
||||
Поддерживаются сжатые файлы. Формат сжатия определяется по расширению файла, либо он может быть задан в секции `COMPRESSION`. Поддерживаются форматы: `'none'`, `'gzip'`, `'deflate'`, `'br'`, `'xz'`, `'zstd'`, `'lz4'`, `'bz2'`.
|
||||
|
||||
Эта функциональность поддерживается [клиентом командной строки](../../interfaces/cli.md) и [clickhouse-local](../../operations/utilities/clickhouse-local.md).
|
||||
|
||||
**Пример**
|
||||
|
||||
Выполните следующие запросы, используя [клиент командной строки](../../interfaces/cli.md):
|
||||
|
||||
```bash
|
||||
echo 1,A > input.csv ; echo 2,B >> input.csv
|
||||
clickhouse-client --query="CREATE TABLE table_from_file (id UInt32, text String) ENGINE=MergeTree() ORDER BY id;"
|
||||
clickhouse-client --query="INSERT INTO table_from_file FROM INFILE 'input.csv' FORMAT CSV;"
|
||||
clickhouse-client --query="SELECT * FROM table_from_file FORMAT PrettyCompact;"
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
```text
|
||||
┌─id─┬─text─┐
|
||||
│ 1 │ A │
|
||||
│ 2 │ B │
|
||||
└────┴──────┘
|
||||
```
|
||||
|
||||
### Вставка в табличную функцию {#inserting-into-table-function}
|
||||
|
||||
Данные могут быть вставлены в таблицы, заданные с помощью [табличных функций](../../sql-reference/table-functions/index.md).
|
||||
|
||||
**Синтаксис**
|
||||
``` sql
|
||||
INSERT INTO [TABLE] FUNCTION table_func ...
|
||||
```
|
||||
|
||||
**Пример**
|
||||
|
||||
Табличная функция [remote](../../sql-reference/table-functions/index.md#remote) используется в следующих запросах:
|
||||
|
||||
``` sql
|
||||
CREATE TABLE simple_table (id UInt32, text String) ENGINE=MergeTree() ORDER BY id;
|
||||
INSERT INTO TABLE FUNCTION remote('localhost', default.simple_table)
|
||||
VALUES (100, 'inserted via remote()');
|
||||
SELECT * FROM simple_table;
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
``` text
|
||||
┌──id─┬─text──────────────────┐
|
||||
│ 100 │ inserted via remote() │
|
||||
└─────┴───────────────────────┘
|
||||
```
|
||||
|
||||
### Замечания о производительности {#zamechaniia-o-proizvoditelnosti}
|
||||
|
||||
`INSERT` сортирует входящие данные по первичному ключу и разбивает их на партиции по ключу партиционирования. Если вы вставляете данные в несколько партиций одновременно, то это может значительно снизить производительность запроса `INSERT`. Чтобы избежать этого:
|
||||
|
||||
@ -25,7 +25,7 @@ SELECT [DISTINCT [ON (column1, column2, ...)]] expr_list
|
||||
[LIMIT [n, ]m] [WITH TIES]
|
||||
[SETTINGS ...]
|
||||
[UNION ALL ...]
|
||||
[INTO OUTFILE filename]
|
||||
[INTO OUTFILE filename [COMPRESSION type] ]
|
||||
[FORMAT format]
|
||||
```
|
||||
|
||||
|
||||
@ -4,10 +4,35 @@ toc_title: INTO OUTFILE
|
||||
|
||||
# Секция INTO OUTFILE {#into-outfile-clause}
|
||||
|
||||
Чтобы перенаправить вывод `SELECT` запроса в указанный файл на стороне клиента, добавьте к нему секцию `INTO OUTFILE filename` (где filename — строковый литерал).
|
||||
Секция `INTO OUTFILE` перенаправляет результат запроса `SELECT` в файл на стороне **клиента**.
|
||||
|
||||
Поддерживаются сжатые файлы. Формат сжатия определяется по расширению файла (по умолчанию используется режим `'auto'`), либо он может быть задан явно в секции `COMPRESSION`.
|
||||
|
||||
**Синтаксис**
|
||||
|
||||
```sql
|
||||
SELECT <expr_list> INTO OUTFILE file_name [COMPRESSION type]
|
||||
```
|
||||
|
||||
`file_name` и `type` задаются в виде строковых литералов. Поддерживаются форматы сжатия: `'none`', `'gzip'`, `'deflate'`, `'br'`, `'xz'`, `'zstd'`, `'lz4'`, `'bz2'`.
|
||||
|
||||
## Детали реализации {#implementation-details}
|
||||
|
||||
- Эта функция доступна только в следующих интерфейсах: [клиент командной строки](../../../interfaces/cli.md) и [clickhouse-local](../../../operations/utilities/clickhouse-local.md). Таким образом, запрос, отправленный через [HTTP интерфейс](../../../interfaces/http.md) вернет ошибку.
|
||||
- Запрос завершится ошибкой, если файл с тем же именем уже существует.
|
||||
- По умолчанию используется [выходной формат](../../../interfaces/formats.md) `TabSeparated` (как в пакетном режиме клиента командной строки).
|
||||
- По умолчанию используется [выходной формат](../../../interfaces/formats.md) `TabSeparated` (как в пакетном режиме клиента командной строки). Его можно изменить в секции [FORMAT](format.md).
|
||||
|
||||
**Пример**
|
||||
|
||||
Выполните следующий запрос, используя [клиент командной строки](../../../interfaces/cli.md):
|
||||
|
||||
```bash
|
||||
clickhouse-client --query="SELECT 1,'ABC' INTO OUTFILE 'select.gz' FORMAT CSV;"
|
||||
zcat select.gz
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
```text
|
||||
1,"ABC"
|
||||
```
|
||||
|
||||
@ -18,6 +18,8 @@ toc_title: "Введение"
|
||||
|
||||
Это один из методов создания таблицы.
|
||||
|
||||
- Запросе [INSERT INTO TABLE FUNCTION](../../sql-reference/statements/insert-into.md#inserting-into-table-function).
|
||||
|
||||
!!! warning "Предупреждение"
|
||||
Если настройка [allow_ddl](../../operations/settings/permissions-for-queries.md#settings_allow_ddl) выключена, то использовать табличные функции невозможно.
|
||||
|
||||
|
||||
@ -86,6 +86,7 @@ git push
|
||||
- **Compiler**: `gcc-9` 或 `clang-10` (或其他架构的`clang-10-xx`, 比如`clang-10-freebsd`).
|
||||
- **Build type**: `Debug` or `RelWithDebInfo` (cmake).
|
||||
- **Sanitizer**: `none` (without sanitizers), `address` (ASan), `memory` (MSan), `undefined` (UBSan), or `thread` (TSan).
|
||||
- **Bundled**: `bundled` 构建使用来自 `contrib` 库, 而 `unbundled` 构建使用系统库.
|
||||
- **Splitted**: `splitted` is a [split build](https://clickhouse.com/docs/en/development/build/#split-build)
|
||||
- **Status**: `成功` 或 `失败`
|
||||
- **Build log**: 链接到构建和文件复制日志, 当构建失败时很有用.
|
||||
|
||||
@ -1 +0,0 @@
|
||||
../../en/development/tests.md
|
||||
335
docs/zh/development/tests.md
Normal file
335
docs/zh/development/tests.md
Normal file
@ -0,0 +1,335 @@
|
||||
# ClickHouse 测试 {#clickhouse-testing}
|
||||
|
||||
## 功能测试 {#functional-tests}
|
||||
|
||||
功能测试使用起来最简单方便. 大多数 ClickHouse 特性都可以通过功能测试进行测试, 并且对于可以通过功能测试进行测试的 ClickHouse 代码的每一个更改, 都必须使用这些特性
|
||||
|
||||
每个功能测试都会向正在运行的 ClickHouse 服务器发送一个或多个查询, 并将结果与参考进行比较.
|
||||
|
||||
测试位于 `查询` 目录中. 有两个子目录: `无状态` 和 `有状态`. 无状态测试在没有任何预加载测试数据的情况下运行查询 - 它们通常在测试本身内即时创建小型合成数据集. 状态测试需要来自 Yandex.Metrica 的预加载测试数据, 它对公众开放.
|
||||
|
||||
每个测试可以是两种类型之一: `.sql` 和 `.sh`. `.sql` 测试是简单的 SQL 脚本, 它通过管道传输到 `clickhouse-client --multiquery --testmode`. `.sh` 测试是一个自己运行的脚本. SQL 测试通常比 `.sh` 测试更可取. 仅当您必须测试某些无法从纯 SQL 中执行的功能时才应使用 `.sh` 测试, 例如将一些输入数据传送到 `clickhouse-client` 或测试 `clickhouse-local`.
|
||||
|
||||
### 在本地运行测试 {#functional-test-locally}
|
||||
|
||||
在本地启动ClickHouse服务器, 监听默认端口(9000). 例如, 要运行测试 `01428_hash_set_nan_key`, 请切换到存储库文件夹并运行以下命令:
|
||||
|
||||
```
|
||||
PATH=$PATH:<path to clickhouse-client> tests/clickhouse-test 01428_hash_set_nan_key
|
||||
```
|
||||
|
||||
有关更多选项, 请参阅`tests/clickhouse-test --help`. 您可以简单地运行所有测试或运行由测试名称中的子字符串过滤的测试子集：`./clickhouse-test substring`. 还有并行或随机顺序运行测试的选项.
|
||||
|
||||
### 添加新测试 {#adding-new-test}
|
||||
|
||||
添加新的测试, 在 `queries/0_stateless` 目录下创建 `.sql` 或 `.sh` 文件, 手动检查, 然后通过以下方式生成`.reference`文件：`clickhouse-client -n --testmode < 00000_test.sql > 00000_test.reference` 或 `./00000_test.sh > ./00000_test.reference`.
|
||||
|
||||
测试应仅使用(创建、删除等)`test` 数据库中假定已预先创建的表; 测试也可以使用临时表.
|
||||
|
||||
### 选择测试名称 {#choosing-test-name}
|
||||
|
||||
测试名称以五位数前缀开头, 后跟描述性名称, 例如 `00422_hash_function_constexpr.sql`. 要选择前缀, 请找到目录中已存在的最大前缀, 并将其加一. 在此期间, 可能会添加一些具有相同数字前缀的其他测试, 但这没关系并且不会导致任何问题, 您以后不必更改它.
|
||||
|
||||
一些测试的名称中标有 `zookeeper`、`shard` 或 `long` . `zookeeper` 用于使用 ZooKeeper 的测试. `shard` 用于需要服务器监听 `127.0.0.*` 的测试; `distributed` 或 `global` 具有相同的含义. `long` 用于运行时间稍长于一秒的测试. Yo你可以分别使用 `--no-zookeeper`、`--no-shard` 和 `--no-long` 选项禁用这些测试组. 如果需要 ZooKeeper 或分布式查询，请确保为您的测试名称添加适当的前缀.
|
||||
|
||||
### 检查必须发生的错误 {#checking-error-must-occur}
|
||||
|
||||
有时您想测试是否因不正确的查询而发生服务器错误. 我们支持在 SQL 测试中对此进行特殊注释, 形式如下:
|
||||
```
|
||||
select x; -- { serverError 49 }
|
||||
```
|
||||
此测试确保服务器返回关于未知列“x”的错误代码为 49. 如果没有错误, 或者错误不同, 则测试失败. 如果您想确保错误发生在客户端, 请改用 `clientError` 注释.
|
||||
|
||||
不要检查错误消息的特定措辞, 它将来可能会发生变化, 并且测试将不必要地中断. 只检查错误代码. 如果现有的错误代码不足以满足您的需求, 请考虑添加一个新的.
|
||||
|
||||
### 测试分布式查询 {#testing-distributed-query}
|
||||
|
||||
如果你想在功能测试中使用分布式查询, 你可以使用 `127.0.0.{1..2}` 的地址, 以便服务器查询自己; 或者您可以在服务器配置文件中使用预定义的测试集群, 例如`test_shard_localhost`. 请记住在测试名称中添加 `shard` 或 `distributed` 字样, 以便它以正确的配置在 CI 中运行, 其中服务器配置为支持分布式查询.
|
||||
|
||||
|
||||
## 已知错误 {#known-bugs}
|
||||
|
||||
如果我们知道一些可以通过功能测试轻松重现的错误, 我们将准备好的功能测试放在 `tests/queries/bugs` 目录中. 修复错误后, 这些测试将移至 `tests/queries/0_stateless` .
|
||||
|
||||
## 集成测试 {#integration-tests}
|
||||
|
||||
集成测试允许在集群配置中测试 ClickHouse 以及 ClickHouse 与其他服务器(如 MySQL、Postgres、MongoDB)的交互. 它们可以用来模拟网络分裂、丢包等情况. 这些测试在Docker下运行, 并使用各种软件创建多个容器.
|
||||
|
||||
有关如何运行这些测试, 请参阅 `tests/integration/README.md` .
|
||||
|
||||
注意, ClickHouse与第三方驱动程序的集成没有经过测试. 另外, 我们目前还没有JDBC和ODBC驱动程序的集成测试.
|
||||
|
||||
## 单元测试 {#unit-tests}
|
||||
|
||||
当您想测试的不是 ClickHouse 整体, 而是单个独立库或类时，单元测试很有用. 您可以使用 `ENABLE_TESTS` CMake 选项启用或禁用测试构建. 单元测试(和其他测试程序)位于代码中的 `tests` 子目录中. 要运行单元测试, 请键入 `ninja test` 。有些测试使用 `gtest` , 但有些程序在测试失败时会返回非零退出码.
|
||||
|
||||
如果代码已经被功能测试覆盖了, 就没有必要进行单元测试(而且功能测试通常更易于使用).
|
||||
|
||||
例如, 您可以通过直接调用可执行文件来运行单独的 gtest 检查:
|
||||
|
||||
```bash
|
||||
$ ./src/unit_tests_dbms --gtest_filter=LocalAddress*
|
||||
```
|
||||
|
||||
## 性能测试 {#performance-tests}
|
||||
|
||||
性能测试允许测量和比较 ClickHouse 的某些孤立部分在合成查询上的性能. 测试位于 `tests/performance`. 每个测试都由带有测试用例描述的 `.xml` 文件表示. 测试使用 `docker/tests/performance-comparison` 工具运行. 请参阅自述文件以进行调用.
|
||||
|
||||
每个测试在循环中运行一个或多个查询(可能带有参数组合). 一些测试可以包含预加载测试数据集的先决条件.
|
||||
|
||||
如果您希望在某些场景中提高ClickHouse的性能，并且如果可以在简单的查询中观察到改进，那么强烈建议编写性能测试。在测试期间使用 `perf top` 或其他perf工具总是有意义的.
|
||||
|
||||
## 测试工具和脚本 {#test-tools-and-scripts}
|
||||
|
||||
`tests` 目录中的一些程序不是准备好的测试，而是测试工具. 例如, 对于 `Lexer`, 有一个工具 `src/Parsers/tests/lexer` , 它只是对标准输入进行标记化并将着色结果写入标准输出. 您可以将这些类型的工具用作代码示例以及用于探索和手动测试.
|
||||
|
||||
## 其他测试 {#miscellaneous-tests}
|
||||
|
||||
在 `tests/external_models` 中有机器学习模型的测试. 这些测试不会更新, 必须转移到集成测试.
|
||||
|
||||
仲裁插入有单独的测试. 该测试在不同的服务器上运行 ClickHouse 集群并模拟各种故障情况：网络分裂、丢包(ClickHouse 节点之间、ClickHouse 和 ZooKeeper 之间、ClickHouse 服务器和客户端之间等)、`kill -9`、`kill -STOP` 和 `kill -CONT` , 比如 [Jepsen](https://aphyr.com/tags/Jepsen). 然后测试检查所有已确认的插入是否已写入并且所有被拒绝的插入均未写入.
|
||||
|
||||
在 ClickHouse 开源之前, Quorum 测试是由单独的团队编写的. 这个团队不再与ClickHouse合作. 测试碰巧是用Java编写的. 由于这些原因, 必须重写仲裁测试并将其转移到集成测试.
|
||||
|
||||
## 手动测试 {#manual-testing}
|
||||
|
||||
当您开发一个新特性时, 手动测试它也是合理的. 您可以按照以下步骤进行操作:
|
||||
|
||||
构建 ClickHouse. 从终端运行 ClickHouse：将目录更改为 `programs/clickhouse-server` 并使用 `./clickhouse-server` 运行它. 默认情况下, 它将使用当前目录中的配置(`config.xml`、`users.xml` 和`config.d` 和`users.d` 目录中的文件). 要连接到 ClickHouse 服务器, 请运行 `programs/clickhouse-client/clickhouse-client` .
|
||||
|
||||
请注意, 所有 clickhouse 工具(服务器、客户端等)都只是指向名为 `clickhouse` 的单个二进制文件的符号链接. 你可以在 `programs/clickhouse` 找到这个二进制文件. 所有工具也可以作为 `clickhouse tool` 而不是 `clickhouse-tool` 调用.
|
||||
|
||||
或者, 您可以安装 ClickHouse 包: 从 Yandex 存储库稳定发布, 或者您可以在 ClickHouse 源根目录中使用 `./release` 为自己构建包. 然后使用 `sudo service clickhouse-server start` 启动服务器(或停止以停止服务器). 在 `/etc/clickhouse-server/clickhouse-server.log` 中查找日志.
|
||||
|
||||
当您的系统上已经安装了 ClickHouse 时，您可以构建一个新的 `clickhouse` 二进制文件并替换现有的二进制文件:
|
||||
|
||||
``` bash
|
||||
$ sudo service clickhouse-server stop
|
||||
$ sudo cp ./clickhouse /usr/bin/
|
||||
$ sudo service clickhouse-server start
|
||||
```
|
||||
|
||||
您也可以停止系统 clickhouse-server 并使用相同的配置运行您自己的服务器, 但登录到终端:
|
||||
|
||||
``` bash
|
||||
$ sudo service clickhouse-server stop
|
||||
$ sudo -u clickhouse /usr/bin/clickhouse server --config-file /etc/clickhouse-server/config.xml
|
||||
```
|
||||
|
||||
使用 gdb 的示例:
|
||||
|
||||
``` bash
|
||||
$ sudo -u clickhouse gdb --args /usr/bin/clickhouse server --config-file /etc/clickhouse-server/config.xml
|
||||
```
|
||||
|
||||
如果系统 clickhouse-server 已经在运行并且你不想停止它, 你可以在你的 `config.xml` 中更改端口号(或在 `config.d` 目录中的文件中覆盖它们), 提供适当的数据路径, 并运行它.
|
||||
|
||||
`clickhouse` 二进制文件几乎没有依赖关系, 可以在广泛的 Linux 发行版中使用. 要在服务器上快速而肮脏地测试您的更改, 您可以简单地将新构建的 `clickhouse` 二进制文件 `scp` 到您的服务器, 然后按照上面的示例运行它.
|
||||
|
||||
## 测试环境 {#testing-environment}
|
||||
|
||||
在发布稳定版之前, 我们将其部署在测试环境中.测试环境是一个集群，处理 [Yandex.Metrica](https://metrica.yandex.com/) 数据的 1/39 部分. 我们与 Yandex.Metrica 团队共享我们的测试环境. ClickHouse无需在现有数据上停机即可升级. 我们首先看到的是, 数据被成功地处理了, 没有滞后于实时, 复制继续工作, Yandex.Metrica 团队没有发现任何问题. 第一次检查可以通过以下方式进行:
|
||||
|
||||
``` sql
|
||||
SELECT hostName() AS h, any(version()), any(uptime()), max(UTCEventTime), count() FROM remote('example01-01-{1..3}t', merge, hits) WHERE EventDate >= today() - 2 GROUP BY h ORDER BY h;
|
||||
```
|
||||
|
||||
在某些情况下, 我们还会部署到 Yandex 中我们朋友团队的测试环境：Market、Cloud 等. 此外, 我们还有一些用于开发目的的硬件服务器.
|
||||
|
||||
## 负载测试 {#load-testing}
|
||||
|
||||
部署到测试环境后, 我们使用来自生产集群的查询运行负载测试. 这是手动完成的.
|
||||
|
||||
确保您在生产集群上启用了 `query_log`.
|
||||
|
||||
收集一天或更长时间的查询日志:
|
||||
|
||||
``` bash
|
||||
$ clickhouse-client --query="SELECT DISTINCT query FROM system.query_log WHERE event_date = today() AND query LIKE '%ym:%' AND query NOT LIKE '%system.query_log%' AND type = 2 AND is_initial_query" > queries.tsv
|
||||
```
|
||||
|
||||
这是一个复杂的例子. `type = 2` 将过滤成功执行的查询. `query LIKE '%ym:%'` 是从 Yandex.Metrica 中选择相关查询. `is_initial_query` 是只选择客户端发起的查询, 而不是 ClickHouse 本身(作为分布式查询处理的一部分).
|
||||
|
||||
`scp` 将此日志记录到您的测试集群并按如下方式运行它:
|
||||
|
||||
``` bash
|
||||
$ clickhouse benchmark --concurrency 16 < queries.tsv
|
||||
```
|
||||
|
||||
(可能你还想指定一个 `--user`)
|
||||
|
||||
然后把它留到晚上或周末, 去休息一下.
|
||||
|
||||
您应该检查 `clickhouse-server` 没有崩溃, 内存占用是有限的, 且性能不会随着时间的推移而降低.
|
||||
|
||||
由于查询和环境的高度可变性, 没有记录和比较精确的查询执行时间.
|
||||
|
||||
## 构建测试 {#build-tests}
|
||||
|
||||
构建测试允许检查在各种可选配置和一些外部系统上的构建是否被破坏. 这些测试也是自动化的.
|
||||
|
||||
示例:
|
||||
- Darwin x86_64 (Mac OS X) 交叉编译
|
||||
- FreeBSD x86_64 交叉编译
|
||||
- Linux AArch64 交叉编译
|
||||
- 使用系统包中的库在 Ubuntu 上构建（不鼓励）
|
||||
- 使用库的共享链接构建（不鼓励）
|
||||
|
||||
例如, 使用系统包构建是不好的做法, 因为我们无法保证系统将拥有哪个确切版本的包. 但这确实是 Debian 维护者所需要的. 出于这个原因, 我们至少必须支持这种构建变体. 另一个例子: 共享链接是一个常见的麻烦来源, 但对于一些爱好者来说是需要的.
|
||||
|
||||
虽然我们无法对所有构建变体运行所有测试, 但我们希望至少检查各种构建变体没有被破坏. 为此, 我们使用构建测试.
|
||||
|
||||
我们还测试了那些太长而无法编译或需要太多RAM的没有翻译单元.
|
||||
|
||||
我们还测试没有太大的堆栈帧.
|
||||
|
||||
## 协议兼容性测试 {#testing-for-protocol-compatibility}
|
||||
|
||||
当我们扩展 ClickHouse 网络协议时, 我们手动测试旧的 clickhouse-client 与新的 clickhouse-server 一起工作, 而新的 clickhouse-client 与旧的 clickhouse-server 一起工作(只需从相应的包中运行二进制文件).
|
||||
|
||||
我们还使用集成测试自动测试一些案例:
|
||||
- 旧版本ClickHouse写入的数据是否可以被新版本成功读取;
|
||||
- 在具有不同 ClickHouse 版本的集群中执行分布式查询.
|
||||
|
||||
## 编译器的帮助 {#help-from-the-compiler}
|
||||
|
||||
主要的 ClickHouse 代码(位于 `dbms` 目录中)是用 `-Wall -Wextra -Werror` 和一些额外的启用警告构建的. 虽然没有为第三方库启用这些选项.
|
||||
|
||||
Clang 有更多有用的警告 - 你可以用 `-Weverything` 寻找它们并选择一些东西来默认构建.
|
||||
|
||||
对于生产构建, 使用 clang, 但我们也测试 make gcc 构建. 对于开发, clang 通常使用起来更方便. 您可以使用调试模式在自己的机器上构建(以节省笔记本电脑的电池), 但请注意, 由于更好的控制流和过程间分析, 编译器能够使用 `-O3` 生成更多警告. 在调试模式下使用 clang 构建时, 使用调试版本的 `libc++` 允许在运行时捕获更多错误.
|
||||
|
||||
## 地址清理器 {#sanitizers}
|
||||
|
||||
### 地址清理器
|
||||
我们在ASan上运行功能测试、集成测试、压力测试和单元测试.
|
||||
|
||||
### 线程清理器
|
||||
我们在TSan下运行功能测试、集成测试、压力测试和单元测试.
|
||||
|
||||
### 内存清理器
|
||||
我们在MSan上运行功能测试、集成测试、压力测试和单元测试.
|
||||
|
||||
### 未定义的行为清理器
|
||||
我们在UBSan下运行功能测试、集成测试、压力测试和单元测试. 某些第三方库的代码未针对 UB 进行清理.
|
||||
|
||||
### Valgrind (Memcheck)
|
||||
我们曾经在 Valgrind 下通宵运行功能测试, 但不再这样做了. 这需要几个小时. 目前在`re2`库中有一个已知的误报, 见[这篇文章](https://research.swtch.com/sparse).
|
||||
|
||||
## 模糊测试 {#fuzzing}
|
||||
|
||||
ClickHouse 模糊测试是使用 [libFuzzer](https://llvm.org/docs/LibFuzzer.html) 和随机 SQL 查询实现的. 所有模糊测试都应使用sanitizers(地址和未定义)进行.
|
||||
|
||||
LibFuzzer 用于库代码的隔离模糊测试. Fuzzer 作为测试代码的一部分实现, 并具有 `_fuzzer` 名称后缀.
|
||||
Fuzzer 示例可以在 `src/Parsers/tests/lexer_fuzzer.cpp` 中找到. LibFuzzer 特定的配置、字典和语料库存储在 `tests/fuzz` 中.
|
||||
我们鼓励您为处理用户输入的每个功能编写模糊测试.
|
||||
|
||||
默认情况下不构建模糊器. 要构建模糊器, 应设置` -DENABLE_FUZZING=1` 和 `-DENABLE_TESTS=1` 选项.
|
||||
我们建议在构建模糊器时禁用 Jemalloc. 用于将 ClickHouse fuzzing 集成到 Google OSS-Fuzz 的配置可以在 `docker/fuzz` 中找到.
|
||||
|
||||
我们还使用简单的模糊测试来生成随机SQL查询, 并检查服务器在执行这些查询时是否会死亡.
|
||||
你可以在 `00746_sql_fuzzy.pl` 中找到它. 这个测试应该连续运行(通宵或更长时间).
|
||||
|
||||
我们还使用复杂的基于 AST 的查询模糊器, 它能够找到大量的极端情况. 它在查询 AST 中进行随机排列和替换. 它会记住先前测试中的 AST 节点, 以使用它们对后续测试进行模糊测试, 同时以随机顺序处理它们. 您可以在 [这篇博客文章](https://clickhouse.com/blog/en/2021/fuzzing-clickhouse/) 中了解有关此模糊器的更多信息.
|
||||
|
||||
## 压力测试 {#stress-test}
|
||||
|
||||
压力测试是另一种模糊测试. 它使用单个服务器以随机顺序并行运行所有功能测试. 不检查测试结果.
|
||||
|
||||
经检查:
|
||||
- 服务器不会崩溃，不会触发调试或清理程序陷阱;
|
||||
- 没有死锁;
|
||||
- 数据库结构一致;
|
||||
- 服务器可以在测试后成功停止并重新启动，没有异常;
|
||||
|
||||
有五种变体 (Debug, ASan, TSan, MSan, UBSan).
|
||||
|
||||
## 线程模糊器 {#thread-fuzzer}
|
||||
|
||||
Thread Fuzzer(请不要与 Thread Sanitizer 混淆)是另一种允许随机化线程执行顺序的模糊测试. 它有助于找到更多特殊情况.
|
||||
|
||||
## 安全审计 {#security-audit}
|
||||
|
||||
Yandex安全团队的人员从安全的角度对ClickHouse的功能做了一些基本的概述.
|
||||
|
||||
## 静态分析仪 {#static-analyzers}
|
||||
|
||||
我们在每次提交的基础上运行 `clang-tidy` 和 `PVS-Studio`. `clang-static-analyzer` 检查也被启用. `clang-tidy` 也用于一些样式检查.
|
||||
|
||||
我们已经评估了 `clang-tidy`、`Coverity`、`cppcheck`、`PVS-Studio`、`tscancode`、`CodeQL`. 您将在 `tests/instructions/` 目录中找到使用说明. 你也可以阅读[俄文文章](https://habr.com/company/yandex/blog/342018/).
|
||||
|
||||
如果你使用 `CLion` 作为 IDE, 你可以利用一些开箱即用的 `clang-tidy` 检查
|
||||
|
||||
我们还使用 `shellcheck` 对shell脚本进行静态分析.
|
||||
|
||||
## 硬化 {#hardening}
|
||||
|
||||
在调试版本中, 我们使用自定义分配器执行用户级分配的 ASLR.
|
||||
|
||||
我们还手动保护在分配后预期为只读的内存区域.
|
||||
|
||||
在调试构建中, 我们还需要对libc进行自定义, 以确保不会调用 "有害的" (过时的、不安全的、非线程安全的)函数.
|
||||
|
||||
Debug 断言被广泛使用.
|
||||
|
||||
在调试版本中，如果抛出带有 "逻辑错误" 代码(暗示错误)的异常, 则程序会过早终止. 它允许在发布版本中使用异常, 但在调试版本中使其成为断言.
|
||||
|
||||
jemalloc 的调试版本用于调试版本.
|
||||
libc++ 的调试版本用于调试版本.
|
||||
|
||||
## 运行时完整性检查
|
||||
|
||||
对存储在磁盘上的数据是校验和. MergeTree 表中的数据同时以三种方式进行校验和*(压缩数据块、未压缩数据块、跨块的总校验和). 客户端和服务器之间或服务器之间通过网络传输的数据也会进行校验和. 复制确保副本上的数据位相同.
|
||||
|
||||
需要防止硬件故障(存储介质上的位腐烂、服务器上 RAM 中的位翻转、网络控制器 RAM 中的位翻转、网络交换机 RAM 中的位翻转、客户端 RAM 中的位翻转、线路上的位翻转). 请注意，比特位操作很常见, 即使对于 ECC RAM 和 TCP 校验和(如果您每天设法运行数千台处理 PB 数据的服务器, 也可能发生比特位操作. [观看视频(俄语)](https://www.youtube.com/watch?v=ooBAQIe0KlQ).
|
||||
|
||||
ClickHouse 提供诊断功能, 可帮助运维工程师找到故障硬件.
|
||||
|
||||
\* 它并不慢.
|
||||
|
||||
## 代码风格 {#code-style}
|
||||
|
||||
[此处](style.md)描述了代码样式规则.
|
||||
|
||||
要检查一些常见的样式违规，您可以使用 `utils/check-style` 脚本.
|
||||
|
||||
要强制使用正确的代码样式, 您可以使用 `clang-format`. 文件 `.clang-format` 位于源根目录. 它大多与我们的实际代码风格相对应. 但是不建议将 `clang-format` 应用于现有文件, 因为它会使格式变得更糟. 您可以使用可以在 clang 源代码库中找到的 `clang-format-diff` 工具.
|
||||
|
||||
或者, 您可以尝试使用 `uncrustify` 工具来重新格式化您的代码. 配置位于源根目录中的 `uncrustify.cfg` 中. 它比 `clang-format` 测试更少.
|
||||
|
||||
`CLion` 有自己的代码格式化程序, 必须根据我们的代码风格进行调整.
|
||||
|
||||
我们还使用 `codespell` 来查找代码中的拼写错误.它也是自动化的.
|
||||
|
||||
## Metrica B2B 测试 {#metrica-b2b-tests}
|
||||
|
||||
每个 ClickHouse 版本都使用 Yandex Metrica 和 AppMetrica 引擎进行测试. ClickHouse 的测试版和稳定版部署在 VM 上, 并使用 Metrica 引擎的小副本运行, 该引擎处理输入数据的固定样本. 然后将两个 Metrica 引擎实例的结果放在一起比较.
|
||||
|
||||
这些测试由单独的团队自动化. 由于移动部件数量众多, 测试在大多数情况下都因完全不相关的原因而失败, 这些原因很难弄清楚. 这些测试很可能对我们有负面价值. 尽管如此, 这些测试在数百次中被证明是有用的.
|
||||
|
||||
## 测试覆盖率 {#test-coverage}
|
||||
|
||||
我们还跟踪测试覆盖率, 但仅针对功能测试和 clickhouse-server. 它每天进行.
|
||||
|
||||
## Tests for Tests
|
||||
|
||||
有自动检测薄片测试. 它运行所有新测试100次(用于功能测试)或10次(用于集成测试). 如果至少有一次测试失败，它就被认为是脆弱的.
|
||||
|
||||
## Testflows
|
||||
|
||||
[Testflows](https://testflows.com/) 是一个企业级的测试框架. Altinity 使用它进行一些测试, 我们在 CI 中运行这些测试.
|
||||
|
||||
## Yandex 检查 (only for Yandex employees)
|
||||
|
||||
这些检查将ClickHouse代码导入到Yandex内部的单一存储库中, 所以ClickHouse代码库可以被Yandex的其他产品(YT和YDB)用作库. 请注意, clickhouse-server本身并不是由内部回购构建的, Yandex应用程序使用的是未经修改的开源构建的.
|
||||
|
||||
## 测试自动化 {#test-automation}
|
||||
|
||||
我们使用 Yandex 内部 CI 和名为 "Sandbox" 的作业自动化系统运行测试.
|
||||
|
||||
在每次提交的基础上, 构建作业和测试都在沙箱中运行. 生成的包和测试结果发布在GitHub上, 可以通过直接链接下载. 产物要保存几个月. 当你在GitHub上发送一个pull请求时, 我们会把它标记为 "可以测试" , 我们的CI系统会为你构建ClickHouse包(发布、调试、使用地址清理器等).
|
||||
|
||||
由于时间和计算能力的限制, 我们不使用 Travis CI.
|
||||
我们不用Jenkins. 以前用过, 现在我们很高兴不用Jenkins了.
|
||||
|
||||
[原始文章](https://clickhouse.com/docs/en/development/tests/) <!--hide-->
|
||||
@ -4,11 +4,53 @@ toc_title: WHERE
|
||||
|
||||
# WHERE {#select-where}
|
||||
|
||||
`WHERE` 子句允许过滤从 [FROM](../../../sql-reference/statements/select/from.md) 子句 `SELECT`.
|
||||
`WHERE` 子句允许过滤来自`SELECT`的子句 [FROM](../../../sql-reference/statements/select/from.md) 的数据.
|
||||
|
||||
如果有一个 `WHERE` 子句，它必须包含一个表达式与 `UInt8` 类型。 这通常是一个带有比较和逻辑运算符的表达式。 此表达式计算结果为0的行将从进一步的转换或结果中解释出来。
|
||||
如果有一个 `WHERE` 子句，它必须包含一个表达式与 `UInt8` 类型。 这通常是一个带有比较和逻辑运算符的表达式。 表达式计算结果为0的行将被排除在在进一步的转换或结果之外。
|
||||
|
||||
`WHERE` 如果基础表引擎支持，则根据使用索引和分区修剪的能力评估表达式。
|
||||
如果基础表引擎支持，`WHERE`表达式会使用索引和分区进行剪枝。
|
||||
|
||||
!!! note "注"
|
||||
有一个叫做过滤优化 [prewhere](../../../sql-reference/statements/select/prewhere.md) 的东西.
|
||||
|
||||
如果需要测试一个 [NULL](../../../sql-reference/syntax.md#null-literal) 值，请使用 [IS NULL](../../operators/index.md#operator-is-null) and [IS NOT NULL](../../operators/index.md#is-not-null) 运算符或 [isNull](../../../sql-reference/functions/functions-for-nulls.md#isnull) 和 [isNotNull](../../../sql-reference/functions/functions-for-nulls.md#isnotnull) 函数。否则带有 NULL 的表达式永远不会通过。
|
||||
|
||||
**示例**
|
||||
|
||||
在 [numbers table](../../../sql-reference/table-functions/numbers.md) 表上执行下述语句以找到为3的倍数且大于10的数字：
|
||||
To find numbers that are multiples of 3 and are greater than 10 execute the following query on the :
|
||||
|
||||
``` sql
|
||||
SELECT number FROM numbers(20) WHERE (number > 10) AND (number % 3 == 0);
|
||||
```
|
||||
|
||||
结果:
|
||||
|
||||
``` text
|
||||
┌─number─┐
|
||||
│ 12 │
|
||||
│ 15 │
|
||||
│ 18 │
|
||||
└────────┘
|
||||
```
|
||||
|
||||
带有 `NULL` 值的查询:
|
||||
|
||||
``` sql
|
||||
CREATE TABLE t_null(x Int8, y Nullable(Int8)) ENGINE=MergeTree() ORDER BY x;
|
||||
INSERT INTO t_null VALUES (1, NULL), (2, 3);
|
||||
|
||||
SELECT * FROM t_null WHERE y IS NULL;
|
||||
SELECT * FROM t_null WHERE y != 0;
|
||||
```
|
||||
|
||||
结果:
|
||||
|
||||
``` text
|
||||
┌─x─┬────y─┐
|
||||
│ 1 │ ᴺᵁᴸᴸ │
|
||||
└───┴──────┘
|
||||
┌─x─┬─y─┐
|
||||
│ 2 │ 3 │
|
||||
└───┴───┘
|
||||
```
|
||||
|
||||
@ -1 +0,0 @@
|
||||
../../../en/whats-new/changelog/2017.md
|
||||
261
docs/zh/whats-new/changelog/2017.md
Normal file
261
docs/zh/whats-new/changelog/2017.md
Normal file
@ -0,0 +1,261 @@
|
||||
### ClickHouse 版本 1.1.54327, 2017-12-21 {#clickhouse-release-1-1-54327-2017-12-21}
|
||||
|
||||
此版本包含先前版本 1.1.54318 的错误修复:
|
||||
|
||||
- 修复了可能导致数据丢失的复制中可能出现的竞争条件的错误. 此问题影响1.1.54310和1.1.54318版本. 如果将这些版本的任意一个与 Replicated 表一起使用，则强烈建议进行更新. 此问题显示在日志中的警告消息中，例如 `Part ... from own log does not exist.` 即使您没有在日志中看到这些消息，该问题也是相关的.
|
||||
|
||||
### ClickHouse 版本 1.1.54318, 2017-11-30 {#clickhouse-release-1-1-54318-2017-11-30}
|
||||
|
||||
此版本包含先前版本 1.1.54310 的错误修复:
|
||||
|
||||
- 修复了在 SummingMergeTree 引擎中合并期间不正确的行删除.
|
||||
- 修复了未复制的 MergeTree 引擎中的内存泄漏.
|
||||
- 修复了在 MergeTree 引擎中频繁插入导致性能下降的问题.
|
||||
- 修复了导致复制队列停止运行的问题.
|
||||
- 修复了服务器日志的轮换和归档.
|
||||
|
||||
### ClickHouse 版本 1.1.54310, 2017-11-01 {#clickhouse-release-1-1-54310-2017-11-01}
|
||||
|
||||
#### 新特征: {#new-features}
|
||||
|
||||
- MergeTree 系列表引擎的自定义分区键.
|
||||
- [Kafka](https://clickhouse.com/docs/en/operations/table_engines/kafka/) 表引擎.
|
||||
- 增加了对加载[CatBoost](https://catboost.yandex/)模型的支持, 并将它们应用到存储在ClickHouse中的数据.
|
||||
- 添加了对 UTC 非整数偏移时区的支持
|
||||
- 添加了对具有时间间隔的算术运算的支持.
|
||||
- Date 和 DateTime 类型的值范围扩展到 2105 年.
|
||||
- 添加了 `CREATE MATERIALIZED VIEW x TO y` 查询(指定用于存储物化视图数据的现有表).
|
||||
- 添加了不带参数的`ATTACH TABLE` 查询.
|
||||
- SummingMergeTree 表中名称以 -Map 结尾的嵌套列的处理逻辑被提取到 sumMap 聚合函数中. 您现在可以明确指定此类列.
|
||||
- IP 树字典的最大大小增加到 128M 条目.
|
||||
- 添加了 getSizeOfEnumType 函数.
|
||||
- 添加了 sumWithOverflow 聚合函数.
|
||||
- 添加了对 Cap'n Proto 输入格式的支持.
|
||||
- 您现在可以在使用 zstd 算法时自定义压缩级别.
|
||||
|
||||
#### 向后不兼容的变化: {#backward-incompatible-changes}
|
||||
|
||||
- 不允许使用内存以外的引擎创建临时表.
|
||||
- 不允许使用 View 或 MaterializedView 引擎显式创建表.
|
||||
- 在表创建期间，新的检查验证采样键表达式是否包含在主键中.
|
||||
|
||||
#### Bug 修复: {#bug-fixes}
|
||||
|
||||
- 修复了同步插入分布式表时的挂断问题.
|
||||
- 修复了复制表中部件的非原子添加和删除.
|
||||
- 插入物化视图的数据不会进行不必要的重复数据删除.
|
||||
- 对本地副本滞后且远程副本不可用的分布式表执行查询不再导致错误.
|
||||
- 用户不再需要访问 `default` 数据库的权限来创建临时表.
|
||||
- 修复了指定不带参数的 Array 类型时崩溃的问题.
|
||||
- 修复了包含服务器日志的磁盘卷已满时的挂断问题.
|
||||
- 修复了 Unix 纪元第一周 toRelativeWeekNum 函数中的溢出问题.
|
||||
|
||||
#### Build 改进: {#build-improvements}
|
||||
|
||||
- 更新了多个第三方库(尤其是 Poco)并转换为 git 子模块.
|
||||
|
||||
### ClickHouse 版本 1.1.54304, 2017-10-19 {#clickhouse-release-1-1-54304-2017-10-19}
|
||||
|
||||
#### 新特征: {#new-features-1}
|
||||
|
||||
- 本机协议中的 TLS 支持(要启用，请在 `config.xml` 中设置 `tcp_ssl_port`).
|
||||
|
||||
#### Bug 修复: {#bug-fixes-1}
|
||||
|
||||
- 复制表的`ALTER` 现在尝试尽快开始运行.
|
||||
- 修复了使用设置 `preferred_block_size_bytes=0.` 读取数据时崩溃的问题.
|
||||
- 修复了按下 `Page Down` 时 `clickhouse-client` 崩溃的问题.
|
||||
- 使用 `GLOBAL IN` 和 `UNION ALL` 正确解释某些复杂的查询.
|
||||
- `FREEZE PARTITION` 现在总是以原子方式工作.
|
||||
- 空 POST 请求现在返回代码为 411 的响应.
|
||||
- 修正了像 `CAST(1 AS Nullable(UInt8)).` 这样的表达式的解释错误.
|
||||
- 修正了从 `MergeTree` 表中读取 `Array(Nullable(String))` 列时的错误.
|
||||
- 修复了在解析诸如 `SELECT dummy AS dummy, dummy AS b` 之类的查询时崩溃的问题.
|
||||
- 用户使用无效的 `users.xml` 正确更新.
|
||||
- 可执行字典返回非零响应代码时的正确处理.
|
||||
|
||||
### ClickHouse 版本 1.1.54292, 2017-09-20 {#clickhouse-release-1-1-54292-2017-09-20}
|
||||
|
||||
#### 新特征: {#new-features-2}
|
||||
|
||||
- 添加了用于处理坐标平面上的坐标的 `pointInPolygon` 函数.
|
||||
- 添加了用于计算数组总和的 `sumMap` 聚合函数, 类似于 `SummingMergeTree` .
|
||||
- 添加了 `trunc` 功能. 改进了舍入函数(`round`、`floor`、`ceil`、`roundToExp2`)的性能并更正了它们工作方式的逻辑. 更改了分数和负数的 `roundToExp2` 函数的逻辑.
|
||||
- ClickHouse 可执行文件现在较少依赖于 libc 版本. 同一个 ClickHouse 可执行文件可以在各种 Linux 系统上运行. 使用编译查询时仍然存在依赖性(使用设置 `compile = 1` , 默认情况下不使用).
|
||||
- 减少动态编译查询所需的时间.
|
||||
|
||||
#### Bug 修复: {#bug-fixes-2}
|
||||
|
||||
- 修复了有时会产生 `part ... intersects previous part` 消息和削弱副本一致性的错误.
|
||||
- 修复了关闭期间 ZooKeeper 不可用导致服务器锁定的错误.
|
||||
- 恢复副本时删除了过多的日志记录.
|
||||
- 修复了 UNION ALL 实现中的错误.
|
||||
- 修复了如果块中的第一列具有 Array 类型时在 concat 函数中发生的错误.
|
||||
- 进度现在在 system.merges 表中可以正确显示.
|
||||
|
||||
### ClickHouse 版本 1.1.54289, 2017-09-13 {#clickhouse-release-1-1-54289-2017-09-13}
|
||||
|
||||
#### 新特征: {#new-features-3}
|
||||
|
||||
- 用于服务器管理的 `SYSTEM` 查询: `SYSTEM RELOAD DICTIONARY`, `SYSTEM RELOAD DICTIONARIES`, `SYSTEM DROP DNS CACHE`, `SYSTEM SHUTDOWN`, `SYSTEM KILL`.
|
||||
- 添加了用于处理数组的函数: `concat`, `arraySlice`, `arrayPushBack`, `arrayPushFront`, `arrayPopBack`, `arrayPopFront`.
|
||||
- 为 ZooKeeper 配置添加了 `root` 和 `identity` 参数. 这将允许您隔离同一 ZooKeeper 集群上的各个用户.
|
||||
- 添加了聚合函数 `groupBitAnd` 、 `groupBitOr` 和 `groupBitXor` (为了兼容性，它们也可以在名称 `BIT_AND` 、`BIT_OR`和`BIT_XOR` 下使用).
|
||||
- 可以通过在文件系统中指定套接字来从 MySQL 加载外部字典.
|
||||
- 可以通过 SSL 从 MySQL 加载外部字典 (`ssl_cert`, `ssl_key`, `ssl_ca` 参数).
|
||||
- 添加了 `max_network_bandwidth_for_user` 设置以限制每个用户查询的总体带宽使用.
|
||||
- 支持临时表的 `DROP TABLE`.
|
||||
- 支持从 `CSV` 和 `JSONEachRow` 格式读取 Unix 时间戳格式的 `DateTime` 值.
|
||||
- 现在默认排除分布式查询中的滞后副本(默认阈值为 5 分钟).
|
||||
- 在 ALTER 期间使用 FIFO 锁定：对于连续运行的查询，ALTER 查询不会无限期阻塞.
|
||||
- 在配置文件中设置 `umask` 的选项.
|
||||
- 使用 `DISTINCT` 提高查询的性能.
|
||||
|
||||
#### Bug 修复: {#bug-fixes-3}
|
||||
|
||||
- 改进了在 ZooKeeper 中删除旧节点的过程. 以前, 如果插入非常频繁, 旧节点有时不会被删除, 从而导致服务器关闭缓慢等.
|
||||
- 修复了为 ZooKeeper 连接选择主机时的随机化问题.
|
||||
- 如果副本是本地主机, 则修复了在分布式查询中排除滞后副本的问题.
|
||||
- 修复了在 `嵌套` 结构中的元素上运行 `ALTER MODIFY` 后, `ReplicatedMergeTree` 表中的数据部分可能被破坏的错误.
|
||||
- 修复了可能导致 SELECT 查询 `hang` 的错误.
|
||||
- 分布式 DDL 查询的改进.
|
||||
- 修复了查询 `CREATE TABLE ... AS <materialized view>`.
|
||||
- 解决了对 Buffer 表的 `ALTER ... CLEAR COLUMN IN PARTITION` 查询中的死锁.
|
||||
- 修复了使用 `JSONEachRow` 和 `TSKV` 格式时 `Enum` 的无效默认值 (0 而不是最小值).
|
||||
- 解决了使用带有 `可执行` 源的字典时出现僵尸进程的问题.
|
||||
- 修复了 HEAD 查询的段错误.
|
||||
|
||||
#### 改进了开发和组装ClickHouse的工作流: {#improved-workflow-for-developing-and-assembling-clickhouse}
|
||||
|
||||
- 您可以使用 `pbuilder` 来构建 ClickHouse.
|
||||
- 你可以使用 `libc++` 代替 `libstdc++` 在 Linux 上构建.
|
||||
- 添加了使用静态代码分析工具的说明: `Coverage`, `clang-tidy`, `cppcheck`.
|
||||
|
||||
#### 升级时请注意: {#please-note-when-upgrading}
|
||||
|
||||
- 现在有更高的 MergeTree 设置默认值 `max_bytes_to_merge_at_max_space_in_pool` (要合并的数据部分的最大总大小, 以字节为单位): 它已从 100 GiB 增加到 150 GiB. 这可能会导致在服务器升级后运行大型合并, 从而导致磁盘子系统负载增加. 如果服务器上的可用空间小于正在运行的合并总量的两倍, 这将导致所有其他合并停止运行, 包括小数据部分的合并. 因此, INSERT 查询将失败并显示消息"合并的处理速度明显慢于插入." , 使用 `SELECT * FROM system.merges` 查询来监控情况. 您还可以在 `system.metrics` 表或 Graphite 中检查 `DiskSpaceReservedForMerge` 指标. 您不需要做任何事情来解决这个问题, 因为一旦大型合并完成, 问题就会自行解决. 如果您发现这不可接受, 您可以恢复 `max_bytes_to_merge_at_max_space_in_pool` 设置的先前值. 为此, 请转到 config.xml 中的 `<merge_tree>` 部分, 设置 ``` <merge_tree>``<max_bytes_to_merge_at_max_space_in_pool>107374182400</max_bytes_to_merge_at_max_space_in_pool> ``` 并重新启动服务器.
|
||||
|
||||
### ClickHouse 版本 1.1.54284, 2017-08-29 {#clickhouse-release-1-1-54284-2017-08-29}
|
||||
|
||||
- 这是先前 1.1.54282 版本的错误修复版本. 它修复了 ZooKeeper 中部分目录中的泄漏.
|
||||
|
||||
### ClickHouse 版本 1.1.54282, 2017-08-23 {#clickhouse-release-1-1-54282-2017-08-23}
|
||||
|
||||
此版本包含先前版本 1.1.54276 的错误修复:
|
||||
|
||||
- 修复了插入分布式表时的 `DB::Exception: Assertion violation: !_path.empty()`.
|
||||
- 如果输入数据以 ';' 开头, 则在以 RowBinary 格式插入时固定解析.
|
||||
- 某些聚合函数 (例如 `groupArray()` ) 的运行时编译期间的错误.
|
||||
|
||||
### ClickHouse 版本 1.1.54276, 2017-08-16 {#clickhouse-release-1-1-54276-2017-08-16}
|
||||
|
||||
#### 新特征: {#new-features-4}
|
||||
|
||||
- SELECT 查询添加了一个可选的 WITH 部分. 示例查询：`WITH 1+1 AS a SELECT a, a*a` .
|
||||
- NSERT 可以在分布式表中同步执行：只有在所有数据都保存在所有分片上后才返回 OK. 这是通过设置 `insert_distributed_sync=1` 激活的.
|
||||
- 添加了用于处理 16 字节标识符的 UUID 数据类型.
|
||||
- 添加了 CHAR、FLOAT 和其他类型的别名以与 Tableau 兼容.
|
||||
- 新增 toYYYYMM, toYYYYMMDD, toYYYYMMDDhhmmss 时间转数字功能.
|
||||
- 您可以使用 IP 地址 (与主机名一起) 来识别集群 DDL 查询的服务器.
|
||||
- 在函数 `substring(str, pos, len)` 中添加了对非常量参数和负偏移量的支持.
|
||||
- 为 `groupArray(max_size)(column)` 聚合函数增加了max_size参数, 并优化了其性能.
|
||||
|
||||
#### Main Changes: {#main-changes}
|
||||
|
||||
- 安全改进：所有服务器文件都使用 0640 权限创建(可以通过 `<umask>` 配置参数更改).
|
||||
- 改进了语法无效查询的错误消息.
|
||||
- 合并大段 MergeTree 数据时显着减少内存消耗并提高性能.
|
||||
- 显着提高了 ReplacingMergeTree 引擎的数据合并性能.
|
||||
- 通过组合多个源插入提高了从分布式表进行异步插入的性能. 要启用此功能, 请使用设置 `distributed_directory_monitor_batch_inserts=1` .
|
||||
|
||||
#### Backward Incompatible Changes: {#backward-incompatible-changes-1}
|
||||
|
||||
- 更改了数组 `groupArray(array_column)` 函数聚合状态的二进制格式.
|
||||
|
||||
#### Complete List of Changes: {#complete-list-of-changes}
|
||||
|
||||
- 添加了 `output_format_json_quote_denormals` 设置, 可以以 JSON 格式输出 nan 和 inf 值.
|
||||
- 从分布式表读取时优化流分配.
|
||||
- 如果值不变, 可以在只读模式下配置设置.
|
||||
- 加了检索 MergeTree 引擎的非整数粒度的功能, 以满足对 `preferred_block_size_bytes` 设置中指定的块大小的限制. 目的是在处理来自大列的表的查询时减少RAM的消耗并增加缓存局部性.
|
||||
- 有效地使用包含像 `toStartOfHour(x)` 这样的表达式的索引来处理像 `toStartOfHour(x) op сonstexpr` 这样的条件.
|
||||
- 添加了 MergeTree 引擎的新设置(config.xml 中的 merge_tree 部分):
|
||||
- `replicad_deduplication_window_seconds` 设置允许在复制表中删除重复插入的秒数.
|
||||
- `cleanup_delay_period` 设置启动清理以删除过时数据的频率.
|
||||
- `Replicationd_can_become_leader` 可以防止副本成为领导者(并分配合并).
|
||||
- 加速清理以从 ZooKeeper 中删除过时的数据.
|
||||
- 集群 DDL 查询的多项改进和修复. 特别有趣的是新设置 `distributed_ddl_task_timeout`, 它限制了等待集群中服务器响应的时间. 如果 ddl 请求没有在所有主机上执行，响应将包含超时错误并且请求将以异步模式执行.
|
||||
- 改进了服务器日志中堆栈跟踪的显示.
|
||||
- 为压缩方法添加了 "none" 值.
|
||||
- 您可以在 config.xml 中使用多个dictionaries_config 部分.
|
||||
- 可以通过文件系统中的套接字连接到 MySQL.
|
||||
- `system.parts` 表有一个新列, 其中包含有关标记大小的信息(以字节为单位).
|
||||
|
||||
#### Bug 修复: {#bug-fixes-4}
|
||||
|
||||
- 使用 Merge 表的分布式表现在可以正确用于带有 `_table` 字段条件的 SELECT 查询.
|
||||
- 修复了检查数据部分时 ReplicatedMergeTree 中罕见的竞争条件.
|
||||
- 修复了启动服务器时 `leader election` 可能会冻结的问题.
|
||||
- 使用数据源的本地副本时，将忽略 `max_replica_delay_for_distributed_queries` 设置. 这已被修复.
|
||||
- 修复了尝试清理不存在的列时 `ALTER TABLE CLEAR COLUMN IN PARTITION` 的错误行为.
|
||||
- 修复了 multiIf 函数中使用空数组或字符串时的异常.
|
||||
- 修复了反序列化本机格式时过多的内存分配.
|
||||
- 修复了 Trie 词典的错误自动更新.
|
||||
- 修复了在使用 SAMPLE 时从合并表中使用 GROUP BY 子句运行查询时的异常.
|
||||
- 修复了 `distributed_aggregation_memory_efficient=1` 时 GROUP BY 的崩溃.
|
||||
- 现在可以在 IN 和 JOIN 右侧指定 `database.table`.
|
||||
- 太多线程用于并行聚合. 这已被修复.
|
||||
- 修复了 `if` 函数如何与 FixedString 参数一起工作.
|
||||
- 对于权重为 0 的分片, SELECT 在分布式表中工作不正确. 这已得到修复.
|
||||
- 运行 `CREATE VIEW IF EXISTS 不再导致崩溃` .
|
||||
- 修复了设置 `input_format_skip_unknown_fields=1` 且存在负数时的错误行为.
|
||||
- 修复了如果字典中有一些无效数据, `dictGetHierarchy()` 函数中的无限循环.
|
||||
- 修复了使用 IN 或 JOIN 子句和合并表中的子查询运行分布式查询时的 `Syntax error: unexpected (...)` 错误.
|
||||
- 修复了对字典表中 SELECT 查询的错误解释.
|
||||
- 修复了在超过 20 亿元素的 IN 和 JOIN 子句中使用数组时的 "Cannot mremap" 错误.
|
||||
- 修复了以 MySQL 为源的字典的故障转移.
|
||||
|
||||
#### 改进了开发和组装ClickHouse的工作流: {#improved-workflow-for-developing-and-assembling-clickhouse-1}
|
||||
|
||||
- 可以在 Arcadia 中组装 Builds.
|
||||
- 可以使用 gcc 7 编译 ClickHouse.
|
||||
- 使用 ccache+distcc 的并行构建现在更快了.
|
||||
|
||||
### ClickHouse 版本 1.1.54245, 2017-07-04 {#clickhouse-release-1-1-54245-2017-07-04}
|
||||
|
||||
#### 新特征: {#new-features-5}
|
||||
|
||||
- 分布式 DDL(例如, `REATE TABLE ON CLUSTER`).
|
||||
- 复制查询 `ALTER TABLE CLEAR COLUMN IN PARTITION.` .
|
||||
- 字典表引擎 (以表的形式访问字典数据).
|
||||
- 字典数据库引擎 (这种类型的数据库自动为所有连接的外部字典提供字典表).
|
||||
- 您可以通过向源发送请求来检查字典的更新.
|
||||
- 合格的列名.
|
||||
- 使用双引号引用标识符.
|
||||
- HTTP 接口中的会话.
|
||||
- 复制表的 OPTIMIZE 查询不仅可以在领导者上运行.
|
||||
|
||||
#### 向后不兼容的变化: {#backward-incompatible-changes-2}
|
||||
|
||||
- 删除了 SET GLOBAL.
|
||||
|
||||
#### 次要更改: {#minor-changes}
|
||||
|
||||
- 目前在触发警报后，日志会打印完整的堆栈跟踪.
|
||||
- 放宽了启动时损坏/额外数据部分数量的验证(误报太多).
|
||||
|
||||
#### Bug 修复: {#bug-fixes-5}
|
||||
|
||||
- 修复了插入分布式表时的错误连接"卡住"问题.
|
||||
- GLOBAL IN 现在适用于来自查看分布式表的合并表的查询.
|
||||
- 在 Google Compute Engine 虚拟机上检测到的内核数不正确. 这已被修复.
|
||||
- 缓存外部字典的可执行源的工作方式发生了变化.
|
||||
- 修正了包含空字符的字符串的比较.
|
||||
- 修复了 Float32 主键字段与常量的比较.
|
||||
- 以前，对字段大小的错误估计可能会导致分配过大.
|
||||
- 修复了使用 ALTER 查询添加到表中的 Nullable 列时发生的崩溃.
|
||||
- 修复了当行数小于 LIMIT 时按 Nullable 列排序时崩溃的问题.
|
||||
- 修复了仅由常量值组成的 ORDER BY 子查询.
|
||||
- 以前，复制表在 DROP TABLE 失败后可能保持无效状态.
|
||||
- 结果为空的标量子查询的别名不再丢失.
|
||||
- 现在，如果 .so 文件损坏，使用编译的查询不会因错误而失败.
|
||||
@ -1 +0,0 @@
|
||||
../../../en/whats-new/changelog/2018.md
|
||||
1056
docs/zh/whats-new/changelog/2018.md
Normal file
1056
docs/zh/whats-new/changelog/2018.md
Normal file
File diff suppressed because it is too large
Load Diff
@ -1 +0,0 @@
|
||||
../../../en/whats-new/changelog/2019.md
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user