mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-26 09:32:01 +00:00
Merge remote-tracking branch 'ClickHouse/master' into docs-editing-query-cache-settings
This commit is contained in:
Robert Schulze
commit
a3487c9c47
6
.github/ISSUE_TEMPLATE/85_bug-report.md
vendored
6
.github/ISSUE_TEMPLATE/85_bug-report.md
vendored
@ -17,7 +17,7 @@ assignees: ''
|
||||
|
||||
> A link to reproducer in [https://fiddle.clickhouse.com/](https://fiddle.clickhouse.com/).
|
||||
|
||||
**Does it reproduce on recent release?**
|
||||
**Does it reproduce on the most recent release?**
|
||||
|
||||
[The list of releases](https://github.com/ClickHouse/ClickHouse/blob/master/utils/list-versions/version_date.tsv)
|
||||
|
||||
@ -34,11 +34,11 @@ assignees: ''
|
||||
**How to reproduce**
|
||||
|
||||
* Which ClickHouse server version to use
|
||||
* Which interface to use, if matters
|
||||
* Which interface to use, if it matters
|
||||
* Non-default settings, if any
|
||||
* `CREATE TABLE` statements for all tables involved
|
||||
* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
|
||||
* Queries to run that lead to unexpected result
|
||||
* Queries to run that lead to an unexpected result
|
||||
|
||||
**Expected behavior**
|
||||
|
||||
|
||||
15
.github/workflows/backport_branches.yml
vendored
15
.github/workflows/backport_branches.yml
vendored
@ -138,19 +138,26 @@ jobs:
|
||||
############################################################################################
|
||||
##################################### Docker images #######################################
|
||||
############################################################################################
|
||||
DockerServerImages:
|
||||
DockerServerImage:
|
||||
needs: [RunConfig, BuilderDebRelease, BuilderDebAarch64]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Docker server and keeper images
|
||||
test_name: Docker server image
|
||||
runner_type: style-checker
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
checkout_depth: 0 # It MUST BE THE SAME for all dependencies and the job itself
|
||||
run_command: |
|
||||
cd "$GITHUB_WORKSPACE/tests/ci"
|
||||
python3 docker_server.py --release-type head --no-push \
|
||||
--image-repo clickhouse/clickhouse-server --image-path docker/server --allow-build-reuse
|
||||
DockerKeeperImage:
|
||||
needs: [RunConfig, BuilderDebRelease, BuilderDebAarch64]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Docker keeper image
|
||||
runner_type: style-checker
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
run_command: |
|
||||
python3 docker_server.py --release-type head --no-push \
|
||||
--image-repo clickhouse/clickhouse-keeper --image-path docker/keeper --allow-build-reuse
|
||||
############################################################################################
|
||||
|
||||
2
.github/workflows/jepsen.yml
vendored
2
.github/workflows/jepsen.yml
vendored
@ -8,13 +8,13 @@ on: # yamllint disable-line rule:truthy
|
||||
schedule:
|
||||
- cron: '0 */6 * * *'
|
||||
workflow_dispatch:
|
||||
workflow_call:
|
||||
jobs:
|
||||
KeeperJepsenRelease:
|
||||
uses: ./.github/workflows/reusable_simple_job.yml
|
||||
with:
|
||||
test_name: Jepsen keeper check
|
||||
runner_type: style-checker
|
||||
report_required: true
|
||||
run_command: |
|
||||
python3 jepsen_check.py keeper
|
||||
# ServerJepsenRelease:
|
||||
|
||||
138
.github/workflows/master.yml
vendored
138
.github/workflows/master.yml
vendored
@ -15,6 +15,8 @@ jobs:
|
||||
outputs:
|
||||
data: ${{ steps.runconfig.outputs.CI_DATA }}
|
||||
steps:
|
||||
- name: DebugInfo
|
||||
uses: hmarr/debug-action@a701ed95a46e6f2fb0df25e1a558c16356fae35a
|
||||
- name: Check out repository code
|
||||
uses: ClickHouse/checkout@v1
|
||||
with:
|
||||
@ -33,11 +35,9 @@ jobs:
|
||||
- name: PrepareRunConfig
|
||||
id: runconfig
|
||||
run: |
|
||||
echo "::group::configure CI run"
|
||||
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --rebuild-all-binaries --outfile ${{ runner.temp }}/ci_run_data.json
|
||||
echo "::endgroup::"
|
||||
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --outfile ${{ runner.temp }}/ci_run_data.json
|
||||
|
||||
echo "::group::CI run configure results"
|
||||
echo "::group::CI configuration"
|
||||
python3 -m json.tool ${{ runner.temp }}/ci_run_data.json
|
||||
echo "::endgroup::"
|
||||
|
||||
@ -55,7 +55,6 @@ jobs:
|
||||
uses: ./.github/workflows/reusable_docker.yml
|
||||
with:
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
set_latest: true
|
||||
StyleCheck:
|
||||
needs: [RunConfig, BuildDockers]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
@ -242,22 +241,28 @@ jobs:
|
||||
############################################################################################
|
||||
##################################### Docker images #######################################
|
||||
############################################################################################
|
||||
DockerServerImages:
|
||||
DockerServerImage:
|
||||
needs: [RunConfig, BuilderDebRelease, BuilderDebAarch64]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Docker server and keeper images
|
||||
test_name: Docker server image
|
||||
runner_type: style-checker
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
# FIXME: avoid using 0 checkout
|
||||
checkout_depth: 0 # It MUST BE THE SAME for all dependencies and the job itself
|
||||
run_command: |
|
||||
cd "$GITHUB_WORKSPACE/tests/ci"
|
||||
python3 docker_server.py --release-type head \
|
||||
--image-repo clickhouse/clickhouse-server --image-path docker/server
|
||||
--image-repo clickhouse/clickhouse-server --image-path docker/server --allow-build-reuse
|
||||
DockerKeeperImage:
|
||||
needs: [RunConfig, BuilderDebRelease, BuilderDebAarch64]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Docker keeper image
|
||||
runner_type: style-checker
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
run_command: |
|
||||
python3 docker_server.py --release-type head \
|
||||
--image-repo clickhouse/clickhouse-keeper --image-path docker/keeper
|
||||
--image-repo clickhouse/clickhouse-keeper --image-path docker/keeper --allow-build-reuse
|
||||
############################################################################################
|
||||
##################################### BUILD REPORTER #######################################
|
||||
############################################################################################
|
||||
@ -313,7 +318,6 @@ jobs:
|
||||
run_command: |
|
||||
python3 build_report_check.py "$CHECK_NAME"
|
||||
MarkReleaseReady:
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
needs:
|
||||
- BuilderBinDarwin
|
||||
- BuilderBinDarwinAarch64
|
||||
@ -323,8 +327,6 @@ jobs:
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
uses: ClickHouse/checkout@v1
|
||||
with:
|
||||
clear-repository: true
|
||||
- name: Mark Commit Release Ready
|
||||
run: |
|
||||
cd "$GITHUB_WORKSPACE/tests/ci"
|
||||
@ -363,14 +365,6 @@ jobs:
|
||||
test_name: Stateless tests (release)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FunctionalStatelessTestReleaseDatabaseOrdinary:
|
||||
needs: [RunConfig, BuilderDebRelease]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateless tests (release, DatabaseOrdinary)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FunctionalStatelessTestReleaseDatabaseReplicated:
|
||||
needs: [RunConfig, BuilderDebRelease]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
@ -395,6 +389,22 @@ jobs:
|
||||
test_name: Stateless tests (release, s3 storage)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FunctionalStatelessTestS3Debug:
|
||||
needs: [RunConfig, BuilderDebDebug]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateless tests (debug, s3 storage)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FunctionalStatelessTestS3Tsan:
|
||||
needs: [RunConfig, BuilderDebTsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateless tests (tsan, s3 storage)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FunctionalStatelessTestAarch64:
|
||||
needs: [RunConfig, BuilderDebAarch64]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
@ -503,6 +513,55 @@ jobs:
|
||||
test_name: Stateful tests (debug)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
# Parallel replicas
|
||||
FunctionalStatefulTestDebugParallelReplicas:
|
||||
needs: [RunConfig, BuilderDebDebug]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateful tests (debug, ParallelReplicas)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FunctionalStatefulTestUBsanParallelReplicas:
|
||||
needs: [RunConfig, BuilderDebUBsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateful tests (ubsan, ParallelReplicas)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FunctionalStatefulTestMsanParallelReplicas:
|
||||
needs: [RunConfig, BuilderDebMsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateful tests (msan, ParallelReplicas)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FunctionalStatefulTestTsanParallelReplicas:
|
||||
needs: [RunConfig, BuilderDebTsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateful tests (tsan, ParallelReplicas)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FunctionalStatefulTestAsanParallelReplicas:
|
||||
needs: [RunConfig, BuilderDebAsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateful tests (asan, ParallelReplicas)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FunctionalStatefulTestReleaseParallelReplicas:
|
||||
needs: [RunConfig, BuilderDebRelease]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateful tests (release, ParallelReplicas)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
##############################################################################################
|
||||
########################### ClickBench #######################################################
|
||||
##############################################################################################
|
||||
@ -710,6 +769,28 @@ jobs:
|
||||
runner_type: func-tester-aarch64
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
##############################################################################################
|
||||
############################ SQLLOGIC TEST ###################################################
|
||||
##############################################################################################
|
||||
SQLLogicTestRelease:
|
||||
needs: [RunConfig, BuilderDebRelease]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Sqllogic test (release)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
##############################################################################################
|
||||
##################################### SQL TEST ###############################################
|
||||
##############################################################################################
|
||||
SQLTest:
|
||||
needs: [RunConfig, BuilderDebRelease]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: SQLTest
|
||||
runner_type: fuzzer-unit-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
##############################################################################################
|
||||
###################################### SQLANCER FUZZERS ######################################
|
||||
##############################################################################################
|
||||
SQLancerTestRelease:
|
||||
@ -734,7 +815,6 @@ jobs:
|
||||
- MarkReleaseReady
|
||||
- FunctionalStatelessTestDebug
|
||||
- FunctionalStatelessTestRelease
|
||||
- FunctionalStatelessTestReleaseDatabaseOrdinary
|
||||
- FunctionalStatelessTestReleaseDatabaseReplicated
|
||||
- FunctionalStatelessTestReleaseAnalyzer
|
||||
- FunctionalStatelessTestReleaseS3
|
||||
@ -743,6 +823,8 @@ jobs:
|
||||
- FunctionalStatelessTestTsan
|
||||
- FunctionalStatelessTestMsan
|
||||
- FunctionalStatelessTestUBsan
|
||||
- FunctionalStatelessTestS3Debug
|
||||
- FunctionalStatelessTestS3Tsan
|
||||
- FunctionalStatefulTestDebug
|
||||
- FunctionalStatefulTestRelease
|
||||
- FunctionalStatefulTestAarch64
|
||||
@ -750,6 +832,12 @@ jobs:
|
||||
- FunctionalStatefulTestTsan
|
||||
- FunctionalStatefulTestMsan
|
||||
- FunctionalStatefulTestUBsan
|
||||
- FunctionalStatefulTestDebugParallelReplicas
|
||||
- FunctionalStatefulTestUBsanParallelReplicas
|
||||
- FunctionalStatefulTestMsanParallelReplicas
|
||||
- FunctionalStatefulTestTsanParallelReplicas
|
||||
- FunctionalStatefulTestAsanParallelReplicas
|
||||
- FunctionalStatefulTestReleaseParallelReplicas
|
||||
- StressTestDebug
|
||||
- StressTestAsan
|
||||
- StressTestTsan
|
||||
@ -775,6 +863,8 @@ jobs:
|
||||
- UnitTestsReleaseClang
|
||||
- SQLancerTestRelease
|
||||
- SQLancerTestDebug
|
||||
- SQLLogicTestRelease
|
||||
- SQLTest
|
||||
runs-on: [self-hosted, style-checker]
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
|
||||
2
.github/workflows/nightly.yml
vendored
2
.github/workflows/nightly.yml
vendored
@ -28,7 +28,7 @@ jobs:
|
||||
id: runconfig
|
||||
run: |
|
||||
echo "::group::configure CI run"
|
||||
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --skip-jobs --rebuild-all-docker --outfile ${{ runner.temp }}/ci_run_data.json
|
||||
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --skip-jobs --outfile ${{ runner.temp }}/ci_run_data.json
|
||||
echo "::endgroup::"
|
||||
|
||||
echo "::group::CI run configure results"
|
||||
|
||||
61
.github/workflows/pull_request.yml
vendored
61
.github/workflows/pull_request.yml
vendored
@ -22,6 +22,8 @@ jobs:
|
||||
outputs:
|
||||
data: ${{ steps.runconfig.outputs.CI_DATA }}
|
||||
steps:
|
||||
- name: DebugInfo
|
||||
uses: hmarr/debug-action@a701ed95a46e6f2fb0df25e1a558c16356fae35a
|
||||
- name: Check out repository code
|
||||
uses: ClickHouse/checkout@v1
|
||||
with:
|
||||
@ -44,11 +46,9 @@ jobs:
|
||||
- name: PrepareRunConfig
|
||||
id: runconfig
|
||||
run: |
|
||||
echo "::group::configure CI run"
|
||||
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --outfile ${{ runner.temp }}/ci_run_data.json
|
||||
echo "::endgroup::"
|
||||
|
||||
echo "::group::CI run configure results"
|
||||
echo "::group::CI configuration"
|
||||
python3 -m json.tool ${{ runner.temp }}/ci_run_data.json
|
||||
echo "::endgroup::"
|
||||
|
||||
@ -67,6 +67,7 @@ jobs:
|
||||
DOCKER_TAG=$(echo '${{ toJson(fromJson(steps.runconfig.outputs.CI_DATA).docker_data.images) }}' | tr -d '\n')
|
||||
export DOCKER_TAG=$DOCKER_TAG
|
||||
python3 ./tests/ci/style_check.py --no-push
|
||||
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ runner.temp }}/ci_run_data.json --post --job-name 'Style check'
|
||||
BuildDockers:
|
||||
needs: [RunConfig]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
@ -103,7 +104,7 @@ jobs:
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Fast tests
|
||||
test_name: Fast test
|
||||
runner_type: builder
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
run_command: |
|
||||
@ -272,19 +273,26 @@ jobs:
|
||||
############################################################################################
|
||||
##################################### Docker images #######################################
|
||||
############################################################################################
|
||||
DockerServerImages:
|
||||
DockerServerImage:
|
||||
needs: [RunConfig, BuilderDebRelease, BuilderDebAarch64]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Docker server and keeper images
|
||||
test_name: Docker server image
|
||||
runner_type: style-checker
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
checkout_depth: 0 # It MUST BE THE SAME for all dependencies and the job itself
|
||||
run_command: |
|
||||
cd "$GITHUB_WORKSPACE/tests/ci"
|
||||
python3 docker_server.py --release-type head --no-push \
|
||||
--image-repo clickhouse/clickhouse-server --image-path docker/server --allow-build-reuse
|
||||
DockerKeeperImage:
|
||||
needs: [RunConfig, BuilderDebRelease, BuilderDebAarch64]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Docker keeper image
|
||||
runner_type: style-checker
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
run_command: |
|
||||
python3 docker_server.py --release-type head --no-push \
|
||||
--image-repo clickhouse/clickhouse-keeper --image-path docker/keeper --allow-build-reuse
|
||||
############################################################################################
|
||||
@ -483,11 +491,11 @@ jobs:
|
||||
run_command: |
|
||||
TEMP_PATH="${TEMP_PATH}/integration" \
|
||||
python3 integration_test_check.py "Integration $CHECK_NAME" \
|
||||
--validate-bugfix --post-commit-status=file || echo 'ignore exit code'
|
||||
--validate-bugfix --post-commit-status=file || echo "ignore exit code"
|
||||
|
||||
TEMP_PATH="${TEMP_PATH}/stateless" \
|
||||
python3 functional_test_check.py "Stateless $CHECK_NAME" "$KILL_TIMEOUT" \
|
||||
--validate-bugfix --post-commit-status=file || echo 'ignore exit code'
|
||||
--validate-bugfix --post-commit-status=file || echo "ignore exit code"
|
||||
|
||||
python3 bugfix_validate_check.py "${TEMP_PATH}/stateless/functional_commit_status.tsv" "${TEMP_PATH}/integration/integration_commit_status.tsv"
|
||||
##############################################################################################
|
||||
@ -777,6 +785,15 @@ jobs:
|
||||
test_name: Integration tests (release)
|
||||
runner_type: stress-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
IntegrationTestsAarch64:
|
||||
needs: [RunConfig, BuilderDebAarch64]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Integration tests (aarch64)
|
||||
# FIXME: there is no stress-tester for aarch64. func-tester-aarch64 is ok?
|
||||
runner_type: func-tester-aarch64
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
IntegrationTestsFlakyCheck:
|
||||
needs: [RunConfig, BuilderDebAsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
@ -796,7 +813,7 @@ jobs:
|
||||
test_name: Unit tests (asan)
|
||||
runner_type: fuzzer-unit-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
UnitTestsReleaseClang:
|
||||
UnitTestsRelease:
|
||||
needs: [RunConfig, BuilderBinRelease]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
@ -916,6 +933,7 @@ jobs:
|
||||
- IntegrationTestsAnalyzerAsan
|
||||
- IntegrationTestsTsan
|
||||
- IntegrationTestsRelease
|
||||
- IntegrationTestsAarch64
|
||||
- IntegrationTestsFlakyCheck
|
||||
- PerformanceComparisonX86
|
||||
- PerformanceComparisonAarch
|
||||
@ -923,7 +941,7 @@ jobs:
|
||||
- UnitTestsTsan
|
||||
- UnitTestsMsan
|
||||
- UnitTestsUBsan
|
||||
- UnitTestsReleaseClang
|
||||
- UnitTestsRelease
|
||||
- CompatibilityCheckX86
|
||||
- CompatibilityCheckAarch64
|
||||
- SQLancerTestRelease
|
||||
@ -966,18 +984,25 @@ jobs:
|
||||
#############################################################################################
|
||||
###################################### JEPSEN TESTS #########################################
|
||||
#############################################################################################
|
||||
# This is special test NOT INCLUDED in FinishCheck
|
||||
# When it's skipped, all dependent tasks will be skipped too.
|
||||
# DO NOT add it there
|
||||
Jepsen:
|
||||
# This is special test NOT INCLUDED in FinishCheck
|
||||
# When it's skipped, all dependent tasks will be skipped too.
|
||||
# DO NOT add it there
|
||||
if: ${{ !failure() && !cancelled() && contains(github.event.pull_request.labels.*.name, 'jepsen-test') }}
|
||||
# we need concurrency as the job uses dedicated instances in the cloud
|
||||
concurrency:
|
||||
group: jepsen
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
needs: [RunConfig, BuilderBinRelease]
|
||||
uses: ./.github/workflows/jepsen.yml
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: ClickHouse Keeper Jepsen
|
||||
runner_type: style-checker
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
#############################################################################################
|
||||
####################################### libFuzzer ###########################################
|
||||
#############################################################################################
|
||||
libFuzzer:
|
||||
if: ${{ !failure() && !cancelled() && contains(github.event.pull_request.labels.*.name, 'libFuzzer') }}
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
needs: [RunConfig, StyleCheck]
|
||||
uses: ./.github/workflows/libfuzzer.yml
|
||||
with:
|
||||
|
||||
40
.github/workflows/release_branches.yml
vendored
40
.github/workflows/release_branches.yml
vendored
@ -41,7 +41,7 @@ jobs:
|
||||
id: runconfig
|
||||
run: |
|
||||
echo "::group::configure CI run"
|
||||
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --rebuild-all-binaries --outfile ${{ runner.temp }}/ci_run_data.json
|
||||
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --outfile ${{ runner.temp }}/ci_run_data.json
|
||||
echo "::endgroup::"
|
||||
echo "::group::CI run configure results"
|
||||
python3 -m json.tool ${{ runner.temp }}/ci_run_data.json
|
||||
@ -91,6 +91,8 @@ jobs:
|
||||
build_name: package_release
|
||||
checkout_depth: 0
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
# always rebuild on release branches to be able to publish from any commit
|
||||
force: true
|
||||
BuilderDebAarch64:
|
||||
needs: [RunConfig, BuildDockers]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
@ -99,6 +101,8 @@ jobs:
|
||||
build_name: package_aarch64
|
||||
checkout_depth: 0
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
# always rebuild on release branches to be able to publish from any commit
|
||||
force: true
|
||||
BuilderDebAsan:
|
||||
needs: [RunConfig, BuildDockers]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
@ -142,6 +146,8 @@ jobs:
|
||||
build_name: binary_darwin
|
||||
checkout_depth: 0
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
# always rebuild on release branches to be able to publish from any commit
|
||||
force: true
|
||||
BuilderBinDarwinAarch64:
|
||||
needs: [RunConfig, BuildDockers]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
@ -150,22 +156,31 @@ jobs:
|
||||
build_name: binary_darwin_aarch64
|
||||
checkout_depth: 0
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
# always rebuild on release branches to be able to publish from any commit
|
||||
force: true
|
||||
############################################################################################
|
||||
##################################### Docker images #######################################
|
||||
############################################################################################
|
||||
DockerServerImages:
|
||||
DockerServerImage:
|
||||
needs: [RunConfig, BuilderDebRelease, BuilderDebAarch64]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Docker server and keeper images
|
||||
test_name: Docker server image
|
||||
runner_type: style-checker
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
checkout_depth: 0
|
||||
run_command: |
|
||||
cd "$GITHUB_WORKSPACE/tests/ci"
|
||||
python3 docker_server.py --release-type head --no-push \
|
||||
--image-repo clickhouse/clickhouse-server --image-path docker/server --allow-build-reuse
|
||||
DockerKeeperImage:
|
||||
needs: [RunConfig, BuilderDebRelease, BuilderDebAarch64]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Docker keeper image
|
||||
runner_type: style-checker
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
run_command: |
|
||||
python3 docker_server.py --release-type head --no-push \
|
||||
--image-repo clickhouse/clickhouse-keeper --image-path docker/keeper --allow-build-reuse
|
||||
############################################################################################
|
||||
@ -199,13 +214,8 @@ jobs:
|
||||
if: ${{ !cancelled() }}
|
||||
needs:
|
||||
- RunConfig
|
||||
- BuilderDebRelease
|
||||
- BuilderDebAarch64
|
||||
- BuilderDebAsan
|
||||
- BuilderDebTsan
|
||||
- BuilderDebUBsan
|
||||
- BuilderDebMsan
|
||||
- BuilderDebDebug
|
||||
- BuilderBinDarwin
|
||||
- BuilderBinDarwinAarch64
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: ClickHouse special build check
|
||||
@ -218,7 +228,6 @@ jobs:
|
||||
run_command: |
|
||||
python3 build_report_check.py "$CHECK_NAME"
|
||||
MarkReleaseReady:
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
needs:
|
||||
- BuilderBinDarwin
|
||||
- BuilderBinDarwinAarch64
|
||||
@ -228,8 +237,6 @@ jobs:
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
uses: ClickHouse/checkout@v1
|
||||
with:
|
||||
clear-repository: true
|
||||
- name: Mark Commit Release Ready
|
||||
run: |
|
||||
cd "$GITHUB_WORKSPACE/tests/ci"
|
||||
@ -456,7 +463,8 @@ jobs:
|
||||
FinishCheck:
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
needs:
|
||||
- DockerServerImages
|
||||
- DockerServerImage
|
||||
- DockerKeeperImage
|
||||
- BuilderReport
|
||||
- BuilderSpecialReport
|
||||
- MarkReleaseReady
|
||||
|
||||
18
.github/workflows/reusable_build.yml
vendored
18
.github/workflows/reusable_build.yml
vendored
@ -26,6 +26,10 @@ name: Build ClickHouse
|
||||
description: json ci data
|
||||
type: string
|
||||
required: true
|
||||
force:
|
||||
description: disallow job skipping
|
||||
type: boolean
|
||||
default: false
|
||||
additional_envs:
|
||||
description: additional ENV variables to setup the job
|
||||
type: string
|
||||
@ -33,7 +37,7 @@ name: Build ClickHouse
|
||||
jobs:
|
||||
Build:
|
||||
name: Build-${{inputs.build_name}}
|
||||
if: contains(fromJson(inputs.data).jobs_data.jobs_to_do, inputs.build_name)
|
||||
if: ${{ contains(fromJson(inputs.data).jobs_data.jobs_to_do, inputs.build_name) || inputs.force }}
|
||||
env:
|
||||
GITHUB_JOB_OVERRIDDEN: Build-${{inputs.build_name}}
|
||||
runs-on: [self-hosted, '${{inputs.runner_type}}']
|
||||
@ -58,6 +62,7 @@ jobs:
|
||||
- name: Apply sparse checkout for contrib # in order to check that it doesn't break build
|
||||
# This step is done in GITHUB_WORKSPACE,
|
||||
# because it's broken in REPO_COPY for some reason
|
||||
# See also update-submodules.sh
|
||||
if: ${{ env.BUILD_SPARSE_CHECKOUT == 'true' }}
|
||||
run: |
|
||||
rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed'
|
||||
@ -72,15 +77,20 @@ jobs:
|
||||
- name: Pre
|
||||
run: |
|
||||
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --pre --job-name '${{inputs.build_name}}'
|
||||
- name: Build
|
||||
- name: Run
|
||||
run: |
|
||||
python3 "$GITHUB_WORKSPACE/tests/ci/build_check.py" "$BUILD_NAME"
|
||||
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" \
|
||||
--infile ${{ toJson(inputs.data) }} \
|
||||
--job-name "$BUILD_NAME" \
|
||||
--run \
|
||||
${{ inputs.force && '--force' || '' }}
|
||||
- name: Post
|
||||
# it still be build report to upload for failed build job
|
||||
if: always()
|
||||
if: ${{ !cancelled() }}
|
||||
run: |
|
||||
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --post --job-name '${{inputs.build_name}}'
|
||||
- name: Mark as done
|
||||
if: ${{ !cancelled() }}
|
||||
run: |
|
||||
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --mark-success --job-name '${{inputs.build_name}}'
|
||||
- name: Clean
|
||||
|
||||
16
.github/workflows/reusable_docker.yml
vendored
16
.github/workflows/reusable_docker.yml
vendored
@ -46,7 +46,7 @@ jobs:
|
||||
needs: [DockerBuildAmd64, DockerBuildAarch64]
|
||||
runs-on: [self-hosted, style-checker]
|
||||
if: |
|
||||
!failure() && !cancelled() && toJson(fromJson(inputs.data).docker_data.missing_multi) != '[]'
|
||||
!failure() && !cancelled() && (toJson(fromJson(inputs.data).docker_data.missing_multi) != '[]' || inputs.set_latest)
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
uses: ClickHouse/checkout@v1
|
||||
@ -55,14 +55,12 @@ jobs:
|
||||
- name: Build images
|
||||
run: |
|
||||
cd "$GITHUB_WORKSPACE/tests/ci"
|
||||
FLAG_LATEST=''
|
||||
if [ "${{ inputs.set_latest }}" == "true" ]; then
|
||||
FLAG_LATEST='--set-latest'
|
||||
echo "latest tag will be set for resulting manifests"
|
||||
python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 \
|
||||
--image-tags '${{ toJson(fromJson(inputs.data).docker_data.images) }}' \
|
||||
--missing-images '${{ toJson(fromJson(inputs.data).docker_data.missing_multi) }}' \
|
||||
--set-latest
|
||||
else
|
||||
python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 \
|
||||
--image-tags '${{ toJson(fromJson(inputs.data).docker_data.images) }}' \
|
||||
--missing-images '${{ toJson(fromJson(inputs.data).docker_data.missing_multi) }}'
|
||||
fi
|
||||
python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 \
|
||||
--image-tags '${{ toJson(fromJson(inputs.data).docker_data.images) }}' \
|
||||
--missing-images '${{ toJson(fromJson(inputs.data).docker_data.missing_multi) }}' \
|
||||
$FLAG_LATEST
|
||||
|
||||
18
.github/workflows/reusable_simple_job.yml
vendored
18
.github/workflows/reusable_simple_job.yml
vendored
@ -34,12 +34,16 @@ name: Simple job
|
||||
working-directory:
|
||||
description: sets custom working directory
|
||||
type: string
|
||||
default: ""
|
||||
default: "$GITHUB_WORKSPACE/tests/ci"
|
||||
git_ref:
|
||||
description: commit to use, merge commit for pr or head
|
||||
required: false
|
||||
type: string
|
||||
default: ${{ github.event.after }} # no merge commit
|
||||
report_required:
|
||||
description: set to true if job report with the commit status required
|
||||
type: boolean
|
||||
default: false
|
||||
secrets:
|
||||
secret_envs:
|
||||
description: if given, it's passed to the environments
|
||||
@ -58,6 +62,8 @@ jobs:
|
||||
env:
|
||||
GITHUB_JOB_OVERRIDDEN: ${{inputs.test_name}}
|
||||
steps:
|
||||
- name: DebugInfo
|
||||
uses: hmarr/debug-action@a701ed95a46e6f2fb0df25e1a558c16356fae35a
|
||||
- name: Check out repository code
|
||||
uses: ClickHouse/checkout@v1
|
||||
with:
|
||||
@ -79,12 +85,12 @@ jobs:
|
||||
job_type: test
|
||||
- name: Run
|
||||
run: |
|
||||
if [ -n '${{ inputs.working-directory }}' ]; then
|
||||
cd "${{ inputs.working-directory }}"
|
||||
else
|
||||
cd "$GITHUB_WORKSPACE/tests/ci"
|
||||
fi
|
||||
cd "${{ inputs.working-directory }}"
|
||||
${{ inputs.run_command }}
|
||||
- name: Post
|
||||
if: ${{ inputs.report_required }}
|
||||
run: |
|
||||
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --post --job-name '${{inputs.test_name}}'
|
||||
- name: Clean
|
||||
if: always()
|
||||
uses: ./.github/actions/clean
|
||||
|
||||
22
.github/workflows/reusable_test.yml
vendored
22
.github/workflows/reusable_test.yml
vendored
@ -38,7 +38,7 @@ name: Testing workflow
|
||||
working-directory:
|
||||
description: sets custom working directory
|
||||
type: string
|
||||
default: ""
|
||||
default: "$GITHUB_WORKSPACE/tests/ci"
|
||||
secrets:
|
||||
secret_envs:
|
||||
description: if given, it's passed to the environments
|
||||
@ -96,22 +96,18 @@ jobs:
|
||||
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --pre --job-name '${{inputs.test_name}}'
|
||||
- name: Run
|
||||
run: |
|
||||
if [ -n "${{ inputs.working-directory }}" ]; then
|
||||
cd "${{ inputs.working-directory }}"
|
||||
else
|
||||
cd "$GITHUB_WORKSPACE/tests/ci"
|
||||
fi
|
||||
if [ -n "$(echo '${{ inputs.run_command }}' | tr -d '\n')" ]; then
|
||||
echo "Running command from workflow input"
|
||||
${{ inputs.run_command }}
|
||||
else
|
||||
echo "Running command from job config"
|
||||
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --run --job-name '${{inputs.test_name}}'
|
||||
fi
|
||||
cd "${{ inputs.working-directory }}"
|
||||
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" \
|
||||
--infile ${{ toJson(inputs.data) }} \
|
||||
--job-name '${{inputs.test_name}}' \
|
||||
--run \
|
||||
--run-command '''${{inputs.run_command}}'''
|
||||
- name: Post run
|
||||
if: ${{ !cancelled() }}
|
||||
run: |
|
||||
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --post --job-name '${{inputs.test_name}}'
|
||||
- name: Mark as done
|
||||
if: ${{ !cancelled() }}
|
||||
run: |
|
||||
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --mark-success --job-name '${{inputs.test_name}}' --batch ${{matrix.batch}}
|
||||
- name: Clean
|
||||
|
||||
2
.github/workflows/tags_stable.yml
vendored
2
.github/workflows/tags_stable.yml
vendored
@ -55,7 +55,7 @@ jobs:
|
||||
python3 ./utils/security-generator/generate_security.py > SECURITY.md
|
||||
git diff HEAD
|
||||
- name: Create Pull Request
|
||||
uses: peter-evans/create-pull-request@v3
|
||||
uses: peter-evans/create-pull-request@v6
|
||||
with:
|
||||
author: "robot-clickhouse <robot-clickhouse@users.noreply.github.com>"
|
||||
token: ${{ secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN }}
|
||||
|
||||
16
.gitmessage
16
.gitmessage
@ -1,9 +1,19 @@
|
||||
|
||||
|
||||
## To avoid merge commit in CI run (add a leading space to apply):
|
||||
#no-merge-commit
|
||||
### CI modificators (add a leading space to apply):
|
||||
|
||||
## Running specified job (add a leading space to apply):
|
||||
## To avoid a merge commit in CI:
|
||||
#no_merge_commit
|
||||
|
||||
## To discard CI cache:
|
||||
#no_ci_cache
|
||||
|
||||
## To run specified set of tests in CI:
|
||||
#ci_set_<SET_NAME>
|
||||
#ci_set_reduced
|
||||
#ci_set_arm
|
||||
|
||||
## To run specified job in CI:
|
||||
#job_<JOB NAME>
|
||||
#job_stateless_tests_release
|
||||
#job_package_debug
|
||||
|
||||
11
.gitmodules
vendored
11
.gitmodules
vendored
@ -99,7 +99,7 @@
|
||||
url = https://github.com/awslabs/aws-c-event-stream
|
||||
[submodule "aws-c-common"]
|
||||
path = contrib/aws-c-common
|
||||
url = https://github.com/ClickHouse/aws-c-common
|
||||
url = https://github.com/awslabs/aws-c-common.git
|
||||
[submodule "aws-checksums"]
|
||||
path = contrib/aws-checksums
|
||||
url = https://github.com/awslabs/aws-checksums
|
||||
@ -245,6 +245,12 @@
|
||||
[submodule "contrib/idxd-config"]
|
||||
path = contrib/idxd-config
|
||||
url = https://github.com/intel/idxd-config
|
||||
[submodule "contrib/QAT-ZSTD-Plugin"]
|
||||
path = contrib/QAT-ZSTD-Plugin
|
||||
url = https://github.com/intel/QAT-ZSTD-Plugin
|
||||
[submodule "contrib/qatlib"]
|
||||
path = contrib/qatlib
|
||||
url = https://github.com/intel/qatlib
|
||||
[submodule "contrib/wyhash"]
|
||||
path = contrib/wyhash
|
||||
url = https://github.com/wangyi-fudan/wyhash
|
||||
@ -360,3 +366,6 @@
|
||||
[submodule "contrib/sqids-cpp"]
|
||||
path = contrib/sqids-cpp
|
||||
url = https://github.com/sqids/sqids-cpp.git
|
||||
[submodule "contrib/idna"]
|
||||
path = contrib/idna
|
||||
url = https://github.com/ada-url/idna.git
|
||||
|
||||
2304
CHANGELOG.md
2304
CHANGELOG.md
File diff suppressed because it is too large
Load Diff
@ -254,10 +254,17 @@ endif()
|
||||
|
||||
include(cmake/cpu_features.cmake)
|
||||
|
||||
# Asynchronous unwind tables are needed for Query Profiler.
|
||||
# They are already by default on some platforms but possibly not on all platforms.
|
||||
# Enable it explicitly.
|
||||
set (COMPILER_FLAGS "${COMPILER_FLAGS} -fasynchronous-unwind-tables")
|
||||
|
||||
# Query Profiler doesn't work on MacOS for several reasons
|
||||
# - PHDR cache is not available
|
||||
# - We use native functionality to get stacktraces which is not async signal safe
|
||||
# and thus we don't need to generate asynchronous unwind tables
|
||||
if (NOT OS_DARWIN)
|
||||
# Asynchronous unwind tables are needed for Query Profiler.
|
||||
# They are already by default on some platforms but possibly not on all platforms.
|
||||
# Enable it explicitly.
|
||||
set (COMPILER_FLAGS "${COMPILER_FLAGS} -fasynchronous-unwind-tables")
|
||||
endif()
|
||||
|
||||
# Reproducible builds.
|
||||
if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
|
||||
|
||||
@ -33,11 +33,11 @@ curl https://clickhouse.com/ | sh
|
||||
|
||||
## Upcoming Events
|
||||
|
||||
Keep an eye out for upcoming meetups around the world. Somewhere else you want us to be? Please feel free to reach out to tyler <at> clickhouse <dot> com.
|
||||
Keep an eye out for upcoming meetups around the world. Somewhere else you want us to be? Please feel free to reach out to tyler `<at>` clickhouse `<dot>` com.
|
||||
|
||||
## Recent Recordings
|
||||
* **Recent Meetup Videos**: [Meetup Playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3iNDUzpY1S3L_iV4nARda_U) Whenever possible recordings of the ClickHouse Community Meetups are edited and presented as individual talks. Current featuring "Modern SQL in 2023", "Fast, Concurrent, and Consistent Asynchronous INSERTS in ClickHouse", and "Full-Text Indices: Design and Experiments"
|
||||
* **Recording available**: [**v23.10 Release Webinar**](https://www.youtube.com/watch?v=PGQS6uPb970) All the features of 23.10, one convenient video! Watch it now!
|
||||
* **Recording available**: [**v24.1 Release Webinar**](https://www.youtube.com/watch?v=pBF9g0wGAGs) All the features of 24.1, one convenient video! Watch it now!
|
||||
* **All release webinar recordings**: [YouTube playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3jAlSy1JxyP8zluvXaN3nxU)
|
||||
|
||||
|
||||
|
||||
@ -13,9 +13,10 @@ The following versions of ClickHouse server are currently being supported with s
|
||||
|
||||
| Version | Supported |
|
||||
|:-|:-|
|
||||
| 24.1 | ✔️ |
|
||||
| 23.12 | ✔️ |
|
||||
| 23.11 | ✔️ |
|
||||
| 23.10 | ✔️ |
|
||||
| 23.10 | ❌ |
|
||||
| 23.9 | ❌ |
|
||||
| 23.8 | ✔️ |
|
||||
| 23.7 | ❌ |
|
||||
|
||||
@ -17,6 +17,7 @@ set (SRCS
|
||||
getMemoryAmount.cpp
|
||||
getPageSize.cpp
|
||||
getThreadId.cpp
|
||||
int8_to_string.cpp
|
||||
JSON.cpp
|
||||
mremap.cpp
|
||||
phdr_cache.cpp
|
||||
|
||||
@ -99,7 +99,7 @@ public:
|
||||
};
|
||||
}
|
||||
|
||||
constexpr DB::UInt64 max_uint_mask = std::numeric_limits<DB::UInt64>::max();
|
||||
constexpr UInt64 max_uint_mask = std::numeric_limits<UInt64>::max();
|
||||
|
||||
namespace std
|
||||
{
|
||||
@ -114,8 +114,8 @@ namespace std
|
||||
{
|
||||
size_t operator()(const DB::Decimal128 & x) const
|
||||
{
|
||||
return std::hash<DB::Int64>()(x.value >> 64)
|
||||
^ std::hash<DB::Int64>()(x.value & max_uint_mask);
|
||||
return std::hash<Int64>()(x.value >> 64)
|
||||
^ std::hash<Int64>()(x.value & max_uint_mask);
|
||||
}
|
||||
};
|
||||
|
||||
@ -134,8 +134,8 @@ namespace std
|
||||
size_t operator()(const DB::Decimal256 & x) const
|
||||
{
|
||||
// FIXME temp solution
|
||||
return std::hash<DB::Int64>()(static_cast<DB::Int64>(x.value >> 64 & max_uint_mask))
|
||||
^ std::hash<DB::Int64>()(static_cast<DB::Int64>(x.value & max_uint_mask));
|
||||
return std::hash<Int64>()(static_cast<Int64>(x.value >> 64 & max_uint_mask))
|
||||
^ std::hash<Int64>()(static_cast<Int64>(x.value & max_uint_mask));
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <bit>
|
||||
#include <cstring>
|
||||
#include <algorithm>
|
||||
#include <type_traits>
|
||||
|
||||
@ -1,8 +1,11 @@
|
||||
#include <stdexcept>
|
||||
#include <fstream>
|
||||
#include <base/getMemoryAmount.h>
|
||||
|
||||
#include <base/getPageSize.h>
|
||||
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
#include <stdexcept>
|
||||
|
||||
#include <unistd.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/param.h>
|
||||
@ -11,6 +14,80 @@
|
||||
#endif
|
||||
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
std::optional<uint64_t> getCgroupsV2MemoryLimit()
|
||||
{
|
||||
#if defined(OS_LINUX)
|
||||
const std::filesystem::path default_cgroups_mount = "/sys/fs/cgroup";
|
||||
|
||||
/// This file exists iff the host has cgroups v2 enabled.
|
||||
std::ifstream controllers_file(default_cgroups_mount / "cgroup.controllers");
|
||||
if (!controllers_file.is_open())
|
||||
return {};
|
||||
|
||||
/// Make sure that the memory controller is enabled.
|
||||
/// - cgroup.controllers defines which controllers *can* be enabled.
|
||||
/// - cgroup.subtree_control defines which controllers *are* enabled.
|
||||
/// (see https://docs.kernel.org/admin-guide/cgroup-v2.html)
|
||||
/// Caveat: nested groups may disable controllers. For simplicity, check only the top-level group.
|
||||
/// ReadBufferFromFile subtree_control_file(default_cgroups_mount / "cgroup.subtree_control");
|
||||
/// std::string subtree_control;
|
||||
/// readString(subtree_control, subtree_control_file);
|
||||
/// if (subtree_control.find("memory") == std::string::npos)
|
||||
/// return {};
|
||||
std::ifstream subtree_control_file(default_cgroups_mount / "cgroup.subtree_control");
|
||||
std::stringstream subtree_control_buf;
|
||||
subtree_control_buf << subtree_control_file.rdbuf();
|
||||
std::string subtree_control = subtree_control_buf.str();
|
||||
if (subtree_control.find("memory") == std::string::npos)
|
||||
return {};
|
||||
|
||||
/// Identify the cgroup the process belongs to
|
||||
/// All PIDs assigned to a cgroup are in /sys/fs/cgroups/{cgroup_name}/cgroup.procs
|
||||
/// A simpler way to get the membership is:
|
||||
std::ifstream cgroup_name_file("/proc/self/cgroup");
|
||||
if (!cgroup_name_file.is_open())
|
||||
return {};
|
||||
|
||||
std::stringstream cgroup_name_buf;
|
||||
cgroup_name_buf << cgroup_name_file.rdbuf();
|
||||
std::string cgroup_name = cgroup_name_buf.str();
|
||||
if (!cgroup_name.empty() && cgroup_name.back() == '\n')
|
||||
cgroup_name.pop_back(); /// remove trailing newline, if any
|
||||
/// With cgroups v2, there will be a *single* line with prefix "0::/"
|
||||
const std::string v2_prefix = "0::/";
|
||||
if (!cgroup_name.starts_with(v2_prefix))
|
||||
return {};
|
||||
cgroup_name = cgroup_name.substr(v2_prefix.length());
|
||||
|
||||
std::filesystem::path current_cgroup = cgroup_name.empty() ? default_cgroups_mount : (default_cgroups_mount / cgroup_name);
|
||||
|
||||
/// Open the bottom-most nested memory limit setting file. If there is no such file at the current
|
||||
/// level, try again at the parent level as memory settings are inherited.
|
||||
while (current_cgroup != default_cgroups_mount.parent_path())
|
||||
{
|
||||
std::ifstream setting_file(current_cgroup / "memory.max");
|
||||
if (setting_file.is_open())
|
||||
{
|
||||
uint64_t value;
|
||||
if (setting_file >> value)
|
||||
return {value};
|
||||
else
|
||||
return {}; /// e.g. the cgroups default "max"
|
||||
}
|
||||
current_cgroup = current_cgroup.parent_path();
|
||||
}
|
||||
|
||||
return {};
|
||||
#else
|
||||
return {};
|
||||
#endif
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/** Returns the size of physical memory (RAM) in bytes.
|
||||
* Returns 0 on unsupported platform
|
||||
*/
|
||||
@ -26,34 +103,27 @@ uint64_t getMemoryAmountOrZero()
|
||||
|
||||
uint64_t memory_amount = num_pages * page_size;
|
||||
|
||||
#if defined(OS_LINUX)
|
||||
// Try to lookup at the Cgroup limit
|
||||
|
||||
// CGroups v2
|
||||
std::ifstream cgroupv2_limit("/sys/fs/cgroup/memory.max");
|
||||
if (cgroupv2_limit.is_open())
|
||||
{
|
||||
uint64_t memory_limit = 0;
|
||||
cgroupv2_limit >> memory_limit;
|
||||
if (memory_limit > 0 && memory_limit < memory_amount)
|
||||
memory_amount = memory_limit;
|
||||
}
|
||||
/// Respect the memory limit set by cgroups v2.
|
||||
auto limit_v2 = getCgroupsV2MemoryLimit();
|
||||
if (limit_v2.has_value() && *limit_v2 < memory_amount)
|
||||
memory_amount = *limit_v2;
|
||||
else
|
||||
{
|
||||
// CGroups v1
|
||||
std::ifstream cgroup_limit("/sys/fs/cgroup/memory/memory.limit_in_bytes");
|
||||
if (cgroup_limit.is_open())
|
||||
/// Cgroups v1 were replaced by v2 in 2015. The only reason we keep supporting v1 is that the transition to v2
|
||||
/// has been slow. Caveat : Hierarchical groups as in v2 are not supported for v1, the location of the memory
|
||||
/// limit (virtual) file is hard-coded.
|
||||
/// TODO: check at the end of 2024 if we can get rid of v1.
|
||||
std::ifstream limit_file_v1("/sys/fs/cgroup/memory/memory.limit_in_bytes");
|
||||
if (limit_file_v1.is_open())
|
||||
{
|
||||
uint64_t memory_limit = 0; // in case of read error
|
||||
cgroup_limit >> memory_limit;
|
||||
if (memory_limit > 0 && memory_limit < memory_amount)
|
||||
memory_amount = memory_limit;
|
||||
uint64_t limit_v1;
|
||||
if (limit_file_v1 >> limit_v1)
|
||||
if (limit_v1 < memory_amount)
|
||||
memory_amount = limit_v1;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
return memory_amount;
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
9
base/base/int8_to_string.cpp
Normal file
9
base/base/int8_to_string.cpp
Normal file
@ -0,0 +1,9 @@
|
||||
#include <base/int8_to_string.h>
|
||||
|
||||
namespace std
|
||||
{
|
||||
std::string to_string(Int8 v) /// NOLINT (cert-dcl58-cpp)
|
||||
{
|
||||
return to_string(int8_t{v});
|
||||
}
|
||||
}
|
||||
17
base/base/int8_to_string.h
Normal file
17
base/base/int8_to_string.h
Normal file
@ -0,0 +1,17 @@
|
||||
#pragma once
|
||||
|
||||
#include <base/defines.h>
|
||||
#include <base/types.h>
|
||||
|
||||
#include <fmt/format.h>
|
||||
|
||||
template <>
|
||||
struct fmt::formatter<Int8> : fmt::formatter<int8_t>
|
||||
{
|
||||
};
|
||||
|
||||
|
||||
namespace std
|
||||
{
|
||||
std::string to_string(Int8 v); /// NOLINT (cert-dcl58-cpp)
|
||||
}
|
||||
@ -64,19 +64,14 @@ using ComparatorWrapper = Comparator;
|
||||
|
||||
#include <miniselect/floyd_rivest_select.h>
|
||||
|
||||
template <typename RandomIt>
|
||||
void nth_element(RandomIt first, RandomIt nth, RandomIt last)
|
||||
template <typename RandomIt, typename Compare>
|
||||
void nth_element(RandomIt first, RandomIt nth, RandomIt last, Compare compare)
|
||||
{
|
||||
using value_type = typename std::iterator_traits<RandomIt>::value_type;
|
||||
using comparator = std::less<value_type>;
|
||||
|
||||
comparator compare;
|
||||
ComparatorWrapper<comparator> compare_wrapper = compare;
|
||||
|
||||
#ifndef NDEBUG
|
||||
::shuffle(first, last);
|
||||
#endif
|
||||
|
||||
ComparatorWrapper<Compare> compare_wrapper = compare;
|
||||
::miniselect::floyd_rivest_select(first, nth, last, compare_wrapper);
|
||||
|
||||
#ifndef NDEBUG
|
||||
@ -87,6 +82,15 @@ void nth_element(RandomIt first, RandomIt nth, RandomIt last)
|
||||
#endif
|
||||
}
|
||||
|
||||
template <typename RandomIt>
|
||||
void nth_element(RandomIt first, RandomIt nth, RandomIt last)
|
||||
{
|
||||
using value_type = typename std::iterator_traits<RandomIt>::value_type;
|
||||
using comparator = std::less<value_type>;
|
||||
|
||||
::nth_element(first, nth, last, comparator());
|
||||
}
|
||||
|
||||
template <typename RandomIt, typename Compare>
|
||||
void partial_sort(RandomIt first, RandomIt middle, RandomIt last, Compare compare)
|
||||
{
|
||||
|
||||
@ -3,40 +3,34 @@
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
using Int8 = int8_t;
|
||||
using Int16 = int16_t;
|
||||
using Int32 = int32_t;
|
||||
using Int64 = int64_t;
|
||||
|
||||
#ifndef __cpp_char8_t
|
||||
using char8_t = unsigned char;
|
||||
#endif
|
||||
|
||||
/// This is needed for more strict aliasing. https://godbolt.org/z/xpJBSb https://stackoverflow.com/a/57453713
|
||||
/// Using char8_t more strict aliasing (https://stackoverflow.com/a/57453713)
|
||||
using UInt8 = char8_t;
|
||||
|
||||
/// Same for using signed _BitInt(8) (there isn't a signed char8_t, which would be more convenient)
|
||||
/// See https://godbolt.org/z/fafnWEnnf
|
||||
#pragma clang diagnostic push
|
||||
#pragma clang diagnostic ignored "-Wbit-int-extension"
|
||||
using Int8 = signed _BitInt(8);
|
||||
#pragma clang diagnostic pop
|
||||
|
||||
namespace std
|
||||
{
|
||||
template <>
|
||||
struct hash<Int8> /// NOLINT (cert-dcl58-cpp)
|
||||
{
|
||||
size_t operator()(const Int8 x) const { return std::hash<int8_t>()(int8_t{x}); }
|
||||
};
|
||||
}
|
||||
|
||||
using UInt16 = uint16_t;
|
||||
using UInt32 = uint32_t;
|
||||
using UInt64 = uint64_t;
|
||||
|
||||
using String = std::string;
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
using UInt8 = ::UInt8;
|
||||
using UInt16 = ::UInt16;
|
||||
using UInt32 = ::UInt32;
|
||||
using UInt64 = ::UInt64;
|
||||
|
||||
using Int8 = ::Int8;
|
||||
using Int16 = ::Int16;
|
||||
using Int32 = ::Int32;
|
||||
using Int64 = ::Int64;
|
||||
using Int16 = int16_t;
|
||||
using Int32 = int32_t;
|
||||
using Int64 = int64_t;
|
||||
|
||||
using Float32 = float;
|
||||
using Float64 = double;
|
||||
|
||||
using String = std::string;
|
||||
|
||||
}
|
||||
|
||||
@ -6,6 +6,7 @@
|
||||
|
||||
#include "throwError.h"
|
||||
|
||||
#include <bit>
|
||||
#include <cmath>
|
||||
#include <cfloat>
|
||||
#include <cassert>
|
||||
|
||||
@ -26,6 +26,11 @@
|
||||
#include "Poco/StreamUtil.h"
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
class ReadBufferFromIStream;
|
||||
}
|
||||
|
||||
namespace Poco
|
||||
{
|
||||
|
||||
@ -120,6 +125,8 @@ protected:
|
||||
openmode getMode() const { return _mode; }
|
||||
|
||||
private:
|
||||
friend class DB::ReadBufferFromIStream;
|
||||
|
||||
virtual int readFromDevice(char_type * /*buffer*/, std::streamsize /*length*/) { return 0; }
|
||||
|
||||
virtual int writeToDevice(const char_type * /*buffer*/, std::streamsize /*length*/) { return 0; }
|
||||
|
||||
@ -22,6 +22,7 @@
|
||||
#include <cstddef>
|
||||
#include <map>
|
||||
#include <vector>
|
||||
|
||||
#include "Poco/Channel.h"
|
||||
#include "Poco/Format.h"
|
||||
#include "Poco/Foundation.h"
|
||||
@ -33,7 +34,8 @@ namespace Poco
|
||||
|
||||
|
||||
class Exception;
|
||||
|
||||
class Logger;
|
||||
using LoggerPtr = std::shared_ptr<Logger>;
|
||||
|
||||
class Foundation_API Logger : public Channel
|
||||
/// Logger is a special Channel that acts as the main
|
||||
@ -870,21 +872,21 @@ public:
|
||||
/// If the Logger does not yet exist, it is created, based
|
||||
/// on its parent logger.
|
||||
|
||||
static Logger & unsafeGet(const std::string & name);
|
||||
/// Returns a reference to the Logger with the given name.
|
||||
static LoggerPtr getShared(const std::string & name, bool should_be_owned_by_shared_ptr_if_created = true);
|
||||
/// Returns a shared pointer to the Logger with the given name.
|
||||
/// If the Logger does not yet exist, it is created, based
|
||||
/// on its parent logger.
|
||||
///
|
||||
/// WARNING: This method is not thread safe. You should
|
||||
/// probably use get() instead.
|
||||
/// The only time this method should be used is during
|
||||
/// program initialization, when only one thread is running.
|
||||
|
||||
static Logger & create(const std::string & name, Channel * pChannel, int level = Message::PRIO_INFORMATION);
|
||||
/// Creates and returns a reference to a Logger with the
|
||||
/// given name. The Logger's Channel and log level as set as
|
||||
/// specified.
|
||||
|
||||
static LoggerPtr createShared(const std::string & name, Channel * pChannel, int level = Message::PRIO_INFORMATION);
|
||||
/// Creates and returns a shared pointer to a Logger with the
|
||||
/// given name. The Logger's Channel and log level as set as
|
||||
/// specified.
|
||||
|
||||
static Logger & root();
|
||||
/// Returns a reference to the root logger, which is the ultimate
|
||||
/// ancestor of all Loggers.
|
||||
@ -893,13 +895,6 @@ public:
|
||||
/// Returns a pointer to the Logger with the given name if it
|
||||
/// exists, or a null pointer otherwise.
|
||||
|
||||
static void destroy(const std::string & name);
|
||||
/// Destroys the logger with the specified name. Does nothing
|
||||
/// if the logger is not found.
|
||||
///
|
||||
/// After a logger has been destroyed, all references to it
|
||||
/// become invalid.
|
||||
|
||||
static void shutdown();
|
||||
/// Shuts down the logging framework and releases all
|
||||
/// Loggers.
|
||||
@ -928,9 +923,17 @@ public:
|
||||
|
||||
static const std::string ROOT; /// The name of the root logger ("").
|
||||
|
||||
protected:
|
||||
typedef std::map<std::string, Logger *> LoggerMap;
|
||||
public:
|
||||
struct LoggerEntry
|
||||
{
|
||||
Poco::Logger * logger;
|
||||
bool owned_by_shared_ptr = false;
|
||||
};
|
||||
|
||||
using LoggerMap = std::unordered_map<std::string, LoggerEntry>;
|
||||
using LoggerMapIterator = LoggerMap::iterator;
|
||||
|
||||
protected:
|
||||
Logger(const std::string & name, Channel * pChannel, int level);
|
||||
~Logger();
|
||||
|
||||
@ -938,11 +941,16 @@ protected:
|
||||
void log(const std::string & text, Message::Priority prio, const char * file, int line);
|
||||
|
||||
static std::string format(const std::string & fmt, int argc, std::string argv[]);
|
||||
static Logger & parent(const std::string & name);
|
||||
static void add(Logger * pLogger);
|
||||
static Logger * find(const std::string & name);
|
||||
|
||||
private:
|
||||
static std::pair<Logger::LoggerMapIterator, bool> unsafeGet(const std::string & name, bool get_shared);
|
||||
static Logger * unsafeGetRawPtr(const std::string & name);
|
||||
static std::pair<LoggerMapIterator, bool> unsafeCreate(const std::string & name, Channel * pChannel, int level = Message::PRIO_INFORMATION);
|
||||
static Logger & parent(const std::string & name);
|
||||
static std::pair<LoggerMapIterator, bool> add(Logger * pLogger);
|
||||
static std::optional<LoggerMapIterator> find(const std::string & name);
|
||||
static Logger * findRawPtr(const std::string & name);
|
||||
|
||||
Logger();
|
||||
Logger(const Logger &);
|
||||
Logger & operator=(const Logger &);
|
||||
@ -950,9 +958,6 @@ private:
|
||||
std::string _name;
|
||||
Channel * _pChannel;
|
||||
std::atomic_int _level;
|
||||
|
||||
static LoggerMap * _pLoggerMap;
|
||||
static Mutex _mapMtx;
|
||||
};
|
||||
|
||||
|
||||
|
||||
@ -38,15 +38,15 @@ public:
|
||||
/// Creates the RefCountedObject.
|
||||
/// The initial reference count is one.
|
||||
|
||||
void duplicate() const;
|
||||
/// Increments the object's reference count.
|
||||
size_t duplicate() const;
|
||||
/// Increments the object's reference count, returns reference count before call.
|
||||
|
||||
void release() const throw();
|
||||
size_t release() const throw();
|
||||
/// Decrements the object's reference count
|
||||
/// and deletes the object if the count
|
||||
/// reaches zero.
|
||||
/// reaches zero, returns reference count before call.
|
||||
|
||||
int referenceCount() const;
|
||||
size_t referenceCount() const;
|
||||
/// Returns the reference count.
|
||||
|
||||
protected:
|
||||
@ -57,36 +57,40 @@ private:
|
||||
RefCountedObject(const RefCountedObject &);
|
||||
RefCountedObject & operator=(const RefCountedObject &);
|
||||
|
||||
mutable AtomicCounter _counter;
|
||||
mutable std::atomic<size_t> _counter;
|
||||
};
|
||||
|
||||
|
||||
//
|
||||
// inlines
|
||||
//
|
||||
inline int RefCountedObject::referenceCount() const
|
||||
inline size_t RefCountedObject::referenceCount() const
|
||||
{
|
||||
return _counter.value();
|
||||
return _counter.load(std::memory_order_acquire);
|
||||
}
|
||||
|
||||
|
||||
inline void RefCountedObject::duplicate() const
|
||||
inline size_t RefCountedObject::duplicate() const
|
||||
{
|
||||
++_counter;
|
||||
return _counter.fetch_add(1, std::memory_order_acq_rel);
|
||||
}
|
||||
|
||||
|
||||
inline void RefCountedObject::release() const throw()
|
||||
inline size_t RefCountedObject::release() const throw()
|
||||
{
|
||||
size_t reference_count_before = _counter.fetch_sub(1, std::memory_order_acq_rel);
|
||||
|
||||
try
|
||||
{
|
||||
if (--_counter == 0)
|
||||
if (reference_count_before == 1)
|
||||
delete this;
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
poco_unexpected();
|
||||
}
|
||||
|
||||
return reference_count_before;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -20,12 +20,31 @@
|
||||
#include "Poco/NumberParser.h"
|
||||
#include "Poco/String.h"
|
||||
|
||||
#include <cassert>
|
||||
#include <mutex>
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
std::mutex & getLoggerMutex()
|
||||
{
|
||||
auto get_logger_mutex_placeholder_memory = []()
|
||||
{
|
||||
static char buffer[sizeof(std::mutex)]{};
|
||||
return buffer;
|
||||
};
|
||||
|
||||
static std::mutex * logger_mutex = new (get_logger_mutex_placeholder_memory()) std::mutex();
|
||||
return *logger_mutex;
|
||||
}
|
||||
|
||||
Poco::Logger::LoggerMap * _pLoggerMap = nullptr;
|
||||
|
||||
}
|
||||
|
||||
namespace Poco {
|
||||
|
||||
|
||||
Logger::LoggerMap* Logger::_pLoggerMap = 0;
|
||||
Mutex Logger::_mapMtx;
|
||||
const std::string Logger::ROOT;
|
||||
|
||||
|
||||
@ -73,7 +92,7 @@ void Logger::setProperty(const std::string& name, const std::string& value)
|
||||
setChannel(LoggingRegistry::defaultRegistry().channelForName(value));
|
||||
else if (name == "level")
|
||||
setLevel(value);
|
||||
else
|
||||
else
|
||||
Channel::setProperty(name, value);
|
||||
}
|
||||
|
||||
@ -112,17 +131,17 @@ void Logger::dump(const std::string& msg, const void* buffer, std::size_t length
|
||||
|
||||
void Logger::setLevel(const std::string& name, int level)
|
||||
{
|
||||
Mutex::ScopedLock lock(_mapMtx);
|
||||
std::lock_guard<std::mutex> lock(getLoggerMutex());
|
||||
|
||||
if (_pLoggerMap)
|
||||
{
|
||||
std::string::size_type len = name.length();
|
||||
for (LoggerMap::iterator it = _pLoggerMap->begin(); it != _pLoggerMap->end(); ++it)
|
||||
for (auto & it : *_pLoggerMap)
|
||||
{
|
||||
if (len == 0 ||
|
||||
(it->first.compare(0, len, name) == 0 && (it->first.length() == len || it->first[len] == '.')))
|
||||
if (len == 0 ||
|
||||
(it.first.compare(0, len, name) == 0 && (it.first.length() == len || it.first[len] == '.')))
|
||||
{
|
||||
it->second->setLevel(level);
|
||||
it.second.logger->setLevel(level);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -131,17 +150,17 @@ void Logger::setLevel(const std::string& name, int level)
|
||||
|
||||
void Logger::setChannel(const std::string& name, Channel* pChannel)
|
||||
{
|
||||
Mutex::ScopedLock lock(_mapMtx);
|
||||
std::lock_guard<std::mutex> lock(getLoggerMutex());
|
||||
|
||||
if (_pLoggerMap)
|
||||
{
|
||||
std::string::size_type len = name.length();
|
||||
for (LoggerMap::iterator it = _pLoggerMap->begin(); it != _pLoggerMap->end(); ++it)
|
||||
for (auto & it : *_pLoggerMap)
|
||||
{
|
||||
if (len == 0 ||
|
||||
(it->first.compare(0, len, name) == 0 && (it->first.length() == len || it->first[len] == '.')))
|
||||
(it.first.compare(0, len, name) == 0 && (it.first.length() == len || it.first[len] == '.')))
|
||||
{
|
||||
it->second->setChannel(pChannel);
|
||||
it.second.logger->setChannel(pChannel);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -150,17 +169,17 @@ void Logger::setChannel(const std::string& name, Channel* pChannel)
|
||||
|
||||
void Logger::setProperty(const std::string& loggerName, const std::string& propertyName, const std::string& value)
|
||||
{
|
||||
Mutex::ScopedLock lock(_mapMtx);
|
||||
std::lock_guard<std::mutex> lock(getLoggerMutex());
|
||||
|
||||
if (_pLoggerMap)
|
||||
{
|
||||
std::string::size_type len = loggerName.length();
|
||||
for (LoggerMap::iterator it = _pLoggerMap->begin(); it != _pLoggerMap->end(); ++it)
|
||||
for (auto & it : *_pLoggerMap)
|
||||
{
|
||||
if (len == 0 ||
|
||||
(it->first.compare(0, len, loggerName) == 0 && (it->first.length() == len || it->first[len] == '.')))
|
||||
(it.first.compare(0, len, loggerName) == 0 && (it.first.length() == len || it.first[len] == '.')))
|
||||
{
|
||||
it->second->setProperty(propertyName, value);
|
||||
it.second.logger->setProperty(propertyName, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -280,108 +299,200 @@ void Logger::formatDump(std::string& message, const void* buffer, std::size_t le
|
||||
}
|
||||
|
||||
|
||||
Logger& Logger::get(const std::string& name)
|
||||
namespace
|
||||
{
|
||||
Mutex::ScopedLock lock(_mapMtx);
|
||||
|
||||
return unsafeGet(name);
|
||||
struct LoggerDeleter
|
||||
{
|
||||
void operator()(Poco::Logger * logger)
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(getLoggerMutex());
|
||||
|
||||
/// If logger infrastructure is destroyed just decrement logger reference count
|
||||
if (!_pLoggerMap)
|
||||
{
|
||||
logger->release();
|
||||
return;
|
||||
}
|
||||
|
||||
auto it = _pLoggerMap->find(logger->name());
|
||||
assert(it != _pLoggerMap->end());
|
||||
|
||||
/** If reference count is 1, this means this shared pointer owns logger
|
||||
* and need destroy it.
|
||||
*/
|
||||
size_t reference_count_before_release = logger->release();
|
||||
if (reference_count_before_release == 1)
|
||||
{
|
||||
assert(it->second.owned_by_shared_ptr);
|
||||
_pLoggerMap->erase(it);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
inline LoggerPtr makeLoggerPtr(Logger & logger, bool owned_by_shared_ptr)
|
||||
{
|
||||
if (owned_by_shared_ptr)
|
||||
return LoggerPtr(&logger, LoggerDeleter());
|
||||
|
||||
return LoggerPtr(std::shared_ptr<void>{}, &logger);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
Logger& Logger::unsafeGet(const std::string& name)
|
||||
Logger& Logger::get(const std::string& name)
|
||||
{
|
||||
Logger* pLogger = find(name);
|
||||
if (!pLogger)
|
||||
std::lock_guard<std::mutex> lock(getLoggerMutex());
|
||||
|
||||
auto [it, inserted] = unsafeGet(name, false /*get_shared*/);
|
||||
return *it->second.logger;
|
||||
}
|
||||
|
||||
|
||||
LoggerPtr Logger::getShared(const std::string & name, bool should_be_owned_by_shared_ptr_if_created)
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(getLoggerMutex());
|
||||
auto [it, inserted] = unsafeGet(name, true /*get_shared*/);
|
||||
|
||||
/** If during `unsafeGet` logger was created, then this shared pointer owns it.
|
||||
* If logger was already created, then this shared pointer does not own it.
|
||||
*/
|
||||
if (inserted && should_be_owned_by_shared_ptr_if_created)
|
||||
it->second.owned_by_shared_ptr = true;
|
||||
|
||||
return makeLoggerPtr(*it->second.logger, it->second.owned_by_shared_ptr);
|
||||
}
|
||||
|
||||
|
||||
std::pair<Logger::LoggerMapIterator, bool> Logger::unsafeGet(const std::string& name, bool get_shared)
|
||||
{
|
||||
std::optional<Logger::LoggerMapIterator> optional_logger_it = find(name);
|
||||
|
||||
if (optional_logger_it)
|
||||
{
|
||||
auto & logger_it = *optional_logger_it;
|
||||
|
||||
if (logger_it->second.owned_by_shared_ptr)
|
||||
{
|
||||
logger_it->second.logger->duplicate();
|
||||
|
||||
if (!get_shared)
|
||||
logger_it->second.owned_by_shared_ptr = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (!optional_logger_it)
|
||||
{
|
||||
Logger * logger = nullptr;
|
||||
|
||||
if (name == ROOT)
|
||||
{
|
||||
pLogger = new Logger(name, 0, Message::PRIO_INFORMATION);
|
||||
logger = new Logger(name, nullptr, Message::PRIO_INFORMATION);
|
||||
}
|
||||
else
|
||||
{
|
||||
Logger& par = parent(name);
|
||||
pLogger = new Logger(name, par.getChannel(), par.getLevel());
|
||||
logger = new Logger(name, par.getChannel(), par.getLevel());
|
||||
}
|
||||
add(pLogger);
|
||||
|
||||
return add(logger);
|
||||
}
|
||||
return *pLogger;
|
||||
|
||||
return std::make_pair(*optional_logger_it, false);
|
||||
}
|
||||
|
||||
|
||||
Logger * Logger::unsafeGetRawPtr(const std::string & name)
|
||||
{
|
||||
return unsafeGet(name, false /*get_shared*/).first->second.logger;
|
||||
}
|
||||
|
||||
|
||||
Logger& Logger::create(const std::string& name, Channel* pChannel, int level)
|
||||
{
|
||||
Mutex::ScopedLock lock(_mapMtx);
|
||||
std::lock_guard<std::mutex> lock(getLoggerMutex());
|
||||
|
||||
if (find(name)) throw ExistsException();
|
||||
Logger* pLogger = new Logger(name, pChannel, level);
|
||||
add(pLogger);
|
||||
return *pLogger;
|
||||
return *unsafeCreate(name, pChannel, level).first->second.logger;
|
||||
}
|
||||
|
||||
LoggerPtr Logger::createShared(const std::string & name, Channel * pChannel, int level)
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(getLoggerMutex());
|
||||
|
||||
auto [it, inserted] = unsafeCreate(name, pChannel, level);
|
||||
it->second.owned_by_shared_ptr = true;
|
||||
|
||||
return makeLoggerPtr(*it->second.logger, it->second.owned_by_shared_ptr);
|
||||
}
|
||||
|
||||
Logger& Logger::root()
|
||||
{
|
||||
Mutex::ScopedLock lock(_mapMtx);
|
||||
std::lock_guard<std::mutex> lock(getLoggerMutex());
|
||||
|
||||
return unsafeGet(ROOT);
|
||||
return *unsafeGetRawPtr(ROOT);
|
||||
}
|
||||
|
||||
|
||||
Logger* Logger::has(const std::string& name)
|
||||
{
|
||||
Mutex::ScopedLock lock(_mapMtx);
|
||||
std::lock_guard<std::mutex> lock(getLoggerMutex());
|
||||
|
||||
return find(name);
|
||||
auto optional_it = find(name);
|
||||
if (!optional_it)
|
||||
return nullptr;
|
||||
|
||||
return (*optional_it)->second.logger;
|
||||
}
|
||||
|
||||
|
||||
void Logger::shutdown()
|
||||
{
|
||||
Mutex::ScopedLock lock(_mapMtx);
|
||||
std::lock_guard<std::mutex> lock(getLoggerMutex());
|
||||
|
||||
if (_pLoggerMap)
|
||||
{
|
||||
for (LoggerMap::iterator it = _pLoggerMap->begin(); it != _pLoggerMap->end(); ++it)
|
||||
for (auto & it : *_pLoggerMap)
|
||||
{
|
||||
it->second->release();
|
||||
if (it.second.owned_by_shared_ptr)
|
||||
continue;
|
||||
|
||||
it.second.logger->release();
|
||||
}
|
||||
|
||||
delete _pLoggerMap;
|
||||
_pLoggerMap = 0;
|
||||
_pLoggerMap = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Logger* Logger::find(const std::string& name)
|
||||
std::optional<Logger::LoggerMapIterator> Logger::find(const std::string& name)
|
||||
{
|
||||
if (_pLoggerMap)
|
||||
{
|
||||
LoggerMap::iterator it = _pLoggerMap->find(name);
|
||||
if (it != _pLoggerMap->end())
|
||||
return it->second;
|
||||
return it;
|
||||
|
||||
return {};
|
||||
}
|
||||
return 0;
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
|
||||
void Logger::destroy(const std::string& name)
|
||||
Logger * Logger::findRawPtr(const std::string & name)
|
||||
{
|
||||
Mutex::ScopedLock lock(_mapMtx);
|
||||
auto optional_it = find(name);
|
||||
if (!optional_it)
|
||||
return nullptr;
|
||||
|
||||
if (_pLoggerMap)
|
||||
{
|
||||
LoggerMap::iterator it = _pLoggerMap->find(name);
|
||||
if (it != _pLoggerMap->end())
|
||||
{
|
||||
it->second->release();
|
||||
_pLoggerMap->erase(it);
|
||||
}
|
||||
}
|
||||
return (*optional_it)->second.logger;
|
||||
}
|
||||
|
||||
|
||||
void Logger::names(std::vector<std::string>& names)
|
||||
{
|
||||
Mutex::ScopedLock lock(_mapMtx);
|
||||
std::lock_guard<std::mutex> lock(getLoggerMutex());
|
||||
|
||||
names.clear();
|
||||
if (_pLoggerMap)
|
||||
@ -394,19 +505,27 @@ void Logger::names(std::vector<std::string>& names)
|
||||
}
|
||||
|
||||
|
||||
std::pair<Logger::LoggerMapIterator, bool> Logger::unsafeCreate(const std::string & name, Channel * pChannel, int level)
|
||||
{
|
||||
if (find(name)) throw ExistsException();
|
||||
Logger* pLogger = new Logger(name, pChannel, level);
|
||||
return add(pLogger);
|
||||
}
|
||||
|
||||
|
||||
Logger& Logger::parent(const std::string& name)
|
||||
{
|
||||
std::string::size_type pos = name.rfind('.');
|
||||
if (pos != std::string::npos)
|
||||
{
|
||||
std::string pname = name.substr(0, pos);
|
||||
Logger* pParent = find(pname);
|
||||
Logger* pParent = findRawPtr(pname);
|
||||
if (pParent)
|
||||
return *pParent;
|
||||
else
|
||||
return parent(pname);
|
||||
}
|
||||
else return unsafeGet(ROOT);
|
||||
else return *unsafeGetRawPtr(ROOT);
|
||||
}
|
||||
|
||||
|
||||
@ -474,11 +593,14 @@ namespace
|
||||
}
|
||||
|
||||
|
||||
void Logger::add(Logger* pLogger)
|
||||
std::pair<Logger::LoggerMapIterator, bool> Logger::add(Logger* pLogger)
|
||||
{
|
||||
if (!_pLoggerMap)
|
||||
_pLoggerMap = new LoggerMap;
|
||||
_pLoggerMap->insert(LoggerMap::value_type(pLogger->name(), pLogger));
|
||||
_pLoggerMap = new Logger::LoggerMap;
|
||||
|
||||
auto result = _pLoggerMap->emplace(pLogger->name(), LoggerEntry{pLogger, false /*owned_by_shared_ptr*/});
|
||||
assert(result.second);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -2,11 +2,11 @@
|
||||
|
||||
# NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION,
|
||||
# only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes.
|
||||
SET(VERSION_REVISION 54482)
|
||||
SET(VERSION_REVISION 54483)
|
||||
SET(VERSION_MAJOR 24)
|
||||
SET(VERSION_MINOR 1)
|
||||
SET(VERSION_MINOR 2)
|
||||
SET(VERSION_PATCH 1)
|
||||
SET(VERSION_GITHASH a2faa65b080a587026c86844f3a20c74d23a86f8)
|
||||
SET(VERSION_DESCRIBE v24.1.1.1-testing)
|
||||
SET(VERSION_STRING 24.1.1.1)
|
||||
SET(VERSION_GITHASH 5a024dfc0936e062770d0cfaad0805b57c1fba17)
|
||||
SET(VERSION_DESCRIBE v24.2.1.1-testing)
|
||||
SET(VERSION_STRING 24.2.1.1)
|
||||
# end of autochange
|
||||
|
||||
@ -79,6 +79,10 @@ if (SANITIZE_COVERAGE)
|
||||
|
||||
# But the actual coverage will be enabled on per-library basis: for ClickHouse code, but not for 3rd-party.
|
||||
set (COVERAGE_FLAGS "-fsanitize-coverage=trace-pc-guard,pc-table")
|
||||
endif()
|
||||
|
||||
set (WITHOUT_COVERAGE_FLAGS "-fno-profile-instr-generate -fno-coverage-mapping -fno-sanitize-coverage=trace-pc-guard,pc-table")
|
||||
set (WITHOUT_COVERAGE_FLAGS "-fno-profile-instr-generate -fno-coverage-mapping -fno-sanitize-coverage=trace-pc-guard,pc-table")
|
||||
set (WITHOUT_COVERAGE_FLAGS_LIST -fno-profile-instr-generate -fno-coverage-mapping -fno-sanitize-coverage=trace-pc-guard,pc-table)
|
||||
else()
|
||||
set (WITHOUT_COVERAGE_FLAGS "")
|
||||
set (WITHOUT_COVERAGE_FLAGS_LIST "")
|
||||
endif()
|
||||
|
||||
27
contrib/CMakeLists.txt
vendored
27
contrib/CMakeLists.txt
vendored
@ -154,6 +154,7 @@ add_contrib (libpqxx-cmake libpqxx)
|
||||
add_contrib (libpq-cmake libpq)
|
||||
add_contrib (nuraft-cmake NuRaft)
|
||||
add_contrib (fast_float-cmake fast_float)
|
||||
add_contrib (idna-cmake idna)
|
||||
add_contrib (datasketches-cpp-cmake datasketches-cpp)
|
||||
add_contrib (incbin-cmake incbin)
|
||||
add_contrib (sqids-cpp-cmake sqids-cpp)
|
||||
@ -171,9 +172,9 @@ add_contrib (s2geometry-cmake s2geometry)
|
||||
add_contrib (c-ares-cmake c-ares)
|
||||
|
||||
if (OS_LINUX AND ARCH_AMD64 AND ENABLE_SSE42)
|
||||
option (ENABLE_QPL "Enable Intel® Query Processing Library" ${ENABLE_LIBRARIES})
|
||||
option (ENABLE_QPL "Enable Intel® Query Processing Library (QPL)" ${ENABLE_LIBRARIES})
|
||||
elseif(ENABLE_QPL)
|
||||
message (${RECONFIGURE_MESSAGE_LEVEL} "QPL library is only supported on x86_64 arch with SSE 4.2 or higher")
|
||||
message (${RECONFIGURE_MESSAGE_LEVEL} "QPL library is only supported on x86_64 with SSE 4.2 or higher")
|
||||
endif()
|
||||
if (ENABLE_QPL)
|
||||
add_contrib (idxd-config-cmake idxd-config)
|
||||
@ -182,6 +183,28 @@ else()
|
||||
message(STATUS "Not using QPL")
|
||||
endif ()
|
||||
|
||||
if (OS_LINUX AND ARCH_AMD64)
|
||||
option (ENABLE_QATLIB "Enable Intel® QuickAssist Technology Library (QATlib)" ${ENABLE_LIBRARIES})
|
||||
elseif(ENABLE_QATLIB)
|
||||
message (${RECONFIGURE_MESSAGE_LEVEL} "QATLib is only supported on x86_64")
|
||||
endif()
|
||||
if (ENABLE_QATLIB)
|
||||
option (ENABLE_QAT_USDM_DRIVER "A User Space DMA-able Memory (USDM) component which allocates/frees DMA-able memory" OFF)
|
||||
option (ENABLE_QAT_OUT_OF_TREE_BUILD "Using out-of-tree driver, user needs to customize ICP_ROOT variable" OFF)
|
||||
set(ICP_ROOT "" CACHE STRING "ICP_ROOT variable to define the path of out-of-tree driver package")
|
||||
if (ENABLE_QAT_OUT_OF_TREE_BUILD)
|
||||
if (ICP_ROOT STREQUAL "")
|
||||
message(FATAL_ERROR "Please define the path of out-of-tree driver package with -DICP_ROOT=xxx or disable out-of-tree build with -DENABLE_QAT_OUT_OF_TREE_BUILD=OFF; \
|
||||
If you want out-of-tree build but have no package available, please download and build ICP package from: https://www.intel.com/content/www/us/en/download/765501.html")
|
||||
endif ()
|
||||
else()
|
||||
add_contrib (qatlib-cmake qatlib) # requires: isa-l
|
||||
endif ()
|
||||
add_contrib (QAT-ZSTD-Plugin-cmake QAT-ZSTD-Plugin)
|
||||
else()
|
||||
message(STATUS "Not using QATLib")
|
||||
endif ()
|
||||
|
||||
add_contrib (morton-nd-cmake morton-nd)
|
||||
if (ARCH_S390X)
|
||||
add_contrib(crc32-s390x-cmake crc32-s390x)
|
||||
|
||||
2
contrib/NuRaft
vendored
2
contrib/NuRaft
vendored
@ -1 +1 @@
|
||||
Subproject commit b7ea89b817a18dc0eafc1f909d568869f02d2d04
|
||||
Subproject commit 1278e32bb0d5dc489f947e002bdf8c71b0ddaa63
|
||||
1
contrib/QAT-ZSTD-Plugin
vendored
Submodule
1
contrib/QAT-ZSTD-Plugin
vendored
Submodule
@ -0,0 +1 @@
|
||||
Subproject commit e5a134e12d2ea8a5b0f3b83c5b1c325fda4eb0a8
|
||||
85
contrib/QAT-ZSTD-Plugin-cmake/CMakeLists.txt
Normal file
85
contrib/QAT-ZSTD-Plugin-cmake/CMakeLists.txt
Normal file
@ -0,0 +1,85 @@
|
||||
# Intel® QuickAssist Technology ZSTD Plugin (QAT ZSTD Plugin) is a plugin to Zstandard*(ZSTD*) for accelerating compression by QAT.
|
||||
# ENABLE_QAT_OUT_OF_TREE_BUILD = 1 means kernel don't have native support, user will build and install driver from external package: https://www.intel.com/content/www/us/en/download/765501.html
|
||||
# meanwhile, user need to set ICP_ROOT environment variable which point to the root directory of QAT driver source tree.
|
||||
# ENABLE_QAT_OUT_OF_TREE_BUILD = 0 means kernel has built-in qat driver, QAT-ZSTD-PLUGIN just has dependency on qatlib.
|
||||
|
||||
if (ENABLE_QAT_OUT_OF_TREE_BUILD)
|
||||
message(STATUS "Intel QATZSTD out-of-tree build, ICP_ROOT:${ICP_ROOT}")
|
||||
|
||||
set(QATZSTD_SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/QAT-ZSTD-Plugin/src")
|
||||
set(QATZSTD_SRC "${QATZSTD_SRC_DIR}/qatseqprod.c")
|
||||
set(ZSTD_LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/zstd/lib")
|
||||
set(QAT_INCLUDE_DIR "${ICP_ROOT}/quickassist/include")
|
||||
set(QAT_DC_INCLUDE_DIR "${ICP_ROOT}/quickassist/include/dc")
|
||||
set(QAT_AL_INCLUDE_DIR "${ICP_ROOT}/quickassist/lookaside/access_layer/include")
|
||||
set(QAT_USDM_INCLUDE_DIR "${ICP_ROOT}/quickassist/utilities/libusdm_drv")
|
||||
set(USDM_LIBRARY "${ICP_ROOT}/build/libusdm_drv_s.so")
|
||||
set(QAT_S_LIBRARY "${ICP_ROOT}/build/libqat_s.so")
|
||||
if (ENABLE_QAT_USDM_DRIVER)
|
||||
add_definitions(-DENABLE_USDM_DRV)
|
||||
endif()
|
||||
add_library(_qatzstd_plugin ${QATZSTD_SRC})
|
||||
target_link_libraries (_qatzstd_plugin PUBLIC ${USDM_LIBRARY} ${QAT_S_LIBRARY})
|
||||
target_include_directories(_qatzstd_plugin
|
||||
SYSTEM PUBLIC "${QATZSTD_SRC_DIR}"
|
||||
PRIVATE ${QAT_INCLUDE_DIR}
|
||||
${QAT_DC_INCLUDE_DIR}
|
||||
${QAT_AL_INCLUDE_DIR}
|
||||
${QAT_USDM_INCLUDE_DIR}
|
||||
${ZSTD_LIBRARY_DIR})
|
||||
target_compile_definitions(_qatzstd_plugin PRIVATE -DDEBUGLEVEL=0 PUBLIC -DENABLE_ZSTD_QAT_CODEC)
|
||||
add_library (ch_contrib::qatzstd_plugin ALIAS _qatzstd_plugin)
|
||||
else () # In-tree build
|
||||
message(STATUS "Intel QATZSTD in-tree build")
|
||||
set(QATZSTD_SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/QAT-ZSTD-Plugin/src")
|
||||
set(QATZSTD_SRC "${QATZSTD_SRC_DIR}/qatseqprod.c")
|
||||
set(ZSTD_LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/zstd/lib")
|
||||
|
||||
# please download&build ICP package from: https://www.intel.com/content/www/us/en/download/765501.html
|
||||
set(ICP_ROOT "${ClickHouse_SOURCE_DIR}/contrib/qatlib")
|
||||
set(QAT_INCLUDE_DIR "${ICP_ROOT}/quickassist/include")
|
||||
set(QAT_DC_INCLUDE_DIR "${ICP_ROOT}/quickassist/include/dc")
|
||||
set(QAT_AL_INCLUDE_DIR "${ICP_ROOT}/quickassist/lookaside/access_layer/include")
|
||||
set(QAT_USDM_INCLUDE_DIR "${ICP_ROOT}/quickassist/utilities/libusdm_drv")
|
||||
set(USDM_LIBRARY "${ICP_ROOT}/build/libusdm_drv_s.so")
|
||||
set(QAT_S_LIBRARY "${ICP_ROOT}/build/libqat_s.so")
|
||||
set(LIBQAT_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/qatlib")
|
||||
set(LIBQAT_HEADER_DIR "${CMAKE_CURRENT_BINARY_DIR}/include")
|
||||
|
||||
file(MAKE_DIRECTORY
|
||||
"${LIBQAT_HEADER_DIR}/qat"
|
||||
)
|
||||
file(COPY "${LIBQAT_ROOT_DIR}/quickassist/include/cpa.h"
|
||||
DESTINATION "${LIBQAT_HEADER_DIR}/qat/"
|
||||
)
|
||||
file(COPY "${LIBQAT_ROOT_DIR}/quickassist/include/dc/cpa_dc.h"
|
||||
DESTINATION "${LIBQAT_HEADER_DIR}/qat/"
|
||||
)
|
||||
file(COPY "${LIBQAT_ROOT_DIR}/quickassist/lookaside/access_layer/include/icp_sal_poll.h"
|
||||
DESTINATION "${LIBQAT_HEADER_DIR}/qat/"
|
||||
)
|
||||
file(COPY "${LIBQAT_ROOT_DIR}/quickassist/lookaside/access_layer/include/icp_sal_user.h"
|
||||
DESTINATION "${LIBQAT_HEADER_DIR}/qat/"
|
||||
)
|
||||
file(COPY "${LIBQAT_ROOT_DIR}/quickassist/utilities/libusdm_drv/qae_mem.h"
|
||||
DESTINATION "${LIBQAT_HEADER_DIR}/qat/"
|
||||
)
|
||||
|
||||
if (ENABLE_QAT_USDM_DRIVER)
|
||||
add_definitions(-DENABLE_USDM_DRV)
|
||||
endif()
|
||||
|
||||
add_library(_qatzstd_plugin ${QATZSTD_SRC})
|
||||
target_link_libraries (_qatzstd_plugin PUBLIC ch_contrib::qatlib ch_contrib::usdm)
|
||||
target_include_directories(_qatzstd_plugin PRIVATE
|
||||
${QAT_INCLUDE_DIR}
|
||||
${QAT_DC_INCLUDE_DIR}
|
||||
${QAT_AL_INCLUDE_DIR}
|
||||
${QAT_USDM_INCLUDE_DIR}
|
||||
${ZSTD_LIBRARY_DIR}
|
||||
${LIBQAT_HEADER_DIR})
|
||||
target_compile_definitions(_qatzstd_plugin PRIVATE -DDEBUGLEVEL=0 PUBLIC -DENABLE_ZSTD_QAT_CODEC -DINTREE)
|
||||
target_include_directories(_qatzstd_plugin SYSTEM PUBLIC $<BUILD_INTERFACE:${QATZSTD_SRC_DIR}> $<INSTALL_INTERFACE:include>)
|
||||
add_library (ch_contrib::qatzstd_plugin ALIAS _qatzstd_plugin)
|
||||
endif ()
|
||||
|
||||
2
contrib/avro
vendored
2
contrib/avro
vendored
@ -1 +1 @@
|
||||
Subproject commit 2fb8a8a6ec0eab9109b68abf3b4857e8c476b918
|
||||
Subproject commit d43acc84d3d455b016f847d6666fbc3cd27f16a9
|
||||
2
contrib/aws
vendored
2
contrib/aws
vendored
@ -1 +1 @@
|
||||
Subproject commit ca02358dcc7ce3ab733dd4cbcc32734eecfa4ee3
|
||||
Subproject commit 9eb5097a0abfa837722cca7a5114a25837817bf2
|
||||
2
contrib/aws-c-auth
vendored
2
contrib/aws-c-auth
vendored
@ -1 +1 @@
|
||||
Subproject commit 97133a2b5dbca1ccdf88cd6f44f39d0531d27d12
|
||||
Subproject commit baeffa791d9d1cf61460662a6d9ac2186aaf05df
|
||||
2
contrib/aws-c-cal
vendored
2
contrib/aws-c-cal
vendored
@ -1 +1 @@
|
||||
Subproject commit 85dd7664b786a389c6fb1a6f031ab4bb2282133d
|
||||
Subproject commit 9453687ff5493ba94eaccf8851200565c4364c77
|
||||
2
contrib/aws-c-common
vendored
2
contrib/aws-c-common
vendored
@ -1 +1 @@
|
||||
Subproject commit 45dcb2849c891dba2100b270b4676765c92949ff
|
||||
Subproject commit 80f21b3cac5ac51c6b8a62c7d2a5ef58a75195ee
|
||||
2
contrib/aws-c-compression
vendored
2
contrib/aws-c-compression
vendored
@ -1 +1 @@
|
||||
Subproject commit b517b7decd0dac30be2162f5186c250221c53aff
|
||||
Subproject commit 99ec79ee2970f1a045d4ced1501b97ee521f2f85
|
||||
2
contrib/aws-c-event-stream
vendored
2
contrib/aws-c-event-stream
vendored
@ -1 +1 @@
|
||||
Subproject commit 2f9b60c42f90840ec11822acda3d8cdfa97a773d
|
||||
Subproject commit 08f24e384e5be20bcffa42b49213d24dad7881ae
|
||||
2
contrib/aws-c-http
vendored
2
contrib/aws-c-http
vendored
@ -1 +1 @@
|
||||
Subproject commit dd34461987947672444d0bc872c5a733dfdb9711
|
||||
Subproject commit a082f8a2067e4a31db73f1d4ffd702a8dc0f7089
|
||||
2
contrib/aws-c-io
vendored
2
contrib/aws-c-io
vendored
@ -1 +1 @@
|
||||
Subproject commit d58ed4f272b1cb4f89ac9196526ceebe5f2b0d89
|
||||
Subproject commit 11ce3c750a1dac7b04069fc5bff89e97e91bad4d
|
||||
2
contrib/aws-c-mqtt
vendored
2
contrib/aws-c-mqtt
vendored
@ -1 +1 @@
|
||||
Subproject commit 33c3455cec82b16feb940e12006cefd7b3ef4194
|
||||
Subproject commit 6d36cd3726233cb757468d0ea26f6cd8dad151ec
|
||||
2
contrib/aws-c-s3
vendored
2
contrib/aws-c-s3
vendored
@ -1 +1 @@
|
||||
Subproject commit d7bfe602d6925948f1fff95784e3613cca6a3900
|
||||
Subproject commit de36fee8fe7ab02f10987877ae94a805bf440c1f
|
||||
2
contrib/aws-c-sdkutils
vendored
2
contrib/aws-c-sdkutils
vendored
@ -1 +1 @@
|
||||
Subproject commit 208a701fa01e99c7c8cc3dcebc8317da71362972
|
||||
Subproject commit fd8c0ba2e233997eaaefe82fb818b8b444b956d3
|
||||
2
contrib/aws-checksums
vendored
2
contrib/aws-checksums
vendored
@ -1 +1 @@
|
||||
Subproject commit ad53be196a25bbefa3700a01187fdce573a7d2d0
|
||||
Subproject commit 321b805559c8e911be5bddba13fcbd222a3e2d3a
|
||||
@ -25,6 +25,7 @@ include("${ClickHouse_SOURCE_DIR}/contrib/aws-cmake/AwsFeatureTests.cmake")
|
||||
include("${ClickHouse_SOURCE_DIR}/contrib/aws-cmake/AwsThreadAffinity.cmake")
|
||||
include("${ClickHouse_SOURCE_DIR}/contrib/aws-cmake/AwsThreadName.cmake")
|
||||
include("${ClickHouse_SOURCE_DIR}/contrib/aws-cmake/AwsSIMD.cmake")
|
||||
include("${ClickHouse_SOURCE_DIR}/contrib/aws-crt-cpp/cmake/AwsGetVersion.cmake")
|
||||
|
||||
|
||||
# Gather sources and options.
|
||||
@ -35,6 +36,8 @@ set(AWS_PUBLIC_COMPILE_DEFS)
|
||||
set(AWS_PRIVATE_COMPILE_DEFS)
|
||||
set(AWS_PRIVATE_LIBS)
|
||||
|
||||
list(APPEND AWS_PRIVATE_COMPILE_DEFS "-DINTEL_NO_ITTNOTIFY_API")
|
||||
|
||||
if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
|
||||
list(APPEND AWS_PRIVATE_COMPILE_DEFS "-DDEBUG_BUILD")
|
||||
endif()
|
||||
@ -85,14 +88,20 @@ file(GLOB AWS_SDK_CORE_SRC
|
||||
"${AWS_SDK_CORE_DIR}/source/external/cjson/*.cpp"
|
||||
"${AWS_SDK_CORE_DIR}/source/external/tinyxml2/*.cpp"
|
||||
"${AWS_SDK_CORE_DIR}/source/http/*.cpp"
|
||||
"${AWS_SDK_CORE_DIR}/source/http/crt/*.cpp"
|
||||
"${AWS_SDK_CORE_DIR}/source/http/standard/*.cpp"
|
||||
"${AWS_SDK_CORE_DIR}/source/internal/*.cpp"
|
||||
"${AWS_SDK_CORE_DIR}/source/monitoring/*.cpp"
|
||||
"${AWS_SDK_CORE_DIR}/source/net/*.cpp"
|
||||
"${AWS_SDK_CORE_DIR}/source/net/linux-shared/*.cpp"
|
||||
"${AWS_SDK_CORE_DIR}/source/platform/linux-shared/*.cpp"
|
||||
"${AWS_SDK_CORE_DIR}/source/smithy/tracing/*.cpp"
|
||||
"${AWS_SDK_CORE_DIR}/source/utils/*.cpp"
|
||||
"${AWS_SDK_CORE_DIR}/source/utils/base64/*.cpp"
|
||||
"${AWS_SDK_CORE_DIR}/source/utils/component-registry/*.cpp"
|
||||
"${AWS_SDK_CORE_DIR}/source/utils/crypto/*.cpp"
|
||||
"${AWS_SDK_CORE_DIR}/source/utils/crypto/openssl/*.cpp"
|
||||
"${AWS_SDK_CORE_DIR}/source/utils/crypto/factory/*.cpp"
|
||||
"${AWS_SDK_CORE_DIR}/source/utils/crypto/openssl/*.cpp"
|
||||
"${AWS_SDK_CORE_DIR}/source/utils/event/*.cpp"
|
||||
"${AWS_SDK_CORE_DIR}/source/utils/json/*.cpp"
|
||||
"${AWS_SDK_CORE_DIR}/source/utils/logging/*.cpp"
|
||||
@ -115,9 +124,8 @@ OPTION(USE_AWS_MEMORY_MANAGEMENT "Aws memory management" OFF)
|
||||
configure_file("${AWS_SDK_CORE_DIR}/include/aws/core/SDKConfig.h.in"
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/include/aws/core/SDKConfig.h" @ONLY)
|
||||
|
||||
list(APPEND AWS_PUBLIC_COMPILE_DEFS "-DAWS_SDK_VERSION_MAJOR=1")
|
||||
list(APPEND AWS_PUBLIC_COMPILE_DEFS "-DAWS_SDK_VERSION_MINOR=10")
|
||||
list(APPEND AWS_PUBLIC_COMPILE_DEFS "-DAWS_SDK_VERSION_PATCH=36")
|
||||
aws_get_version(AWS_CRT_CPP_VERSION_MAJOR AWS_CRT_CPP_VERSION_MINOR AWS_CRT_CPP_VERSION_PATCH FULL_VERSION GIT_HASH)
|
||||
configure_file("${AWS_CRT_DIR}/include/aws/crt/Config.h.in" "${AWS_CRT_DIR}/include/aws/crt/Config.h" @ONLY)
|
||||
|
||||
list(APPEND AWS_SOURCES ${AWS_SDK_CORE_SRC} ${AWS_SDK_CORE_NET_SRC} ${AWS_SDK_CORE_PLATFORM_SRC})
|
||||
|
||||
@ -176,6 +184,7 @@ file(GLOB AWS_COMMON_SRC
|
||||
"${AWS_COMMON_DIR}/source/*.c"
|
||||
"${AWS_COMMON_DIR}/source/external/*.c"
|
||||
"${AWS_COMMON_DIR}/source/posix/*.c"
|
||||
"${AWS_COMMON_DIR}/source/linux/*.c"
|
||||
)
|
||||
|
||||
file(GLOB AWS_COMMON_ARCH_SRC
|
||||
|
||||
2
contrib/aws-crt-cpp
vendored
2
contrib/aws-crt-cpp
vendored
@ -1 +1 @@
|
||||
Subproject commit 8a301b7e842f1daed478090c869207300972379f
|
||||
Subproject commit f532d6abc0d2b0d8b5d6fe9e7c51eaedbe4afbd0
|
||||
2
contrib/aws-s2n-tls
vendored
2
contrib/aws-s2n-tls
vendored
@ -1 +1 @@
|
||||
Subproject commit 71f4794b7580cf780eb4aca77d69eded5d3c7bb4
|
||||
Subproject commit 9a1e75454023e952b366ce1eab9c54007250119f
|
||||
2
contrib/azure
vendored
2
contrib/azure
vendored
@ -1 +1 @@
|
||||
Subproject commit 060c54dfb0abe869c065143303a9d3e9c54c29e3
|
||||
Subproject commit e71395e44f309f97b5a486f5c2c59b82f85dd2d2
|
||||
@ -44,12 +44,14 @@ set (SRCS_IOSTREAMS
|
||||
"${LIBRARY_DIR}/libs/iostreams/src/gzip.cpp"
|
||||
"${LIBRARY_DIR}/libs/iostreams/src/mapped_file.cpp"
|
||||
"${LIBRARY_DIR}/libs/iostreams/src/zlib.cpp"
|
||||
"${LIBRARY_DIR}/libs/iostreams/src/zstd.cpp"
|
||||
)
|
||||
|
||||
add_library (_boost_iostreams ${SRCS_IOSTREAMS})
|
||||
add_library (boost::iostreams ALIAS _boost_iostreams)
|
||||
target_include_directories (_boost_iostreams PRIVATE ${LIBRARY_DIR})
|
||||
target_link_libraries (_boost_iostreams PRIVATE ch_contrib::zlib)
|
||||
target_link_libraries (_boost_iostreams PRIVATE ch_contrib::zstd)
|
||||
|
||||
# program_options
|
||||
|
||||
|
||||
@ -1,8 +1,5 @@
|
||||
if (NOT ENABLE_LIBRARIES)
|
||||
set(DEFAULT_ENABLE_RUST FALSE)
|
||||
elseif((CMAKE_TOOLCHAIN_FILE MATCHES "darwin") AND (CMAKE_TOOLCHAIN_FILE MATCHES "aarch64"))
|
||||
message(STATUS "Rust is not available on aarch64-apple-darwin")
|
||||
set(DEFAULT_ENABLE_RUST FALSE)
|
||||
else()
|
||||
list (APPEND CMAKE_MODULE_PATH "${ClickHouse_SOURCE_DIR}/contrib/corrosion/cmake")
|
||||
find_package(Rust)
|
||||
@ -19,27 +16,30 @@ message(STATUS "Checking Rust toolchain for current target")
|
||||
|
||||
# See https://doc.rust-lang.org/nightly/rustc/platform-support.html
|
||||
|
||||
if((CMAKE_TOOLCHAIN_FILE MATCHES "linux/toolchain-x86_64") AND (CMAKE_TOOLCHAIN_FILE MATCHES "musl"))
|
||||
set(Rust_CARGO_TARGET "x86_64-unknown-linux-musl")
|
||||
elseif(CMAKE_TOOLCHAIN_FILE MATCHES "linux/toolchain-x86_64")
|
||||
set(Rust_CARGO_TARGET "x86_64-unknown-linux-gnu")
|
||||
elseif((CMAKE_TOOLCHAIN_FILE MATCHES "linux/toolchain-aarch64") AND (CMAKE_TOOLCHAIN_FILE MATCHES "musl"))
|
||||
set(Rust_CARGO_TARGET "aarch64-unknown-linux-musl")
|
||||
elseif(CMAKE_TOOLCHAIN_FILE MATCHES "linux/toolchain-aarch64")
|
||||
set(Rust_CARGO_TARGET "aarch64-unknown-linux-gnu")
|
||||
elseif((CMAKE_TOOLCHAIN_FILE MATCHES "darwin") AND (CMAKE_TOOLCHAIN_FILE MATCHES "x86_64"))
|
||||
set(Rust_CARGO_TARGET "x86_64-apple-darwin")
|
||||
elseif((CMAKE_TOOLCHAIN_FILE MATCHES "freebsd") AND (CMAKE_TOOLCHAIN_FILE MATCHES "x86_64"))
|
||||
set(Rust_CARGO_TARGET "x86_64-unknown-freebsd")
|
||||
elseif(CMAKE_TOOLCHAIN_FILE MATCHES "linux/toolchain-riscv64")
|
||||
set(Rust_CARGO_TARGET "riscv64gc-unknown-linux-gnu")
|
||||
endif()
|
||||
|
||||
if(CMAKE_TOOLCHAIN_FILE MATCHES "ppc64le")
|
||||
set(Rust_CARGO_TARGET "powerpc64le-unknown-linux-gnu")
|
||||
endif()
|
||||
|
||||
message(STATUS "Switched Rust target to ${Rust_CARGO_TARGET}")
|
||||
if(DEFINED CMAKE_TOOLCHAIN_FILE)
|
||||
if(CMAKE_TOOLCHAIN_FILE MATCHES "ppc64le")
|
||||
set(Rust_CARGO_TARGET "powerpc64le-unknown-linux-gnu")
|
||||
elseif((CMAKE_TOOLCHAIN_FILE MATCHES "linux/toolchain-x86_64") AND (CMAKE_TOOLCHAIN_FILE MATCHES "musl"))
|
||||
set(Rust_CARGO_TARGET "x86_64-unknown-linux-musl")
|
||||
elseif(CMAKE_TOOLCHAIN_FILE MATCHES "linux/toolchain-x86_64")
|
||||
set(Rust_CARGO_TARGET "x86_64-unknown-linux-gnu")
|
||||
elseif((CMAKE_TOOLCHAIN_FILE MATCHES "linux/toolchain-aarch64") AND (CMAKE_TOOLCHAIN_FILE MATCHES "musl"))
|
||||
set(Rust_CARGO_TARGET "aarch64-unknown-linux-musl")
|
||||
elseif(CMAKE_TOOLCHAIN_FILE MATCHES "linux/toolchain-aarch64")
|
||||
set(Rust_CARGO_TARGET "aarch64-unknown-linux-gnu")
|
||||
elseif((CMAKE_TOOLCHAIN_FILE MATCHES "darwin") AND (CMAKE_TOOLCHAIN_FILE MATCHES "x86_64"))
|
||||
set(Rust_CARGO_TARGET "x86_64-apple-darwin")
|
||||
elseif((CMAKE_TOOLCHAIN_FILE MATCHES "darwin") AND (CMAKE_TOOLCHAIN_FILE MATCHES "aarch64"))
|
||||
set(Rust_CARGO_TARGET "aarch64-apple-darwin")
|
||||
elseif((CMAKE_TOOLCHAIN_FILE MATCHES "freebsd") AND (CMAKE_TOOLCHAIN_FILE MATCHES "x86_64"))
|
||||
set(Rust_CARGO_TARGET "x86_64-unknown-freebsd")
|
||||
elseif(CMAKE_TOOLCHAIN_FILE MATCHES "linux/toolchain-riscv64")
|
||||
set(Rust_CARGO_TARGET "riscv64gc-unknown-linux-gnu")
|
||||
else()
|
||||
message(FATAL_ERROR "Unsupported rust target")
|
||||
endif()
|
||||
message(STATUS "Switched Rust target to ${Rust_CARGO_TARGET}")
|
||||
endif ()
|
||||
|
||||
# FindRust.cmake
|
||||
list(APPEND CMAKE_MODULE_PATH "${ClickHouse_SOURCE_DIR}/contrib/corrosion/cmake")
|
||||
|
||||
2
contrib/curl
vendored
2
contrib/curl
vendored
@ -1 +1 @@
|
||||
Subproject commit d755a5f7c009dd63a61b2c745180d8ba937cbfeb
|
||||
Subproject commit 7161cb17c01dcff1dc5bf89a18437d9d729f1ecd
|
||||
1
contrib/idna
vendored
Submodule
1
contrib/idna
vendored
Submodule
@ -0,0 +1 @@
|
||||
Subproject commit 3c8be01d42b75649f1ac9b697d0ef757eebfe667
|
||||
24
contrib/idna-cmake/CMakeLists.txt
Normal file
24
contrib/idna-cmake/CMakeLists.txt
Normal file
@ -0,0 +1,24 @@
|
||||
option(ENABLE_IDNA "Enable idna support" ${ENABLE_LIBRARIES})
|
||||
if ((NOT ENABLE_IDNA))
|
||||
message (STATUS "Not using idna")
|
||||
return()
|
||||
endif()
|
||||
set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/idna")
|
||||
|
||||
set (SRCS
|
||||
"${LIBRARY_DIR}/src/idna.cpp"
|
||||
"${LIBRARY_DIR}/src/mapping.cpp"
|
||||
"${LIBRARY_DIR}/src/mapping_tables.cpp"
|
||||
"${LIBRARY_DIR}/src/normalization.cpp"
|
||||
"${LIBRARY_DIR}/src/normalization_tables.cpp"
|
||||
"${LIBRARY_DIR}/src/punycode.cpp"
|
||||
"${LIBRARY_DIR}/src/to_ascii.cpp"
|
||||
"${LIBRARY_DIR}/src/to_unicode.cpp"
|
||||
"${LIBRARY_DIR}/src/unicode_transcoding.cpp"
|
||||
"${LIBRARY_DIR}/src/validity.cpp"
|
||||
)
|
||||
|
||||
add_library (_idna ${SRCS})
|
||||
target_include_directories(_idna PUBLIC "${LIBRARY_DIR}/include")
|
||||
|
||||
add_library (ch_contrib::idna ALIAS _idna)
|
||||
@ -34,9 +34,9 @@ if (OS_LINUX)
|
||||
# avoid spurious latencies and additional work associated with
|
||||
# MADV_DONTNEED. See
|
||||
# https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation.
|
||||
set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000")
|
||||
set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000")
|
||||
else()
|
||||
set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000")
|
||||
set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000")
|
||||
endif()
|
||||
# CACHE variable is empty to allow changing defaults without the necessity
|
||||
# to purge cache
|
||||
@ -161,6 +161,9 @@ target_include_directories(_jemalloc SYSTEM PRIVATE
|
||||
|
||||
target_compile_definitions(_jemalloc PRIVATE -DJEMALLOC_NO_PRIVATE_NAMESPACE)
|
||||
|
||||
# Because our coverage callbacks call malloc, and recursive call of malloc could not work.
|
||||
target_compile_options(_jemalloc PRIVATE ${WITHOUT_COVERAGE_FLAGS_LIST})
|
||||
|
||||
if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
|
||||
target_compile_definitions(_jemalloc PRIVATE
|
||||
-DJEMALLOC_DEBUG=1
|
||||
|
||||
@ -33,7 +33,6 @@ set(SRCS
|
||||
"${LIBCXX_SOURCE_DIR}/src/optional.cpp"
|
||||
"${LIBCXX_SOURCE_DIR}/src/random.cpp"
|
||||
"${LIBCXX_SOURCE_DIR}/src/random_shuffle.cpp"
|
||||
"${LIBCXX_SOURCE_DIR}/src/regex.cpp"
|
||||
"${LIBCXX_SOURCE_DIR}/src/ryu/d2fixed.cpp"
|
||||
"${LIBCXX_SOURCE_DIR}/src/ryu/d2s.cpp"
|
||||
"${LIBCXX_SOURCE_DIR}/src/ryu/f2s.cpp"
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
option (ENABLE_SSH "Enable support for SSH keys and protocol" ON)
|
||||
option (ENABLE_SSH "Enable support for SSH keys and protocol" ${ENABLE_LIBRARIES})
|
||||
|
||||
if (NOT ENABLE_SSH)
|
||||
message(STATUS "Not using SSH")
|
||||
|
||||
@ -1,6 +1,10 @@
|
||||
#include <libunwind.h>
|
||||
|
||||
/// On MacOS this function will be replaced with a dynamic symbol
|
||||
/// from the system library.
|
||||
#if !defined(OS_DARWIN)
|
||||
int backtrace(void ** buffer, int size)
|
||||
{
|
||||
return unw_backtrace(buffer, size);
|
||||
}
|
||||
#endif
|
||||
|
||||
2
contrib/libuv
vendored
2
contrib/libuv
vendored
@ -1 +1 @@
|
||||
Subproject commit 3a85b2eb3d83f369b8a8cafd329d7e9dc28f60cf
|
||||
Subproject commit 4482964660c77eec1166cd7d14fb915e3dbd774a
|
||||
2
contrib/llvm-project
vendored
2
contrib/llvm-project
vendored
@ -1 +1 @@
|
||||
Subproject commit 1834e42289c58402c804a87be4d489892b88f3ec
|
||||
Subproject commit d2142eed98046a47ff7112e3cc1e197c8a5cd80f
|
||||
@ -1,5 +1,6 @@
|
||||
if (APPLE OR SANITIZE STREQUAL "undefined" OR SANITIZE STREQUAL "memory")
|
||||
# llvm-tblgen, that is used during LLVM build, doesn't work with UBSan.
|
||||
if (APPLE OR SANITIZE STREQUAL "memory")
|
||||
# llvm-tblgen, that is used during LLVM build, will throw MSAN errors when running (breaking the build)
|
||||
# TODO: Retest when upgrading LLVM or build only llvm-tblgen without sanitizers
|
||||
set (ENABLE_EMBEDDED_COMPILER_DEFAULT OFF)
|
||||
set (ENABLE_DWARF_PARSER_DEFAULT OFF)
|
||||
else()
|
||||
|
||||
1
contrib/qatlib
vendored
Submodule
1
contrib/qatlib
vendored
Submodule
@ -0,0 +1 @@
|
||||
Subproject commit abe15d7bfc083117bfbb4baee0b49ffcd1c03c5c
|
||||
213
contrib/qatlib-cmake/CMakeLists.txt
Normal file
213
contrib/qatlib-cmake/CMakeLists.txt
Normal file
@ -0,0 +1,213 @@
|
||||
# Intel® QuickAssist Technology Library (QATlib).
|
||||
|
||||
message(STATUS "Intel QATlib ON")
|
||||
set(LIBQAT_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/qatlib")
|
||||
set(LIBQAT_DIR "${LIBQAT_ROOT_DIR}/quickassist/lookaside/access_layer/src")
|
||||
set(LIBOSAL_DIR "${LIBQAT_ROOT_DIR}/quickassist/utilities/osal/src")
|
||||
set(OPENSSL_DIR "${ClickHouse_SOURCE_DIR}/contrib/openssl")
|
||||
|
||||
# Build 3 libraries: _qatmgr, _osal, _qatlib
|
||||
# Produce ch_contrib::qatlib by linking these libraries.
|
||||
|
||||
# _qatmgr
|
||||
|
||||
SET(LIBQATMGR_sources ${LIBQAT_DIR}/qat_direct/vfio/qat_mgr_client.c
|
||||
${LIBQAT_DIR}/qat_direct/vfio/qat_mgr_lib.c
|
||||
${LIBQAT_DIR}/qat_direct/vfio/qat_log.c
|
||||
${LIBQAT_DIR}/qat_direct/vfio/vfio_lib.c
|
||||
${LIBQAT_DIR}/qat_direct/vfio/adf_pfvf_proto.c
|
||||
${LIBQAT_DIR}/qat_direct/vfio/adf_pfvf_vf_msg.c
|
||||
${LIBQAT_DIR}/qat_direct/vfio/adf_vfio_pf.c)
|
||||
|
||||
add_library(_qatmgr ${LIBQATMGR_sources})
|
||||
|
||||
target_include_directories(_qatmgr PRIVATE
|
||||
${LIBQAT_ROOT_DIR}/quickassist/lookaside/access_layer/src/qat_direct/vfio
|
||||
${LIBQAT_ROOT_DIR}/quickassist/lookaside/access_layer/include
|
||||
${LIBQAT_ROOT_DIR}/quickassist/include
|
||||
${LIBQAT_ROOT_DIR}/quickassist/utilities/osal/include
|
||||
${LIBQAT_ROOT_DIR}/quickassist/utilities/osal/src/linux/user_space/include
|
||||
${LIBQAT_ROOT_DIR}/quickassist/qat/drivers/crypto/qat/qat_common
|
||||
${LIBQAT_ROOT_DIR}/quickassist/lookaside/access_layer/src/qat_direct/include
|
||||
${LIBQAT_ROOT_DIR}/quickassist/lookaside/access_layer/src/qat_direct/common/include
|
||||
${ClickHouse_SOURCE_DIR}/contrib/sysroot/linux-x86_64-musl/include)
|
||||
|
||||
target_compile_definitions(_qatmgr PRIVATE -DUSER_SPACE)
|
||||
target_compile_options(_qatmgr PRIVATE -Wno-error=int-conversion)
|
||||
|
||||
# _osal
|
||||
|
||||
SET(LIBOSAL_sources
|
||||
${LIBOSAL_DIR}/linux/user_space/OsalSemaphore.c
|
||||
${LIBOSAL_DIR}/linux/user_space/OsalThread.c
|
||||
${LIBOSAL_DIR}/linux/user_space/OsalMutex.c
|
||||
${LIBOSAL_DIR}/linux/user_space/OsalSpinLock.c
|
||||
${LIBOSAL_DIR}/linux/user_space/OsalAtomic.c
|
||||
${LIBOSAL_DIR}/linux/user_space/OsalServices.c
|
||||
${LIBOSAL_DIR}/linux/user_space/OsalUsrKrnProxy.c
|
||||
${LIBOSAL_DIR}/linux/user_space/OsalCryptoInterface.c)
|
||||
|
||||
add_library(_osal ${LIBOSAL_sources})
|
||||
|
||||
target_include_directories(_osal PRIVATE
|
||||
${LIBQAT_ROOT_DIR}/quickassist/utilities/osal/src/linux/user_space
|
||||
${LIBQAT_ROOT_DIR}/quickassist/utilities/osal/include
|
||||
${LIBQAT_ROOT_DIR}/quickassist/utilities/osal/src/linux/user_space/include
|
||||
${OPENSSL_DIR}/include
|
||||
${ClickHouse_SOURCE_DIR}/contrib/openssl-cmake/linux_x86_64/include
|
||||
${ClickHouse_SOURCE_DIR}/contrib/qatlib-cmake/include)
|
||||
|
||||
target_compile_definitions(_osal PRIVATE -DOSAL_ENSURE_ON -DUSE_OPENSSL)
|
||||
|
||||
# _qatlib
|
||||
SET(LIBQAT_sources
|
||||
${LIBQAT_DIR}/common/compression/dc_buffers.c
|
||||
${LIBQAT_DIR}/common/compression/dc_chain.c
|
||||
${LIBQAT_DIR}/common/compression/dc_datapath.c
|
||||
${LIBQAT_DIR}/common/compression/dc_dp.c
|
||||
${LIBQAT_DIR}/common/compression/dc_header_footer.c
|
||||
${LIBQAT_DIR}/common/compression/dc_header_footer_lz4.c
|
||||
${LIBQAT_DIR}/common/compression/dc_session.c
|
||||
${LIBQAT_DIR}/common/compression/dc_stats.c
|
||||
${LIBQAT_DIR}/common/compression/dc_err_sim.c
|
||||
${LIBQAT_DIR}/common/compression/dc_ns_datapath.c
|
||||
${LIBQAT_DIR}/common/compression/dc_ns_header_footer.c
|
||||
${LIBQAT_DIR}/common/compression/dc_crc32.c
|
||||
${LIBQAT_DIR}/common/compression/dc_crc64.c
|
||||
${LIBQAT_DIR}/common/compression/dc_xxhash32.c
|
||||
${LIBQAT_DIR}/common/compression/icp_sal_dc_err_sim.c
|
||||
${LIBQAT_DIR}/common/crypto/asym/diffie_hellman/lac_dh_control_path.c
|
||||
${LIBQAT_DIR}/common/crypto/asym/diffie_hellman/lac_dh_data_path.c
|
||||
${LIBQAT_DIR}/common/crypto/asym/diffie_hellman/lac_dh_interface_check.c
|
||||
${LIBQAT_DIR}/common/crypto/asym/diffie_hellman/lac_dh_stats.c
|
||||
${LIBQAT_DIR}/common/crypto/asym/dsa/lac_dsa.c
|
||||
${LIBQAT_DIR}/common/crypto/asym/dsa/lac_dsa_interface_check.c
|
||||
${LIBQAT_DIR}/common/crypto/asym/ecc/lac_ec.c
|
||||
${LIBQAT_DIR}/common/crypto/asym/ecc/lac_ec_common.c
|
||||
${LIBQAT_DIR}/common/crypto/asym/ecc/lac_ec_montedwds.c
|
||||
${LIBQAT_DIR}/common/crypto/asym/ecc/lac_ec_nist_curves.c
|
||||
${LIBQAT_DIR}/common/crypto/asym/ecc/lac_ecdh.c
|
||||
${LIBQAT_DIR}/common/crypto/asym/ecc/lac_ecdsa.c
|
||||
${LIBQAT_DIR}/common/crypto/asym/ecc/lac_ecsm2.c
|
||||
${LIBQAT_DIR}/common/crypto/asym/ecc/lac_kpt_ecdsa.c
|
||||
${LIBQAT_DIR}/common/crypto/asym/large_number/lac_ln.c
|
||||
${LIBQAT_DIR}/common/crypto/asym/large_number/lac_ln_interface_check.c
|
||||
${LIBQAT_DIR}/common/crypto/asym/pke_common/lac_pke_mmp.c
|
||||
${LIBQAT_DIR}/common/crypto/asym/pke_common/lac_pke_qat_comms.c
|
||||
${LIBQAT_DIR}/common/crypto/asym/pke_common/lac_pke_utils.c
|
||||
${LIBQAT_DIR}/common/crypto/asym/prime/lac_prime.c
|
||||
${LIBQAT_DIR}/common/crypto/asym/prime/lac_prime_interface_check.c
|
||||
${LIBQAT_DIR}/common/crypto/asym/rsa/lac_rsa.c
|
||||
${LIBQAT_DIR}/common/crypto/asym/rsa/lac_rsa_control_path.c
|
||||
${LIBQAT_DIR}/common/crypto/asym/rsa/lac_rsa_decrypt.c
|
||||
${LIBQAT_DIR}/common/crypto/asym/rsa/lac_rsa_encrypt.c
|
||||
${LIBQAT_DIR}/common/crypto/asym/rsa/lac_rsa_interface_check.c
|
||||
${LIBQAT_DIR}/common/crypto/asym/rsa/lac_rsa_keygen.c
|
||||
${LIBQAT_DIR}/common/crypto/asym/rsa/lac_rsa_stats.c
|
||||
${LIBQAT_DIR}/common/crypto/asym/rsa/lac_kpt_rsa_decrypt.c
|
||||
${LIBQAT_DIR}/common/crypto/sym/drbg/lac_sym_drbg_api.c
|
||||
${LIBQAT_DIR}/common/crypto/sym/key/lac_sym_key.c
|
||||
${LIBQAT_DIR}/common/crypto/sym/lac_sym_alg_chain.c
|
||||
${LIBQAT_DIR}/common/crypto/sym/lac_sym_api.c
|
||||
${LIBQAT_DIR}/common/crypto/sym/lac_sym_auth_enc.c
|
||||
${LIBQAT_DIR}/common/crypto/sym/lac_sym_cb.c
|
||||
${LIBQAT_DIR}/common/crypto/sym/lac_sym_cipher.c
|
||||
${LIBQAT_DIR}/common/crypto/sym/lac_sym_compile_check.c
|
||||
${LIBQAT_DIR}/common/crypto/sym/lac_sym_dp.c
|
||||
${LIBQAT_DIR}/common/crypto/sym/lac_sym_hash.c
|
||||
${LIBQAT_DIR}/common/crypto/sym/lac_sym_partial.c
|
||||
${LIBQAT_DIR}/common/crypto/sym/lac_sym_queue.c
|
||||
${LIBQAT_DIR}/common/crypto/sym/lac_sym_stats.c
|
||||
${LIBQAT_DIR}/common/crypto/sym/nrbg/lac_sym_nrbg_api.c
|
||||
${LIBQAT_DIR}/common/crypto/sym/qat/lac_sym_qat.c
|
||||
${LIBQAT_DIR}/common/crypto/sym/qat/lac_sym_qat_cipher.c
|
||||
${LIBQAT_DIR}/common/crypto/sym/qat/lac_sym_qat_constants_table.c
|
||||
${LIBQAT_DIR}/common/crypto/sym/qat/lac_sym_qat_hash.c
|
||||
${LIBQAT_DIR}/common/crypto/sym/qat/lac_sym_qat_hash_defs_lookup.c
|
||||
${LIBQAT_DIR}/common/crypto/sym/qat/lac_sym_qat_key.c
|
||||
${LIBQAT_DIR}/common/crypto/sym/lac_sym_hash_sw_precomputes.c
|
||||
${LIBQAT_DIR}/common/crypto/kpt/provision/lac_kpt_provision.c
|
||||
${LIBQAT_DIR}/common/ctrl/sal_compression.c
|
||||
${LIBQAT_DIR}/common/ctrl/sal_create_services.c
|
||||
${LIBQAT_DIR}/common/ctrl/sal_ctrl_services.c
|
||||
${LIBQAT_DIR}/common/ctrl/sal_list.c
|
||||
${LIBQAT_DIR}/common/ctrl/sal_crypto.c
|
||||
${LIBQAT_DIR}/common/ctrl/sal_dc_chain.c
|
||||
${LIBQAT_DIR}/common/ctrl/sal_instances.c
|
||||
${LIBQAT_DIR}/common/qat_comms/sal_qat_cmn_msg.c
|
||||
${LIBQAT_DIR}/common/utils/lac_buffer_desc.c
|
||||
${LIBQAT_DIR}/common/utils/lac_log_message.c
|
||||
${LIBQAT_DIR}/common/utils/lac_mem.c
|
||||
${LIBQAT_DIR}/common/utils/lac_mem_pools.c
|
||||
${LIBQAT_DIR}/common/utils/lac_sw_responses.c
|
||||
${LIBQAT_DIR}/common/utils/lac_sync.c
|
||||
${LIBQAT_DIR}/common/utils/sal_service_state.c
|
||||
${LIBQAT_DIR}/common/utils/sal_statistics.c
|
||||
${LIBQAT_DIR}/common/utils/sal_misc_error_stats.c
|
||||
${LIBQAT_DIR}/common/utils/sal_string_parse.c
|
||||
${LIBQAT_DIR}/common/utils/sal_user_process.c
|
||||
${LIBQAT_DIR}/common/utils/sal_versions.c
|
||||
${LIBQAT_DIR}/common/device/sal_dev_info.c
|
||||
${LIBQAT_DIR}/user/sal_user.c
|
||||
${LIBQAT_DIR}/user/sal_user_dyn_instance.c
|
||||
${LIBQAT_DIR}/qat_direct/common/adf_process_proxy.c
|
||||
${LIBQAT_DIR}/qat_direct/common/adf_user_cfg.c
|
||||
${LIBQAT_DIR}/qat_direct/common/adf_user_device.c
|
||||
${LIBQAT_DIR}/qat_direct/common/adf_user_dyn.c
|
||||
${LIBQAT_DIR}/qat_direct/common/adf_user_ETring_mgr_dp.c
|
||||
${LIBQAT_DIR}/qat_direct/common/adf_user_init.c
|
||||
${LIBQAT_DIR}/qat_direct/common/adf_user_ring.c
|
||||
${LIBQAT_DIR}/qat_direct/common/adf_user_transport_ctrl.c
|
||||
${LIBQAT_DIR}/qat_direct/vfio/adf_vfio_cfg.c
|
||||
${LIBQAT_DIR}/qat_direct/vfio/adf_vfio_ring.c
|
||||
${LIBQAT_DIR}/qat_direct/vfio/adf_vfio_user_bundles.c
|
||||
${LIBQAT_DIR}/qat_direct/vfio/adf_vfio_user_proxy.c
|
||||
${LIBQAT_DIR}/common/compression/dc_crc_base.c)
|
||||
|
||||
add_library(_qatlib ${LIBQAT_sources})
|
||||
|
||||
target_include_directories(_qatlib PRIVATE
|
||||
${CMAKE_SYSROOT}/usr/include
|
||||
${LIBQAT_ROOT_DIR}/quickassist/lookaside/access_layer/include
|
||||
${LIBQAT_ROOT_DIR}/quickassist/utilities/libusdm_drv
|
||||
${LIBQAT_ROOT_DIR}/quickassist/utilities/osal/include
|
||||
${LIBOSAL_DIR}/linux/user_space/include
|
||||
${LIBQAT_ROOT_DIR}/quickassist/include
|
||||
${LIBQAT_ROOT_DIR}/quickassist/include/lac
|
||||
${LIBQAT_ROOT_DIR}/quickassist/include/dc
|
||||
${LIBQAT_ROOT_DIR}/quickassist/qat/drivers/crypto/qat/qat_common
|
||||
${LIBQAT_ROOT_DIR}/quickassist/lookaside/access_layer/src/common/compression/include
|
||||
${LIBQAT_ROOT_DIR}/quickassist/lookaside/access_layer/src/common/crypto/sym/include
|
||||
${LIBQAT_ROOT_DIR}/quickassist/lookaside/access_layer/src/common/crypto/asym/include
|
||||
${LIBQAT_ROOT_DIR}/quickassist/lookaside/firmware/include
|
||||
${LIBQAT_ROOT_DIR}/quickassist/lookaside/access_layer/src/common/include
|
||||
${LIBQAT_ROOT_DIR}/quickassist/lookaside/access_layer/src/qat_direct/include
|
||||
${LIBQAT_ROOT_DIR}/quickassist/lookaside/access_layer/src/qat_direct/common/include
|
||||
${LIBQAT_ROOT_DIR}/quickassist/lookaside/access_layer/src/qat_direct/vfio
|
||||
${LIBQAT_ROOT_DIR}/quickassist/utilities/osal/src/linux/user_space
|
||||
${LIBQAT_ROOT_DIR}/quickassist/utilities/osal/src/linux/user_space/include
|
||||
${ClickHouse_SOURCE_DIR}/contrib/sysroot/linux-x86_64-musl/include)
|
||||
|
||||
target_link_libraries(_qatlib PRIVATE _qatmgr _osal OpenSSL::SSL ch_contrib::isal)
|
||||
target_compile_definitions(_qatlib PRIVATE -DUSER_SPACE -DLAC_BYTE_ORDER=__LITTLE_ENDIAN -DOSAL_ENSURE_ON)
|
||||
target_link_options(_qatlib PRIVATE -pie -z relro -z now -z noexecstack)
|
||||
target_compile_options(_qatlib PRIVATE -march=native)
|
||||
add_library (ch_contrib::qatlib ALIAS _qatlib)
|
||||
|
||||
# _usdm
|
||||
|
||||
set(LIBUSDM_DIR "${ClickHouse_SOURCE_DIR}/contrib/qatlib/quickassist/utilities/libusdm_drv")
|
||||
set(LIBUSDM_sources
|
||||
${LIBUSDM_DIR}/user_space/vfio/qae_mem_utils_vfio.c
|
||||
${LIBUSDM_DIR}/user_space/qae_mem_utils_common.c
|
||||
${LIBUSDM_DIR}/user_space/vfio/qae_mem_hugepage_utils_vfio.c)
|
||||
|
||||
add_library(_usdm ${LIBUSDM_sources})
|
||||
|
||||
target_include_directories(_usdm PRIVATE
|
||||
${ClickHouse_SOURCE_DIR}/contrib/sysroot/linux-x86_64-musl/include
|
||||
${LIBUSDM_DIR}
|
||||
${LIBUSDM_DIR}/include
|
||||
${LIBUSDM_DIR}/user_space)
|
||||
|
||||
add_library (ch_contrib::usdm ALIAS _usdm)
|
||||
14
contrib/qatlib-cmake/include/mqueue.h
Normal file
14
contrib/qatlib-cmake/include/mqueue.h
Normal file
@ -0,0 +1,14 @@
|
||||
/* This is a workaround for a build conflict issue
|
||||
1. __GLIBC_PREREQ (referenced in OsalServices.c) is only defined in './sysroot/linux-x86_64/include/features.h'
|
||||
2. mqueue.h only exist under './sysroot/linux-x86_64-musl/'
|
||||
This cause target_include_directories for _osal has a conflict between './sysroot/linux-x86_64/include' and './sysroot/linux-x86_64-musl/'
|
||||
hence create mqueue.h separately under ./qatlib-cmake/include as an alternative.
|
||||
*/
|
||||
|
||||
/* Major and minor version number of the GNU C library package. Use
|
||||
these macros to test for features in specific releases. */
|
||||
#define __GLIBC__ 2
|
||||
#define __GLIBC_MINOR__ 27
|
||||
|
||||
#define __GLIBC_PREREQ(maj, min) \
|
||||
((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min))
|
||||
2
contrib/rocksdb
vendored
2
contrib/rocksdb
vendored
@ -1 +1 @@
|
||||
Subproject commit 66e3cbec31400ed3a23deb878c5d7f56f990f0ae
|
||||
Subproject commit dead55e60b873d5f70f0e9458fbbba2b2180f430
|
||||
2
contrib/simdjson
vendored
2
contrib/simdjson
vendored
@ -1 +1 @@
|
||||
Subproject commit 1075e8609c4afa253162d441437af929c29e31bb
|
||||
Subproject commit 6060be2fdf62edf4a8f51a8b0883d57d09397b30
|
||||
2
contrib/sqids-cpp
vendored
2
contrib/sqids-cpp
vendored
@ -1 +1 @@
|
||||
Subproject commit 3756e537d4d48cc0dd4176801fe19f99601439b0
|
||||
Subproject commit a471f53672e98d49223f598528a533b07b085c61
|
||||
8
contrib/update-submodules.sh
vendored
8
contrib/update-submodules.sh
vendored
@ -6,9 +6,15 @@ SCRIPT_DIR=$(dirname "${SCRIPT_PATH}")
|
||||
GIT_DIR=$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel)
|
||||
cd $GIT_DIR
|
||||
|
||||
# Exclude from contribs some garbage subdirs that we don't need.
|
||||
# It reduces the checked out files size about 3 times and therefore speeds up indexing in IDEs and searching.
|
||||
# NOTE .git/ still contains everything that we don't check out (although, it's compressed)
|
||||
# See also https://git-scm.com/docs/git-sparse-checkout
|
||||
contrib/sparse-checkout/setup-sparse-checkout.sh
|
||||
|
||||
git submodule init
|
||||
git submodule sync
|
||||
|
||||
# NOTE: do not use --remote for `git submodule update`[1] command, since the submodule references to the specific commit SHA1 in the subproject.
|
||||
# It may cause unexpected behavior. Instead you need to commit a new SHA1 for a submodule.
|
||||
#
|
||||
@ -18,7 +24,7 @@ git config --file .gitmodules --get-regexp '.*path' | sed 's/[^ ]* //' | xargs -
|
||||
# We don't want to depend on any third-party CMake files.
|
||||
# To check it, find and delete them.
|
||||
grep -o -P '"contrib/[^"]+"' .gitmodules |
|
||||
grep -v -P 'contrib/(llvm-project|google-protobuf|grpc|abseil-cpp|corrosion)' |
|
||||
grep -v -P 'contrib/(llvm-project|google-protobuf|grpc|abseil-cpp|corrosion|aws-crt-cpp)' |
|
||||
xargs -I@ find @ \
|
||||
-'(' -name 'CMakeLists.txt' -or -name '*.cmake' -')' -and -not -name '*.h.cmake' \
|
||||
-delete
|
||||
|
||||
@ -62,7 +62,6 @@
|
||||
"dependent": []
|
||||
},
|
||||
"docker/test/integration/runner": {
|
||||
"only_amd64": true,
|
||||
"name": "clickhouse/integration-tests-runner",
|
||||
"dependent": []
|
||||
},
|
||||
|
||||
@ -34,7 +34,7 @@ RUN arch=${TARGETARCH:-amd64} \
|
||||
# lts / testing / prestable / etc
|
||||
ARG REPO_CHANNEL="stable"
|
||||
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
|
||||
ARG VERSION="23.12.1.1368"
|
||||
ARG VERSION="24.1.2.5"
|
||||
ARG PACKAGES="clickhouse-keeper"
|
||||
ARG DIRECT_DOWNLOAD_URLS=""
|
||||
|
||||
|
||||
@ -72,7 +72,7 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \
|
||||
zstd \
|
||||
zip \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
|
||||
|
||||
# Download toolchain and SDK for Darwin
|
||||
RUN curl -sL -O https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX11.0.sdk.tar.xz
|
||||
|
||||
@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \
|
||||
# lts / testing / prestable / etc
|
||||
ARG REPO_CHANNEL="stable"
|
||||
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
|
||||
ARG VERSION="23.12.1.1368"
|
||||
ARG VERSION="24.1.2.5"
|
||||
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
|
||||
ARG DIRECT_DOWNLOAD_URLS=""
|
||||
|
||||
|
||||
@ -23,14 +23,11 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list
|
||||
tzdata \
|
||||
wget \
|
||||
&& apt-get clean \
|
||||
&& rm -rf \
|
||||
/var/lib/apt/lists/* \
|
||||
/var/cache/debconf \
|
||||
/tmp/*
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
|
||||
|
||||
ARG REPO_CHANNEL="stable"
|
||||
ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
|
||||
ARG VERSION="23.12.1.1368"
|
||||
ARG VERSION="24.1.2.5"
|
||||
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
|
||||
|
||||
# set non-empty deb_location_url url to create a docker image
|
||||
|
||||
@ -41,21 +41,18 @@ readarray -t DISKS_PATHS < <(clickhouse extract-from-config --config-file "$CLIC
|
||||
readarray -t DISKS_METADATA_PATHS < <(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key='storage_configuration.disks.*.metadata_path' || true)
|
||||
|
||||
CLICKHOUSE_USER="${CLICKHOUSE_USER:-default}"
|
||||
CLICKHOUSE_PASSWORD_FILE="${CLICKHOUSE_PASSWORD_FILE:-}"
|
||||
if [[ -n "${CLICKHOUSE_PASSWORD_FILE}" && -f "${CLICKHOUSE_PASSWORD_FILE}" ]]; then
|
||||
CLICKHOUSE_PASSWORD="$(cat "${CLICKHOUSE_PASSWORD_FILE}")"
|
||||
fi
|
||||
CLICKHOUSE_PASSWORD="${CLICKHOUSE_PASSWORD:-}"
|
||||
CLICKHOUSE_DB="${CLICKHOUSE_DB:-}"
|
||||
CLICKHOUSE_ACCESS_MANAGEMENT="${CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT:-0}"
|
||||
|
||||
for dir in "$DATA_DIR" \
|
||||
"$ERROR_LOG_DIR" \
|
||||
"$LOG_DIR" \
|
||||
"$TMP_DIR" \
|
||||
"$USER_PATH" \
|
||||
"$FORMAT_SCHEMA_PATH" \
|
||||
"${DISKS_PATHS[@]}" \
|
||||
"${DISKS_METADATA_PATHS[@]}"
|
||||
do
|
||||
function create_directory_and_do_chown() {
|
||||
local dir=$1
|
||||
# check if variable not empty
|
||||
[ -z "$dir" ] && continue
|
||||
[ -z "$dir" ] && return
|
||||
# ensure directories exist
|
||||
if [ "$DO_CHOWN" = "1" ]; then
|
||||
mkdir="mkdir"
|
||||
@ -77,6 +74,23 @@ do
|
||||
chown -R "$USER:$GROUP" "$dir"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
create_directory_and_do_chown "$DATA_DIR"
|
||||
|
||||
# Change working directory to $DATA_DIR in case there're paths relative to $DATA_DIR, also avoids running
|
||||
# clickhouse-server at root directory.
|
||||
cd "$DATA_DIR"
|
||||
|
||||
for dir in "$ERROR_LOG_DIR" \
|
||||
"$LOG_DIR" \
|
||||
"$TMP_DIR" \
|
||||
"$USER_PATH" \
|
||||
"$FORMAT_SCHEMA_PATH" \
|
||||
"${DISKS_PATHS[@]}" \
|
||||
"${DISKS_METADATA_PATHS[@]}"
|
||||
do
|
||||
create_directory_and_do_chown "$dir"
|
||||
done
|
||||
|
||||
# if clickhouse user is defined - create it (user "default" already exists out of box)
|
||||
|
||||
@ -13,7 +13,10 @@ RUN apt-get update \
|
||||
zstd \
|
||||
locales \
|
||||
sudo \
|
||||
--yes --no-install-recommends
|
||||
--yes --no-install-recommends \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
|
||||
|
||||
|
||||
# Sanitizer options for services (clickhouse-server)
|
||||
# Set resident memory limit for TSAN to 45GiB (46080MiB) to avoid OOMs in Stress tests
|
||||
|
||||
@ -20,9 +20,11 @@ RUN apt-get update \
|
||||
pv \
|
||||
jq \
|
||||
zstd \
|
||||
--yes --no-install-recommends
|
||||
--yes --no-install-recommends \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
|
||||
|
||||
RUN pip3 install numpy scipy pandas Jinja2
|
||||
RUN pip3 install numpy==1.26.3 scipy==1.12.0 pandas==1.5.3 Jinja2==3.1.3
|
||||
|
||||
ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.4.20200302/clickhouse-odbc-1.1.4-Linux.tar.gz"
|
||||
|
||||
@ -31,12 +33,14 @@ RUN mkdir -p /tmp/clickhouse-odbc-tmp \
|
||||
&& cp /tmp/clickhouse-odbc-tmp/lib64/*.so /usr/local/lib/ \
|
||||
&& odbcinst -i -d -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbcinst.ini.sample \
|
||||
&& odbcinst -i -s -l -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbc.ini.sample \
|
||||
&& rm -rf /tmp/clickhouse-odbc-tmp \
|
||||
&& rm -rf /tmp/clickhouse-odbc-tmp
|
||||
|
||||
# Give suid to gdb to grant it attach permissions
|
||||
# chmod 777 to make the container user independent
|
||||
RUN chmod u+s /usr/bin/gdb \
|
||||
&& mkdir -p /var/lib/clickhouse \
|
||||
&& chmod 777 /var/lib/clickhouse
|
||||
|
||||
# chmod 777 to make the container user independent
|
||||
|
||||
ENV TZ=Europe/Amsterdam
|
||||
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
|
||||
|
||||
|
||||
@ -211,6 +211,17 @@ function build
|
||||
echo "build_clickhouse_fasttest_binary: [ OK ] $BUILD_SECONDS_ELAPSED sec." \
|
||||
| ts '%Y-%m-%d %H:%M:%S' \
|
||||
| tee "$FASTTEST_OUTPUT/test_result.txt"
|
||||
|
||||
(
|
||||
# This query should fail, and print stacktrace with proper symbol names (even on a stripped binary)
|
||||
clickhouse_output=$(programs/clickhouse-stripped --stacktrace -q 'select' 2>&1 || :)
|
||||
if [[ $clickhouse_output =~ DB::LocalServer::main ]]; then
|
||||
echo "stripped_clickhouse_shows_symbols_names: [ OK ] 0 sec."
|
||||
else
|
||||
echo -e "stripped_clickhouse_shows_symbols_names: [ FAIL ] 0 sec. - clickhouse output:\n\n$clickhouse_output\n"
|
||||
fi
|
||||
) | ts '%Y-%m-%d %H:%M:%S' | tee -a "$FASTTEST_OUTPUT/test_result.txt"
|
||||
|
||||
if [ "$COPY_CLICKHOUSE_BINARY_TO_OUTPUT" -eq "1" ]; then
|
||||
mkdir -p "$FASTTEST_OUTPUT/binaries/"
|
||||
cp programs/clickhouse "$FASTTEST_OUTPUT/binaries/clickhouse"
|
||||
|
||||
@ -29,7 +29,7 @@ RUN apt-get update \
|
||||
wget \
|
||||
&& apt-get autoremove --yes \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
|
||||
|
||||
RUN pip3 install Jinja2
|
||||
|
||||
|
||||
@ -242,7 +242,7 @@ quit
|
||||
--create-query-fuzzer-runs=50 \
|
||||
--queries-file $(ls -1 ch/tests/queries/0_stateless/*.sql | sort -R) \
|
||||
$NEW_TESTS_OPT \
|
||||
> >(tail -n 100000 > fuzzer.log) \
|
||||
> fuzzer.log \
|
||||
2>&1 &
|
||||
fuzzer_pid=$!
|
||||
echo "Fuzzer pid is $fuzzer_pid"
|
||||
@ -389,7 +389,8 @@ fi
|
||||
rg --text -F '<Fatal>' server.log > fatal.log ||:
|
||||
dmesg -T > dmesg.log ||:
|
||||
|
||||
zstd --threads=0 server.log
|
||||
zstd --threads=0 --rm server.log
|
||||
zstd --threads=0 --rm fuzzer.log
|
||||
|
||||
cat > report.html <<EOF ||:
|
||||
<!DOCTYPE html>
|
||||
@ -413,7 +414,7 @@ p.links a { padding: 5px; margin: 3px; background: #FFF; line-height: 2; white-s
|
||||
<h1>AST Fuzzer for PR <a href="https://github.com/ClickHouse/ClickHouse/pull/${PR_TO_TEST}">#${PR_TO_TEST}</a> @ ${SHA_TO_TEST}</h1>
|
||||
<p class="links">
|
||||
<a href="run.log">run.log</a>
|
||||
<a href="fuzzer.log">fuzzer.log</a>
|
||||
<a href="fuzzer.log.zst">fuzzer.log.zst</a>
|
||||
<a href="server.log.zst">server.log.zst</a>
|
||||
<a href="main.log">main.log</a>
|
||||
<a href="dmesg.log">dmesg.log</a>
|
||||
|
||||
@ -10,13 +10,13 @@ ENV \
|
||||
init=/lib/systemd/systemd
|
||||
|
||||
# install systemd packages
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
sudo \
|
||||
systemd \
|
||||
&& \
|
||||
apt-get clean && \
|
||||
rm -rf /var/lib/apt/lists
|
||||
\
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
|
||||
|
||||
# configure systemd
|
||||
# remove systemd 'wants' triggers
|
||||
|
||||
@ -1,31 +1,27 @@
|
||||
FROM ubuntu:20.04
|
||||
MAINTAINER lgbo-ustc <lgbo.ustc@gmail.com>
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y wget openjdk-8-jre
|
||||
|
||||
RUN wget https://archive.apache.org/dist/hadoop/common/hadoop-3.1.0/hadoop-3.1.0.tar.gz && \
|
||||
tar -xf hadoop-3.1.0.tar.gz && rm -rf hadoop-3.1.0.tar.gz
|
||||
RUN wget https://apache.apache.org/dist/hive/hive-2.3.9/apache-hive-2.3.9-bin.tar.gz && \
|
||||
tar -xf apache-hive-2.3.9-bin.tar.gz && rm -rf apache-hive-2.3.9-bin.tar.gz
|
||||
RUN apt install -y vim
|
||||
|
||||
RUN apt install -y openssh-server openssh-client
|
||||
|
||||
RUN apt install -y mysql-server
|
||||
|
||||
RUN mkdir -p /root/.ssh && \
|
||||
ssh-keygen -t rsa -b 2048 -P '' -f /root/.ssh/id_rsa && \
|
||||
cat /root/.ssh/id_rsa.pub > /root/.ssh/authorized_keys && \
|
||||
cp /root/.ssh/id_rsa /etc/ssh/ssh_host_rsa_key && \
|
||||
cp /root/.ssh/id_rsa.pub /etc/ssh/ssh_host_rsa_key.pub
|
||||
|
||||
RUN wget https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.27.tar.gz &&\
|
||||
tar -xf mysql-connector-java-8.0.27.tar.gz && \
|
||||
mv mysql-connector-java-8.0.27/mysql-connector-java-8.0.27.jar /apache-hive-2.3.9-bin/lib/ && \
|
||||
rm -rf mysql-connector-java-8.0.27.tar.gz mysql-connector-java-8.0.27
|
||||
|
||||
RUN apt install -y iputils-ping net-tools
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y wget openjdk-8-jre \
|
||||
&& wget https://archive.apache.org/dist/hadoop/common/hadoop-3.1.0/hadoop-3.1.0.tar.gz \
|
||||
&& tar -xf hadoop-3.1.0.tar.gz && rm -rf hadoop-3.1.0.tar.gz \
|
||||
&& wget https://apache.apache.org/dist/hive/hive-2.3.9/apache-hive-2.3.9-bin.tar.gz \
|
||||
&& tar -xf apache-hive-2.3.9-bin.tar.gz && rm -rf apache-hive-2.3.9-bin.tar.gz \
|
||||
&& apt install -y vim \
|
||||
&& apt install -y openssh-server openssh-client \
|
||||
&& apt install -y mysql-server \
|
||||
&& mkdir -p /root/.ssh \
|
||||
&& ssh-keygen -t rsa -b 2048 -P '' -f /root/.ssh/id_rsa \
|
||||
&& cat /root/.ssh/id_rsa.pub > /root/.ssh/authorized_keys \
|
||||
&& cp /root/.ssh/id_rsa /etc/ssh/ssh_host_rsa_key \
|
||||
&& cp /root/.ssh/id_rsa.pub /etc/ssh/ssh_host_rsa_key.pub \
|
||||
&& wget https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.27.tar.gz \
|
||||
&& tar -xf mysql-connector-java-8.0.27.tar.gz \
|
||||
&& mv mysql-connector-java-8.0.27/mysql-connector-java-8.0.27.jar /apache-hive-2.3.9-bin/lib/ \
|
||||
&& rm -rf mysql-connector-java-8.0.27.tar.gz mysql-connector-java-8.0.27 \
|
||||
&& apt install -y iputils-ping net-tools \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
|
||||
|
||||
ENV JAVA_HOME=/usr
|
||||
ENV HADOOP_HOME=/hadoop-3.1.0
|
||||
@ -44,4 +40,3 @@ COPY demo_data.txt /
|
||||
ENV PATH=/apache-hive-2.3.9-bin/bin:/hadoop-3.1.0/bin:/hadoop-3.1.0/sbin:$PATH
|
||||
RUN service ssh start && sed s/HOSTNAME/$HOSTNAME/ /hadoop-3.1.0/etc/hadoop/core-site.xml.template > /hadoop-3.1.0/etc/hadoop/core-site.xml && hdfs namenode -format
|
||||
COPY start.sh /
|
||||
|
||||
|
||||
@ -3,14 +3,10 @@
|
||||
|
||||
FROM ubuntu:18.04
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y software-properties-common build-essential openjdk-8-jdk curl
|
||||
|
||||
RUN rm -rf \
|
||||
/var/lib/apt/lists/* \
|
||||
/var/cache/debconf \
|
||||
/tmp/* \
|
||||
RUN apt-get clean
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y software-properties-common build-essential openjdk-8-jdk curl \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
|
||||
|
||||
ARG ver=42.2.12
|
||||
RUN curl -L -o /postgresql-java-${ver}.jar https://repo1.maven.org/maven2/org/postgresql/postgresql/${ver}/postgresql-${ver}.jar
|
||||
|
||||
@ -37,11 +37,8 @@ RUN apt-get update \
|
||||
libkrb5-dev \
|
||||
krb5-user \
|
||||
g++ \
|
||||
&& rm -rf \
|
||||
/var/lib/apt/lists/* \
|
||||
/var/cache/debconf \
|
||||
/tmp/* \
|
||||
&& apt-get clean
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
|
||||
|
||||
ENV TZ=Etc/UTC
|
||||
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
|
||||
@ -62,6 +59,8 @@ RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
|
||||
&& dockerd --version; docker --version
|
||||
|
||||
|
||||
# kazoo 2.10.0 is broken
|
||||
# https://s3.amazonaws.com/clickhouse-test-reports/59337/524625a1d2f4cc608a3f1059e3df2c30f353a649/integration_tests__asan__analyzer__[5_6].html
|
||||
RUN python3 -m pip install --no-cache-dir \
|
||||
PyMySQL \
|
||||
aerospike==11.1.0 \
|
||||
@ -70,7 +69,7 @@ RUN python3 -m pip install --no-cache-dir \
|
||||
azure-storage-blob \
|
||||
boto3 \
|
||||
cassandra-driver \
|
||||
confluent-kafka==1.9.2 \
|
||||
confluent-kafka==2.3.0 \
|
||||
delta-spark==2.3.0 \
|
||||
dict2xml \
|
||||
dicttoxml \
|
||||
@ -79,7 +78,7 @@ RUN python3 -m pip install --no-cache-dir \
|
||||
grpcio \
|
||||
grpcio-tools \
|
||||
kafka-python \
|
||||
kazoo \
|
||||
kazoo==2.9.0 \
|
||||
lz4 \
|
||||
minio \
|
||||
nats-py \
|
||||
|
||||
@ -23,13 +23,15 @@ if [ -f /sys/fs/cgroup/cgroup.controllers ]; then
|
||||
> /sys/fs/cgroup/cgroup.subtree_control
|
||||
fi
|
||||
|
||||
# In case of test hung it is convenient to use pytest --pdb to debug it,
|
||||
# and on hung you can simply press Ctrl-C and it will spawn a python pdb,
|
||||
# but on SIGINT dockerd will exit, so ignore it to preserve the daemon.
|
||||
trap '' INT
|
||||
# Binding to an IP address without --tlsverify is deprecated. Startup is intentionally being slowed
|
||||
# unless --tls=false or --tlsverify=false is set
|
||||
dockerd --host=unix:///var/run/docker.sock --tls=false --host=tcp://0.0.0.0:2375 --default-address-pool base=172.17.0.0/12,size=24 &>/ClickHouse/tests/integration/dockerd.log &
|
||||
#
|
||||
# In case of test hung it is convenient to use pytest --pdb to debug it,
|
||||
# and on hung you can simply press Ctrl-C and it will spawn a python pdb,
|
||||
# but on SIGINT dockerd will exit, so we spawn new session to ignore SIGINT by
|
||||
# docker.
|
||||
# Note, that if you will run it via runner, it will send SIGINT to docker anyway.
|
||||
setsid dockerd --host=unix:///var/run/docker.sock --tls=false --host=tcp://0.0.0.0:2375 --default-address-pool base=172.17.0.0/12,size=24 &>/ClickHouse/tests/integration/dockerd.log &
|
||||
|
||||
set +e
|
||||
reties=0
|
||||
|
||||
@ -24,7 +24,10 @@ RUN mkdir "/root/.ssh"
|
||||
RUN touch "/root/.ssh/known_hosts"
|
||||
|
||||
# install java
|
||||
RUN apt-get update && apt-get install default-jre default-jdk libjna-java libjna-jni ssh gnuplot graphviz --yes --no-install-recommends
|
||||
RUN apt-get update && \
|
||||
apt-get install default-jre default-jdk libjna-java libjna-jni ssh gnuplot graphviz --yes --no-install-recommends \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
|
||||
|
||||
# install clojure
|
||||
RUN curl -O "https://download.clojure.org/install/linux-install-${CLOJURE_VERSION}.sh" && \
|
||||
|
||||
@ -27,7 +27,7 @@ RUN apt-get update \
|
||||
wget \
|
||||
&& apt-get autoremove --yes \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
|
||||
|
||||
RUN pip3 install Jinja2
|
||||
|
||||
|
||||
@ -37,7 +37,7 @@ RUN apt-get update \
|
||||
&& apt-get purge --yes python3-dev g++ \
|
||||
&& apt-get autoremove --yes \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
|
||||
|
||||
COPY run.sh /
|
||||
|
||||
|
||||
@ -31,7 +31,9 @@ RUN mkdir "/root/.ssh"
|
||||
RUN touch "/root/.ssh/known_hosts"
|
||||
|
||||
# install java
|
||||
RUN apt-get update && apt-get install default-jre default-jdk libjna-java libjna-jni ssh gnuplot graphviz --yes --no-install-recommends
|
||||
RUN apt-get update && apt-get install default-jre default-jdk libjna-java libjna-jni ssh gnuplot graphviz --yes --no-install-recommends \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
|
||||
|
||||
# install clojure
|
||||
RUN curl -O "https://download.clojure.org/install/linux-install-${CLOJURE_VERSION}.sh" && \
|
||||
|
||||
@ -5,9 +5,10 @@ FROM ubuntu:22.04
|
||||
ARG apt_archive="http://archive.ubuntu.com"
|
||||
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
|
||||
|
||||
RUN apt-get update --yes && \
|
||||
env DEBIAN_FRONTEND=noninteractive apt-get install wget git default-jdk maven python3 --yes --no-install-recommends && \
|
||||
apt-get clean
|
||||
RUN apt-get update --yes \
|
||||
&& env DEBIAN_FRONTEND=noninteractive apt-get install wget git default-jdk maven python3 --yes --no-install-recommends \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
|
||||
|
||||
# We need to get the repository's HEAD each time despite, so we invalidate layers' cache
|
||||
ARG CACHE_INVALIDATOR=0
|
||||
|
||||
@ -15,7 +15,8 @@ RUN apt-get update --yes \
|
||||
unixodbc-dev \
|
||||
odbcinst \
|
||||
sudo \
|
||||
&& apt-get clean
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
|
||||
|
||||
RUN pip3 install \
|
||||
numpy \
|
||||
|
||||
@ -11,7 +11,8 @@ RUN apt-get update --yes \
|
||||
python3-dev \
|
||||
python3-pip \
|
||||
sudo \
|
||||
&& apt-get clean
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
|
||||
|
||||
RUN pip3 install \
|
||||
pyyaml \
|
||||
|
||||
@ -9,16 +9,9 @@ RUN apt-get update -y \
|
||||
python3-requests \
|
||||
nodejs \
|
||||
npm \
|
||||
&& apt-get clean
|
||||
|
||||
COPY s3downloader /s3downloader
|
||||
|
||||
ENV S3_URL="https://clickhouse-datasets.s3.amazonaws.com"
|
||||
ENV DATASETS="hits visits"
|
||||
|
||||
# The following is already done in clickhouse/stateless-test
|
||||
# RUN npm install -g azurite
|
||||
# RUN npm install tslib
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
|
||||
|
||||
COPY create.sql /
|
||||
COPY run.sh /
|
||||
CMD ["/bin/bash", "/run.sh"]
|
||||
|
||||
333
docker/test/stateful/create.sql
Normal file
333
docker/test/stateful/create.sql
Normal file
@ -0,0 +1,333 @@
|
||||
ATTACH TABLE datasets.hits_v1 UUID '78ebf6a1-d987-4579-b3ec-00c1a087b1f3'
|
||||
(
|
||||
WatchID UInt64,
|
||||
JavaEnable UInt8,
|
||||
Title String,
|
||||
GoodEvent Int16,
|
||||
EventTime DateTime,
|
||||
EventDate Date,
|
||||
CounterID UInt32,
|
||||
ClientIP UInt32,
|
||||
ClientIP6 FixedString(16),
|
||||
RegionID UInt32,
|
||||
UserID UInt64,
|
||||
CounterClass Int8,
|
||||
OS UInt8,
|
||||
UserAgent UInt8,
|
||||
URL String,
|
||||
Referer String,
|
||||
URLDomain String,
|
||||
RefererDomain String,
|
||||
Refresh UInt8,
|
||||
IsRobot UInt8,
|
||||
RefererCategories Array(UInt16),
|
||||
URLCategories Array(UInt16),
|
||||
URLRegions Array(UInt32),
|
||||
RefererRegions Array(UInt32),
|
||||
ResolutionWidth UInt16,
|
||||
ResolutionHeight UInt16,
|
||||
ResolutionDepth UInt8,
|
||||
FlashMajor UInt8,
|
||||
FlashMinor UInt8,
|
||||
FlashMinor2 String,
|
||||
NetMajor UInt8,
|
||||
NetMinor UInt8,
|
||||
UserAgentMajor UInt16,
|
||||
UserAgentMinor FixedString(2),
|
||||
CookieEnable UInt8,
|
||||
JavascriptEnable UInt8,
|
||||
IsMobile UInt8,
|
||||
MobilePhone UInt8,
|
||||
MobilePhoneModel String,
|
||||
Params String,
|
||||
IPNetworkID UInt32,
|
||||
TraficSourceID Int8,
|
||||
SearchEngineID UInt16,
|
||||
SearchPhrase String,
|
||||
AdvEngineID UInt8,
|
||||
IsArtifical UInt8,
|
||||
WindowClientWidth UInt16,
|
||||
WindowClientHeight UInt16,
|
||||
ClientTimeZone Int16,
|
||||
ClientEventTime DateTime,
|
||||
SilverlightVersion1 UInt8,
|
||||
SilverlightVersion2 UInt8,
|
||||
SilverlightVersion3 UInt32,
|
||||
SilverlightVersion4 UInt16,
|
||||
PageCharset String,
|
||||
CodeVersion UInt32,
|
||||
IsLink UInt8,
|
||||
IsDownload UInt8,
|
||||
IsNotBounce UInt8,
|
||||
FUniqID UInt64,
|
||||
HID UInt32,
|
||||
IsOldCounter UInt8,
|
||||
IsEvent UInt8,
|
||||
IsParameter UInt8,
|
||||
DontCountHits UInt8,
|
||||
WithHash UInt8,
|
||||
HitColor FixedString(1),
|
||||
UTCEventTime DateTime,
|
||||
Age UInt8,
|
||||
Sex UInt8,
|
||||
Income UInt8,
|
||||
Interests UInt16,
|
||||
Robotness UInt8,
|
||||
GeneralInterests Array(UInt16),
|
||||
RemoteIP UInt32,
|
||||
RemoteIP6 FixedString(16),
|
||||
WindowName Int32,
|
||||
OpenerName Int32,
|
||||
HistoryLength Int16,
|
||||
BrowserLanguage FixedString(2),
|
||||
BrowserCountry FixedString(2),
|
||||
SocialNetwork String,
|
||||
SocialAction String,
|
||||
HTTPError UInt16,
|
||||
SendTiming Int32,
|
||||
DNSTiming Int32,
|
||||
ConnectTiming Int32,
|
||||
ResponseStartTiming Int32,
|
||||
ResponseEndTiming Int32,
|
||||
FetchTiming Int32,
|
||||
RedirectTiming Int32,
|
||||
DOMInteractiveTiming Int32,
|
||||
DOMContentLoadedTiming Int32,
|
||||
DOMCompleteTiming Int32,
|
||||
LoadEventStartTiming Int32,
|
||||
LoadEventEndTiming Int32,
|
||||
NSToDOMContentLoadedTiming Int32,
|
||||
FirstPaintTiming Int32,
|
||||
RedirectCount Int8,
|
||||
SocialSourceNetworkID UInt8,
|
||||
SocialSourcePage String,
|
||||
ParamPrice Int64,
|
||||
ParamOrderID String,
|
||||
ParamCurrency FixedString(3),
|
||||
ParamCurrencyID UInt16,
|
||||
GoalsReached Array(UInt32),
|
||||
OpenstatServiceName String,
|
||||
OpenstatCampaignID String,
|
||||
OpenstatAdID String,
|
||||
OpenstatSourceID String,
|
||||
UTMSource String,
|
||||
UTMMedium String,
|
||||
UTMCampaign String,
|
||||
UTMContent String,
|
||||
UTMTerm String,
|
||||
FromTag String,
|
||||
HasGCLID UInt8,
|
||||
RefererHash UInt64,
|
||||
URLHash UInt64,
|
||||
CLID UInt32,
|
||||
YCLID UInt64,
|
||||
ShareService String,
|
||||
ShareURL String,
|
||||
ShareTitle String,
|
||||
"ParsedParams.Key1" Array(String),
|
||||
"ParsedParams.Key2" Array(String),
|
||||
"ParsedParams.Key3" Array(String),
|
||||
"ParsedParams.Key4" Array(String),
|
||||
"ParsedParams.Key5" Array(String),
|
||||
"ParsedParams.ValueDouble" Array(Float64),
|
||||
IslandID FixedString(16),
|
||||
RequestNum UInt32,
|
||||
RequestTry UInt8
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(EventDate)
|
||||
ORDER BY (CounterID, EventDate, intHash32(UserID))
|
||||
SAMPLE BY intHash32(UserID)
|
||||
SETTINGS disk = disk(type = cache, path = '/var/lib/clickhouse/filesystem_caches/', max_size = '4G',
|
||||
disk = disk(type = web, endpoint = 'https://clickhouse-datasets-web.s3.us-east-1.amazonaws.com/'));
|
||||
|
||||
ATTACH TABLE datasets.visits_v1 UUID '5131f834-711f-4168-98a5-968b691a104b'
|
||||
(
|
||||
CounterID UInt32,
|
||||
StartDate Date,
|
||||
Sign Int8,
|
||||
IsNew UInt8,
|
||||
VisitID UInt64,
|
||||
UserID UInt64,
|
||||
StartTime DateTime,
|
||||
Duration UInt32,
|
||||
UTCStartTime DateTime,
|
||||
PageViews Int32,
|
||||
Hits Int32,
|
||||
IsBounce UInt8,
|
||||
Referer String,
|
||||
StartURL String,
|
||||
RefererDomain String,
|
||||
StartURLDomain String,
|
||||
EndURL String,
|
||||
LinkURL String,
|
||||
IsDownload UInt8,
|
||||
TraficSourceID Int8,
|
||||
SearchEngineID UInt16,
|
||||
SearchPhrase String,
|
||||
AdvEngineID UInt8,
|
||||
PlaceID Int32,
|
||||
RefererCategories Array(UInt16),
|
||||
URLCategories Array(UInt16),
|
||||
URLRegions Array(UInt32),
|
||||
RefererRegions Array(UInt32),
|
||||
IsYandex UInt8,
|
||||
GoalReachesDepth Int32,
|
||||
GoalReachesURL Int32,
|
||||
GoalReachesAny Int32,
|
||||
SocialSourceNetworkID UInt8,
|
||||
SocialSourcePage String,
|
||||
MobilePhoneModel String,
|
||||
ClientEventTime DateTime,
|
||||
RegionID UInt32,
|
||||
ClientIP UInt32,
|
||||
ClientIP6 FixedString(16),
|
||||
RemoteIP UInt32,
|
||||
RemoteIP6 FixedString(16),
|
||||
IPNetworkID UInt32,
|
||||
SilverlightVersion3 UInt32,
|
||||
CodeVersion UInt32,
|
||||
ResolutionWidth UInt16,
|
||||
ResolutionHeight UInt16,
|
||||
UserAgentMajor UInt16,
|
||||
UserAgentMinor UInt16,
|
||||
WindowClientWidth UInt16,
|
||||
WindowClientHeight UInt16,
|
||||
SilverlightVersion2 UInt8,
|
||||
SilverlightVersion4 UInt16,
|
||||
FlashVersion3 UInt16,
|
||||
FlashVersion4 UInt16,
|
||||
ClientTimeZone Int16,
|
||||
OS UInt8,
|
||||
UserAgent UInt8,
|
||||
ResolutionDepth UInt8,
|
||||
FlashMajor UInt8,
|
||||
FlashMinor UInt8,
|
||||
NetMajor UInt8,
|
||||
NetMinor UInt8,
|
||||
MobilePhone UInt8,
|
||||
SilverlightVersion1 UInt8,
|
||||
Age UInt8,
|
||||
Sex UInt8,
|
||||
Income UInt8,
|
||||
JavaEnable UInt8,
|
||||
CookieEnable UInt8,
|
||||
JavascriptEnable UInt8,
|
||||
IsMobile UInt8,
|
||||
BrowserLanguage UInt16,
|
||||
BrowserCountry UInt16,
|
||||
Interests UInt16,
|
||||
Robotness UInt8,
|
||||
GeneralInterests Array(UInt16),
|
||||
Params Array(String),
|
||||
"Goals.ID" Array(UInt32),
|
||||
"Goals.Serial" Array(UInt32),
|
||||
"Goals.EventTime" Array(DateTime),
|
||||
"Goals.Price" Array(Int64),
|
||||
"Goals.OrderID" Array(String),
|
||||
"Goals.CurrencyID" Array(UInt32),
|
||||
WatchIDs Array(UInt64),
|
||||
ParamSumPrice Int64,
|
||||
ParamCurrency FixedString(3),
|
||||
ParamCurrencyID UInt16,
|
||||
ClickLogID UInt64,
|
||||
ClickEventID Int32,
|
||||
ClickGoodEvent Int32,
|
||||
ClickEventTime DateTime,
|
||||
ClickPriorityID Int32,
|
||||
ClickPhraseID Int32,
|
||||
ClickPageID Int32,
|
||||
ClickPlaceID Int32,
|
||||
ClickTypeID Int32,
|
||||
ClickResourceID Int32,
|
||||
ClickCost UInt32,
|
||||
ClickClientIP UInt32,
|
||||
ClickDomainID UInt32,
|
||||
ClickURL String,
|
||||
ClickAttempt UInt8,
|
||||
ClickOrderID UInt32,
|
||||
ClickBannerID UInt32,
|
||||
ClickMarketCategoryID UInt32,
|
||||
ClickMarketPP UInt32,
|
||||
ClickMarketCategoryName String,
|
||||
ClickMarketPPName String,
|
||||
ClickAWAPSCampaignName String,
|
||||
ClickPageName String,
|
||||
ClickTargetType UInt16,
|
||||
ClickTargetPhraseID UInt64,
|
||||
ClickContextType UInt8,
|
||||
ClickSelectType Int8,
|
||||
ClickOptions String,
|
||||
ClickGroupBannerID Int32,
|
||||
OpenstatServiceName String,
|
||||
OpenstatCampaignID String,
|
||||
OpenstatAdID String,
|
||||
OpenstatSourceID String,
|
||||
UTMSource String,
|
||||
UTMMedium String,
|
||||
UTMCampaign String,
|
||||
UTMContent String,
|
||||
UTMTerm String,
|
||||
FromTag String,
|
||||
HasGCLID UInt8,
|
||||
FirstVisit DateTime,
|
||||
PredLastVisit Date,
|
||||
LastVisit Date,
|
||||
TotalVisits UInt32,
|
||||
"TraficSource.ID" Array(Int8),
|
||||
"TraficSource.SearchEngineID" Array(UInt16),
|
||||
"TraficSource.AdvEngineID" Array(UInt8),
|
||||
"TraficSource.PlaceID" Array(UInt16),
|
||||
"TraficSource.SocialSourceNetworkID" Array(UInt8),
|
||||
"TraficSource.Domain" Array(String),
|
||||
"TraficSource.SearchPhrase" Array(String),
|
||||
"TraficSource.SocialSourcePage" Array(String),
|
||||
Attendance FixedString(16),
|
||||
CLID UInt32,
|
||||
YCLID UInt64,
|
||||
NormalizedRefererHash UInt64,
|
||||
SearchPhraseHash UInt64,
|
||||
RefererDomainHash UInt64,
|
||||
NormalizedStartURLHash UInt64,
|
||||
StartURLDomainHash UInt64,
|
||||
NormalizedEndURLHash UInt64,
|
||||
TopLevelDomain UInt64,
|
||||
URLScheme UInt64,
|
||||
OpenstatServiceNameHash UInt64,
|
||||
OpenstatCampaignIDHash UInt64,
|
||||
OpenstatAdIDHash UInt64,
|
||||
OpenstatSourceIDHash UInt64,
|
||||
UTMSourceHash UInt64,
|
||||
UTMMediumHash UInt64,
|
||||
UTMCampaignHash UInt64,
|
||||
UTMContentHash UInt64,
|
||||
UTMTermHash UInt64,
|
||||
FromHash UInt64,
|
||||
WebVisorEnabled UInt8,
|
||||
WebVisorActivity UInt32,
|
||||
"ParsedParams.Key1" Array(String),
|
||||
"ParsedParams.Key2" Array(String),
|
||||
"ParsedParams.Key3" Array(String),
|
||||
"ParsedParams.Key4" Array(String),
|
||||
"ParsedParams.Key5" Array(String),
|
||||
"ParsedParams.ValueDouble" Array(Float64),
|
||||
"Market.Type" Array(UInt8),
|
||||
"Market.GoalID" Array(UInt32),
|
||||
"Market.OrderID" Array(String),
|
||||
"Market.OrderPrice" Array(Int64),
|
||||
"Market.PP" Array(UInt32),
|
||||
"Market.DirectPlaceID" Array(UInt32),
|
||||
"Market.DirectOrderID" Array(UInt32),
|
||||
"Market.DirectBannerID" Array(UInt32),
|
||||
"Market.GoodID" Array(String),
|
||||
"Market.GoodName" Array(String),
|
||||
"Market.GoodQuantity" Array(Int32),
|
||||
"Market.GoodPrice" Array(Int64),
|
||||
IslandID FixedString(16)
|
||||
)
|
||||
ENGINE = CollapsingMergeTree(Sign)
|
||||
PARTITION BY toYYYYMM(StartDate)
|
||||
ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)
|
||||
SAMPLE BY intHash32(UserID)
|
||||
SETTINGS disk = disk(type = cache, path = '/var/lib/clickhouse/filesystem_caches/', max_size = '4G',
|
||||
disk = disk(type = web, endpoint = 'https://clickhouse-datasets-web.s3.us-east-1.amazonaws.com/'));
|
||||
@ -44,6 +44,9 @@ if [[ -n "$USE_S3_STORAGE_FOR_MERGE_TREE" ]] && [[ "$USE_S3_STORAGE_FOR_MERGE_TR
|
||||
# It is not needed, we will explicitly create tables on s3.
|
||||
# We do not have statefull tests with s3 storage run in public repository, but this is needed for another repository.
|
||||
rm /etc/clickhouse-server/config.d/s3_storage_policy_for_merge_tree_by_default.xml
|
||||
|
||||
rm /etc/clickhouse-server/config.d/storage_metadata_with_full_object_key.xml
|
||||
rm /etc/clickhouse-server/config.d/s3_storage_policy_with_template_object_key.xml
|
||||
fi
|
||||
|
||||
function start()
|
||||
@ -94,21 +97,9 @@ start
|
||||
|
||||
setup_logs_replication
|
||||
|
||||
# shellcheck disable=SC2086 # No quotes because I want to split it into words.
|
||||
/s3downloader --url-prefix "$S3_URL" --dataset-names $DATASETS
|
||||
chmod 777 -R /var/lib/clickhouse
|
||||
clickhouse-client --query "SHOW DATABASES"
|
||||
|
||||
clickhouse-client --query "ATTACH DATABASE datasets ENGINE = Ordinary"
|
||||
|
||||
service clickhouse-server restart
|
||||
|
||||
# Wait for server to start accepting connections
|
||||
for _ in {1..120}; do
|
||||
clickhouse-client --query "SELECT 1" && break
|
||||
sleep 1
|
||||
done
|
||||
|
||||
clickhouse-client --query "CREATE DATABASE datasets"
|
||||
clickhouse-client --multiquery < create.sql
|
||||
clickhouse-client --query "SHOW TABLES FROM datasets"
|
||||
|
||||
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
|
||||
@ -1,126 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import tarfile
|
||||
import logging
|
||||
import argparse
|
||||
import requests
|
||||
import tempfile
|
||||
|
||||
|
||||
DEFAULT_URL = "https://clickhouse-datasets.s3.amazonaws.com"
|
||||
|
||||
AVAILABLE_DATASETS = {
|
||||
"hits": "hits_v1.tar",
|
||||
"visits": "visits_v1.tar",
|
||||
}
|
||||
|
||||
RETRIES_COUNT = 5
|
||||
|
||||
|
||||
def _get_temp_file_name():
|
||||
return os.path.join(
|
||||
tempfile._get_default_tempdir(), next(tempfile._get_candidate_names())
|
||||
)
|
||||
|
||||
|
||||
def build_url(base_url, dataset):
|
||||
return os.path.join(base_url, dataset, "partitions", AVAILABLE_DATASETS[dataset])
|
||||
|
||||
|
||||
def download_with_progress(url, path):
|
||||
logging.info("Downloading from %s to temp path %s", url, path)
|
||||
for i in range(RETRIES_COUNT):
|
||||
try:
|
||||
with open(path, "wb") as f:
|
||||
response = requests.get(url, stream=True)
|
||||
response.raise_for_status()
|
||||
total_length = response.headers.get("content-length")
|
||||
if total_length is None or int(total_length) == 0:
|
||||
logging.info(
|
||||
"No content-length, will download file without progress"
|
||||
)
|
||||
f.write(response.content)
|
||||
else:
|
||||
dl = 0
|
||||
total_length = int(total_length)
|
||||
logging.info("Content length is %ld bytes", total_length)
|
||||
for data in response.iter_content(chunk_size=4096):
|
||||
dl += len(data)
|
||||
f.write(data)
|
||||
if sys.stdout.isatty():
|
||||
done = int(50 * dl / total_length)
|
||||
percent = int(100 * float(dl) / total_length)
|
||||
sys.stdout.write(
|
||||
"\r[{}{}] {}%".format(
|
||||
"=" * done, " " * (50 - done), percent
|
||||
)
|
||||
)
|
||||
sys.stdout.flush()
|
||||
break
|
||||
except Exception as ex:
|
||||
sys.stdout.write("\n")
|
||||
time.sleep(3)
|
||||
logging.info("Exception while downloading %s, retry %s", ex, i + 1)
|
||||
if os.path.exists(path):
|
||||
os.remove(path)
|
||||
else:
|
||||
raise Exception(
|
||||
"Cannot download dataset from {}, all retries exceeded".format(url)
|
||||
)
|
||||
|
||||
sys.stdout.write("\n")
|
||||
logging.info("Downloading finished")
|
||||
|
||||
|
||||
def unpack_to_clickhouse_directory(tar_path, clickhouse_path):
|
||||
logging.info(
|
||||
"Will unpack data from temp path %s to clickhouse db %s",
|
||||
tar_path,
|
||||
clickhouse_path,
|
||||
)
|
||||
with tarfile.open(tar_path, "r") as comp_file:
|
||||
comp_file.extractall(path=clickhouse_path)
|
||||
logging.info("Unpack finished")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Simple tool for dowloading datasets for clickhouse from S3"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--dataset-names",
|
||||
required=True,
|
||||
nargs="+",
|
||||
choices=list(AVAILABLE_DATASETS.keys()),
|
||||
)
|
||||
parser.add_argument("--url-prefix", default=DEFAULT_URL)
|
||||
parser.add_argument("--clickhouse-data-path", default="/var/lib/clickhouse/")
|
||||
|
||||
args = parser.parse_args()
|
||||
datasets = args.dataset_names
|
||||
logging.info("Will fetch following datasets: %s", ", ".join(datasets))
|
||||
for dataset in datasets:
|
||||
logging.info("Processing %s", dataset)
|
||||
temp_archive_path = _get_temp_file_name()
|
||||
try:
|
||||
download_url_for_dataset = build_url(args.url_prefix, dataset)
|
||||
download_with_progress(download_url_for_dataset, temp_archive_path)
|
||||
unpack_to_clickhouse_directory(temp_archive_path, args.clickhouse_data_path)
|
||||
except Exception as ex:
|
||||
logging.info("Some exception occured %s", str(ex))
|
||||
raise
|
||||
finally:
|
||||
logging.info(
|
||||
"Will remove downloaded file %s from filesystem if it exists",
|
||||
temp_archive_path,
|
||||
)
|
||||
if os.path.exists(temp_archive_path):
|
||||
os.remove(temp_archive_path)
|
||||
logging.info("Processing of %s finished", dataset)
|
||||
logging.info("Fetch finished, enjoy your tables!")
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user