diff --git a/.clang-tidy b/.clang-tidy
index 219ac263ab3..896052915f7 100644
--- a/.clang-tidy
+++ b/.clang-tidy
@@ -37,6 +37,7 @@ Checks: [
'-cert-oop54-cpp',
'-cert-oop57-cpp',
+ '-clang-analyzer-optin.core.EnumCastOutOfRange', # https://github.com/abseil/abseil-cpp/issues/1667
'-clang-analyzer-optin.performance.Padding',
'-clang-analyzer-unix.Malloc',
@@ -94,6 +95,7 @@ Checks: [
'-modernize-pass-by-value',
'-modernize-return-braced-init-list',
'-modernize-use-auto',
+ '-modernize-use-constraints', # This is a good check, but clang-tidy crashes, see https://github.com/llvm/llvm-project/issues/91872
'-modernize-use-default-member-init',
'-modernize-use-emplace',
'-modernize-use-nodiscard',
@@ -121,7 +123,8 @@ Checks: [
'-readability-magic-numbers',
'-readability-named-parameter',
'-readability-redundant-declaration',
- '-readability-redundant-inline-specifier',
+ '-readability-redundant-inline-specifier', # useful but incompatible with __attribute((always_inline))__ (aka. ALWAYS_INLINE, base/base/defines.h).
+ # ALWAYS_INLINE only has an effect if combined with `inline`: https://godbolt.org/z/Eefd74qdM
'-readability-redundant-member-init', # Useful but triggers another problem. Imagine a struct S with multiple String members. Structs are often instantiated via designated
# initializer S s{.s1 = [...], .s2 = [...], [...]}. In this case, compiler warning `missing-field-initializers` requires to specify all members which are not in-struct
# initialized (example: s1 in struct S { String s1; String s2{};}; is not in-struct initialized, therefore it must be specified at instantiation time). As explicitly
@@ -132,12 +135,7 @@ Checks: [
'-readability-uppercase-literal-suffix',
'-readability-use-anyofallof',
- '-zircon-*',
-
- # This is a good check, but clang-tidy crashes, see https://github.com/llvm/llvm-project/issues/91872
- '-modernize-use-constraints',
- # https://github.com/abseil/abseil-cpp/issues/1667
- '-clang-analyzer-optin.core.EnumCastOutOfRange'
+ '-zircon-*'
]
WarningsAsErrors: '*'
diff --git a/.github/ISSUE_TEMPLATE/10_question.md b/.github/ISSUE_TEMPLATE/10_question.md
deleted file mode 100644
index 0992bf06217..00000000000
--- a/.github/ISSUE_TEMPLATE/10_question.md
+++ /dev/null
@@ -1,12 +0,0 @@
----
-name: Question
-about: Ask a question about ClickHouse
-title: ''
-labels: question
-assignees: ''
-
----
-
-> Make sure to check documentation https://clickhouse.com/docs/en/ first. If the question is concise and probably has a short answer, asking it in [community Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-1gh9ds7f4-PgDhJAaF8ad5RbWBAAjzFg) is probably the fastest way to find the answer. For more complicated questions, consider asking them on StackOverflow with "clickhouse" tag https://stackoverflow.com/questions/tagged/clickhouse
-
-> If you still prefer GitHub issues, remove all this text and ask your question here.
diff --git a/.github/ISSUE_TEMPLATE/10_question.yaml b/.github/ISSUE_TEMPLATE/10_question.yaml
new file mode 100644
index 00000000000..39d4c27807a
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/10_question.yaml
@@ -0,0 +1,20 @@
+name: Question
+description: Ask a question about ClickHouse
+labels: ["question"]
+body:
+ - type: markdown
+ attributes:
+ value: |
+ > Make sure to check documentation https://clickhouse.com/docs/en/ first. If the question is concise and probably has a short answer, asking it in [community Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-1gh9ds7f4-PgDhJAaF8ad5RbWBAAjzFg) is probably the fastest way to find the answer. For more complicated questions, consider asking them on StackOverflow with "clickhouse" tag https://stackoverflow.com/questions/tagged/clickhouse
+ - type: textarea
+ attributes:
+ label: Company or project name
+ description: Put your company name or project description here.
+ validations:
+ required: false
+ - type: textarea
+ attributes:
+ label: Question
+ description: Please put your question here.
+ validations:
+ required: true
diff --git a/.github/ISSUE_TEMPLATE/20_feature-request.md b/.github/ISSUE_TEMPLATE/20_feature-request.md
index f59dbc2c40f..cf5ac000a23 100644
--- a/.github/ISSUE_TEMPLATE/20_feature-request.md
+++ b/.github/ISSUE_TEMPLATE/20_feature-request.md
@@ -9,6 +9,10 @@ assignees: ''
> (you don't have to strictly follow this form)
+**Company or project name**
+
+> Put your company name or project description here
+
**Use case**
> A clear and concise description of what is the intended usage scenario is.
diff --git a/.github/ISSUE_TEMPLATE/30_unexpected-behaviour.md b/.github/ISSUE_TEMPLATE/30_unexpected-behaviour.md
index 3630d95ba33..73c861886e6 100644
--- a/.github/ISSUE_TEMPLATE/30_unexpected-behaviour.md
+++ b/.github/ISSUE_TEMPLATE/30_unexpected-behaviour.md
@@ -9,6 +9,10 @@ assignees: ''
(you don't have to strictly follow this form)
+**Company or project name**
+
+Put your company name or project description here
+
**Describe the unexpected behaviour**
A clear and concise description of what works not as it is supposed to.
diff --git a/.github/ISSUE_TEMPLATE/35_incomplete_implementation.md b/.github/ISSUE_TEMPLATE/35_incomplete_implementation.md
index 6a014ce3c29..45f752b53ef 100644
--- a/.github/ISSUE_TEMPLATE/35_incomplete_implementation.md
+++ b/.github/ISSUE_TEMPLATE/35_incomplete_implementation.md
@@ -9,6 +9,10 @@ assignees: ''
(you don't have to strictly follow this form)
+**Company or project name**
+
+Put your company name or project description here
+
**Describe the unexpected behaviour**
A clear and concise description of what works not as it is supposed to.
diff --git a/.github/ISSUE_TEMPLATE/45_usability-issue.md b/.github/ISSUE_TEMPLATE/45_usability-issue.md
index b03b11606c1..79f23fe0a14 100644
--- a/.github/ISSUE_TEMPLATE/45_usability-issue.md
+++ b/.github/ISSUE_TEMPLATE/45_usability-issue.md
@@ -9,6 +9,9 @@ assignees: ''
(you don't have to strictly follow this form)
+**Company or project name**
+Put your company name or project description here
+
**Describe the issue**
A clear and concise description of what works not as it is supposed to.
diff --git a/.github/ISSUE_TEMPLATE/50_build-issue.md b/.github/ISSUE_TEMPLATE/50_build-issue.md
index 9b05fbbdd13..5a58add9ad8 100644
--- a/.github/ISSUE_TEMPLATE/50_build-issue.md
+++ b/.github/ISSUE_TEMPLATE/50_build-issue.md
@@ -9,6 +9,10 @@ assignees: ''
> Make sure that `git diff` result is empty and you've just pulled fresh master. Try cleaning up cmake cache. Just in case, official build instructions are published here: https://clickhouse.com/docs/en/development/build/
+**Company or project name**
+
+> Put your company name or project description here
+
**Operating system**
> OS kind or distribution, specific version/release, non-standard kernel if any. If you are trying to build inside virtual machine, please mention it too.
diff --git a/.github/ISSUE_TEMPLATE/60_documentation-issue.md b/.github/ISSUE_TEMPLATE/60_documentation-issue.md
index 557e5ea43c9..5a941977dac 100644
--- a/.github/ISSUE_TEMPLATE/60_documentation-issue.md
+++ b/.github/ISSUE_TEMPLATE/60_documentation-issue.md
@@ -8,6 +8,9 @@ labels: comp-documentation
(you don't have to strictly follow this form)
+**Company or project name**
+Put your company name or project description here
+
**Describe the issue**
A clear and concise description of what's wrong in documentation.
diff --git a/.github/ISSUE_TEMPLATE/70_performance-issue.md b/.github/ISSUE_TEMPLATE/70_performance-issue.md
index d0e549039a6..21eba3f5af1 100644
--- a/.github/ISSUE_TEMPLATE/70_performance-issue.md
+++ b/.github/ISSUE_TEMPLATE/70_performance-issue.md
@@ -9,6 +9,9 @@ assignees: ''
(you don't have to strictly follow this form)
+**Company or project name**
+Put your company name or project description here
+
**Describe the situation**
What exactly works slower than expected?
diff --git a/.github/ISSUE_TEMPLATE/80_backward-compatibility.md b/.github/ISSUE_TEMPLATE/80_backward-compatibility.md
index a13e9508f70..8058f5bcc53 100644
--- a/.github/ISSUE_TEMPLATE/80_backward-compatibility.md
+++ b/.github/ISSUE_TEMPLATE/80_backward-compatibility.md
@@ -9,6 +9,9 @@ assignees: ''
(you don't have to strictly follow this form)
+**Company or project name**
+Put your company name or project description here
+
**Describe the issue**
A clear and concise description of what works not as it is supposed to.
diff --git a/.github/ISSUE_TEMPLATE/85_bug-report.md b/.github/ISSUE_TEMPLATE/85_bug-report.md
index 6bf265260ac..c43473d63ad 100644
--- a/.github/ISSUE_TEMPLATE/85_bug-report.md
+++ b/.github/ISSUE_TEMPLATE/85_bug-report.md
@@ -11,6 +11,10 @@ assignees: ''
> You have to provide the following information whenever possible.
+**Company or project name**
+
+> Put your company name or project description here
+
**Describe what's wrong**
> A clear and concise description of what works not as it is supposed to.
diff --git a/.github/ISSUE_TEMPLATE/96_installation-issues.md b/.github/ISSUE_TEMPLATE/96_installation-issues.md
index e4be8af86b6..5f1b6cfd640 100644
--- a/.github/ISSUE_TEMPLATE/96_installation-issues.md
+++ b/.github/ISSUE_TEMPLATE/96_installation-issues.md
@@ -7,6 +7,10 @@ assignees: ''
---
+**Company or project name**
+
+Put your company name or project description here
+
**I have tried the following solutions**: https://clickhouse.com/docs/en/faq/troubleshooting/#troubleshooting-installation-errors
**Installation type**
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 64dc9049bc2..51a1a6e2df8 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -11,6 +11,7 @@ tests/ci/cancel_and_rerun_workflow_lambda/app.py
- Backward Incompatible Change
- Build/Testing/Packaging Improvement
- Documentation (changelog entry is not required)
+- Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)
- Bug Fix (user-visible misbehavior in an official stable release)
- CI Fix or Improvement (changelog entry is not required)
- Not for changelog (changelog entry is not required)
@@ -41,47 +42,27 @@ At a minimum, the following information should be added (but add more as needed)
> Information about CI checks: https://clickhouse.com/docs/en/development/continuous-integration/
-
- CI Settings
-
-**NOTE:** If your merge the PR with modified CI you **MUST KNOW** what you are doing
-**NOTE:** Checked options will be applied if set before CI RunConfig/PrepareRunConfig step
-
-#### Run these jobs only (required builds will be added automatically):
-- [ ] Integration Tests
-- [ ] Stateless tests
-- [ ] Stateful tests
-- [ ] Unit tests
-- [ ] Performance tests
-- [ ] All with aarch64
-- [ ] All with ASAN
-- [ ] All with TSAN
-- [ ] All with Analyzer
-- [ ] All with Azure
-- [ ] Add your option here
-
-#### Deny these jobs:
-- [ ] Fast test
-- [ ] Integration Tests
-- [ ] Stateless tests
-- [ ] Stateful tests
-- [ ] Performance tests
-- [ ] All with ASAN
-- [ ] All with TSAN
-- [ ] All with MSAN
-- [ ] All with UBSAN
-- [ ] All with Coverage
-- [ ] All with Aarch64
-
-#### Extra options:
-- [ ] do not test (only style check)
-- [ ] disable merge-commit (no merge from master before tests)
-- [ ] disable CI cache (job reuse)
-
-#### Only specified batches in multi-batch jobs:
-- [ ] 1
-- [ ] 2
-- [ ] 3
-- [ ] 4
-
-
+#### CI Settings (Only check the boxes if you know what you are doing):
+- [ ] Allow: All Required Checks
+- [ ] Allow: Stateless tests
+- [ ] Allow: Stateful tests
+- [ ] Allow: Integration Tests
+- [ ] Allow: Performance tests
+- [ ] Allow: All NOT Required Checks
+- [ ] Allow: batch 1, 2 for multi-batch jobs
+- [ ] Allow: batch 3, 4, 5, 6 for multi-batch jobs
+---
+- [ ] Exclude: Style check
+- [ ] Exclude: Fast test
+- [ ] Exclude: Integration Tests
+- [ ] Exclude: Stateless tests
+- [ ] Exclude: Stateful tests
+- [ ] Exclude: Performance tests
+- [ ] Exclude: All with ASAN
+- [ ] Exclude: All with Aarch64
+- [ ] Exclude: All with TSAN, MSAN, UBSAN, Coverage
+---
+- [ ] Do not test
+- [ ] Upload binaries for special builds
+- [ ] Disable merge-commit
+- [ ] Disable CI cache
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index d2ea714e4e4..91dcb6a4968 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -27,15 +27,16 @@ jobs:
run: |
cd "$GITHUB_WORKSPACE/tests/ci"
python3 sync_pr.py --merge || :
- - name: Python unit tests
- run: |
- cd "$GITHUB_WORKSPACE/tests/ci"
- echo "Testing the main ci directory"
- python3 -m unittest discover -s . -p 'test_*.py'
- for dir in *_lambda/; do
- echo "Testing $dir"
- python3 -m unittest discover -s "$dir" -p 'test_*.py'
- done
+# Runs in MQ:
+# - name: Python unit tests
+# run: |
+# cd "$GITHUB_WORKSPACE/tests/ci"
+# echo "Testing the main ci directory"
+# python3 -m unittest discover -s . -p 'test_*.py'
+# for dir in *_lambda/; do
+# echo "Testing $dir"
+# python3 -m unittest discover -s "$dir" -p 'test_*.py'
+# done
- name: PrepareRunConfig
id: runconfig
run: |
@@ -53,13 +54,13 @@ jobs:
- name: Re-create GH statuses for skipped jobs if any
run: |
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ runner.temp }}/ci_run_data.json --update-gh-statuses
- BuildDockers:
- needs: [RunConfig]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_docker.yml
- with:
- data: ${{ needs.RunConfig.outputs.data }}
- # Tested in MQ
+# Runs in MQ:
+# BuildDockers:
+# needs: [RunConfig]
+# if: ${{ !failure() && !cancelled() }}
+# uses: ./.github/workflows/reusable_docker.yml
+# with:
+# data: ${{ needs.RunConfig.outputs.data }}
# StyleCheck:
# needs: [RunConfig, BuildDockers]
# if: ${{ !failure() && !cancelled() }}
@@ -70,262 +71,74 @@ jobs:
# data: ${{ needs.RunConfig.outputs.data }}
# run_command: |
# python3 style_check.py --no-push
- CompatibilityCheckX86:
- needs: [RunConfig, BuilderDebRelease]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
+
+ ################################# Main stages #################################
+ # for main CI chain
+ #
+ Builds_1:
+ needs: [RunConfig]
+ if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Builds_1') }}
+ # using callable wf (reusable_stage.yml) allows grouping all nested jobs under a tab
+ uses: ./.github/workflows/reusable_build_stage.yml
with:
- test_name: Compatibility check (amd64)
- runner_type: style-checker
+ stage: Builds_1
data: ${{ needs.RunConfig.outputs.data }}
- CompatibilityCheckAarch64:
- needs: [RunConfig, BuilderDebAarch64]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
+ Tests_1:
+ needs: [RunConfig, Builds_1]
+ if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_1') }}
+ uses: ./.github/workflows/reusable_test_stage.yml
with:
- test_name: Compatibility check (aarch64)
- runner_type: style-checker
+ stage: Tests_1
data: ${{ needs.RunConfig.outputs.data }}
-#########################################################################################
-#################################### ORDINARY BUILDS ####################################
-#########################################################################################
-# TODO: never skip builds!
- BuilderDebRelease:
- needs: [RunConfig, BuildDockers]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_build.yml
+ Builds_2:
+ needs: [RunConfig, Builds_1]
+ if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Builds_2') }}
+ uses: ./.github/workflows/reusable_build_stage.yml
with:
- build_name: package_release
- checkout_depth: 0
+ stage: Builds_2
data: ${{ needs.RunConfig.outputs.data }}
- BuilderDebReleaseCoverage:
- needs: [RunConfig, BuildDockers]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_build.yml
+ Tests_2:
+ needs: [RunConfig, Builds_2]
+ if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_2') }}
+ uses: ./.github/workflows/reusable_test_stage.yml
with:
- build_name: package_release_coverage
- checkout_depth: 0
+ stage: Tests_2
data: ${{ needs.RunConfig.outputs.data }}
- BuilderDebAarch64:
- needs: [RunConfig, BuildDockers]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_build.yml
+ # stage for jobs that do not prohibit merge
+ Tests_3:
+ # Test_3 should not wait for Test_1/Test_2 and should not be blocked by them on master branch since all jobs need to run there.
+ needs: [RunConfig, Builds_1, Builds_2]
+ if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_3') }}
+ uses: ./.github/workflows/reusable_test_stage.yml
with:
- build_name: package_aarch64
- checkout_depth: 0
+ stage: Tests_3
data: ${{ needs.RunConfig.outputs.data }}
- BuilderBinRelease:
- needs: [RunConfig, BuildDockers]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_build.yml
- with:
- build_name: binary_release
- checkout_depth: 0 # otherwise we will have no info about contributors
- data: ${{ needs.RunConfig.outputs.data }}
- BuilderDebAsan:
- needs: [RunConfig, BuildDockers]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_build.yml
- with:
- build_name: package_asan
- data: ${{ needs.RunConfig.outputs.data }}
- BuilderDebUBsan:
- needs: [RunConfig, BuildDockers]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_build.yml
- with:
- build_name: package_ubsan
- data: ${{ needs.RunConfig.outputs.data }}
- BuilderDebTsan:
- needs: [RunConfig, BuildDockers]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_build.yml
- with:
- build_name: package_tsan
- data: ${{ needs.RunConfig.outputs.data }}
- BuilderDebMsan:
- needs: [RunConfig, BuildDockers]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_build.yml
- with:
- build_name: package_msan
- data: ${{ needs.RunConfig.outputs.data }}
- BuilderDebDebug:
- needs: [RunConfig, BuildDockers]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_build.yml
- with:
- build_name: package_debug
- data: ${{ needs.RunConfig.outputs.data }}
-##########################################################################################
-##################################### SPECIAL BUILDS #####################################
-##########################################################################################
- BuilderBinClangTidy:
- needs: [RunConfig, BuilderDebRelease]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_build.yml
- with:
- build_name: binary_tidy
- data: ${{ needs.RunConfig.outputs.data }}
- BuilderBinDarwin:
- needs: [RunConfig, BuilderDebRelease]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_build.yml
- with:
- build_name: binary_darwin
- data: ${{ needs.RunConfig.outputs.data }}
- checkout_depth: 0
- BuilderBinAarch64:
- needs: [RunConfig, BuilderDebRelease]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_build.yml
- with:
- build_name: binary_aarch64
- data: ${{ needs.RunConfig.outputs.data }}
- checkout_depth: 0
- BuilderBinFreeBSD:
- needs: [RunConfig, BuilderDebRelease]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_build.yml
- with:
- build_name: binary_freebsd
- data: ${{ needs.RunConfig.outputs.data }}
- checkout_depth: 0
- BuilderBinDarwinAarch64:
- needs: [RunConfig, BuilderDebRelease]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_build.yml
- with:
- build_name: binary_darwin_aarch64
- data: ${{ needs.RunConfig.outputs.data }}
- checkout_depth: 0
- BuilderBinPPC64:
- needs: [RunConfig, BuilderDebRelease]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_build.yml
- with:
- build_name: binary_ppc64le
- data: ${{ needs.RunConfig.outputs.data }}
- checkout_depth: 0
- BuilderBinAmd64Compat:
- needs: [RunConfig, BuilderDebRelease]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_build.yml
- with:
- build_name: binary_amd64_compat
- data: ${{ needs.RunConfig.outputs.data }}
- checkout_depth: 0
- BuilderBinAmd64Musl:
- needs: [RunConfig, BuilderDebRelease]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_build.yml
- with:
- build_name: binary_amd64_musl
- data: ${{ needs.RunConfig.outputs.data }}
- checkout_depth: 0
- BuilderBinAarch64V80Compat:
- needs: [RunConfig, BuilderDebRelease]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_build.yml
- with:
- build_name: binary_aarch64_v80compat
- data: ${{ needs.RunConfig.outputs.data }}
- checkout_depth: 0
- BuilderBinRISCV64:
- needs: [RunConfig, BuilderDebRelease]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_build.yml
- with:
- build_name: binary_riscv64
- data: ${{ needs.RunConfig.outputs.data }}
- checkout_depth: 0
- BuilderBinS390X:
- needs: [RunConfig, BuilderDebRelease]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_build.yml
- with:
- build_name: binary_s390x
- data: ${{ needs.RunConfig.outputs.data }}
- checkout_depth: 0
- BuilderBinLoongarch64:
- needs: [RunConfig, BuilderDebRelease]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_build.yml
- with:
- build_name: binary_loongarch64
- data: ${{ needs.RunConfig.outputs.data }}
- checkout_depth: 0
-############################################################################################
-##################################### Docker images #######################################
-############################################################################################
- DockerServerImage:
- needs: [RunConfig, BuilderDebRelease, BuilderDebAarch64]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Docker server image
- runner_type: style-checker
- data: ${{ needs.RunConfig.outputs.data }}
- DockerKeeperImage:
- needs: [RunConfig, BuilderDebRelease, BuilderDebAarch64]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Docker keeper image
- runner_type: style-checker
- data: ${{ needs.RunConfig.outputs.data }}
-############################################################################################
-##################################### BUILD REPORTER #######################################
-############################################################################################
- BuilderReport:
+
+ ################################# Reports #################################
+ # Reports should be run even if Builds_1/2 failed - put them separately in wf (not in Tests_1/2)
+ Builds_1_Report:
# run report check for failed builds to indicate the CI error
- if: ${{ !cancelled() }}
- needs:
- - RunConfig
- - BuilderDebAarch64
- - BuilderDebAsan
- - BuilderDebDebug
- - BuilderDebMsan
- - BuilderDebRelease
- - BuilderDebTsan
- - BuilderDebUBsan
+ if: ${{ !cancelled() && needs.RunConfig.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse build check') }}
+ needs: [RunConfig, Builds_1]
uses: ./.github/workflows/reusable_test.yml
with:
test_name: ClickHouse build check
runner_type: style-checker-aarch64
data: ${{ needs.RunConfig.outputs.data }}
- BuilderSpecialReport:
+ Builds_2_Report:
# run report check for failed builds to indicate the CI error
- if: ${{ !cancelled() }}
- needs:
- - RunConfig
- - BuilderBinAarch64
- - BuilderBinDarwin
- - BuilderBinDarwinAarch64
- - BuilderBinFreeBSD
- - BuilderBinPPC64
- - BuilderBinRISCV64
- - BuilderBinS390X
- - BuilderBinLoongarch64
- - BuilderBinAmd64Compat
- - BuilderBinAarch64V80Compat
- - BuilderBinClangTidy
- - BuilderBinAmd64Musl
- - BuilderDebReleaseCoverage
- - BuilderBinRelease
+ if: ${{ !cancelled() && needs.RunConfig.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse special build check') }}
+ needs: [RunConfig, Builds_2]
uses: ./.github/workflows/reusable_test.yml
with:
test_name: ClickHouse special build check
runner_type: style-checker-aarch64
data: ${{ needs.RunConfig.outputs.data }}
+
MarkReleaseReady:
if: ${{ !failure() && !cancelled() }}
- needs:
- - BuilderBinDarwin
- - BuilderBinDarwinAarch64
- - BuilderDebRelease
- - BuilderDebAarch64
- runs-on: [self-hosted, style-checker]
+ needs: [RunConfig, Builds_1, Builds_2]
+ runs-on: [self-hosted, style-checker-aarch64]
steps:
- name: Debug
run: |
@@ -338,7 +151,7 @@ jobs:
no both ${{ !(contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
EOF
- name: Not ready
- # fail the job to be able restart it
+ # fail the job to be able to restart it
if: ${{ contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure') }}
run: exit 1
- name: Check out repository code
@@ -349,544 +162,14 @@ jobs:
run: |
cd "$GITHUB_WORKSPACE/tests/ci"
python3 mark_release_ready.py
-############################################################################################
-#################################### INSTALL PACKAGES ######################################
-############################################################################################
- InstallPackagesTestRelease:
- needs: [RunConfig, BuilderDebRelease]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Install packages (amd64)
- runner_type: style-checker
- data: ${{ needs.RunConfig.outputs.data }}
- run_command: |
- python3 install_check.py "$CHECK_NAME"
- InstallPackagesTestAarch64:
- needs: [RunConfig, BuilderDebAarch64]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Install packages (arm64)
- runner_type: style-checker-aarch64
- data: ${{ needs.RunConfig.outputs.data }}
- run_command: |
- python3 install_check.py "$CHECK_NAME"
-##############################################################################################
-########################### FUNCTIONAl STATELESS TESTS #######################################
-##############################################################################################
- FunctionalStatelessTestRelease:
- needs: [RunConfig, BuilderDebRelease]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Stateless tests (release)
- runner_type: func-tester
- data: ${{ needs.RunConfig.outputs.data }}
- FunctionalStatelessTestReleaseAnalyzerS3Replicated:
- needs: [RunConfig, BuilderDebRelease]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Stateless tests (release, old analyzer, s3, DatabaseReplicated)
- runner_type: func-tester
- data: ${{ needs.RunConfig.outputs.data }}
- FunctionalStatelessTestS3Debug:
- needs: [RunConfig, BuilderDebDebug]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Stateless tests (debug, s3 storage)
- runner_type: func-tester
- data: ${{ needs.RunConfig.outputs.data }}
- FunctionalStatelessTestS3Tsan:
- needs: [RunConfig, BuilderDebTsan]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Stateless tests (tsan, s3 storage)
- runner_type: func-tester
- data: ${{ needs.RunConfig.outputs.data }}
- FunctionalStatelessTestAarch64:
- needs: [RunConfig, BuilderDebAarch64]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Stateless tests (aarch64)
- runner_type: func-tester-aarch64
- data: ${{ needs.RunConfig.outputs.data }}
- FunctionalStatelessTestAsan:
- needs: [RunConfig, BuilderDebAsan]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Stateless tests (asan)
- runner_type: func-tester
- data: ${{ needs.RunConfig.outputs.data }}
- FunctionalStatelessTestTsan:
- needs: [RunConfig, BuilderDebTsan]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Stateless tests (tsan)
- runner_type: func-tester
- data: ${{ needs.RunConfig.outputs.data }}
- FunctionalStatelessTestMsan:
- needs: [RunConfig, BuilderDebMsan]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Stateless tests (msan)
- runner_type: func-tester
- data: ${{ needs.RunConfig.outputs.data }}
- FunctionalStatelessTestUBsan:
- needs: [RunConfig, BuilderDebUBsan]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Stateless tests (ubsan)
- runner_type: func-tester
- data: ${{ needs.RunConfig.outputs.data }}
- FunctionalStatelessTestDebug:
- needs: [RunConfig, BuilderDebDebug]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Stateless tests (debug)
- runner_type: func-tester
- data: ${{ needs.RunConfig.outputs.data }}
- FunctionalStatelessTestAsanAzure:
- needs: [RunConfig, BuilderDebAsan]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Stateless tests (azure, asan)
- runner_type: func-tester
- data: ${{ needs.RunConfig.outputs.data }}
-##############################################################################################
-############################ FUNCTIONAl STATEFUL TESTS #######################################
-##############################################################################################
- FunctionalStatefulTestRelease:
- needs: [RunConfig, BuilderDebRelease]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Stateful tests (release)
- runner_type: func-tester
- data: ${{ needs.RunConfig.outputs.data }}
- FunctionalStatefulTestAarch64:
- needs: [RunConfig, BuilderDebAarch64]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Stateful tests (aarch64)
- runner_type: func-tester-aarch64
- data: ${{ needs.RunConfig.outputs.data }}
- FunctionalStatefulTestAsan:
- needs: [RunConfig, BuilderDebAsan]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Stateful tests (asan)
- runner_type: func-tester
- data: ${{ needs.RunConfig.outputs.data }}
- FunctionalStatefulTestTsan:
- needs: [RunConfig, BuilderDebTsan]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Stateful tests (tsan)
- runner_type: func-tester
- data: ${{ needs.RunConfig.outputs.data }}
- FunctionalStatefulTestMsan:
- needs: [RunConfig, BuilderDebMsan]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Stateful tests (msan)
- runner_type: func-tester
- data: ${{ needs.RunConfig.outputs.data }}
- FunctionalStatefulTestUBsan:
- needs: [RunConfig, BuilderDebUBsan]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Stateful tests (ubsan)
- runner_type: func-tester
- data: ${{ needs.RunConfig.outputs.data }}
- FunctionalStatefulTestDebug:
- needs: [RunConfig, BuilderDebDebug]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Stateful tests (debug)
- runner_type: func-tester
- data: ${{ needs.RunConfig.outputs.data }}
- # Parallel replicas
- FunctionalStatefulTestDebugParallelReplicas:
- needs: [RunConfig, BuilderDebDebug]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Stateful tests (debug, ParallelReplicas)
- runner_type: func-tester
- data: ${{ needs.RunConfig.outputs.data }}
- FunctionalStatefulTestUBsanParallelReplicas:
- needs: [RunConfig, BuilderDebUBsan]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Stateful tests (ubsan, ParallelReplicas)
- runner_type: func-tester
- data: ${{ needs.RunConfig.outputs.data }}
- FunctionalStatefulTestMsanParallelReplicas:
- needs: [RunConfig, BuilderDebMsan]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Stateful tests (msan, ParallelReplicas)
- runner_type: func-tester
- data: ${{ needs.RunConfig.outputs.data }}
- FunctionalStatefulTestTsanParallelReplicas:
- needs: [RunConfig, BuilderDebTsan]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Stateful tests (tsan, ParallelReplicas)
- runner_type: func-tester
- data: ${{ needs.RunConfig.outputs.data }}
- FunctionalStatefulTestAsanParallelReplicas:
- needs: [RunConfig, BuilderDebAsan]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Stateful tests (asan, ParallelReplicas)
- runner_type: func-tester
- data: ${{ needs.RunConfig.outputs.data }}
- FunctionalStatefulTestReleaseParallelReplicas:
- needs: [RunConfig, BuilderDebRelease]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Stateful tests (release, ParallelReplicas)
- runner_type: func-tester
- data: ${{ needs.RunConfig.outputs.data }}
-##############################################################################################
-########################### ClickBench #######################################################
-##############################################################################################
- ClickBenchAMD64:
- needs: [RunConfig, BuilderDebRelease]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: ClickBench (amd64)
- runner_type: func-tester
- data: ${{ needs.RunConfig.outputs.data }}
- run_command: |
- python3 clickbench.py "$CHECK_NAME"
- ClickBenchAarch64:
- needs: [RunConfig, BuilderDebAarch64]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: ClickBench (aarch64)
- runner_type: func-tester-aarch64
- data: ${{ needs.RunConfig.outputs.data }}
- run_command: |
- python3 clickbench.py "$CHECK_NAME"
-##############################################################################################
-######################################### STRESS TESTS #######################################
-##############################################################################################
- StressTestAsan:
- needs: [RunConfig, BuilderDebAsan]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Stress test (asan)
- runner_type: stress-tester
- data: ${{ needs.RunConfig.outputs.data }}
- StressTestTsan:
- needs: [RunConfig, BuilderDebTsan]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Stress test (tsan)
- runner_type: stress-tester
- data: ${{ needs.RunConfig.outputs.data }}
- StressTestTsanAzure:
- needs: [RunConfig, BuilderDebTsan]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Stress test (azure, tsan)
- runner_type: stress-tester
- data: ${{ needs.RunConfig.outputs.data }}
- StressTestMsan:
- needs: [RunConfig, BuilderDebMsan]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Stress test (msan)
- runner_type: stress-tester
- data: ${{ needs.RunConfig.outputs.data }}
- StressTestUBsan:
- needs: [RunConfig, BuilderDebUBsan]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Stress test (ubsan)
- runner_type: stress-tester
- data: ${{ needs.RunConfig.outputs.data }}
- StressTestDebug:
- needs: [RunConfig, BuilderDebDebug]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Stress test (debug)
- runner_type: stress-tester
- data: ${{ needs.RunConfig.outputs.data }}
-#############################################################################################
-############################# INTEGRATION TESTS #############################################
-#############################################################################################
- IntegrationTestsAsan:
- needs: [RunConfig, BuilderDebAsan]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Integration tests (asan)
- runner_type: stress-tester
- data: ${{ needs.RunConfig.outputs.data }}
- IntegrationTestsAnalyzerAsan:
- needs: [RunConfig, BuilderDebAsan]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Integration tests (asan, old analyzer)
- runner_type: stress-tester
- data: ${{ needs.RunConfig.outputs.data }}
- IntegrationTestsTsan:
- needs: [RunConfig, BuilderDebTsan]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Integration tests (tsan)
- runner_type: stress-tester
- data: ${{ needs.RunConfig.outputs.data }}
- IntegrationTestsRelease:
- needs: [RunConfig, BuilderDebRelease]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Integration tests (release)
- runner_type: stress-tester
- data: ${{ needs.RunConfig.outputs.data }}
-##############################################################################################
-##################################### AST FUZZERS ############################################
-##############################################################################################
- ASTFuzzerTestAsan:
- needs: [RunConfig, BuilderDebAsan]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: AST fuzzer (asan)
- runner_type: fuzzer-unit-tester
- data: ${{ needs.RunConfig.outputs.data }}
- ASTFuzzerTestTsan:
- needs: [RunConfig, BuilderDebTsan]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: AST fuzzer (tsan)
- runner_type: fuzzer-unit-tester
- data: ${{ needs.RunConfig.outputs.data }}
- ASTFuzzerTestUBSan:
- needs: [RunConfig, BuilderDebUBsan]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: AST fuzzer (ubsan)
- runner_type: fuzzer-unit-tester
- data: ${{ needs.RunConfig.outputs.data }}
- ASTFuzzerTestMSan:
- needs: [RunConfig, BuilderDebMsan]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: AST fuzzer (msan)
- runner_type: fuzzer-unit-tester
- data: ${{ needs.RunConfig.outputs.data }}
- ASTFuzzerTestDebug:
- needs: [RunConfig, BuilderDebDebug]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: AST fuzzer (debug)
- runner_type: fuzzer-unit-tester
- data: ${{ needs.RunConfig.outputs.data }}
-#############################################################################################
-#################################### UNIT TESTS #############################################
-#############################################################################################
- UnitTestsAsan:
- needs: [RunConfig, BuilderDebAsan]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Unit tests (asan)
- runner_type: fuzzer-unit-tester
- data: ${{ needs.RunConfig.outputs.data }}
- UnitTestsReleaseClang:
- needs: [RunConfig, BuilderBinRelease]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Unit tests (release)
- runner_type: fuzzer-unit-tester
- data: ${{ needs.RunConfig.outputs.data }}
- UnitTestsTsan:
- needs: [RunConfig, BuilderDebTsan]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Unit tests (tsan)
- runner_type: fuzzer-unit-tester
- data: ${{ needs.RunConfig.outputs.data }}
- UnitTestsMsan:
- needs: [RunConfig, BuilderDebMsan]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Unit tests (msan)
- runner_type: fuzzer-unit-tester
- data: ${{ needs.RunConfig.outputs.data }}
- UnitTestsUBsan:
- needs: [RunConfig, BuilderDebUBsan]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Unit tests (ubsan)
- runner_type: fuzzer-unit-tester
- data: ${{ needs.RunConfig.outputs.data }}
-#############################################################################################
-#################################### PERFORMANCE TESTS ######################################
-#############################################################################################
- PerformanceComparisonX86:
- needs: [RunConfig, BuilderDebRelease]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Performance Comparison
- runner_type: stress-tester
- data: ${{ needs.RunConfig.outputs.data }}
- PerformanceComparisonAarch:
- needs: [RunConfig, BuilderDebAarch64]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Performance Comparison Aarch64
- runner_type: func-tester-aarch64
- data: ${{ needs.RunConfig.outputs.data }}
-##############################################################################################
-############################ SQLLOGIC TEST ###################################################
-##############################################################################################
- SQLLogicTestRelease:
- needs: [RunConfig, BuilderDebRelease]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: Sqllogic test (release)
- runner_type: func-tester
- data: ${{ needs.RunConfig.outputs.data }}
-##############################################################################################
-##################################### SQL TEST ###############################################
-##############################################################################################
- SQLTest:
- needs: [RunConfig, BuilderDebRelease]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: SQLTest
- runner_type: fuzzer-unit-tester
- data: ${{ needs.RunConfig.outputs.data }}
-##############################################################################################
-###################################### SQLANCER FUZZERS ######################################
-##############################################################################################
- SQLancerTestRelease:
- needs: [RunConfig, BuilderDebRelease]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: SQLancer (release)
- runner_type: fuzzer-unit-tester
- data: ${{ needs.RunConfig.outputs.data }}
- SQLancerTestDebug:
- needs: [RunConfig, BuilderDebDebug]
- if: ${{ !failure() && !cancelled() }}
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: SQLancer (debug)
- runner_type: fuzzer-unit-tester
- data: ${{ needs.RunConfig.outputs.data }}
FinishCheck:
- if: ${{ !failure() && !cancelled() }}
- needs:
- - MarkReleaseReady
- - FunctionalStatelessTestDebug
- - FunctionalStatelessTestRelease
- - FunctionalStatelessTestReleaseAnalyzerS3Replicated
- - FunctionalStatelessTestAarch64
- - FunctionalStatelessTestAsan
- - FunctionalStatelessTestTsan
- - FunctionalStatelessTestMsan
- - FunctionalStatelessTestUBsan
- - FunctionalStatelessTestS3Debug
- - FunctionalStatelessTestS3Tsan
- - FunctionalStatefulTestDebug
- - FunctionalStatefulTestRelease
- - FunctionalStatefulTestAarch64
- - FunctionalStatefulTestAsan
- - FunctionalStatefulTestTsan
- - FunctionalStatefulTestMsan
- - FunctionalStatefulTestUBsan
- - FunctionalStatefulTestDebugParallelReplicas
- - FunctionalStatefulTestUBsanParallelReplicas
- - FunctionalStatefulTestMsanParallelReplicas
- - FunctionalStatefulTestTsanParallelReplicas
- - FunctionalStatefulTestAsanParallelReplicas
- - FunctionalStatefulTestReleaseParallelReplicas
- - StressTestDebug
- - StressTestAsan
- - StressTestTsan
- - StressTestMsan
- - StressTestUBsan
- - IntegrationTestsAsan
- - IntegrationTestsAnalyzerAsan
- - IntegrationTestsTsan
- - IntegrationTestsRelease
- - PerformanceComparisonX86
- - PerformanceComparisonAarch
- - CompatibilityCheckX86
- - CompatibilityCheckAarch64
- - ASTFuzzerTestDebug
- - ASTFuzzerTestAsan
- - ASTFuzzerTestTsan
- - ASTFuzzerTestMSan
- - ASTFuzzerTestUBSan
- - UnitTestsAsan
- - UnitTestsTsan
- - UnitTestsMsan
- - UnitTestsUBsan
- - UnitTestsReleaseClang
- - SQLancerTestRelease
- - SQLancerTestDebug
- - SQLLogicTestRelease
- - SQLTest
- runs-on: [self-hosted, style-checker]
+ if: ${{ !cancelled() }}
+ needs: [RunConfig, Builds_1, Builds_2, Builds_1_Report, Builds_2_Report, Tests_1, Tests_2, Tests_3]
+ runs-on: [self-hosted, style-checker-aarch64]
steps:
- name: Check out repository code
uses: ClickHouse/checkout@v1
- with:
- clear-repository: true
- name: Finish label
run: |
cd "$GITHUB_WORKSPACE/tests/ci"
diff --git a/.github/workflows/merge_queue.yml b/.github/workflows/merge_queue.yml
index 97aa0db4cdb..c8b2452829b 100644
--- a/.github/workflows/merge_queue.yml
+++ b/.github/workflows/merge_queue.yml
@@ -20,7 +20,7 @@ jobs:
uses: ClickHouse/checkout@v1
with:
clear-repository: true # to ensure correct digests
- fetch-depth: 0 # to get version
+ fetch-depth: 0 # to get a version
filter: tree:0
- name: Cancel PR workflow
run: |
@@ -60,7 +60,7 @@ jobs:
uses: ./.github/workflows/reusable_test.yml
with:
test_name: Style check
- runner_type: style-checker
+ runner_type: style-checker-aarch64
run_command: |
python3 style_check.py
data: ${{ needs.RunConfig.outputs.data }}
@@ -80,12 +80,28 @@ jobs:
run_command: |
python3 fast_test_check.py
+ Builds_1:
+ needs: [RunConfig, BuildDockers]
+ if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Builds_1') }}
+ # using callable wf (reusable_stage.yml) allows grouping all nested jobs under a tab
+ uses: ./.github/workflows/reusable_build_stage.yml
+ with:
+ stage: Builds_1
+ data: ${{ needs.RunConfig.outputs.data }}
+ Tests_1:
+ needs: [RunConfig, Builds_1]
+ if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_1') }}
+ uses: ./.github/workflows/reusable_test_stage.yml
+ with:
+ stage: Tests_1
+ data: ${{ needs.RunConfig.outputs.data }}
+
################################# Stage Final #################################
#
FinishCheck:
if: ${{ !failure() && !cancelled() }}
- needs: [RunConfig, BuildDockers, StyleCheck, FastTest]
- runs-on: [self-hosted, style-checker]
+ needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Tests_1]
+ runs-on: [self-hosted, style-checker-aarch64]
steps:
- name: Check out repository code
uses: ClickHouse/checkout@v1
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index f20e987db97..e4deaf9f35e 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -31,8 +31,14 @@ jobs:
uses: ClickHouse/checkout@v1
with:
clear-repository: true # to ensure correct digests
- fetch-depth: 0 # to get version
+ fetch-depth: 0 # to get a version
filter: tree:0
+ - name: Cancel previous Sync PR workflow
+ run: |
+ python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --cancel-previous-run
+ - name: Set pending Sync status
+ run: |
+ python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --set-pending-status
- name: Labels check
run: |
cd "$GITHUB_WORKSPACE/tests/ci"
@@ -75,7 +81,7 @@ jobs:
uses: ./.github/workflows/reusable_test.yml
with:
test_name: Style check
- runner_type: style-checker
+ runner_type: style-checker-aarch64
run_command: |
python3 style_check.py
data: ${{ needs.RunConfig.outputs.data }}
@@ -95,13 +101,13 @@ jobs:
run_command: |
python3 fast_test_check.py
- ################################# Main statges #################################
+ ################################# Main stages #################################
# for main CI chain
#
Builds_1:
needs: [RunConfig, StyleCheck, FastTest]
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Builds_1') }}
- # using callable wf (reusable_stage.yml) allows to group all nested jobs under a tab
+ # using callable wf (reusable_stage.yml) allows grouping all nested jobs under a tab
uses: ./.github/workflows/reusable_build_stage.yml
with:
stage: Builds_1
@@ -109,7 +115,6 @@ jobs:
Tests_1:
needs: [RunConfig, Builds_1]
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_1') }}
- # using callable wf (reusable_stage.yml) allows to group all nested jobs under a tab
uses: ./.github/workflows/reusable_test_stage.yml
with:
stage: Tests_1
@@ -117,7 +122,6 @@ jobs:
Builds_2:
needs: [RunConfig, Builds_1]
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Builds_2') }}
- # using callable wf (reusable_stage.yml) allows to group all nested jobs under a tab
uses: ./.github/workflows/reusable_build_stage.yml
with:
stage: Builds_2
@@ -125,14 +129,13 @@ jobs:
Tests_2:
needs: [RunConfig, Builds_2]
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_2') }}
- # using callable wf (reusable_stage.yml) allows to group all nested jobs under a tab
uses: ./.github/workflows/reusable_test_stage.yml
with:
stage: Tests_2
data: ${{ needs.RunConfig.outputs.data }}
# stage for jobs that do not prohibit merge
Tests_3:
- needs: [RunConfig, Tests_1, Tests_2]
+ needs: [RunConfig, Builds_1, Tests_1, Builds_2, Tests_2]
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_3') }}
uses: ./.github/workflows/reusable_test_stage.yml
with:
@@ -177,9 +180,9 @@ jobs:
################################# Stage Final #################################
#
FinishCheck:
- if: ${{ !failure() && !cancelled() }}
+ if: ${{ !cancelled() }}
needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_1_Report, Builds_2_Report, Tests_1, Tests_2, Tests_3]
- runs-on: [self-hosted, style-checker]
+ runs-on: [self-hosted, style-checker-aarch64]
steps:
- name: Check out repository code
uses: ClickHouse/checkout@v1
@@ -189,13 +192,6 @@ jobs:
run: |
cd "$GITHUB_WORKSPACE/tests/ci"
python3 finish_check.py
- # FIXME: merge on approval does not work with MQ. Could be fixed by using defaul GH's automerge after some corrections in Mergeable Check status
- # - name: Auto merge if approved
- # if: ${{ github.event_name != 'merge_group' }}
- # run: |
- # cd "$GITHUB_WORKSPACE/tests/ci"
- # python3 merge_pr.py --check-approved
-
#############################################################################################
###################################### JEPSEN TESTS #########################################
@@ -213,5 +209,5 @@ jobs:
uses: ./.github/workflows/reusable_test.yml
with:
test_name: ClickHouse Keeper Jepsen
- runner_type: style-checker
+ runner_type: style-checker-aarch64
data: ${{ needs.RunConfig.outputs.data }}
diff --git a/.github/workflows/reusable_build.yml b/.github/workflows/reusable_build.yml
index 80d78d93e1b..5e254d785ec 100644
--- a/.github/workflows/reusable_build.yml
+++ b/.github/workflows/reusable_build.yml
@@ -33,6 +33,10 @@ name: Build ClickHouse
additional_envs:
description: additional ENV variables to setup the job
type: string
+ secrets:
+ secret_envs:
+ description: if given, it's passed to the environments
+ required: false
jobs:
Build:
@@ -54,6 +58,7 @@ jobs:
run: |
cat >> "$GITHUB_ENV" << 'EOF'
${{inputs.additional_envs}}
+ ${{secrets.secret_envs}}
DOCKER_TAG< 1 && format('-{0}',matrix.batch) || '' }}
strategy:
- fail-fast: false # we always wait for entire matrix
+ fail-fast: false # we always wait for the entire matrix
matrix:
batch: ${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].batches }}
steps:
diff --git a/.github/workflows/reusable_test_stage.yml b/.github/workflows/reusable_test_stage.yml
index d7bd55fab43..8926b43d372 100644
--- a/.github/workflows/reusable_test_stage.yml
+++ b/.github/workflows/reusable_test_stage.yml
@@ -10,6 +10,10 @@ name: StageWF
description: ci data
type: string
required: true
+ secrets:
+ secret_envs:
+ description: if given, it's passed to the environments
+ required: false
jobs:
s:
@@ -23,3 +27,5 @@ jobs:
test_name: ${{ matrix.job_name_and_runner_type.job_name }}
runner_type: ${{ matrix.job_name_and_runner_type.runner_type }}
data: ${{ inputs.data }}
+ secrets:
+ secret_envs: ${{ secrets.secret_envs }}
diff --git a/.gitignore b/.gitignore
index db3f77d7d1e..4bc162c1b0f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -21,6 +21,9 @@
*.stderr
*.stdout
+# llvm-xray logs
+xray-log.*
+
/docs/build
/docs/publish
/docs/edit
diff --git a/.gitmessage b/.gitmessage
deleted file mode 100644
index 89ee7d35d23..00000000000
--- a/.gitmessage
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
-### CI modificators (add a leading space to apply) ###
-
-## To avoid a merge commit in CI:
-#no_merge_commit
-
-## To discard CI cache:
-#no_ci_cache
-
-## To not test (only style check):
-#do_not_test
-
-## To run specified set of tests in CI:
-#ci_set_
-#ci_set_reduced
-#ci_set_arm
-#ci_set_integration
-#ci_set_old_analyzer
-
-## To run specified job in CI:
-#job_
-#job_stateless_tests_release
-#job_package_debug
-#job_integration_tests_asan
-
-## To run only specified batches for multi-batch job(s)
-#batch_2
-#batch_1_2_3
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 207b88f7860..4891b79e4c7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
### Table of Contents
+**[ClickHouse release v24.5, 2024-05-30](#245)**
**[ClickHouse release v24.4, 2024-04-30](#244)**
**[ClickHouse release v24.3 LTS, 2024-03-26](#243)**
**[ClickHouse release v24.2, 2024-02-29](#242)**
@@ -7,6 +8,162 @@
# 2024 Changelog
+### ClickHouse release 24.5, 2024-05-30
+
+#### Backward Incompatible Change
+* Renamed "inverted indexes" to "full-text indexes" which is a less technical / more user-friendly name. This also changes internal table metadata and breaks tables with existing (experimental) inverted indexes. Please make sure to drop such indexes before upgrade and re-create them after upgrade. [#62884](https://github.com/ClickHouse/ClickHouse/pull/62884) ([Robert Schulze](https://github.com/rschu1ze)).
+* Usage of functions `neighbor`, `runningAccumulate`, `runningDifferenceStartingWithFirstValue`, `runningDifference` deprecated (because it is error-prone). Proper window functions should be used instead. To enable them back, set `allow_deprecated_error_prone_window_functions = 1` or set `compatibility = '24.4'` or lower. [#63132](https://github.com/ClickHouse/ClickHouse/pull/63132) ([Nikita Taranov](https://github.com/nickitat)).
+* Queries from `system.columns` will work faster if there is a large number of columns, but many databases or tables are not granted for `SHOW TABLES`. Note that in previous versions, if you grant `SHOW COLUMNS` to individual columns without granting `SHOW TABLES` to the corresponding tables, the `system.columns` table will show these columns, but in a new version, it will skip the table entirely. Remove trace log messages "Access granted" and "Access denied" that slowed down queries. [#63439](https://github.com/ClickHouse/ClickHouse/pull/63439) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+
+#### New Feature
+* Adds the `Form` format to read/write a single record in the `application/x-www-form-urlencoded` format. [#60199](https://github.com/ClickHouse/ClickHouse/pull/60199) ([Shaun Struwig](https://github.com/Blargian)).
+* Added possibility to compress in CROSS JOIN. [#60459](https://github.com/ClickHouse/ClickHouse/pull/60459) ([p1rattttt](https://github.com/p1rattttt)).
+* Added possibility to do `CROSS JOIN` in temporary files if the size exceeds limits. [#63432](https://github.com/ClickHouse/ClickHouse/pull/63432) ([p1rattttt](https://github.com/p1rattttt)).
+* Support join with inequal conditions which involve columns from both left and right table. e.g. `t1.y < t2.y`. To enable, `SET allow_experimental_join_condition = 1`. [#60920](https://github.com/ClickHouse/ClickHouse/pull/60920) ([lgbo](https://github.com/lgbo-ustc)).
+* Maps can now have `Float32`, `Float64`, `Array(T)`, `Map(K, V)` and `Tuple(T1, T2, ...)` as keys. Closes [#54537](https://github.com/ClickHouse/ClickHouse/issues/54537). [#59318](https://github.com/ClickHouse/ClickHouse/pull/59318) ([李扬](https://github.com/taiyang-li)).
+* Introduce bulk loading to `EmbeddedRocksDB` by creating and ingesting SST file instead of relying on rocksdb build-in memtable. This help to increase importing speed, especially for long-running insert query to StorageEmbeddedRocksDB tables. Also, introduce `EmbeddedRocksDB` table settings. [#59163](https://github.com/ClickHouse/ClickHouse/pull/59163) [#63324](https://github.com/ClickHouse/ClickHouse/pull/63324) ([Duc Canh Le](https://github.com/canhld94)).
+* User can now parse CRLF with TSV format using a setting `input_format_tsv_crlf_end_of_line`. Closes [#56257](https://github.com/ClickHouse/ClickHouse/issues/56257). [#59747](https://github.com/ClickHouse/ClickHouse/pull/59747) ([Shaun Struwig](https://github.com/Blargian)).
+* A new setting `input_format_force_null_for_omitted_fields` that forces NULL values for omitted fields. [#60887](https://github.com/ClickHouse/ClickHouse/pull/60887) ([Constantine Peresypkin](https://github.com/pkit)).
+* Earlier our S3 storage and s3 table function didn't support selecting from archive container files, such as tarballs, zip, 7z. Now they allow to iterate over files inside archives in S3. [#62259](https://github.com/ClickHouse/ClickHouse/pull/62259) ([Daniil Ivanik](https://github.com/divanik)).
+* Support for conditional function `clamp`. [#62377](https://github.com/ClickHouse/ClickHouse/pull/62377) ([skyoct](https://github.com/skyoct)).
+* Add `NPy` output format. [#62430](https://github.com/ClickHouse/ClickHouse/pull/62430) ([豪肥肥](https://github.com/HowePa)).
+* `Raw` format as a synonym for `TSVRaw`. [#63394](https://github.com/ClickHouse/ClickHouse/pull/63394) ([Unalian](https://github.com/Unalian)).
+* Added a new SQL function `generateUUIDv7` to generate version 7 UUIDs aka. timestamp-based UUIDs with random component. Also added a new function `UUIDToNum` to extract bytes from a UUID and a new function `UUIDv7ToDateTime` to extract timestamp component from a UUID version 7. [#62852](https://github.com/ClickHouse/ClickHouse/pull/62852) ([Alexey Petrunyaka](https://github.com/pet74alex)).
+* On Linux and MacOS, if the program has stdout redirected to a file with a compression extension, use the corresponding compression method instead of nothing (making it behave similarly to `INTO OUTFILE`). [#63662](https://github.com/ClickHouse/ClickHouse/pull/63662) ([v01dXYZ](https://github.com/v01dXYZ)).
+* Change warning on high number of attached tables to differentiate tables, views and dictionaries. [#64180](https://github.com/ClickHouse/ClickHouse/pull/64180) ([Francisco J. Jurado Moreno](https://github.com/Beetelbrox)).
+* Provide support for `azureBlobStorage` function in ClickHouse server to use Azure Workload identity to authenticate against Azure blob storage. If `use_workload_identity` parameter is set in config, [workload identity](https://github.com/Azure/azure-sdk-for-cpp/tree/main/sdk/identity/azure-identity#authenticate-azure-hosted-applications) is used for authentication. [#57881](https://github.com/ClickHouse/ClickHouse/pull/57881) ([Vinay Suryadevara](https://github.com/vinay92-ch)).
+* Add TTL information in the `system.parts_columns` table. [#63200](https://github.com/ClickHouse/ClickHouse/pull/63200) ([litlig](https://github.com/litlig)).
+
+#### Experimental Features
+* Implement `Dynamic` data type that allows to store values of any type inside it without knowing all of them in advance. `Dynamic` type is available under a setting `allow_experimental_dynamic_type`. Reference: [#54864](https://github.com/ClickHouse/ClickHouse/issues/54864). [#63058](https://github.com/ClickHouse/ClickHouse/pull/63058) ([Kruglov Pavel](https://github.com/Avogar)).
+* Allowed to create `MaterializedMySQL` database without connection to MySQL. [#63397](https://github.com/ClickHouse/ClickHouse/pull/63397) ([Kirill](https://github.com/kirillgarbar)).
+* Automatically mark a replica of Replicated database as lost and start recovery if some DDL task fails more than `max_retries_before_automatic_recovery` (100 by default) times in a row with the same error. Also, fixed a bug that could cause skipping DDL entries when an exception is thrown during an early stage of entry execution. [#63549](https://github.com/ClickHouse/ClickHouse/pull/63549) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Account failed files in `s3queue_tracked_file_ttl_sec` and `s3queue_traked_files_limit` for `StorageS3Queue`. [#63638](https://github.com/ClickHouse/ClickHouse/pull/63638) ([Kseniia Sumarokova](https://github.com/kssenii)).
+
+#### Performance Improvement
+* Less contention in filesystem cache (part 4). Allow to keep filesystem cache not filled to the limit by doing additional eviction in the background (controlled by `keep_free_space_size(elements)_ratio`). This allows to release pressure from space reservation for queries (on `tryReserve` method). Also this is done in a lock free way as much as possible, e.g. should not block normal cache usage. [#61250](https://github.com/ClickHouse/ClickHouse/pull/61250) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Skip merging of newly created projection blocks during `INSERT`-s. [#59405](https://github.com/ClickHouse/ClickHouse/pull/59405) ([Nikita Taranov](https://github.com/nickitat)).
+* Process string functions `...UTF8` 'asciily' if input strings are all ascii chars. Inspired by https://github.com/apache/doris/pull/29799. Overall speed up by 1.07x~1.62x. Notice that peak memory usage had been decreased in some cases. [#61632](https://github.com/ClickHouse/ClickHouse/pull/61632) ([李扬](https://github.com/taiyang-li)).
+* Improved performance of selection (`{}`) globs in StorageS3. [#62120](https://github.com/ClickHouse/ClickHouse/pull/62120) ([Andrey Zvonov](https://github.com/zvonand)).
+* HostResolver has each IP address several times. If remote host has several IPs and by some reason (firewall rules for example) access on some IPs allowed and on others forbidden, than only first record of forbidden IPs marked as failed, and in each try these IPs have a chance to be chosen (and failed again). Even if fix this, every 120 seconds DNS cache dropped, and IPs can be chosen again. [#62652](https://github.com/ClickHouse/ClickHouse/pull/62652) ([Anton Ivashkin](https://github.com/ianton-ru)).
+* Add a new configuration`prefer_merge_sort_block_bytes` to control the memory usage and speed up sorting 2 times when merging when there are many columns. [#62904](https://github.com/ClickHouse/ClickHouse/pull/62904) ([LiuNeng](https://github.com/liuneng1994)).
+* `clickhouse-local` will start faster. In previous versions, it was not deleting temporary directories by mistake. Now it will. This closes [#62941](https://github.com/ClickHouse/ClickHouse/issues/62941). [#63074](https://github.com/ClickHouse/ClickHouse/pull/63074) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Micro-optimizations for the new analyzer. [#63429](https://github.com/ClickHouse/ClickHouse/pull/63429) ([Raúl Marín](https://github.com/Algunenano)).
+* Index analysis will work if `DateTime` is compared to `DateTime64`. This closes [#63441](https://github.com/ClickHouse/ClickHouse/issues/63441). [#63443](https://github.com/ClickHouse/ClickHouse/pull/63443) [#63532](https://github.com/ClickHouse/ClickHouse/pull/63532) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Speed up indices of type `set` a little (around 1.5 times) by removing garbage. [#64098](https://github.com/ClickHouse/ClickHouse/pull/64098) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Remove copying data when writing to the filesystem cache. [#63401](https://github.com/ClickHouse/ClickHouse/pull/63401) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Now backups with azure blob storage will use multicopy. [#64116](https://github.com/ClickHouse/ClickHouse/pull/64116) ([alesapin](https://github.com/alesapin)).
+* Allow to use native copy for azure even with different containers. [#64154](https://github.com/ClickHouse/ClickHouse/pull/64154) ([alesapin](https://github.com/alesapin)).
+* Finally enable native copy for azure. [#64182](https://github.com/ClickHouse/ClickHouse/pull/64182) ([alesapin](https://github.com/alesapin)).
+
+#### Improvement
+* Allow using `clickhouse-local` and its shortcuts `clickhouse` and `ch` with a query or queries file as a positional argument. Examples: `ch "SELECT 1"`, `ch --param_test Hello "SELECT {test:String}"`, `ch query.sql`. This closes [#62361](https://github.com/ClickHouse/ClickHouse/issues/62361). [#63081](https://github.com/ClickHouse/ClickHouse/pull/63081) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Enable plain_rewritable metadata for local and Azure (azure_blob_storage) object storages. [#63365](https://github.com/ClickHouse/ClickHouse/pull/63365) ([Julia Kartseva](https://github.com/jkartseva)).
+* Support English-style Unicode quotes, e.g. “Hello”, ‘world’. This is questionable in general but helpful when you type your query in a word processor, such as Google Docs. This closes [#58634](https://github.com/ClickHouse/ClickHouse/issues/58634). [#63381](https://github.com/ClickHouse/ClickHouse/pull/63381) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Allow trailing commas in the columns list in the INSERT query. For example, `INSERT INTO test (a, b, c, ) VALUES ...`. [#63803](https://github.com/ClickHouse/ClickHouse/pull/63803) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Better exception messages for the `Regexp` format. [#63804](https://github.com/ClickHouse/ClickHouse/pull/63804) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Allow trailing commas in the `Values` format. For example, this query is allowed: `INSERT INTO test (a, b, c) VALUES (4, 5, 6,);`. [#63810](https://github.com/ClickHouse/ClickHouse/pull/63810) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Make rabbitmq nack broken messages. Closes [#45350](https://github.com/ClickHouse/ClickHouse/issues/45350). [#60312](https://github.com/ClickHouse/ClickHouse/pull/60312) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix a crash in asynchronous stack unwinding (such as when using the sampling query profiler) while interpreting debug info. This closes [#60460](https://github.com/ClickHouse/ClickHouse/issues/60460). [#60468](https://github.com/ClickHouse/ClickHouse/pull/60468) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Distinct messages for s3 error 'no key' for cases disk and storage. [#61108](https://github.com/ClickHouse/ClickHouse/pull/61108) ([Sema Checherinda](https://github.com/CheSema)).
+* The progress bar will work for trivial queries with LIMIT from `system.zeros`, `system.zeros_mt` (it already works for `system.numbers` and `system.numbers_mt`), and the `generateRandom` table function. As a bonus, if the total number of records is greater than the `max_rows_to_read` limit, it will throw an exception earlier. This closes [#58183](https://github.com/ClickHouse/ClickHouse/issues/58183). [#61823](https://github.com/ClickHouse/ClickHouse/pull/61823) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Support for "Merge Key" in YAML configurations (this is a weird feature of YAML, please never mind). [#62685](https://github.com/ClickHouse/ClickHouse/pull/62685) ([Azat Khuzhin](https://github.com/azat)).
+* Enhance error message when non-deterministic function is used with Replicated source. [#62896](https://github.com/ClickHouse/ClickHouse/pull/62896) ([Grégoire Pineau](https://github.com/lyrixx)).
+* Fix interserver secret for Distributed over Distributed from `remote`. [#63013](https://github.com/ClickHouse/ClickHouse/pull/63013) ([Azat Khuzhin](https://github.com/azat)).
+* Support `include_from` for YAML files. However, you should better use `config.d` [#63106](https://github.com/ClickHouse/ClickHouse/pull/63106) ([Eduard Karacharov](https://github.com/korowa)).
+* Keep previous data in terminal after picking from skim suggestions. [#63261](https://github.com/ClickHouse/ClickHouse/pull/63261) ([FlameFactory](https://github.com/FlameFactory)).
+* Width of fields (in Pretty formats or the `visibleWidth` function) now correctly ignores ANSI escape sequences. [#63270](https://github.com/ClickHouse/ClickHouse/pull/63270) ([Shaun Struwig](https://github.com/Blargian)).
+* Update the usage of error code `NUMBER_OF_ARGUMENTS_DOESNT_MATCH` by more accurate error codes when appropriate. [#63406](https://github.com/ClickHouse/ClickHouse/pull/63406) ([Yohann Jardin](https://github.com/yohannj)).
+* `os_user` and `client_hostname` are now correctly set up for queries for command line suggestions in clickhouse-client. This closes [#63430](https://github.com/ClickHouse/ClickHouse/issues/63430). [#63433](https://github.com/ClickHouse/ClickHouse/pull/63433) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Automatically correct `max_block_size` to the default value if it is zero. [#63587](https://github.com/ClickHouse/ClickHouse/pull/63587) ([Antonio Andelic](https://github.com/antonio2368)).
+* Add a build_id ALIAS column to trace_log to facilitate auto renaming upon detecting binary changes. This is to address [#52086](https://github.com/ClickHouse/ClickHouse/issues/52086). [#63656](https://github.com/ClickHouse/ClickHouse/pull/63656) ([Zimu Li](https://github.com/woodlzm)).
+* Enable truncate operation for object storage disks. [#63693](https://github.com/ClickHouse/ClickHouse/pull/63693) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
+* The loading of the keywords list is now dependent on the server revision and will be disabled for the old versions of ClickHouse server. CC @azat. [#63786](https://github.com/ClickHouse/ClickHouse/pull/63786) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Clickhouse disks have to read server setting to obtain actual metadata format version. [#63831](https://github.com/ClickHouse/ClickHouse/pull/63831) ([Sema Checherinda](https://github.com/CheSema)).
+* Disable pretty format restrictions (`output_format_pretty_max_rows`/`output_format_pretty_max_value_width`) when stdout is not TTY. [#63942](https://github.com/ClickHouse/ClickHouse/pull/63942) ([Azat Khuzhin](https://github.com/azat)).
+* Exception handling now works when ClickHouse is used inside AWS Lambda. Author: [Alexey Coolnev](https://github.com/acoolnev). [#64014](https://github.com/ClickHouse/ClickHouse/pull/64014) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Throw `CANNOT_DECOMPRESS` instread of `CORRUPTED_DATA` on invalid compressed data passed via HTTP. [#64036](https://github.com/ClickHouse/ClickHouse/pull/64036) ([vdimir](https://github.com/vdimir)).
+* A tip for a single large number in Pretty formats now works for Nullable and LowCardinality. This closes [#61993](https://github.com/ClickHouse/ClickHouse/issues/61993). [#64084](https://github.com/ClickHouse/ClickHouse/pull/64084) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Add metrics, logs, and thread names around parts filtering with indices. [#64130](https://github.com/ClickHouse/ClickHouse/pull/64130) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Ignore `allow_suspicious_primary_key` on `ATTACH` and verify on `ALTER`. [#64202](https://github.com/ClickHouse/ClickHouse/pull/64202) ([Azat Khuzhin](https://github.com/azat)).
+
+#### Build/Testing/Packaging Improvement
+* ClickHouse is built with clang-18. A lot of new checks from clang-tidy-18 have been enabled. [#60469](https://github.com/ClickHouse/ClickHouse/pull/60469) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Experimentally support loongarch64 as a new platform for ClickHouse. [#63733](https://github.com/ClickHouse/ClickHouse/pull/63733) ([qiangxuhui](https://github.com/qiangxuhui)).
+* The Dockerfile is reviewed by the docker official library in https://github.com/docker-library/official-images/pull/15846. [#63400](https://github.com/ClickHouse/ClickHouse/pull/63400) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Information about every symbol in every translation unit will be collected in the CI database for every build in the CI. This closes [#63494](https://github.com/ClickHouse/ClickHouse/issues/63494). [#63495](https://github.com/ClickHouse/ClickHouse/pull/63495) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Update Apache Datasketches library. It resolves [#63858](https://github.com/ClickHouse/ClickHouse/issues/63858). [#63923](https://github.com/ClickHouse/ClickHouse/pull/63923) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Enable GRPC support for aarch64 linux while cross-compiling binary. [#64072](https://github.com/ClickHouse/ClickHouse/pull/64072) ([alesapin](https://github.com/alesapin)).
+* Fix unwind on SIGSEGV on aarch64 (due to small stack for signal) [#64058](https://github.com/ClickHouse/ClickHouse/pull/64058) ([Azat Khuzhin](https://github.com/azat)).
+
+#### Bug Fix
+* Disabled `enable_vertical_final` setting by default. This feature should not be used because it has a bug: [#64543](https://github.com/ClickHouse/ClickHouse/issues/64543). [#64544](https://github.com/ClickHouse/ClickHouse/pull/64544) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fix making backup when multiple shards are used [#57684](https://github.com/ClickHouse/ClickHouse/pull/57684) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Fix passing projections/indexes/primary key from columns list from CREATE query into inner table of MV [#59183](https://github.com/ClickHouse/ClickHouse/pull/59183) ([Azat Khuzhin](https://github.com/azat)).
+* Fix boundRatio incorrect merge [#60532](https://github.com/ClickHouse/ClickHouse/pull/60532) ([Tao Wang](https://github.com/wangtZJU)).
+* Fix crash when calling some functions on const low-cardinality columns [#61966](https://github.com/ClickHouse/ClickHouse/pull/61966) ([Michael Kolupaev](https://github.com/al13n321)).
+* Fix queries with FINAL give wrong result when table does not use adaptive granularity [#62432](https://github.com/ClickHouse/ClickHouse/pull/62432) ([Duc Canh Le](https://github.com/canhld94)).
+* Improve detection of cgroups v2 support for memory controllers [#62903](https://github.com/ClickHouse/ClickHouse/pull/62903) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix subsequent use of external tables in client [#62964](https://github.com/ClickHouse/ClickHouse/pull/62964) ([Azat Khuzhin](https://github.com/azat)).
+* Fix crash with untuple and unresolved lambda [#63131](https://github.com/ClickHouse/ClickHouse/pull/63131) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix premature server listen for connections [#63181](https://github.com/ClickHouse/ClickHouse/pull/63181) ([alesapin](https://github.com/alesapin)).
+* Fix intersecting parts when restarting after a DROP PART command [#63202](https://github.com/ClickHouse/ClickHouse/pull/63202) ([Han Fei](https://github.com/hanfei1991)).
+* Correctly load SQL security defaults during startup [#63209](https://github.com/ClickHouse/ClickHouse/pull/63209) ([pufit](https://github.com/pufit)).
+* JOIN filter push down filter join fix [#63234](https://github.com/ClickHouse/ClickHouse/pull/63234) ([Maksim Kita](https://github.com/kitaisreal)).
+* Fix infinite loop in AzureObjectStorage::listObjects [#63257](https://github.com/ClickHouse/ClickHouse/pull/63257) ([Julia Kartseva](https://github.com/jkartseva)).
+* CROSS join ignore join_algorithm setting [#63273](https://github.com/ClickHouse/ClickHouse/pull/63273) ([vdimir](https://github.com/vdimir)).
+* Fix finalize WriteBufferToFileSegment and StatusFile [#63346](https://github.com/ClickHouse/ClickHouse/pull/63346) ([vdimir](https://github.com/vdimir)).
+* Fix logical error during SELECT query after ALTER in rare case [#63353](https://github.com/ClickHouse/ClickHouse/pull/63353) ([alesapin](https://github.com/alesapin)).
+* Fix `X-ClickHouse-Timezone` header with `session_timezone` [#63377](https://github.com/ClickHouse/ClickHouse/pull/63377) ([Andrey Zvonov](https://github.com/zvonand)).
+* Fix debug assert when using grouping WITH ROLLUP and LowCardinality types [#63398](https://github.com/ClickHouse/ClickHouse/pull/63398) ([Raúl Marín](https://github.com/Algunenano)).
+* Small fixes for group_by_use_nulls [#63405](https://github.com/ClickHouse/ClickHouse/pull/63405) ([vdimir](https://github.com/vdimir)).
+* Fix backup/restore of projection part in case projection was removed from table metadata, but part still has projection [#63426](https://github.com/ClickHouse/ClickHouse/pull/63426) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix mysql dictionary source [#63481](https://github.com/ClickHouse/ClickHouse/pull/63481) ([vdimir](https://github.com/vdimir)).
+* Insert QueryFinish on AsyncInsertFlush with no data [#63483](https://github.com/ClickHouse/ClickHouse/pull/63483) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix: empty used_dictionaries in system.query_log [#63487](https://github.com/ClickHouse/ClickHouse/pull/63487) ([Eduard Karacharov](https://github.com/korowa)).
+* Make `MergeTreePrefetchedReadPool` safer [#63513](https://github.com/ClickHouse/ClickHouse/pull/63513) ([Antonio Andelic](https://github.com/antonio2368)).
+* Fix crash on exit with sentry enabled (due to openssl destroyed before sentry) [#63548](https://github.com/ClickHouse/ClickHouse/pull/63548) ([Azat Khuzhin](https://github.com/azat)).
+* Fix Array and Map support with Keyed hashing [#63628](https://github.com/ClickHouse/ClickHouse/pull/63628) ([Salvatore Mesoraca](https://github.com/aiven-sal)).
+* Fix filter pushdown for Parquet and maybe StorageMerge [#63642](https://github.com/ClickHouse/ClickHouse/pull/63642) ([Michael Kolupaev](https://github.com/al13n321)).
+* Prevent conversion to Replicated if zookeeper path already exists [#63670](https://github.com/ClickHouse/ClickHouse/pull/63670) ([Kirill](https://github.com/kirillgarbar)).
+* Analyzer: views read only necessary columns [#63688](https://github.com/ClickHouse/ClickHouse/pull/63688) ([Maksim Kita](https://github.com/kitaisreal)).
+* Analyzer: Forbid WINDOW redefinition [#63694](https://github.com/ClickHouse/ClickHouse/pull/63694) ([Dmitry Novik](https://github.com/novikd)).
+* flatten_nested was broken with the experimental Replicated database. [#63695](https://github.com/ClickHouse/ClickHouse/pull/63695) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix [#63653](https://github.com/ClickHouse/ClickHouse/issues/63653) [#63722](https://github.com/ClickHouse/ClickHouse/pull/63722) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Allow cast from Array(Nothing) to Map(Nothing, Nothing) [#63753](https://github.com/ClickHouse/ClickHouse/pull/63753) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix ILLEGAL_COLUMN in partial_merge join [#63755](https://github.com/ClickHouse/ClickHouse/pull/63755) ([vdimir](https://github.com/vdimir)).
+* Fix: remove redundant distinct with window functions [#63776](https://github.com/ClickHouse/ClickHouse/pull/63776) ([Igor Nikonov](https://github.com/devcrafter)).
+* Fix possible crash with SYSTEM UNLOAD PRIMARY KEY [#63778](https://github.com/ClickHouse/ClickHouse/pull/63778) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix a query with duplicating cycling alias. [#63791](https://github.com/ClickHouse/ClickHouse/pull/63791) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Make `TokenIterator` lazy as it should be [#63801](https://github.com/ClickHouse/ClickHouse/pull/63801) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Add `endpoint_subpath` S3 URI setting [#63806](https://github.com/ClickHouse/ClickHouse/pull/63806) ([Julia Kartseva](https://github.com/jkartseva)).
+* Fix deadlock in `ParallelReadBuffer` [#63814](https://github.com/ClickHouse/ClickHouse/pull/63814) ([Antonio Andelic](https://github.com/antonio2368)).
+* JOIN filter push down equivalent columns fix [#63819](https://github.com/ClickHouse/ClickHouse/pull/63819) ([Maksim Kita](https://github.com/kitaisreal)).
+* Remove data from all disks after DROP with Lazy database. [#63848](https://github.com/ClickHouse/ClickHouse/pull/63848) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
+* Fix incorrect result when reading from MV with parallel replicas and new analyzer [#63861](https://github.com/ClickHouse/ClickHouse/pull/63861) ([Nikita Taranov](https://github.com/nickitat)).
+* Fixes in `find_super_nodes` and `find_big_family` command of keeper-client [#63862](https://github.com/ClickHouse/ClickHouse/pull/63862) ([Alexander Gololobov](https://github.com/davenger)).
+* Update lambda execution name [#63864](https://github.com/ClickHouse/ClickHouse/pull/63864) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix SIGSEGV due to CPU/Real profiler [#63865](https://github.com/ClickHouse/ClickHouse/pull/63865) ([Azat Khuzhin](https://github.com/azat)).
+* Fix `EXPLAIN CURRENT TRANSACTION` query [#63926](https://github.com/ClickHouse/ClickHouse/pull/63926) ([Anton Popov](https://github.com/CurtizJ)).
+* Fix analyzer: there's turtles all the way down... [#63930](https://github.com/ClickHouse/ClickHouse/pull/63930) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Allow certain ALTER TABLE commands for `plain_rewritable` disk [#63933](https://github.com/ClickHouse/ClickHouse/pull/63933) ([Julia Kartseva](https://github.com/jkartseva)).
+* Recursive CTE distributed fix [#63939](https://github.com/ClickHouse/ClickHouse/pull/63939) ([Maksim Kita](https://github.com/kitaisreal)).
+* Analyzer: Fix COLUMNS resolve [#63962](https://github.com/ClickHouse/ClickHouse/pull/63962) ([Dmitry Novik](https://github.com/novikd)).
+* LIMIT BY and skip_unused_shards with analyzer [#63983](https://github.com/ClickHouse/ClickHouse/pull/63983) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* A fix for some trash (experimental Kusto) [#63992](https://github.com/ClickHouse/ClickHouse/pull/63992) ([Yong Wang](https://github.com/kashwy)).
+* Deserialize untrusted binary inputs in a safer way [#64024](https://github.com/ClickHouse/ClickHouse/pull/64024) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix query analysis for queries with the setting `final` = 1 for Distributed tables over tables from other than the MergeTree family. [#64037](https://github.com/ClickHouse/ClickHouse/pull/64037) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Add missing settings to recoverLostReplica [#64040](https://github.com/ClickHouse/ClickHouse/pull/64040) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix SQL security access checks with analyzer [#64079](https://github.com/ClickHouse/ClickHouse/pull/64079) ([pufit](https://github.com/pufit)).
+* Fix analyzer: only interpolate expression should be used for DAG [#64096](https://github.com/ClickHouse/ClickHouse/pull/64096) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Fix azure backup writing multipart blocks by 1 MiB (read buffer size) instead of `max_upload_part_size` (in non-native copy case) [#64117](https://github.com/ClickHouse/ClickHouse/pull/64117) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Correctly fallback during backup copy [#64153](https://github.com/ClickHouse/ClickHouse/pull/64153) ([Antonio Andelic](https://github.com/antonio2368)).
+* Prevent LOGICAL_ERROR on CREATE TABLE as Materialized View [#64174](https://github.com/ClickHouse/ClickHouse/pull/64174) ([Raúl Marín](https://github.com/Algunenano)).
+* Query Cache: Consider identical queries against different databases as different [#64199](https://github.com/ClickHouse/ClickHouse/pull/64199) ([Robert Schulze](https://github.com/rschu1ze)).
+* Ignore `text_log` for Keeper [#64218](https://github.com/ClickHouse/ClickHouse/pull/64218) ([Antonio Andelic](https://github.com/antonio2368)).
+* Fix Logical error: Bad cast for Buffer table with prewhere. [#64388](https://github.com/ClickHouse/ClickHouse/pull/64388) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+
+
### ClickHouse release 24.4, 2024-04-30
#### Upgrade Notes
@@ -506,7 +663,7 @@
* Improve the operation of `sumMapFiltered` with NaN values. NaN values are now placed at the end (instead of randomly) and considered different from any values. `-0` is now also treated as equal to `0`; since 0 values are discarded, `-0` values are discarded too. [#58959](https://github.com/ClickHouse/ClickHouse/pull/58959) ([Raúl Marín](https://github.com/Algunenano)).
* The function `visibleWidth` will behave according to the docs. In previous versions, it simply counted code points after string serialization, like the `lengthUTF8` function, but didn't consider zero-width and combining characters, full-width characters, tabs, and deletes. Now the behavior is changed accordingly. If you want to keep the old behavior, set `function_visible_width_behavior` to `0`, or set `compatibility` to `23.12` or lower. [#59022](https://github.com/ClickHouse/ClickHouse/pull/59022) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* `Kusto` dialect is disabled until these two bugs will be fixed: [#59037](https://github.com/ClickHouse/ClickHouse/issues/59037) and [#59036](https://github.com/ClickHouse/ClickHouse/issues/59036). [#59305](https://github.com/ClickHouse/ClickHouse/pull/59305) ([Alexey Milovidov](https://github.com/alexey-milovidov)). Any attempt to use `Kusto` will result in exception.
-* More efficient implementation of the `FINAL` modifier no longer guarantees preserving the order even if `max_threads = 1`. If you counted on the previous behavior, set `enable_vertical_final` to 0 or `compatibility` to `23.12`.
+* More efficient implementation of the `FINAL` modifier no longer guarantees preserving the order even if `max_threads = 1`. If you counted on the previous behavior, set `enable_vertical_final` to 0 or `compatibility` to `23.12`.
#### New Feature
* Implement Variant data type that represents a union of other data types. Type `Variant(T1, T2, ..., TN)` means that each row of this type has a value of either type `T1` or `T2` or ... or `TN` or none of them (`NULL` value). Variant type is available under a setting `allow_experimental_variant_type`. Reference: [#54864](https://github.com/ClickHouse/ClickHouse/issues/54864). [#58047](https://github.com/ClickHouse/ClickHouse/pull/58047) ([Kruglov Pavel](https://github.com/Avogar)).
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 96ba2961d3a..455adc24182 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -122,6 +122,8 @@ add_library(global-libs INTERFACE)
include (cmake/sanitize.cmake)
+include (cmake/xray_instrumentation.cmake)
+
option(ENABLE_COLORED_BUILD "Enable colors in compiler output" ON)
set (CMAKE_COLOR_MAKEFILE ${ENABLE_COLORED_BUILD}) # works only for the makefile generator
@@ -208,8 +210,6 @@ option(OMIT_HEAVY_DEBUG_SYMBOLS
"Do not generate debugger info for heavy modules (ClickHouse functions and dictionaries, some contrib)"
${OMIT_HEAVY_DEBUG_SYMBOLS_DEFAULT})
-option(USE_DEBUG_HELPERS "Enable debug helpers" ${USE_DEBUG_HELPERS})
-
option(BUILD_STANDALONE_KEEPER "Build keeper as small standalone binary" OFF)
if (NOT BUILD_STANDALONE_KEEPER)
option(CREATE_KEEPER_SYMLINK "Create symlink for clickhouse-keeper to main server binary" ON)
diff --git a/SECURITY.md b/SECURITY.md
index 14c39129db9..8635951dc0e 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -2,22 +2,27 @@
the file is autogenerated by utils/security-generator/generate_security.py
-->
-# Security Policy
+# ClickHouse Security Vulnerability Response Policy
-## Security Announcements
-Security fixes will be announced by posting them in the [security changelog](https://clickhouse.com/docs/en/whats-new/security-changelog/).
+## Security Change Log and Support
-## Scope and Supported Versions
+Details regarding security fixes are publicly reported in our [security changelog](https://clickhouse.com/docs/en/whats-new/security-changelog/). A summary of known security vulnerabilities is shown at the bottom of this page.
-The following versions of ClickHouse server are currently being supported with security updates:
+Vulnerability notifications pre-release or during embargo periods are available to open source users and support customers registered for vulnerability alerts. Refer to our [Embargo Policy](#embargo-policy) below.
+
+The following versions of ClickHouse server are currently supported with security updates:
| Version | Supported |
|:-|:-|
+| 24.5 | ✔️ |
| 24.4 | ✔️ |
| 24.3 | ✔️ |
-| 24.2 | ✔️ |
+| 24.2 | ❌ |
| 24.1 | ❌ |
-| 23.* | ❌ |
+| 23.12 | ❌ |
+| 23.11 | ❌ |
+| 23.10 | ❌ |
+| 23.9 | ❌ |
| 23.8 | ✔️ |
| 23.7 | ❌ |
| 23.6 | ❌ |
@@ -37,7 +42,7 @@ The following versions of ClickHouse server are currently being supported with s
We're extremely grateful for security researchers and users that report vulnerabilities to the ClickHouse Open Source Community. All reports are thoroughly investigated by developers.
-To report a potential vulnerability in ClickHouse please send the details about it to [security@clickhouse.com](mailto:security@clickhouse.com). We do not offer any financial rewards for reporting issues to us using this method. Alternatively, you can also submit your findings through our public bug bounty program hosted by [Bugcrowd](https://bugcrowd.com/clickhouse) and be rewarded for it as per the program scope and rules of engagement.
+To report a potential vulnerability in ClickHouse please send the details about it through our public bug bounty program hosted by [Bugcrowd](https://bugcrowd.com/clickhouse) and be rewarded for it as per the program scope and rules of engagement.
### When Should I Report a Vulnerability?
@@ -59,3 +64,21 @@ As the security issue moves from triage, to identified fix, to release planning
A public disclosure date is negotiated by the ClickHouse maintainers and the bug submitter. We prefer to fully disclose the bug as soon as possible once a user mitigation is available. It is reasonable to delay disclosure when the bug or the fix is not yet fully understood, the solution is not well-tested, or for vendor coordination. The timeframe for disclosure is from immediate (especially if it's already publicly known) to 90 days. For a vulnerability with a straightforward mitigation, we expect the report date to disclosure date to be on the order of 7 days.
+## Embargo Policy
+
+Open source users and support customers may subscribe to receive alerts during the embargo period by visiting [https://trust.clickhouse.com/?product=clickhouseoss](https://trust.clickhouse.com/?product=clickhouseoss), requesting access and subscribing for alerts. Subscribers agree not to make these notifications public, issue communications, share this information with others, or issue public patches before the disclosure date. Accidental disclosures must be reported immediately to trust@clickhouse.com. Failure to follow this policy or repeated leaks may result in removal from the subscriber list.
+
+Participation criteria:
+1. Be a current open source user or support customer with a valid corporate email domain (no @gmail.com, @azure.com, etc.).
+1. Sign up to the ClickHouse OSS Trust Center at [https://trust.clickhouse.com](https://trust.clickhouse.com).
+1. Accept the ClickHouse Security Vulnerability Response Policy as outlined above.
+1. Subscribe to ClickHouse OSS Trust Center alerts.
+
+Removal criteria:
+1. Members may be removed for failure to follow this policy or repeated leaks.
+1. Members may be removed for bounced messages (mail delivery failure).
+1. Members may unsubscribe at any time.
+
+Notification process:
+ClickHouse will post notifications within our OSS Trust Center and notify subscribers. Subscribers must log in to the Trust Center to download the notification. The notification will include the timeframe for public disclosure.
+
diff --git a/base/base/BorrowedObjectPool.h b/base/base/BorrowedObjectPool.h
index 05a23d5835e..f5ef28582b2 100644
--- a/base/base/BorrowedObjectPool.h
+++ b/base/base/BorrowedObjectPool.h
@@ -86,7 +86,7 @@ public:
}
/// Return object into pool. Client must return same object that was borrowed.
- inline void returnObject(T && object_to_return)
+ void returnObject(T && object_to_return)
{
{
std::lock_guard lock(objects_mutex);
@@ -99,20 +99,20 @@ public:
}
/// Max pool size
- inline size_t maxSize() const
+ size_t maxSize() const
{
return max_size;
}
/// Allocated objects size by the pool. If allocatedObjectsSize == maxSize then pool is full.
- inline size_t allocatedObjectsSize() const
+ size_t allocatedObjectsSize() const
{
std::lock_guard lock(objects_mutex);
return allocated_objects_size;
}
/// Returns allocatedObjectsSize == maxSize
- inline bool isFull() const
+ bool isFull() const
{
std::lock_guard lock(objects_mutex);
return allocated_objects_size == max_size;
@@ -120,7 +120,7 @@ public:
/// Borrowed objects size. If borrowedObjectsSize == allocatedObjectsSize and pool is full.
/// Then client will wait during borrowObject function call.
- inline size_t borrowedObjectsSize() const
+ size_t borrowedObjectsSize() const
{
std::lock_guard lock(objects_mutex);
return borrowed_objects_size;
@@ -129,7 +129,7 @@ public:
private:
template
- inline T allocateObjectForBorrowing(const std::unique_lock &, FactoryFunc && func)
+ T allocateObjectForBorrowing(const std::unique_lock &, FactoryFunc && func)
{
++allocated_objects_size;
++borrowed_objects_size;
@@ -137,7 +137,7 @@ private:
return std::forward(func)();
}
- inline T borrowFromObjects(const std::unique_lock &)
+ T borrowFromObjects(const std::unique_lock &)
{
T dst;
detail::moveOrCopyIfThrow(std::move(objects.back()), dst);
diff --git a/base/base/CMakeLists.txt b/base/base/CMakeLists.txt
index 27aa0bd6baf..159502c9735 100644
--- a/base/base/CMakeLists.txt
+++ b/base/base/CMakeLists.txt
@@ -34,15 +34,6 @@ set (SRCS
throwError.cpp
)
-if (USE_DEBUG_HELPERS)
- get_target_property(MAGIC_ENUM_INCLUDE_DIR ch_contrib::magic_enum INTERFACE_INCLUDE_DIRECTORIES)
- # CMake generator expression will do insane quoting when it encounters special character like quotes, spaces, etc.
- # Prefixing "SHELL:" will force it to use the original text.
- set (INCLUDE_DEBUG_HELPERS "SHELL:-I\"${MAGIC_ENUM_INCLUDE_DIR}\" -include \"${ClickHouse_SOURCE_DIR}/base/base/iostream_debug_helpers.h\"")
- # Use generator expression as we don't want to pollute CMAKE_CXX_FLAGS, which will interfere with CMake check system.
- add_compile_options($<$:${INCLUDE_DEBUG_HELPERS}>)
-endif ()
-
add_library (common ${SRCS})
if (WITH_COVERAGE)
diff --git a/base/base/Decimal_fwd.h b/base/base/Decimal_fwd.h
index beb228cea3c..a11e13a479b 100644
--- a/base/base/Decimal_fwd.h
+++ b/base/base/Decimal_fwd.h
@@ -44,6 +44,10 @@ concept is_over_big_int =
|| std::is_same_v
|| std::is_same_v
|| std::is_same_v;
+
+template
+concept is_over_big_decimal = is_decimal && is_over_big_int;
+
}
template <> struct is_signed { static constexpr bool value = true; };
diff --git a/base/base/iostream_debug_helpers.h b/base/base/iostream_debug_helpers.h
deleted file mode 100644
index b23d3d9794d..00000000000
--- a/base/base/iostream_debug_helpers.h
+++ /dev/null
@@ -1,187 +0,0 @@
-#pragma once
-
-#include "demangle.h"
-#include "getThreadId.h"
-#include
-#include
-#include
-#include
-#include
-
-/** Usage:
- *
- * DUMP(variable...)
- */
-
-
-template
-Out & dumpValue(Out &, T &&);
-
-
-/// Catch-all case.
-template
-requires(priority == -1)
-Out & dumpImpl(Out & out, T &&) // NOLINT(cppcoreguidelines-missing-std-forward)
-{
- return out << "{...}";
-}
-
-/// An object, that could be output with operator <<.
-template
-requires(priority == 0)
-Out & dumpImpl(Out & out, T && x, std::decay_t() << std::declval())> * = nullptr) // NOLINT(cppcoreguidelines-missing-std-forward)
-{
- return out << x;
-}
-
-/// A pointer-like object.
-template
-requires(priority == 1
- /// Protect from the case when operator * do effectively nothing (function pointer).
- && !std::is_same_v, std::decay_t())>>)
-Out & dumpImpl(Out & out, T && x, std::decay_t())> * = nullptr) // NOLINT(cppcoreguidelines-missing-std-forward)
-{
- if (!x)
- return out << "nullptr";
- return dumpValue(out, *x);
-}
-
-/// Container.
-template
-requires(priority == 2)
-Out & dumpImpl(Out & out, T && x, std::decay_t()))> * = nullptr) // NOLINT(cppcoreguidelines-missing-std-forward)
-{
- bool first = true;
- out << "{";
- for (const auto & elem : x)
- {
- if (first)
- first = false;
- else
- out << ", ";
- dumpValue(out, elem);
- }
- return out << "}";
-}
-
-
-template
-requires(priority == 3 && std::is_enum_v>)
-Out & dumpImpl(Out & out, T && x) // NOLINT(cppcoreguidelines-missing-std-forward)
-{
- return out << magic_enum::enum_name(x);
-}
-
-/// string and const char * - output not as container or pointer.
-
-template
-requires(priority == 3 && (std::is_same_v, std::string> || std::is_same_v, const char *>))
-Out & dumpImpl(Out & out, T && x) // NOLINT(cppcoreguidelines-missing-std-forward)
-{
- return out << std::quoted(x);
-}
-
-/// UInt8 - output as number, not char.
-
-template
-requires(priority == 3 && std::is_same_v, unsigned char>)
-Out & dumpImpl(Out & out, T && x) // NOLINT(cppcoreguidelines-missing-std-forward)
-{
- return out << int(x);
-}
-
-
-/// Tuple, pair
-template
-Out & dumpTupleImpl(Out & out, T && x) // NOLINT(cppcoreguidelines-missing-std-forward)
-{
- if constexpr (N == 0)
- out << "{";
- else
- out << ", ";
-
- dumpValue(out, std::get(x));
-
- if constexpr (N + 1 == std::tuple_size_v>)
- out << "}";
- else
- dumpTupleImpl(out, x);
-
- return out;
-}
-
-template
-requires(priority == 4)
-Out & dumpImpl(Out & out, T && x, std::decay_t(std::declval()))> * = nullptr) // NOLINT(cppcoreguidelines-missing-std-forward)
-{
- return dumpTupleImpl<0>(out, x);
-}
-
-
-template
-Out & dumpDispatchPriorities(Out & out, T && x, std::decay_t(std::declval(), std::declval()))> *) // NOLINT(cppcoreguidelines-missing-std-forward)
-{
- return dumpImpl(out, x);
-}
-
-// NOLINTNEXTLINE(google-explicit-constructor)
-struct LowPriority { LowPriority(void *) {} };
-
-template
-Out & dumpDispatchPriorities(Out & out, T && x, LowPriority) // NOLINT(cppcoreguidelines-missing-std-forward)
-{
- return dumpDispatchPriorities(out, x, nullptr);
-}
-
-
-template
-Out & dumpValue(Out & out, T && x) // NOLINT(cppcoreguidelines-missing-std-forward)
-{
- return dumpDispatchPriorities<5>(out, x, nullptr);
-}
-
-
-template
-Out & dump(Out & out, const char * name, T && x) // NOLINT(cppcoreguidelines-missing-std-forward)
-{
- // Dumping string literal, printing name and demangled type is irrelevant.
- if constexpr (std::is_same_v>>)
- {
- const auto name_len = strlen(name);
- const auto value_len = strlen(x);
- // `name` is the same as quoted `x`
- if (name_len > 2 && value_len > 0 && name[0] == '"' && name[name_len - 1] == '"'
- && strncmp(name + 1, x, std::min(value_len, name_len) - 1) == 0)
- return out << x;
- }
-
- out << demangle(typeid(x).name()) << " " << name << " = ";
- return dumpValue(out, x) << "; ";
-}
-
-#pragma clang diagnostic ignored "-Wgnu-zero-variadic-macro-arguments"
-
-#define DUMPVAR(VAR) ::dump(std::cerr, #VAR, (VAR));
-#define DUMPHEAD std::cerr << __FILE__ << ':' << __LINE__ << " [ " << getThreadId() << " ] ";
-#define DUMPTAIL std::cerr << '\n';
-
-#define DUMP1(V1) do { DUMPHEAD DUMPVAR(V1) DUMPTAIL } while(0)
-#define DUMP2(V1, V2) do { DUMPHEAD DUMPVAR(V1) DUMPVAR(V2) DUMPTAIL } while(0)
-#define DUMP3(V1, V2, V3) do { DUMPHEAD DUMPVAR(V1) DUMPVAR(V2) DUMPVAR(V3) DUMPTAIL } while(0)
-#define DUMP4(V1, V2, V3, V4) do { DUMPHEAD DUMPVAR(V1) DUMPVAR(V2) DUMPVAR(V3) DUMPVAR(V4) DUMPTAIL } while(0)
-#define DUMP5(V1, V2, V3, V4, V5) do { DUMPHEAD DUMPVAR(V1) DUMPVAR(V2) DUMPVAR(V3) DUMPVAR(V4) DUMPVAR(V5) DUMPTAIL } while(0)
-#define DUMP6(V1, V2, V3, V4, V5, V6) do { DUMPHEAD DUMPVAR(V1) DUMPVAR(V2) DUMPVAR(V3) DUMPVAR(V4) DUMPVAR(V5) DUMPVAR(V6) DUMPTAIL } while(0)
-#define DUMP7(V1, V2, V3, V4, V5, V6, V7) do { DUMPHEAD DUMPVAR(V1) DUMPVAR(V2) DUMPVAR(V3) DUMPVAR(V4) DUMPVAR(V5) DUMPVAR(V6) DUMPVAR(V7) DUMPTAIL } while(0)
-#define DUMP8(V1, V2, V3, V4, V5, V6, V7, V8) do { DUMPHEAD DUMPVAR(V1) DUMPVAR(V2) DUMPVAR(V3) DUMPVAR(V4) DUMPVAR(V5) DUMPVAR(V6) DUMPVAR(V7) DUMPVAR(V8) DUMPTAIL } while(0)
-#define DUMP9(V1, V2, V3, V4, V5, V6, V7, V8, V9) do { DUMPHEAD DUMPVAR(V1) DUMPVAR(V2) DUMPVAR(V3) DUMPVAR(V4) DUMPVAR(V5) DUMPVAR(V6) DUMPVAR(V7) DUMPVAR(V8) DUMPVAR(V9) DUMPTAIL } while(0)
-
-/// https://groups.google.com/forum/#!searchin/kona-dev/variadic$20macro%7Csort:date/kona-dev/XMA-lDOqtlI/GCzdfZsD41sJ
-
-#define VA_NUM_ARGS_IMPL(x1, x2, x3, x4, x5, x6, x7, x8, x9, N, ...) N
-#define VA_NUM_ARGS(...) VA_NUM_ARGS_IMPL(__VA_ARGS__, 9, 8, 7, 6, 5, 4, 3, 2, 1)
-
-#define MAKE_VAR_MACRO_IMPL_CONCAT(PREFIX, NUM_ARGS) PREFIX ## NUM_ARGS
-#define MAKE_VAR_MACRO_IMPL(PREFIX, NUM_ARGS) MAKE_VAR_MACRO_IMPL_CONCAT(PREFIX, NUM_ARGS)
-#define MAKE_VAR_MACRO(PREFIX, ...) MAKE_VAR_MACRO_IMPL(PREFIX, VA_NUM_ARGS(__VA_ARGS__))
-
-#define DUMP(...) MAKE_VAR_MACRO(DUMP, __VA_ARGS__)(__VA_ARGS__)
diff --git a/base/base/tests/CMakeLists.txt b/base/base/tests/CMakeLists.txt
index 81db4f3622f..e69de29bb2d 100644
--- a/base/base/tests/CMakeLists.txt
+++ b/base/base/tests/CMakeLists.txt
@@ -1,2 +0,0 @@
-clickhouse_add_executable (dump_variable dump_variable.cpp)
-target_link_libraries (dump_variable PRIVATE clickhouse_common_io)
diff --git a/base/base/tests/dump_variable.cpp b/base/base/tests/dump_variable.cpp
deleted file mode 100644
index 9addc298ecb..00000000000
--- a/base/base/tests/dump_variable.cpp
+++ /dev/null
@@ -1,70 +0,0 @@
-#include
-
-#include
-#include
-#include
-#include