Merge branch 'master' into dev_materialized_view

.arcignore

@@ -1,12 +0,0 @@
-# .arcignore is the same as .gitignore but for Arc VCS.
-# Arc VCS is a proprietary VCS in Yandex that is very similar to Git
-# from the user perspective but with the following differences:
-# 1. Data is stored in distributed object storage.
-# 2. Local copy works via FUSE without downloading all the objects.
-# For this reason, it is better suited for huge monorepositories that can be found in large companies (e.g. Yandex, Google).
-# As ClickHouse developers, we don't use Arc as a VCS (we use Git).
-# But the ClickHouse source code is also mirrored into internal monorepository and our collegues are using Arc.
-# You can read more about Arc here: https://habr.com/en/company/yandex/blog/482926/
-
-# Repository is synchronized without 3rd-party submodules.
-contrib

.clang-format

@@ -12,6 +12,7 @@ BraceWrapping:
     AfterUnion: true
     BeforeCatch: true
     BeforeElse: true
+    BeforeLambdaBody: true
     IndentBraces: false
 BreakConstructorInitializersBeforeComma: false
 Cpp11BracedListStyle: true

.clang-tidy

@@ -16,7 +16,6 @@ Checks: '-*,
     modernize-make-unique,
     modernize-raw-string-literal,
     modernize-redundant-void-arg,
-    modernize-replace-auto-ptr,
     modernize-replace-random-shuffle,
     modernize-use-bool-literals,
     modernize-use-nullptr,
@@ -33,6 +32,8 @@ Checks: '-*,
     performance-no-automatic-move,
     performance-trivially-destructible,
     performance-unnecessary-copy-initialization,
+    performance-noexcept-move-constructor,
+    performance-move-const-arg,
 
     readability-avoid-const-params-in-decls,
     readability-const-return-type,
@@ -142,6 +143,8 @@ Checks: '-*,
     clang-analyzer-cplusplus.PlacementNewChecker,
     clang-analyzer-cplusplus.SelfAssignment,
     clang-analyzer-deadcode.DeadStores,
+    clang-analyzer-cplusplus.Move,
+    clang-analyzer-optin.cplusplus.UninitializedObject,
     clang-analyzer-optin.cplusplus.VirtualCall,
     clang-analyzer-security.insecureAPI.UncheckedReturn,
     clang-analyzer-security.insecureAPI.bcmp,
@@ -161,6 +164,8 @@ Checks: '-*,
     clang-analyzer-unix.cstring.NullArg,
 
     boost-use-to-string,
+
+    alpha.security.cert.env.InvalidPtr,
 '
 WarningsAsErrors: '*'
 
@@ -203,3 +208,10 @@ CheckOptions:
     value: CamelCase
   - key: readability-identifier-naming.UsingCase
     value: CamelCase
+  - key: modernize-loop-convert.UseCxx20ReverseRanges
+    value: false
+  - key: performance-move-const-arg.CheckTriviallyCopyableMove
+    value: false
+    # Workaround clang-tidy bug: https://github.com/llvm/llvm-project/issues/46097
+  - key: readability-identifier-naming.TypeTemplateParameterIgnoredRegexp
+    value: expr-type

.gitattributes

@@ -1,4 +1,3 @@
 contrib/* linguist-vendored
 *.h linguist-language=C++
-# to avoid frequent conflicts
+tests/queries/0_stateless/data_json/* binary
-tests/queries/0_stateless/arcadia_skip_list.txt text merge=union

.github/CODEOWNERS

@@ -1,3 +1 @@
-docs/* @ClickHouse/docs
+
-docs/zh/* @ClickHouse/docs-zh
-website/* @ClickHouse/docs

.github/ISSUE_TEMPLATE/10_question.md

@@ -7,6 +7,6 @@ assignees: ''
 
 ---
 
-Make sure to check documentation https://clickhouse.yandex/docs/en/ first. If the question is concise and probably has a short answer, asking it in Telegram chat https://telegram.me/clickhouse_en is probably the fastest way to find the answer. For more complicated questions, consider asking them on StackOverflow with "clickhouse" tag https://stackoverflow.com/questions/tagged/clickhouse 
+> Make sure to check documentation https://clickhouse.com/docs/en/ first. If the question is concise and probably has a short answer, asking it in Telegram chat https://telegram.me/clickhouse_en is probably the fastest way to find the answer. For more complicated questions, consider asking them on StackOverflow with "clickhouse" tag https://stackoverflow.com/questions/tagged/clickhouse 
 
-If you still prefer GitHub issues, remove all this text and ask your question here.
+> If you still prefer GitHub issues, remove all this text and ask your question here.

.github/ISSUE_TEMPLATE/20_feature-request.md

@@ -7,16 +7,20 @@ assignees: ''
 
 ---
 
-(you don't have to strictly follow this form)
+> (you don't have to strictly follow this form)
 
 **Use case**
-A clear and concise description of what is the intended usage scenario is.
+
+> A clear and concise description of what is the intended usage scenario is.
 
 **Describe the solution you'd like**
-A clear and concise description of what you want to happen.
+
+> A clear and concise description of what you want to happen.
 
 **Describe alternatives you've considered**
-A clear and concise description of any alternative solutions or features you've considered.
+
+> A clear and concise description of any alternative solutions or features you've considered.
 
 **Additional context**
-Add any other context or screenshots about the feature request here.
+
+> Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/40_bug-report.md

@@ -1,33 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve ClickHouse
-title: ''
-labels: bug
-assignees: ''
-
----
-
-You have to provide the following information whenever possible.
-
-**Describe the bug**
-A clear and concise description of what works not as it is supposed to.
-
-**Does it reproduce on recent release?**
-[The list of releases](https://github.com/ClickHouse/ClickHouse/blob/master/utils/list-versions/version_date.tsv)
-
-**How to reproduce**
-* Which ClickHouse server version to use
-* Which interface to use, if matters
-* Non-default settings, if any
-* `CREATE TABLE` statements for all tables involved
-* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
-* Queries to run that lead to unexpected result
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Error message and/or stacktrace**
-If applicable, add screenshots to help explain your problem.
-
-**Additional context**
-Add any other context about the problem here.

.github/ISSUE_TEMPLATE/50_build-issue.md

@@ -7,10 +7,11 @@ assignees: ''
 
 ---
 
-Make sure that `git diff` result is empty and you've just pulled fresh master. Try cleaning up cmake cache. Just in case, official build instructions are published here: https://clickhouse.yandex/docs/en/development/build/
+> Make sure that `git diff` result is empty and you've just pulled fresh master. Try cleaning up cmake cache. Just in case, official build instructions are published here: https://clickhouse.com/docs/en/development/build/
 
 **Operating system**
-OS kind or distribution, specific version/release, non-standard kernel if any. If you are trying to build inside virtual machine, please mention it too.
+
+> OS kind or distribution, specific version/release, non-standard kernel if any. If you are trying to build inside virtual machine, please mention it too.
 
 **Cmake version**
 

.github/ISSUE_TEMPLATE/85_bug-report.md

@@ -0,0 +1,50 @@
+---
+name: Bug report
+about: Wrong behaviour (visible to users) in official ClickHouse release.
+title: ''
+labels: 'potential bug'
+assignees: ''
+
+---
+
+> You have to provide the following information whenever possible.
+
+**Describe what's wrong**
+
+> A clear and concise description of what works not as it is supposed to.
+
+**Does it reproduce on recent release?**
+
+[The list of releases](https://github.com/ClickHouse/ClickHouse/blob/master/utils/list-versions/version_date.tsv)
+
+**Enable crash reporting**
+
+> If possible, change "enabled" to true in "send_crash_reports" section in `config.xml`:
+
+```
+<send_crash_reports>
+        <!-- Changing <enabled> to true allows sending crash reports to -->
+        <!-- the ClickHouse core developers team via Sentry https://sentry.io -->
+        <enabled>false</enabled>
+```
+
+**How to reproduce**
+
+* Which ClickHouse server version to use
+* Which interface to use, if matters
+* Non-default settings, if any
+* `CREATE TABLE` statements for all tables involved
+* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
+* Queries to run that lead to unexpected result
+
+**Expected behavior**
+
+> A clear and concise description of what you expected to happen.
+
+**Error message and/or stacktrace**
+
+> If applicable, add screenshots to help explain your problem.
+
+**Additional context**
+
+> Add any other context about the problem here.

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,29 +1,16 @@
-I hereby agree to the terms of the CLA available at: https://yandex.ru/legal/cla/?lang=en
+### Changelog category (leave one):
-
-Changelog category (leave one):
 - New Feature
-- Bug Fix
 - Improvement
+- Bug Fix (user-visible misbehaviour in official stable or prestable release)
 - Performance Improvement
 - Backward Incompatible Change
 - Build/Testing/Packaging Improvement
 - Documentation (changelog entry is not required)
-- Other
 - Not for changelog (changelog entry is not required)
 
 
-Changelog entry (a user-readable short description of the changes that goes to CHANGELOG.md):
+### Changelog entry (a user-readable short description of the changes that goes to CHANGELOG.md):
-
 ...
 
 
-Detailed description / Documentation draft:
+> Information about CI checks: https://clickhouse.tech/docs/en/development/continuous-integration/
-
-...
-
-By adding documentation, you'll allow users to try your new feature immediately, not when someone else will have time to document it later. Documentation is necessary for all features that affect user experience in any way. You can add brief documentation draft above, or add documentation right into your patch as Markdown files in [docs](https://github.com/ClickHouse/ClickHouse/tree/master/docs) folder.
-
-If you are doing this for the first time, it's recommended to read the lightweight [Contributing to ClickHouse Documentation](https://github.com/ClickHouse/ClickHouse/tree/master/docs/README.md) guide first.
-
-
-Information about CI checks: https://clickhouse.tech/docs/en/development/continuous-integration/

.github/actionlint.yml

@@ -0,0 +1,9 @@
+self-hosted-runner:
+  labels:
+    - builder
+    - func-tester
+    - func-tester-aarch64
+    - fuzzer-unit-tester
+    - stress-tester
+    - style-checker
+    - style-checker-aarch64

.github/codecov.yml

@@ -1,17 +0,0 @@
-codecov:
-  max_report_age: "off"
-  strict_yaml_branch: "master"
-
-ignore:
-  - "contrib"
-  - "docs"
-  - "benchmark"
-  - "tests"
-  - "docker"
-  - "debian"
-  - "cmake"
-
-comment: false
-
-github_checks:
-  annotations: false

.github/workflows/anchore-analysis.yml

@@ -1,39 +0,0 @@
-# This workflow checks out code, performs an Anchore container image
-# vulnerability and compliance scan, and integrates the results with
-# GitHub Advanced Security code scanning feature. For more information on
-# the Anchore scan action usage and parameters, see
-# https://github.com/anchore/scan-action. For more information on
-# Anchore container image scanning in general, see
-# https://docs.anchore.com.
-
-name: Docker Container Scan (clickhouse-server)
-
-"on":
-  pull_request:
-    paths:
-      - docker/server/Dockerfile
-      - .github/workflows/anchore-analysis.yml
-  schedule:
-    - cron: '0 21 * * *'
-
-jobs:
-  Anchore-Build-Scan:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout the code
-        uses: actions/checkout@v2
-      - name: Build the Docker image
-        run: |
-          cd docker/server
-          perl -pi -e 's|=\$version||g' Dockerfile
-          docker build . --file Dockerfile --tag localbuild/testimage:latest
-      - name: Run the local Anchore scan action itself with GitHub Advanced Security code scanning integration enabled
-        uses: anchore/scan-action@v2
-        id: scan
-        with:
-          image: "localbuild/testimage:latest"
-          acs-report-enable: true
-      - name: Upload Anchore Scan Report
-        uses: github/codeql-action/upload-sarif@v1
-        with:
-          sarif_file: ${{ steps.scan.outputs.sarif }}

.github/workflows/backport.yml

@@ -0,0 +1,43 @@
+name: CherryPick
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+
+concurrency:
+  group: cherry-pick
+on: # yamllint disable-line rule:truthy
+  schedule:
+    - cron: '0 */3 * * *'
+jobs:
+  CherryPick:
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Set envs
+        # https://docs.github.com/en/actions/learn-github-actions/workflow-commands-for-github-actions#multiline-strings
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/cherry_pick
+          ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
+          ${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
+          RCSK
+          REPO_OWNER=ClickHouse
+          REPO_NAME=ClickHouse
+          REPO_TEAM=core
+          EOF
+      - name: Check out repository code
+        uses: actions/checkout@v2
+        with:
+          token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
+          fetch-depth: 0
+      - name: Cherry pick
+        run: |
+          sudo pip install GitPython
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 cherry_pick.py
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill "$(docker ps -q)" ||:
+          docker rm -f "$(docker ps -a -q)" ||:
+          sudo rm -fr "$TEMP_PATH"

.github/workflows/backport_branches.yml

@@ -0,0 +1,560 @@
+name: BackportPR
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+
+on: # yamllint disable-line rule:truthy
+  push:
+    branches:
+      - 'backport/**'
+jobs:
+  PythonUnitTests:
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Python unit tests
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 -m unittest discover -s . -p '*_test.py'
+  DockerHubPushAarch64:
+    runs-on: [self-hosted, style-checker-aarch64]
+    steps:
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Images check
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 docker_images_check.py --suffix aarch64
+      - name: Upload images files to artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: changed_images_aarch64
+          path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json
+  DockerHubPushAmd64:
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Images check
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 docker_images_check.py --suffix amd64
+      - name: Upload images files to artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: changed_images_amd64
+          path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json
+  DockerHubPush:
+    needs: [DockerHubPushAmd64, DockerHubPushAarch64]
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Download changed aarch64 images
+        uses: actions/download-artifact@v2
+        with:
+          name: changed_images_aarch64
+          path: ${{ runner.temp }}
+      - name: Download changed amd64 images
+        uses: actions/download-artifact@v2
+        with:
+          name: changed_images_amd64
+          path: ${{ runner.temp }}
+      - name: Images check
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64
+      - name: Upload images files to artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: changed_images
+          path: ${{ runner.temp }}/changed_images.json
+  CompatibilityCheck:
+    needs: [BuilderDebRelease]
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/compatibility_check
+          REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          EOF
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Download json reports
+        uses: actions/download-artifact@v2
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: CompatibilityCheck
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill "$(docker ps -q)" ||:
+          docker rm -f "$(docker ps -a -q)" ||:
+          sudo rm -fr "$TEMP_PATH"
+#########################################################################################
+#################################### ORDINARY BUILDS ####################################
+#########################################################################################
+  BuilderDebRelease:
+    needs: [DockerHubPush]
+    runs-on: [self-hosted, builder]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/build_check
+          IMAGES_PATH=${{runner.temp}}/images_path
+          REPO_COPY=${{runner.temp}}/build_check/ClickHouse
+          CACHES_PATH=${{runner.temp}}/../ccaches
+          CHECK_NAME=ClickHouse build check (actions)
+          BUILD_NAME=package_release
+          EOF
+      - name: Download changed images
+        uses: actions/download-artifact@v2
+        with:
+          name: changed_images
+          path: ${{ env.IMAGES_PATH }}
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+        with:
+          submodules: 'true'
+          fetch-depth: 0 # otherwise we will have no info about contributors
+      - name: Build
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci" && python3 build_check.py "$CHECK_NAME" "$BUILD_NAME"
+      - name: Upload build URLs to artifacts
+        if: ${{ success() || failure() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill "$(docker ps -q)" ||:
+          docker rm -f "$(docker ps -a -q)" ||:
+          sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
+  BuilderDebAarch64:
+    needs: [DockerHubPush]
+    runs-on: [self-hosted, builder]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/build_check
+          IMAGES_PATH=${{runner.temp}}/images_path
+          REPO_COPY=${{runner.temp}}/build_check/ClickHouse
+          CACHES_PATH=${{runner.temp}}/../ccaches
+          CHECK_NAME=ClickHouse build check (actions)
+          BUILD_NAME=package_aarch64
+          EOF
+      - name: Download changed images
+        uses: actions/download-artifact@v2
+        with:
+          name: changed_images
+          path: ${{ runner.temp }}/images_path
+      - name: Check out repository code
+        uses: actions/checkout@v2
+        with:
+          submodules: 'true'
+          fetch-depth: 0 # otherwise we will have no info about contributors
+      - name: Build
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci" && python3 build_check.py "$CHECK_NAME" "$BUILD_NAME"
+      - name: Upload build URLs to artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ runner.temp }}/build_check/${{ env.BUILD_URLS }}.json
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill "$(docker ps -q)" ||:
+          docker rm -f "$(docker ps -a -q)" ||:
+          sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
+  BuilderDebAsan:
+    needs: [DockerHubPush]
+    runs-on: [self-hosted, builder]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/build_check
+          IMAGES_PATH=${{runner.temp}}/images_path
+          REPO_COPY=${{runner.temp}}/build_check/ClickHouse
+          CACHES_PATH=${{runner.temp}}/../ccaches
+          CHECK_NAME=ClickHouse build check (actions)
+          BUILD_NAME=package_asan
+          EOF
+      - name: Download changed images
+        uses: actions/download-artifact@v2
+        with:
+          name: changed_images
+          path: ${{ env.IMAGES_PATH }}
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+        with:
+          submodules: 'true'
+          fetch-depth: 0 # otherwise we will have no info about contributors
+      - name: Build
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci" && python3 build_check.py "$CHECK_NAME" "$BUILD_NAME"
+      - name: Upload build URLs to artifacts
+        if: ${{ success() || failure() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill "$(docker ps -q)" ||:
+          docker rm -f "$(docker ps -a -q)" ||:
+          sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
+  BuilderDebTsan:
+    needs: [DockerHubPush]
+    runs-on: [self-hosted, builder]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/build_check
+          IMAGES_PATH=${{runner.temp}}/images_path
+          REPO_COPY=${{runner.temp}}/build_check/ClickHouse
+          CACHES_PATH=${{runner.temp}}/../ccaches
+          CHECK_NAME=ClickHouse build check (actions)
+          BUILD_NAME=package_tsan
+          EOF
+      - name: Download changed images
+        uses: actions/download-artifact@v2
+        with:
+          name: changed_images
+          path: ${{ env.IMAGES_PATH }}
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+        with:
+          submodules: 'true'
+          fetch-depth: 0 # otherwise we will have no info about contributors
+      - name: Build
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci" && python3 build_check.py "$CHECK_NAME" "$BUILD_NAME"
+      - name: Upload build URLs to artifacts
+        if: ${{ success() || failure() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill "$(docker ps -q)" ||:
+          docker rm -f "$(docker ps -a -q)" ||:
+          sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
+  BuilderDebDebug:
+    needs: [DockerHubPush]
+    runs-on: [self-hosted, builder]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/build_check
+          IMAGES_PATH=${{runner.temp}}/images_path
+          REPO_COPY=${{runner.temp}}/build_check/ClickHouse
+          CACHES_PATH=${{runner.temp}}/../ccaches
+          CHECK_NAME=ClickHouse build check (actions)
+          BUILD_NAME=package_debug
+          EOF
+      - name: Download changed images
+        uses: actions/download-artifact@v2
+        with:
+          name: changed_images
+          path: ${{ env.IMAGES_PATH }}
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+        with:
+          submodules: 'true'
+          fetch-depth: 0 # otherwise we will have no info about contributors
+      - name: Build
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci" && python3 build_check.py "$CHECK_NAME" "$BUILD_NAME"
+      - name: Upload build URLs to artifacts
+        if: ${{ success() || failure() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill "$(docker ps -q)" ||:
+          docker rm -f "$(docker ps -a -q)" ||:
+          sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
+############################################################################################
+##################################### BUILD REPORTER #######################################
+############################################################################################
+  BuilderReport:
+    needs:
+      - BuilderDebRelease
+      - BuilderDebAarch64
+      - BuilderDebAsan
+      - BuilderDebTsan
+      - BuilderDebDebug
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Set envs
+        run: |
+          DEPENDENCIES=$(cat << 'EOF' | jq '. | length'
+          ${{ toJSON(needs) }}
+          EOF
+          )
+          echo "DEPENDENCIES=$DEPENDENCIES" >> "$GITHUB_ENV"
+          cat >> "$GITHUB_ENV" << 'EOF'
+          CHECK_NAME=ClickHouse build check (actions)
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          TEMP_PATH=${{runner.temp}}/report_check
+          EOF
+      - name: Download json reports
+        uses: actions/download-artifact@v2
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Report Builder
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 build_report_check.py "$CHECK_NAME" "$DEPENDENCIES"
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill "$(docker ps -q)" ||:
+          docker rm -f "$(docker ps -a -q)" ||:
+          sudo rm -fr "$TEMP_PATH"
+##############################################################################################
+########################### FUNCTIONAl STATELESS TESTS #######################################
+##############################################################################################
+  FunctionalStatelessTestAsan:
+    needs: [BuilderDebAsan]
+    runs-on: [self-hosted, func-tester]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/stateless_debug
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          CHECK_NAME=Stateless tests (address, actions)
+          REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
+          KILL_TIMEOUT=10800
+          EOF
+      - name: Download json reports
+        uses: actions/download-artifact@v2
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Functional test
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci"
+          python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill "$(docker ps -q)" ||:
+          docker rm -f "$(docker ps -a -q)" ||:
+          sudo rm -fr "$TEMP_PATH"
+##############################################################################################
+############################ FUNCTIONAl STATEFUL TESTS #######################################
+##############################################################################################
+  FunctionalStatefulTestDebug:
+    needs: [BuilderDebDebug]
+    runs-on: [self-hosted, func-tester]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/stateful_debug
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          CHECK_NAME=Stateful tests (debug, actions)
+          REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse
+          KILL_TIMEOUT=3600
+          EOF
+      - name: Download json reports
+        uses: actions/download-artifact@v2
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Functional test
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci"
+          python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill "$(docker ps -q)" ||:
+          docker rm -f "$(docker ps -a -q)" ||:
+          sudo rm -fr "$TEMP_PATH"
+##############################################################################################
+######################################### STRESS TESTS #######################################
+##############################################################################################
+  StressTestTsan:
+    needs: [BuilderDebTsan]
+    # func testers have 16 cores + 128 GB memory
+    # while stress testers have 36 cores + 72 memory
+    # It would be better to have something like 32 + 128,
+    # but such servers almost unavailable as spot instances.
+    runs-on: [self-hosted, func-tester]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/stress_thread
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          CHECK_NAME=Stress test (thread, actions)
+          REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse
+          EOF
+      - name: Download json reports
+        uses: actions/download-artifact@v2
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Stress test
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci"
+          python3 stress_check.py "$CHECK_NAME"
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill "$(docker ps -q)" ||:
+          docker rm -f "$(docker ps -a -q)" ||:
+          sudo rm -fr "$TEMP_PATH"
+#############################################################################################
+############################# INTEGRATION TESTS #############################################
+#############################################################################################
+  IntegrationTestsRelease:
+    needs: [BuilderDebRelease]
+    runs-on: [self-hosted, stress-tester]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/integration_tests_release
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          CHECK_NAME=Integration tests (release, actions)
+          REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse
+          EOF
+      - name: Download json reports
+        uses: actions/download-artifact@v2
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Integration test
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci"
+          python3 integration_test_check.py "$CHECK_NAME"
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill "$(docker ps -q)" ||:
+          docker rm -f "$(docker ps -a -q)" ||:
+          sudo rm -fr "$TEMP_PATH"
+  FinishCheck:
+    needs:
+      - DockerHubPush
+      - BuilderReport
+      - FunctionalStatelessTestAsan
+      - FunctionalStatefulTestDebug
+      - StressTestTsan
+      - IntegrationTestsRelease
+      - CompatibilityCheck
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Finish label
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 finish_check.py

.github/workflows/cancel.yml

@@ -0,0 +1,19 @@
+name: Cancel
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+
+on: # yamllint disable-line rule:truthy
+  workflow_run:
+    workflows: ["PullRequestCI", "ReleaseCI", "DocsCheck", "BackportPR"]
+    types:
+      - requested
+jobs:
+  cancel:
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - uses: styfle/cancel-workflow-action@0.9.1
+        with:
+          all_but_latest: true
+          workflow_id: ${{ github.event.workflow.id }}

.github/workflows/debug.yml

@@ -0,0 +1,11 @@
+# The CI for each commit, prints envs and content of GITHUB_EVENT_PATH
+name: Debug
+
+'on':
+  [push, pull_request, release]
+
+jobs:
+  DebugInfo:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: hmarr/debug-action@1201a20fc9d278ddddd5f0f46922d06513892491

.github/workflows/docs_check.yml

@@ -0,0 +1,125 @@
+name: DocsCheck
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+
+on: # yamllint disable-line rule:truthy
+  pull_request:
+    types:
+      - synchronize
+      - reopened
+      - opened
+    branches:
+      - master
+    paths:
+      - 'docs/**'
+      - 'website/**'
+jobs:
+  CheckLabels:
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Clear repository
+        run: |
+          sudo rm -rf "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Labels check
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 run_check.py
+  DockerHubPushAarch64:
+    needs: CheckLabels
+    runs-on: [self-hosted, style-checker-aarch64]
+    steps:
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Images check
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 docker_images_check.py --suffix aarch64
+      - name: Upload images files to artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: changed_images_aarch64
+          path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json
+  DockerHubPushAmd64:
+    needs: CheckLabels
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Images check
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 docker_images_check.py --suffix amd64
+      - name: Upload images files to artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: changed_images_amd64
+          path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json
+  DockerHubPush:
+    needs: [DockerHubPushAmd64, DockerHubPushAarch64]
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Download changed aarch64 images
+        uses: actions/download-artifact@v2
+        with:
+          name: changed_images_aarch64
+          path: ${{ runner.temp }}
+      - name: Download changed amd64 images
+        uses: actions/download-artifact@v2
+        with:
+          name: changed_images_amd64
+          path: ${{ runner.temp }}
+      - name: Images check
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64
+      - name: Upload images files to artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: changed_images
+          path: ${{ runner.temp }}/changed_images.json
+  DocsCheck:
+    needs: DockerHubPush
+    runs-on: [self-hosted, func-tester]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/docs_check
+          REPO_COPY=${{runner.temp}}/docs_check/ClickHouse
+          EOF
+      - name: Download changed images
+        uses: actions/download-artifact@v2
+        with:
+          name: changed_images
+          path: ${{ env.TEMP_PATH }}
+      - name: Clear repository
+        run: |
+          sudo rm -rf "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Docs Check
+        run: |
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci"
+          python3 docs_check.py
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill "$(docker ps -q)" ||:
+          docker rm -f "$(docker ps -a -q)" ||:
+          sudo rm -fr "$TEMP_PATH"

.github/workflows/docs_release.yml

@@ -0,0 +1,121 @@
+name: DocsReleaseChecks
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+
+concurrency:
+  group: master-release
+  cancel-in-progress: true
+on: # yamllint disable-line rule:truthy
+  push:
+    branches:
+      - master
+    paths:
+      - 'docs/**'
+      - 'website/**'
+      - 'benchmark/**'
+      - 'docker/**'
+      - '.github/**'
+  workflow_dispatch:
+jobs:
+  DockerHubPushAarch64:
+    runs-on: [self-hosted, style-checker-aarch64]
+    steps:
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Images check
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 docker_images_check.py --suffix aarch64
+      - name: Upload images files to artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: changed_images_aarch64
+          path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json
+  DockerHubPushAmd64:
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Images check
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 docker_images_check.py --suffix amd64
+      - name: Upload images files to artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: changed_images_amd64
+          path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json
+  DockerHubPush:
+    needs: [DockerHubPushAmd64, DockerHubPushAarch64]
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Download changed aarch64 images
+        uses: actions/download-artifact@v2
+        with:
+          name: changed_images_aarch64
+          path: ${{ runner.temp }}
+      - name: Download changed amd64 images
+        uses: actions/download-artifact@v2
+        with:
+          name: changed_images_amd64
+          path: ${{ runner.temp }}
+      - name: Images check
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64
+      - name: Upload images files to artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: changed_images
+          path: ${{ runner.temp }}/changed_images.json
+  DocsRelease:
+    needs: DockerHubPush
+    runs-on: [self-hosted, func-tester]
+    steps:
+      - name: Set envs
+        # https://docs.github.com/en/actions/learn-github-actions/workflow-commands-for-github-actions#multiline-strings
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/docs_release
+          REPO_COPY=${{runner.temp}}/docs_release/ClickHouse
+          CLOUDFLARE_TOKEN=${{secrets.CLOUDFLARE}}
+          ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
+          ${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
+          RCSK
+          EOF
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Download changed images
+        uses: actions/download-artifact@v2
+        with:
+          name: changed_images
+          path: ${{ env.TEMP_PATH }}
+      - name: Docs Release
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci"
+          python3 docs_release.py
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill "$(docker ps -q)" ||:
+          docker rm -f "$(docker ps -a -q)" ||:
+          sudo rm -fr "$TEMP_PATH"

.github/workflows/jepsen.yml

@@ -0,0 +1,44 @@
+name: JepsenWorkflow
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+concurrency:
+  group: jepsen
+on: # yamllint disable-line rule:truthy
+  schedule:
+    - cron: '0 */6 * * *'
+  workflow_run:
+    workflows: ["PullRequestCI"]
+    types:
+      - completed
+  workflow_dispatch:
+jobs:
+  KeeperJepsenRelease:
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/keeper_jepsen
+          REPO_COPY=${{runner.temp}}/keeper_jepsen/ClickHouse
+          EOF
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Jepsen Test
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci"
+          python3 keeper_jepsen_check.py
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill "$(docker ps -q)" ||:
+          docker rm -f "$(docker ps -a -q)" ||:
+          sudo rm -fr "$TEMP_PATH"

.github/workflows/nightly.yml

@@ -0,0 +1,124 @@
+name: NightlyBuilds
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+
+"on":
+  schedule:
+    - cron: '13 3 * * *'
+  workflow_dispatch:
+
+jobs:
+  DockerHubPushAarch64:
+    runs-on: [self-hosted, style-checker-aarch64]
+    steps:
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Images check
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 docker_images_check.py --suffix aarch64 --all
+      - name: Upload images files to artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: changed_images_aarch64
+          path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json
+  DockerHubPushAmd64:
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Images check
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 docker_images_check.py --suffix amd64 --all
+      - name: Upload images files to artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: changed_images_amd64
+          path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json
+  DockerHubPush:
+    needs: [DockerHubPushAmd64, DockerHubPushAarch64]
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Download changed aarch64 images
+        uses: actions/download-artifact@v2
+        with:
+          name: changed_images_aarch64
+          path: ${{ runner.temp }}
+      - name: Download changed amd64 images
+        uses: actions/download-artifact@v2
+        with:
+          name: changed_images_amd64
+          path: ${{ runner.temp }}
+      - name: Images check
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64
+      - name: Upload images files to artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: changed_images
+          path: ${{ runner.temp }}/changed_images.json
+  BuilderCoverity:
+    needs: DockerHubPush
+    runs-on: [self-hosted, builder]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          BUILD_NAME=coverity
+          CACHES_PATH=${{runner.temp}}/../ccaches
+          CHECK_NAME=ClickHouse build check (actions)
+          IMAGES_PATH=${{runner.temp}}/images_path
+          REPO_COPY=${{runner.temp}}/build_check/ClickHouse
+          TEMP_PATH=${{runner.temp}}/build_check
+          EOF
+          echo "COVERITY_TOKEN=${{ secrets.COVERITY_TOKEN }}" >> "$GITHUB_ENV"
+      - name: Download changed images
+        uses: actions/download-artifact@v2
+        with:
+          name: changed_images
+          path: ${{ env.IMAGES_PATH }}
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        id: coverity-checkout
+        uses: actions/checkout@v2
+        with:
+          submodules: 'true'
+          fetch-depth: 0 # otherwise we will have no info about contributors
+      - name: Build
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci" && python3 build_check.py "$CHECK_NAME" "$BUILD_NAME"
+      - name: Upload Coverity Analysis
+        if: ${{ success() || failure() }}
+        run: |
+          curl --form token="${COVERITY_TOKEN}" \
+            --form email='security+coverity@clickhouse.com' \
+            --form file="@$TEMP_PATH/$BUILD_NAME/coverity-scan.tgz" \
+            --form version="${GITHUB_REF#refs/heads/}-${GITHUB_SHA::6}" \
+            --form description="Nighly Scan: $(date +'%Y-%m-%dT%H:%M:%S')" \
+            https://scan.coverity.com/builds?project=ClickHouse%2FClickHouse
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill "$(docker ps -q)" ||:
+          docker rm -f "$(docker ps -a -q)" ||:
+          sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"

.github/workflows/release.yml

@@ -0,0 +1,63 @@
+name: ReleaseWorkflow
+# - Gets artifacts from S3
+# - Sends it to JFROG Artifactory
+# - Adds them to the release assets
+
+on: # yamllint disable-line rule:truthy
+  release:
+    types:
+    - published
+
+jobs:
+  ReleasePublish:
+    runs-on: [self-hosted, style-checker]
+    steps:
+    - name: Set envs
+      run: |
+        cat >> "$GITHUB_ENV" << 'EOF'
+        JFROG_API_KEY=${{ secrets.JFROG_KEY_API_PACKAGES }}
+        TEMP_PATH=${{runner.temp}}/release_packages
+        REPO_COPY=${{runner.temp}}/release_packages/ClickHouse
+        EOF
+    - name: Check out repository code
+      uses: actions/checkout@v2
+    - name: Download packages and push to Artifactory
+      run: |
+        rm -rf "$TEMP_PATH" && mkdir -p "$TEMP_PATH"
+        cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+        cd "$REPO_COPY"
+        python3 ./tests/ci/push_to_artifactory.py --release "${{ github.ref }}" \
+          --commit '${{ github.sha }}' --all
+    - name: Upload packages to release assets
+      uses: svenstaro/upload-release-action@v2
+      with:
+        repo_token: ${{ secrets.GITHUB_TOKEN }}
+        file: ${{runner.temp}}/push_to_artifactory/*
+        overwrite: true
+        tag: ${{ github.ref }}
+        file_glob: true
+  ############################################################################################
+  ##################################### Docker images  #######################################
+  ############################################################################################
+  DockerServerImages:
+    runs-on: [self-hosted, style-checker]
+    steps:
+    - name: Clear repository
+      run: |
+        sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+    - name: Check out repository code
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0  # otherwise we will have no version info
+    - name: Check docker clickhouse/clickhouse-server building
+      run: |
+        cd "$GITHUB_WORKSPACE/tests/ci"
+        python3 docker_server.py --release-type auto --version "${{ github.ref }}"
+        python3 docker_server.py --release-type auto --version "${{ github.ref }}" --no-ubuntu \
+          --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper
+    - name: Cleanup
+      if: always()
+      run: |
+        docker kill "$(docker ps -q)" ||:
+        docker rm -f "$(docker ps -a -q)" ||:
+        sudo rm -fr "$TEMP_PATH"

.github/workflows/tags_stable.yml

@@ -0,0 +1,40 @@
+name: TagsStableWorkflow
+# - Gets artifacts from S3
+# - Sends it to JFROG Artifactory
+# - Adds them to the release assets
+
+on: # yamllint disable-line rule:truthy
+  push:
+    tags:
+    - 'v*-stable'
+    - 'v*-lts'
+
+
+jobs:
+  UpdateVersions:
+    runs-on: [self-hosted, style-checker]
+    steps:
+    - name: Get tag name
+      run: echo "GITHUB_TAG=${GITHUB_REF#refs/tags/}" >> "$GITHUB_ENV"
+    - name: Check out repository code
+      uses: actions/checkout@v2
+      with:
+        ref: master
+    - name: Generate versions
+      run: |
+        git fetch --tags
+        ./utils/list-versions/list-versions.sh > ./utils/list-versions/version_date.tsv
+    - name: Create Pull Request
+      uses: peter-evans/create-pull-request@v3
+      with:
+        author: "robot-clickhouse <robot-clickhouse@users.noreply.github.com>"
+        committer: "robot-clickhouse <robot-clickhouse@users.noreply.github.com>"
+        commit-message: Update version_date.tsv after ${{ env.GITHUB_TAG }}
+        branch: auto/${{ env.GITHUB_TAG }}
+        delete-branch: true
+        title: Update version_date.tsv after ${{ env.GITHUB_TAG }}
+        body: |
+          Update version_date.tsv after ${{ env.GITHUB_TAG }}
+
+          Changelog category (leave one):
+          - Not for changelog (changelog entry is not required)

.github/workflows/woboq.yml

@@ -0,0 +1,42 @@
+name: WoboqBuilder
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+
+concurrency:
+  group: woboq
+on: # yamllint disable-line rule:truthy
+  schedule:
+    - cron: '0 */18 * * *'
+  workflow_dispatch:
+jobs:
+  # don't use dockerhub push because this image updates so rarely
+  WoboqCodebrowser:
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/codebrowser
+          REPO_COPY=${{runner.temp}}/codebrowser/ClickHouse
+          IMAGES_PATH=${{runner.temp}}/images_path
+          EOF
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+        with:
+          submodules: 'true'
+      - name: Codebrowser
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci" && python3 codebrowser_check.py
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill "$(docker ps -q)" ||:
+          docker rm -f "$(docker ps -a -q)" ||:
+          sudo rm -fr "$TEMP_PATH"

.gitignore

@@ -13,6 +13,12 @@
 /build_*
 /build-*
 /tests/venv
+/obj-x86_64-linux-gnu/
+
+# logs
+*.log
+*.stderr
+*.stdout
 
 /docs/build
 /docs/publish
@@ -28,6 +34,9 @@
 /docs/ja/single.md
 /docs/fa/single.md
 /docs/en/development/cmake-in-clickhouse.md
+/docs/ja/development/cmake-in-clickhouse.md
+/docs/zh/development/cmake-in-clickhouse.md
+/docs/ru/development/cmake-in-clickhouse.md
 
 # callgrind files
 callgrind.out.*

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "contrib/poco"]
 	path = contrib/poco
-	url = https://github.com/ClickHouse-Extras/poco.git
+	url = https://github.com/ClickHouse/poco.git
 	branch = clickhouse
 [submodule "contrib/zstd"]
 	path = contrib/zstd
@@ -10,14 +10,14 @@
 	url = https://github.com/lz4/lz4.git
 [submodule "contrib/librdkafka"]
 	path = contrib/librdkafka
-	url = https://github.com/ClickHouse-Extras/librdkafka.git
+	url = https://github.com/ClickHouse/librdkafka.git
 [submodule "contrib/cctz"]
 	path = contrib/cctz
-	url = https://github.com/ClickHouse-Extras/cctz.git
+	url = https://github.com/ClickHouse/cctz.git
 [submodule "contrib/zlib-ng"]
 	path = contrib/zlib-ng
-	url = https://github.com/ClickHouse-Extras/zlib-ng.git
+	url = https://github.com/ClickHouse/zlib-ng.git
-	branch = clickhouse-new
+	branch = clickhouse-2.0.x
 [submodule "contrib/googletest"]
 	path = contrib/googletest
 	url = https://github.com/google/googletest.git
@@ -32,51 +32,51 @@
 	url = https://github.com/google/re2.git
 [submodule "contrib/llvm"]
 	path = contrib/llvm
-	url = https://github.com/ClickHouse-Extras/llvm
+	url = https://github.com/ClickHouse/llvm
 [submodule "contrib/mariadb-connector-c"]
 	path = contrib/mariadb-connector-c
-	url = https://github.com/ClickHouse-Extras/mariadb-connector-c.git
+	url = https://github.com/ClickHouse/mariadb-connector-c.git
 [submodule "contrib/jemalloc"]
 	path = contrib/jemalloc
-	url = https://github.com/ClickHouse-Extras/jemalloc.git
+	url = https://github.com/jemalloc/jemalloc.git
 [submodule "contrib/unixodbc"]
 	path = contrib/unixodbc
-	url = https://github.com/ClickHouse-Extras/UnixODBC.git
+	url = https://github.com/ClickHouse/UnixODBC.git
 [submodule "contrib/protobuf"]
 	path = contrib/protobuf
-	url = https://github.com/ClickHouse-Extras/protobuf.git
+	url = https://github.com/ClickHouse/protobuf.git
 	branch = v3.13.0.1
 [submodule "contrib/boost"]
 	path = contrib/boost
-	url = https://github.com/ClickHouse-Extras/boost.git
+	url = https://github.com/ClickHouse/boost.git
 [submodule "contrib/base64"]
 	path = contrib/base64
-	url = https://github.com/ClickHouse-Extras/Turbo-Base64.git
+	url = https://github.com/ClickHouse/Turbo-Base64.git
 [submodule "contrib/arrow"]
 	path = contrib/arrow
-	url = https://github.com/ClickHouse-Extras/arrow
+	url = https://github.com/ClickHouse/arrow.git
-	branch = clickhouse-arrow-2.0.0
+	branch = blessed/release-6.0.1
 [submodule "contrib/thrift"]
 	path = contrib/thrift
 	url = https://github.com/apache/thrift.git
 [submodule "contrib/libhdfs3"]
 	path = contrib/libhdfs3
-	url = https://github.com/ClickHouse-Extras/libhdfs3.git
+	url = https://github.com/ClickHouse/libhdfs3.git
 [submodule "contrib/libxml2"]
 	path = contrib/libxml2
 	url = https://github.com/GNOME/libxml2.git
 [submodule "contrib/libgsasl"]
 	path = contrib/libgsasl
-	url = https://github.com/ClickHouse-Extras/libgsasl.git
+	url = https://github.com/ClickHouse/libgsasl.git
 [submodule "contrib/libcxx"]
 	path = contrib/libcxx
-	url = https://github.com/ClickHouse-Extras/libcxx.git
+	url = https://github.com/ClickHouse/libcxx.git
 [submodule "contrib/libcxxabi"]
 	path = contrib/libcxxabi
-	url = https://github.com/ClickHouse-Extras/libcxxabi.git
+	url = https://github.com/ClickHouse/libcxxabi.git
 [submodule "contrib/snappy"]
 	path = contrib/snappy
-	url = https://github.com/google/snappy
+	url = https://github.com/ClickHouse/snappy.git
 [submodule "contrib/cppkafka"]
 	path = contrib/cppkafka
 	url = https://github.com/mfontanini/cppkafka.git
@@ -85,107 +85,95 @@
 	url = https://github.com/google/brotli.git
 [submodule "contrib/h3"]
 	path = contrib/h3
-	url = https://github.com/ClickHouse-Extras/h3
+	url = https://github.com/ClickHouse/h3
 [submodule "contrib/hyperscan"]
 	path = contrib/hyperscan
-	url = https://github.com/ClickHouse-Extras/hyperscan.git
+	url = https://github.com/ClickHouse/hyperscan.git
 [submodule "contrib/libunwind"]
 	path = contrib/libunwind
-	url = https://github.com/ClickHouse-Extras/libunwind.git
+	url = https://github.com/ClickHouse/libunwind.git
 [submodule "contrib/simdjson"]
 	path = contrib/simdjson
 	url = https://github.com/simdjson/simdjson.git
 [submodule "contrib/rapidjson"]
 	path = contrib/rapidjson
-	url = https://github.com/ClickHouse-Extras/rapidjson
+	url = https://github.com/ClickHouse/rapidjson
 [submodule "contrib/fastops"]
 	path = contrib/fastops
-	url = https://github.com/ClickHouse-Extras/fastops
+	url = https://github.com/ClickHouse/fastops
 [submodule "contrib/orc"]
 	path = contrib/orc
-	url = https://github.com/apache/orc
+	url = https://github.com/ClickHouse/orc
 [submodule "contrib/sparsehash-c11"]
 	path = contrib/sparsehash-c11
 	url = https://github.com/sparsehash/sparsehash-c11.git
 [submodule "contrib/grpc"]
 	path = contrib/grpc
-	url = https://github.com/ClickHouse-Extras/grpc.git
+	url = https://github.com/ClickHouse/grpc.git
 	branch = v1.33.2
 [submodule "contrib/aws"]
 	path = contrib/aws
-	url = https://github.com/ClickHouse-Extras/aws-sdk-cpp.git
+	url = https://github.com/ClickHouse/aws-sdk-cpp.git
 [submodule "aws-c-event-stream"]
 	path = contrib/aws-c-event-stream
-	url = https://github.com/ClickHouse-Extras/aws-c-event-stream.git
+	url = https://github.com/ClickHouse/aws-c-event-stream.git
 [submodule "aws-c-common"]
 	path = contrib/aws-c-common
-	url = https://github.com/ClickHouse-Extras/aws-c-common.git
+	url = https://github.com/ClickHouse/aws-c-common.git
 [submodule "aws-checksums"]
 	path = contrib/aws-checksums
-	url = https://github.com/ClickHouse-Extras/aws-checksums.git
+	url = https://github.com/ClickHouse/aws-checksums.git
 [submodule "contrib/curl"]
 	path = contrib/curl
 	url = https://github.com/curl/curl.git
 [submodule "contrib/icudata"]
 	path = contrib/icudata
-	url = https://github.com/ClickHouse-Extras/icudata.git
+	url = https://github.com/ClickHouse/icudata.git
 [submodule "contrib/icu"]
 	path = contrib/icu
 	url = https://github.com/unicode-org/icu.git
 [submodule "contrib/flatbuffers"]
 	path = contrib/flatbuffers
-	url = https://github.com/ClickHouse-Extras/flatbuffers.git
+	url = https://github.com/ClickHouse/flatbuffers.git
-[submodule "contrib/libc-headers"]
-	path = contrib/libc-headers
-	url = https://github.com/ClickHouse-Extras/libc-headers.git
 [submodule "contrib/replxx"]
 	path = contrib/replxx
-	url = https://github.com/ClickHouse-Extras/replxx.git
+	url = https://github.com/ClickHouse/replxx.git
 [submodule "contrib/avro"]
 	path = contrib/avro
-	url = https://github.com/ClickHouse-Extras/avro.git
+	url = https://github.com/ClickHouse/avro.git
 	ignore = untracked
 [submodule "contrib/msgpack-c"]
 	path = contrib/msgpack-c
 	url = https://github.com/msgpack/msgpack-c
 [submodule "contrib/libcpuid"]
 	path = contrib/libcpuid
-	url = https://github.com/ClickHouse-Extras/libcpuid.git
+	url = https://github.com/ClickHouse/libcpuid.git
 [submodule "contrib/openldap"]
 	path = contrib/openldap
-	url = https://github.com/ClickHouse-Extras/openldap.git
+	url = https://github.com/ClickHouse/openldap.git
 [submodule "contrib/AMQP-CPP"]
 	path = contrib/AMQP-CPP
-	url = https://github.com/ClickHouse-Extras/AMQP-CPP.git
+	url = https://github.com/ClickHouse/AMQP-CPP.git
 [submodule "contrib/cassandra"]
 	path = contrib/cassandra
-	url = https://github.com/ClickHouse-Extras/cpp-driver.git
+	url = https://github.com/ClickHouse/cpp-driver.git
 	branch = clickhouse
 [submodule "contrib/libuv"]
 	path = contrib/libuv
-	url = https://github.com/ClickHouse-Extras/libuv.git
+	url = https://github.com/ClickHouse/libuv.git
 	branch = clickhouse
 [submodule "contrib/fmtlib"]
 	path = contrib/fmtlib
 	url = https://github.com/fmtlib/fmt.git
-[submodule "contrib/antlr4-runtime"]
-	path = contrib/antlr4-runtime
-	url = https://github.com/ClickHouse-Extras/antlr4-runtime.git
 [submodule "contrib/sentry-native"]
 	path = contrib/sentry-native
-	url = https://github.com/ClickHouse-Extras/sentry-native.git
+	url = https://github.com/ClickHouse/sentry-native.git
-[submodule "contrib/gcem"]
-	path = contrib/gcem
-	url = https://github.com/kthohr/gcem.git
-[submodule "contrib/stats"]
-	path = contrib/stats
-	url = https://github.com/kthohr/stats.git
 [submodule "contrib/krb5"]
 	path = contrib/krb5
-	url = https://github.com/ClickHouse-Extras/krb5
+	url = https://github.com/ClickHouse/krb5
 [submodule "contrib/cyrus-sasl"]
 	path = contrib/cyrus-sasl
-	url = https://github.com/ClickHouse-Extras/cyrus-sasl
+	url = https://github.com/ClickHouse/cyrus-sasl
 	branch = cyrus-sasl-2.1
 [submodule "contrib/croaring"]
 	path = contrib/croaring
@@ -196,35 +184,81 @@
 	url = https://github.com/danlark1/miniselect
 [submodule "contrib/rocksdb"]
 	path = contrib/rocksdb
-	url = https://github.com/ClickHouse-Extras/rocksdb.git 
+	url = https://github.com/ClickHouse/rocksdb.git
 [submodule "contrib/xz"]
 	path = contrib/xz
 	url = https://github.com/xz-mirror/xz
 [submodule "contrib/abseil-cpp"]
 	path = contrib/abseil-cpp
-	url = https://github.com/ClickHouse-Extras/abseil-cpp.git
+	url = https://github.com/abseil/abseil-cpp.git
-	branch = lts_2020_02_25
+	branch = lts_2021_11_02
 [submodule "contrib/dragonbox"]
 	path = contrib/dragonbox
-	url = https://github.com/ClickHouse-Extras/dragonbox.git
+	url = https://github.com/ClickHouse/dragonbox.git
 [submodule "contrib/fast_float"]
 	path = contrib/fast_float
 	url = https://github.com/fastfloat/fast_float
-[submodule "contrib/libpqxx"]
-	path = contrib/libpqxx
-	url = https://github.com/jtv/libpqxx
 [submodule "contrib/libpq"]
 	path = contrib/libpq
-	url = https://github.com/ClickHouse-Extras/libpq
+	url = https://github.com/ClickHouse/libpq
 [submodule "contrib/boringssl"]
 	path = contrib/boringssl
-	url = https://github.com/ClickHouse-Extras/boringssl.git
+	url = https://github.com/ClickHouse/boringssl.git
+	branch = MergeWithUpstream
 [submodule "contrib/NuRaft"]
 	path = contrib/NuRaft
-	url = https://github.com/ClickHouse-Extras/NuRaft.git
+	url = https://github.com/ClickHouse/NuRaft.git
 [submodule "contrib/nanodbc"]
 	path = contrib/nanodbc
-	url = https://github.com/ClickHouse-Extras/nanodbc.git
+	url = https://github.com/ClickHouse/nanodbc.git
 [submodule "contrib/datasketches-cpp"]
 	path = contrib/datasketches-cpp
-	url = https://github.com/ClickHouse-Extras/datasketches-cpp.git
+	url = https://github.com/ClickHouse/datasketches-cpp.git
+[submodule "contrib/yaml-cpp"]
+	path = contrib/yaml-cpp
+	url = https://github.com/ClickHouse/yaml-cpp.git
+[submodule "contrib/cld2"]
+	path = contrib/cld2
+	url = https://github.com/ClickHouse/cld2.git
+[submodule "contrib/libstemmer_c"]
+	path = contrib/libstemmer_c
+	url = https://github.com/ClickHouse/libstemmer_c.git
+[submodule "contrib/wordnet-blast"]
+	path = contrib/wordnet-blast
+	url = https://github.com/ClickHouse/wordnet-blast.git
+[submodule "contrib/lemmagen-c"]
+	path = contrib/lemmagen-c
+	url = https://github.com/ClickHouse/lemmagen-c.git
+[submodule "contrib/libpqxx"]
+	path = contrib/libpqxx
+	url = https://github.com/ClickHouse/libpqxx.git
+[submodule "contrib/sqlite-amalgamation"]
+	path = contrib/sqlite-amalgamation
+	url = https://github.com/azadkuh/sqlite-amalgamation
+[submodule "contrib/s2geometry"]
+	path = contrib/s2geometry
+	url = https://github.com/ClickHouse/s2geometry.git
+[submodule "contrib/bzip2"]
+	path = contrib/bzip2
+	url = https://github.com/ClickHouse/bzip2.git
+[submodule "contrib/magic_enum"]
+	path = contrib/magic_enum
+	url = https://github.com/Neargye/magic_enum
+[submodule "contrib/libprotobuf-mutator"]
+	path = contrib/libprotobuf-mutator
+	url = https://github.com/google/libprotobuf-mutator
+[submodule "contrib/sysroot"]
+	path = contrib/sysroot
+	url = https://github.com/ClickHouse/sysroot.git
+[submodule "contrib/nlp-data"]
+	path = contrib/nlp-data
+	url = https://github.com/ClickHouse/nlp-data.git
+[submodule "contrib/hive-metastore"]
+	path = contrib/hive-metastore
+	url = https://github.com/ClickHouse/hive-metastore
+[submodule "contrib/azure"]
+	path = contrib/azure
+	url = https://github.com/ClickHouse/azure-sdk-for-cpp.git
+[submodule "contrib/minizip-ng"]
+	path = contrib/minizip-ng
+	url = https://github.com/zlib-ng/minizip-ng

.potato.yml

@@ -1,27 +0,0 @@
-# This is the configuration file with settings for Potato.
-# Potato is an internal Yandex technology that allows us to sync internal [Yandex.Tracker](https://yandex.com/tracker/) and GitHub.
-
-# For all PRs where documentation is needed, just add a 'pr-feature' label and we will include it into documentation sprints.
-
-# The project name.
-name: clickhouse
-# Object handlers defines which handlers we use.
-handlers:
-  # The handler for creating an Yandex.Tracker issue.
-  - name: issue-create
-    params:
-      triggers:
-        # The trigger for creating the Yandex.Tracker issue. When the specified event occurs, it transfers PR data to Yandex.Tracker.
-        github:pullRequest:labeled:
-          data:
-          # The Yandex.Tracker queue to create the issue in. Each issue in Tracker belongs to one of the project queues.
-          queue: CLICKHOUSEDOCS
-          # The issue title.
-          summary: '[Potato] Pull Request #{{pullRequest.number}}'
-          # The issue description.
-          description: >
-                {{pullRequest.description}}
-
-                Ссылка на Pull Request: {{pullRequest.webUrl}}
-          # The condition for creating the Yandex.Tracker issue.
-          condition: eventPayload.labels.filter(label => ['pr-feature'].includes(label.name)).length

AUTHORS

@@ -1,2 +1,2 @@
-To see the list of authors who created the source code of ClickHouse, published and distributed by YANDEX LLC as the owner,
+To see the list of authors who created the source code of ClickHouse, published and distributed by ClickHouse, Inc. as the owner,
 run "SELECT * FROM system.contributors;" query on any ClickHouse server.

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.3)
+cmake_minimum_required(VERSION 3.14)
 
 foreach(policy
         CMP0023
@@ -36,7 +36,7 @@ option(FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION
 if(FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION)
     set(RECONFIGURE_MESSAGE_LEVEL FATAL_ERROR)
 else()
-    set(RECONFIGURE_MESSAGE_LEVEL STATUS)
+    set(RECONFIGURE_MESSAGE_LEVEL WARNING)
 endif()
 
 enable_language(C CXX ASM)
@@ -45,6 +45,7 @@ include (cmake/arch.cmake)
 include (cmake/target.cmake)
 include (cmake/tools.cmake)
 include (cmake/analysis.cmake)
+include (cmake/git_status.cmake)
 
 # Ignore export() since we don't use it,
 # but it gets broken with a global targets via link_libraries()
@@ -61,12 +62,12 @@ set(CMAKE_DEBUG_POSTFIX "d" CACHE STRING "Generate debug library name with a pos
 # For more info see https://cmake.org/cmake/help/latest/prop_gbl/USE_FOLDERS.html
 set_property(GLOBAL PROPERTY USE_FOLDERS ON)
 
-# Check that submodules are present only if source was downloaded with git
+# Check that submodules are present
-if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git" AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/boost/boost")
+if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/sysroot/README.md")
     message (FATAL_ERROR "Submodules are not initialized. Run\n\tgit submodule update --init --recursive")
 endif ()
 
-include (cmake/find/ccache.cmake)
+include (cmake/ccache.cmake)
 
 # Take care to add prlimit in command line before ccache, or else ccache thinks that
 # prlimit is compiler, and clang++ is its input file, and refuses to work  with
@@ -79,16 +80,16 @@ include (cmake/find/ccache.cmake)
 # ccache ignore it.
 option(ENABLE_CHECK_HEAVY_BUILDS "Don't allow C++ translation units to compile too long or to take too much memory while compiling." OFF)
 if (ENABLE_CHECK_HEAVY_BUILDS)
-    # set DATA (since RSS does not work since 2.6.x+) to 2G
+    # set DATA (since RSS does not work since 2.6.x+) to 5G
     set (RLIMIT_DATA 5000000000)
     # set VIRT (RLIMIT_AS) to 10G (DATA*10)
     set (RLIMIT_AS 10000000000)
-    # set CPU time limit to 600 seconds
+    # set CPU time limit to 1000 seconds
-    set (RLIMIT_CPU 600)
+    set (RLIMIT_CPU 1000)
 
     # gcc10/gcc10/clang -fsanitize=memory is too heavy
     if (SANITIZE STREQUAL "memory" OR COMPILER_GCC)
-       set (RLIMIT_DATA 10000000000)
+       set (RLIMIT_DATA 10000000000) # 10G
     endif()
 
     set (CMAKE_CXX_COMPILER_LAUNCHER prlimit --as=${RLIMIT_AS} --data=${RLIMIT_DATA} --cpu=${RLIMIT_CPU} ${CMAKE_CXX_COMPILER_LAUNCHER})
@@ -103,9 +104,8 @@ message (STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}")
 string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC)
 
 option(USE_STATIC_LIBRARIES "Disable to use shared libraries" ON)
-option(MAKE_STATIC_LIBRARIES "Disable to make shared libraries" ${USE_STATIC_LIBRARIES})
 
-if (NOT MAKE_STATIC_LIBRARIES)
+if (NOT USE_STATIC_LIBRARIES)
     # DEVELOPER ONLY.
     # Faster linking if turned on.
     option(SPLIT_SHARED_LIBRARIES "Keep all internal libraries as separate .so files")
@@ -114,11 +114,11 @@ if (NOT MAKE_STATIC_LIBRARIES)
         "Make several binaries (clickhouse-server, clickhouse-client etc.) instead of one bundled")
 endif ()
 
-if (MAKE_STATIC_LIBRARIES AND SPLIT_SHARED_LIBRARIES)
+if (USE_STATIC_LIBRARIES AND SPLIT_SHARED_LIBRARIES)
-    message(FATAL_ERROR "Defining SPLIT_SHARED_LIBRARIES=1 without MAKE_STATIC_LIBRARIES=0 has no effect.")
+    message(FATAL_ERROR "Defining SPLIT_SHARED_LIBRARIES=1 without USE_STATIC_LIBRARIES=0 has no effect.")
 endif()
 
-if (NOT MAKE_STATIC_LIBRARIES AND SPLIT_SHARED_LIBRARIES)
+if (NOT USE_STATIC_LIBRARIES AND SPLIT_SHARED_LIBRARIES)
     set(BUILD_SHARED_LIBS 1 CACHE INTERNAL "")
 endif ()
 
@@ -126,13 +126,30 @@ if (USE_STATIC_LIBRARIES)
     list(REVERSE CMAKE_FIND_LIBRARY_SUFFIXES)
 endif ()
 
-# Implies ${WITH_COVERAGE}
 option (ENABLE_FUZZING "Fuzzy testing using libfuzzer" OFF)
 
 if (ENABLE_FUZZING)
+    # Also set WITH_COVERAGE=1 for better fuzzing process
+    # By default this is disabled, because fuzzers are built in CI with the clickhouse itself.
+    # And we don't want to enable coverage for it.
     message (STATUS "Fuzzing instrumentation enabled")
-    set (WITH_COVERAGE ON)
     set (FUZZER "libfuzzer")
+    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -nostdlib++")
+    set (ENABLE_CLICKHOUSE_ODBC_BRIDGE OFF)
+    set (ENABLE_LIBRARIES 0)
+    set (ENABLE_SSL 1)
+    set (USE_UNWIND ON)
+    set (ENABLE_EMBEDDED_COMPILER 0)
+    set (ENABLE_EXAMPLES 0)
+    set (ENABLE_UTILS 0)
+    set (ENABLE_THINLTO 0)
+    set (ENABLE_TCMALLOC 0)
+    set (ENABLE_JEMALLOC 0)
+    set (ENABLE_CHECK_HEAVY_BUILDS 1)
+    set (GLIBC_COMPATIBILITY OFF)
+
+    # For codegen_select_fuzzer
+    set (ENABLE_PROTOBUF 1)
 endif()
 
 # Global libraries
@@ -150,11 +167,10 @@ if (CMAKE_GENERATOR STREQUAL "Ninja" AND NOT DISABLE_COLORED_BUILD)
     set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdiagnostics-color=always")
 endif ()
 
+include (cmake/check_flags.cmake)
 include (cmake/add_warning.cmake)
 
-if (NOT MSVC)
+set (COMMON_WARNING_FLAGS "${COMMON_WARNING_FLAGS} -Wall")    # -Werror and many more is also added inside cmake/warnings.cmake
-    set (COMMON_WARNING_FLAGS "${COMMON_WARNING_FLAGS} -Wall")    # -Werror and many more is also added inside cmake/warnings.cmake
-endif ()
 
 if (COMPILER_CLANG)
     # clang: warning: argument unused during compilation: '-specs=/usr/share/dpkg/no-pie-compile.specs' [-Wunused-command-line-argument]
@@ -163,49 +179,55 @@ if (COMPILER_CLANG)
     if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE")
         set(COMPILER_FLAGS "${COMPILER_FLAGS} -gdwarf-aranges")
     endif ()
+
+    if (HAS_USE_CTOR_HOMING)
+        # For more info see https://blog.llvm.org/posts/2021-04-05-constructor-homing-for-debug-info/
+        if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO")
+            set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xclang -fuse-ctor-homing")
+            set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Xclang -fuse-ctor-homing")
+        endif()
+    endif()
+endif ()
+
+# If compiler has support for -Wreserved-identifier. It is difficult to detect by clang version,
+# because there are two different branches of clang: clang and AppleClang.
+# (AppleClang is not supported by ClickHouse, but some developers have misfortune to use it).
+if (HAS_RESERVED_IDENTIFIER)
+    add_compile_definitions (HAS_RESERVED_IDENTIFIER)
 endif ()
 
 # If turned `ON`, assumes the user has either the system GTest library or the bundled one.
 option(ENABLE_TESTS "Provide unit_test_dbms target with Google.Test unit tests" ON)
 option(ENABLE_EXAMPLES "Build all example programs in 'examples' subdirectories" OFF)
 
-if (OS_LINUX AND (ARCH_AMD64 OR ARCH_AARCH64) AND NOT UNBUNDLED AND MAKE_STATIC_LIBRARIES AND NOT SPLIT_SHARED_LIBRARIES AND CMAKE_VERSION VERSION_GREATER "3.9.0")
+if (OS_LINUX AND (ARCH_AMD64 OR ARCH_AARCH64) AND USE_STATIC_LIBRARIES AND NOT SPLIT_SHARED_LIBRARIES AND NOT USE_MUSL)
     # Only for Linux, x86_64 or aarch64.
     option(GLIBC_COMPATIBILITY "Enable compatibility with older glibc libraries." ON)
 elseif(GLIBC_COMPATIBILITY)
     message (${RECONFIGURE_MESSAGE_LEVEL} "Glibc compatibility cannot be enabled in current configuration")
 endif ()
 
-if (NOT CMAKE_VERSION VERSION_GREATER "3.9.0")
-    message (WARNING "CMake version must be greater than 3.9.0 for production builds.")
-endif ()
-
 # Make sure the final executable has symbols exported
 set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -rdynamic")
 
-if (OS_LINUX)
-    find_program (OBJCOPY_PATH NAMES "llvm-objcopy" "llvm-objcopy-12" "llvm-objcopy-11" "llvm-objcopy-10" "llvm-objcopy-9" "llvm-objcopy-8" "objcopy")
-    if (OBJCOPY_PATH)
-        message(STATUS "Using objcopy: ${OBJCOPY_PATH}.")
-
-        if (ARCH_AMD64)
-            set(OBJCOPY_ARCH_OPTIONS -O elf64-x86-64 -B i386)
-        elseif (ARCH_AARCH64)
-            set(OBJCOPY_ARCH_OPTIONS -O elf64-aarch64 -B aarch64)
-        endif ()
-    else ()
-        message(FATAL_ERROR "Cannot find objcopy.")
-    endif ()
-endif ()
-
 if (OS_DARWIN)
-    set(WHOLE_ARCHIVE -all_load)
+    # The `-all_load` flag forces loading of all symbols from all libraries,
-    set(NO_WHOLE_ARCHIVE -noall_load)
+    # and leads to multiply-defined symbols. This flag allows force loading
+    # from a _specific_ library, which is what we need.
+    set(WHOLE_ARCHIVE -force_load)
+    # The `-noall_load` flag is the default and now obsolete.
+    set(NO_WHOLE_ARCHIVE "-undefined,error") # Effectively, a no-op. Here to avoid empty "-Wl, " sequence to be generated in the command line.
 else ()
     set(WHOLE_ARCHIVE --whole-archive)
     set(NO_WHOLE_ARCHIVE --no-whole-archive)
 endif ()
 
+option(ENABLE_CURL_BUILD "Enable curl, azure, sentry build on by default except MacOS." ON)
+if (OS_DARWIN)
+    # Disable the curl, azure, senry build on MacOS
+    set (ENABLE_CURL_BUILD OFF)
+endif ()
+
 # Ignored if `lld` is used
 option(ADD_GDB_INDEX_FOR_GOLD "Add .gdb-index to resulting binaries for gold linker.")
 
@@ -229,6 +251,13 @@ if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE")
     endif ()
 endif()
 
+if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
+    set(USE_DEBUG_HELPERS ON)
+endif()
+
+option(USE_DEBUG_HELPERS "Enable debug helpers" ${USE_DEBUG_HELPERS})
+option(BUILD_STANDALONE_KEEPER "Build keeper as small standalone binary" OFF)
+
 # Create BuildID when using lld. For other linkers it is created by default.
 if (LINKER_NAME MATCHES "lld$")
     # SHA1 is not cryptographically secure but it is the best what lld is offering.
@@ -238,10 +267,18 @@ endif ()
 # Add a section with the hash of the compiled machine code for integrity checks.
 # Only for official builds, because adding a section can be time consuming (rewrite of several GB).
 # And cross compiled binaries are not supported (since you cannot execute clickhouse hash-binary)
-if (OBJCOPY_PATH AND YANDEX_OFFICIAL_BUILD AND (NOT CMAKE_TOOLCHAIN_FILE))
+if (OBJCOPY_PATH AND CLICKHOUSE_OFFICIAL_BUILD AND (NOT CMAKE_TOOLCHAIN_FILE OR CMAKE_TOOLCHAIN_FILE MATCHES "linux/toolchain-x86_64.cmake$"))
-    set (USE_BINARY_HASH 1)
+    set (USE_BINARY_HASH 1 CACHE STRING "Calculate binary hash and store it in the separate section")
 endif ()
 
+# Allows to build stripped binary in a separate directory
+if (OBJCOPY_PATH AND STRIP_PATH)
+    option(INSTALL_STRIPPED_BINARIES "Build stripped binaries with debug info in separate directory" OFF)
+    if (INSTALL_STRIPPED_BINARIES)
+        set(STRIPPED_BINARIES_OUTPUT "stripped" CACHE STRING "A separate directory for stripped information")
+    endif()
+endif()
+
 cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_PHYSICAL_MEMORY) # Not available under freebsd
 
 
@@ -258,17 +295,25 @@ endif()
 
 include(cmake/cpu_features.cmake)
 
-option(ARCH_NATIVE "Add -march=native compiler flag. This makes your binaries non-portable but more performant code may be generated.")
-
-if (ARCH_NATIVE)
-    set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=native")
-endif ()
-
 # Asynchronous unwind tables are needed for Query Profiler.
 # They are already by default on some platforms but possibly not on all platforms.
 # Enable it explicitly.
 set (COMPILER_FLAGS "${COMPILER_FLAGS} -fasynchronous-unwind-tables")
 
+# Reproducible builds.
+if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
+    set (ENABLE_BUILD_PATH_MAPPING_DEFAULT OFF)
+else ()
+    set (ENABLE_BUILD_PATH_MAPPING_DEFAULT ON)
+endif ()
+
+option (ENABLE_BUILD_PATH_MAPPING "Enable remapping of file source paths in debug info, predefined preprocessor macros, and __builtin_FILE(). It's used to generate reproducible builds. See https://reproducible-builds.org/docs/build-path" ${ENABLE_BUILD_PATH_MAPPING_DEFAULT})
+
+if (ENABLE_BUILD_PATH_MAPPING)
+    set (COMPILER_FLAGS "${COMPILER_FLAGS} -ffile-prefix-map=${CMAKE_SOURCE_DIR}=.")
+    set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} -ffile-prefix-map=${CMAKE_SOURCE_DIR}=.")
+endif ()
+
 if (${CMAKE_VERSION} VERSION_LESS "3.12.4")
     # CMake < 3.12 doesn't support setting 20 as a C++ standard version.
     # We will add C++ standard controlling flag in CMAKE_CXX_FLAGS manually for now.
@@ -302,6 +347,10 @@ if (COMPILER_GCC OR COMPILER_CLANG)
     set(COMPILER_FLAGS "${COMPILER_FLAGS} -falign-functions=32")
 endif ()
 
+if (COMPILER_GCC)
+    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fcoroutines")
+endif ()
+
 # Compiler-specific coverage flags e.g. -fcoverage-mapping for gcc
 option(WITH_COVERAGE "Profile the resulting binary/binaries" OFF)
 
@@ -317,16 +366,19 @@ if (WITH_COVERAGE AND COMPILER_GCC)
     set(WITHOUT_COVERAGE "-fno-profile-arcs -fno-test-coverage")
 endif()
 
-set(COMPILER_FLAGS "${COMPILER_FLAGS}")
+set (COMPILER_FLAGS "${COMPILER_FLAGS}")
+
+# Our built-in unwinder only supports DWARF version up to 4.
+set (DEBUG_INFO_FLAGS "-g -gdwarf-4")
 
 set (CMAKE_BUILD_COLOR_MAKEFILE          ON)
 set (CMAKE_CXX_FLAGS                     "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS} ${PLATFORM_EXTRA_CXX_FLAG} ${COMMON_WARNING_FLAGS} ${CXX_WARNING_FLAGS}")
-set (CMAKE_CXX_FLAGS_RELWITHDEBINFO      "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_CXX_FLAGS_ADD}")
+set (CMAKE_CXX_FLAGS_RELWITHDEBINFO      "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_CXX_FLAGS_ADD}")
-set (CMAKE_CXX_FLAGS_DEBUG               "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_CXX_FLAGS_ADD}")
+set (CMAKE_CXX_FLAGS_DEBUG               "${CMAKE_CXX_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} -fno-inline ${CMAKE_CXX_FLAGS_ADD}")
 
 set (CMAKE_C_FLAGS                       "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} ${COMMON_WARNING_FLAGS} ${CMAKE_C_FLAGS_ADD}")
-set (CMAKE_C_FLAGS_RELWITHDEBINFO        "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_C_FLAGS_ADD}")
+set (CMAKE_C_FLAGS_RELWITHDEBINFO        "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_C_FLAGS_ADD}")
-set (CMAKE_C_FLAGS_DEBUG                 "${CMAKE_C_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_C_FLAGS_ADD}")
+set (CMAKE_C_FLAGS_DEBUG                 "${CMAKE_C_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} -fno-inline ${CMAKE_C_FLAGS_ADD}")
 
 if (COMPILER_CLANG)
     if (OS_DARWIN)
@@ -345,6 +397,8 @@ if (COMPILER_CLANG)
         option(ENABLE_THINLTO "Clang-specific link time optimization" ON)
     endif()
 
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fstrict-vtable-pointers")
+
     # Set new experimental pass manager, it's a performance, build time and binary size win.
     # Can be removed after https://reviews.llvm.org/D66490 merged and released to at least two versions of clang.
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fexperimental-new-pass-manager")
@@ -355,32 +409,13 @@ if (COMPILER_CLANG)
     # completely.
     if (ENABLE_THINLTO AND NOT ENABLE_TESTS AND NOT SANITIZE)
         # Link time optimization
-        set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -flto=thin")
+        set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -flto=thin -fwhole-program-vtables")
-        set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -flto=thin")
+        set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -flto=thin -fwhole-program-vtables")
-        set (CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} -flto=thin")
+        set (CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} -flto=thin -fwhole-program-vtables")
     elseif (ENABLE_THINLTO)
         message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot enable ThinLTO")
     endif ()
 
-    # Always prefer llvm tools when using clang. For instance, we cannot use GNU ar when llvm LTO is enabled
-    find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-12" "llvm-ar-11" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8")
-
-    if (LLVM_AR_PATH)
-        message(STATUS "Using llvm-ar: ${LLVM_AR_PATH}.")
-        set (CMAKE_AR ${LLVM_AR_PATH})
-    else ()
-        message(WARNING "Cannot find llvm-ar. System ar will be used instead. It does not work with ThinLTO.")
-    endif ()
-
-    find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-12" "llvm-ranlib-11" "llvm-ranlib-10" "llvm-ranlib-9" "llvm-ranlib-8")
-
-    if (LLVM_RANLIB_PATH)
-        message(STATUS "Using llvm-ranlib: ${LLVM_RANLIB_PATH}.")
-        set (CMAKE_RANLIB ${LLVM_RANLIB_PATH})
-    else ()
-        message(WARNING "Cannot find llvm-ranlib. System ranlib will be used instead. It does not work with ThinLTO.")
-    endif ()
-
 elseif (ENABLE_THINLTO)
     message (${RECONFIGURE_MESSAGE_LEVEL} "ThinLTO is only available with CLang")
 endif ()
@@ -388,37 +423,21 @@ endif ()
 # Turns on all external libs like s3, kafka, ODBC, ...
 option(ENABLE_LIBRARIES "Enable all external libraries by default" ON)
 
-# We recommend avoiding this mode for production builds because we can't guarantee all needed libraries exist in your
+if (NOT (OS_LINUX OR OS_DARWIN))
-# system.
-# This mode exists for enthusiastic developers who are searching for trouble.
-# Useful for maintainers of OS packages.
-option (UNBUNDLED "Use system libraries instead of ones in contrib/" OFF)
-
-if (UNBUNDLED)
-    set(NOT_UNBUNDLED OFF)
-else ()
-    set(NOT_UNBUNDLED ON)
-endif ()
-
-if (UNBUNDLED OR NOT (OS_LINUX OR OS_DARWIN))
     # Using system libs can cause a lot of warnings in includes (on macro expansion).
     option(WERROR "Enable -Werror compiler option" OFF)
 else ()
     option(WERROR "Enable -Werror compiler option" ON)
 endif ()
 
-if (WERROR)
+# Increase stack size on Musl. We need big stack for our recursive-descend parser.
-    add_warning(error)
+if (USE_MUSL)
-endif ()
+    set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-z,stack-size=2097152")
-
-# Make this extra-checks for correct library dependencies.
-if (OS_LINUX AND NOT SANITIZE)
-    set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-undefined")
-    set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-undefined")
 endif ()
 
 include(cmake/dbms_glob_sources.cmake)
 
+add_library(global-group INTERFACE)
 if (OS_LINUX OR OS_ANDROID)
     include(cmake/linux/default_libs.cmake)
 elseif (OS_DARWIN)
@@ -426,6 +445,18 @@ elseif (OS_DARWIN)
 elseif (OS_FREEBSD)
     include(cmake/freebsd/default_libs.cmake)
 endif ()
+link_libraries(global-group)
+
+if (WERROR)
+    # Don't pollute CMAKE_CXX_FLAGS with -Werror as it will break some CMake checks.
+    # Instead, adopt modern cmake usage requirement.
+    target_compile_options(global-group INTERFACE "-Werror")
+endif ()
+
+# Make this extra-checks for correct library dependencies.
+if (OS_LINUX AND NOT SANITIZE)
+    target_link_options(global-group INTERFACE "-Wl,--no-undefined")
+endif ()
 
 ######################################
 ### Add targets below this comment ###
@@ -433,30 +464,22 @@ endif ()
 
 set (CMAKE_POSTFIX_VARIABLE "CMAKE_${CMAKE_BUILD_TYPE_UC}_POSTFIX")
 
-if (MAKE_STATIC_LIBRARIES)
+if (USE_STATIC_LIBRARIES)
     set (CMAKE_POSITION_INDEPENDENT_CODE OFF)
     if (OS_LINUX AND NOT ARCH_ARM)
         # Slightly more efficient code can be generated
         # It's disabled for ARM because otherwise ClickHouse cannot run on Android.
         set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-pie")
         set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -fno-pie")
-        set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-no-pie")
+        set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -no-pie -Wl,-no-pie")
     endif ()
 else ()
     set (CMAKE_POSITION_INDEPENDENT_CODE ON)
-endif ()
+    # This is required for clang on Arch linux, that uses PIE by default.
-
+    # See enable-SSP-and-PIE-by-default.patch [1].
-# https://github.com/include-what-you-use/include-what-you-use
+    #
-option (USE_INCLUDE_WHAT_YOU_USE "Automatically reduce unneeded includes in source code (external tool)" OFF)
+    #   [1]: https://github.com/archlinux/svntogit-packages/blob/6e681aa860e65ad46a1387081482eb875c2200f2/trunk/enable-SSP-and-PIE-by-default.patch
-
+    set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -no-pie")
-if (USE_INCLUDE_WHAT_YOU_USE)
-    find_program(IWYU_PATH NAMES include-what-you-use iwyu)
-    if (NOT IWYU_PATH)
-        message(FATAL_ERROR "Could not find the program include-what-you-use")
-    endif()
-    if (${CMAKE_VERSION} VERSION_LESS "3.3.0")
-        message(FATAL_ERROR "include-what-you-use requires CMake version at least 3.3.")
-    endif()
 endif ()
 
 if (ENABLE_TESTS)
@@ -477,86 +500,13 @@ endif ()
 message (STATUS
     "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE} ;
     USE_STATIC_LIBRARIES=${USE_STATIC_LIBRARIES}
-    MAKE_STATIC_LIBRARIES=${MAKE_STATIC_LIBRARIES}
     SPLIT_SHARED=${SPLIT_SHARED_LIBRARIES}
-    UNBUNDLED=${UNBUNDLED}
     CCACHE=${CCACHE_FOUND} ${CCACHE_VERSION}")
 
 include (GNUInstallDirs)
-include (cmake/contrib_finder.cmake)
-
-find_contrib_lib(double-conversion) # Must be before parquet
-include (cmake/find/ssl.cmake)
-include (cmake/find/ldap.cmake) # after ssl
-include (cmake/find/icu.cmake)
-include (cmake/find/xz.cmake)
-include (cmake/find/zlib.cmake)
-include (cmake/find/zstd.cmake)
-include (cmake/find/ltdl.cmake) # for odbc
-# openssl, zlib before poco
-include (cmake/find/sparsehash.cmake)
-include (cmake/find/re2.cmake)
-include (cmake/find/krb5.cmake)
-include (cmake/find/libgsasl.cmake)
-include (cmake/find/cyrus-sasl.cmake)
-include (cmake/find/rdkafka.cmake)
-include (cmake/find/libuv.cmake) # for amqpcpp and cassandra
-include (cmake/find/amqpcpp.cmake)
-include (cmake/find/capnp.cmake)
-include (cmake/find/llvm.cmake)
-include (cmake/find/termcap.cmake) # for external static llvm
-include (cmake/find/h3.cmake)
-include (cmake/find/libxml2.cmake)
-include (cmake/find/brotli.cmake)
-include (cmake/find/protobuf.cmake)
-include (cmake/find/grpc.cmake)
-include (cmake/find/pdqsort.cmake)
-include (cmake/find/miniselect.cmake)
-include (cmake/find/hdfs3.cmake) # uses protobuf
-include (cmake/find/poco.cmake)
-include (cmake/find/curl.cmake)
-include (cmake/find/s3.cmake)
-include (cmake/find/base64.cmake)
-include (cmake/find/parquet.cmake)
-include (cmake/find/simdjson.cmake)
-include (cmake/find/fast_float.cmake)
-include (cmake/find/rapidjson.cmake)
-include (cmake/find/fastops.cmake)
-include (cmake/find/odbc.cmake)
-include (cmake/find/nanodbc.cmake)
-include (cmake/find/rocksdb.cmake)
-include (cmake/find/libpqxx.cmake)
-include (cmake/find/nuraft.cmake)
-
-
-if(NOT USE_INTERNAL_PARQUET_LIBRARY)
-    set (ENABLE_ORC OFF CACHE INTERNAL "")
-endif()
-include (cmake/find/orc.cmake)
-
-include (cmake/find/avro.cmake)
-include (cmake/find/msgpack.cmake)
-include (cmake/find/cassandra.cmake)
-include (cmake/find/sentry.cmake)
-include (cmake/find/stats.cmake)
-include (cmake/find/datasketches.cmake)
-
-set (USE_INTERNAL_CITYHASH_LIBRARY ON CACHE INTERNAL "")
-find_contrib_lib(cityhash)
-
-find_contrib_lib(farmhash)
-
-if (ENABLE_TESTS)
-    include (cmake/find/gtest.cmake)
-endif ()
-
-# Need to process before "contrib" dir:
-include (cmake/find/mysqlclient.cmake)
 
 # When testing for memory leaks with Valgrind, don't link tcmalloc or jemalloc.
 
-include (cmake/print_flags.cmake)
-
 if (TARGET global-group)
     install (EXPORT global DESTINATION cmake)
 endif ()
@@ -571,7 +521,7 @@ macro (add_executable target)
     # invoke built-in add_executable
     # explicitly acquire and interpose malloc symbols by clickhouse_malloc
     # if GLIBC_COMPATIBILITY is ON and ENABLE_THINLTO is on than provide memcpy symbol explicitly to neutrialize thinlto's libcall generation.
-    if (GLIBC_COMPATIBILITY AND ENABLE_THINLTO)
+    if (ARCH_AMD64 AND GLIBC_COMPATIBILITY AND ENABLE_THINLTO)
         _add_executable (${ARGV} $<TARGET_OBJECTS:clickhouse_malloc> $<TARGET_OBJECTS:memcpy>)
     else ()
         _add_executable (${ARGV} $<TARGET_OBJECTS:clickhouse_malloc>)
@@ -582,7 +532,23 @@ macro (add_executable target)
         # disabled for TSAN and gcc since libtsan.a provides overrides too
         if (TARGET clickhouse_new_delete)
             # operator::new/delete for executables (MemoryTracker stuff)
-            target_link_libraries (${target} PRIVATE clickhouse_new_delete ${MALLOC_LIBRARIES})
+            target_link_libraries (${target} PRIVATE clickhouse_new_delete)
+        endif()
+
+        # In case of static jemalloc, because zone_register() is located in zone.c and
+        # is never used outside (it is declared as constructor) it is omitted
+        # by the linker, and so jemalloc will not be registered as system
+        # allocator under osx [1], and clickhouse will SIGSEGV.
+        #
+        #   [1]: https://github.com/jemalloc/jemalloc/issues/708
+        #
+        # About symbol name:
+        # - _zone_register not zone_register due to Mach-O binary format,
+        # - _je_zone_register due to JEMALLOC_PRIVATE_NAMESPACE=je_ under OS X.
+        # - but jemalloc-cmake does not run private_namespace.sh
+        #   so symbol name should be _zone_register
+        if (ENABLE_JEMALLOC AND USE_STATIC_LIBRARIES AND OS_DARWIN)
+            set_property(TARGET ${target} APPEND PROPERTY LINK_OPTIONS -u_zone_register)
         endif()
     endif()
 endmacro()
@@ -592,9 +558,7 @@ include_directories(${ConfigIncludePath})
 
 # Add as many warnings as possible for our own code.
 include (cmake/warnings.cmake)
-
+include (cmake/print_flags.cmake)
-# Check if needed compiler flags are supported
-include (cmake/check_flags.cmake)
 
 add_subdirectory (base)
 add_subdirectory (src)
@@ -602,6 +566,4 @@ add_subdirectory (programs)
 add_subdirectory (tests)
 add_subdirectory (utils)
 
-include (cmake/print_include_directories.cmake)
-
 include (cmake/sanitize_target_link_libraries.cmake)

CONTRIBUTING.md

@@ -2,42 +2,16 @@
 
 ClickHouse is an open project, and you can contribute to it in many ways. You can help with ideas, code, or documentation. We appreciate any efforts that help us to make the project better.
 
-Thank you.
+Thank you!
-
-## Technical Info
-
-We have a [developer's guide](https://clickhouse.yandex/docs/en/development/developer_instruction/) for writing code for ClickHouse. Besides this guide, you can find [Overview of ClickHouse Architecture](https://clickhouse.yandex/docs/en/development/architecture/) and instructions on how to build ClickHouse in different environments.
-
-If you want to contribute to documentation, read the [Contributing to ClickHouse Documentation](docs/README.md) guide.
 
 ## Legal Info
 
-In order for us (YANDEX LLC) to accept patches and other contributions from you, you may adopt our Yandex Contributor License Agreement (the "**CLA**"). The current version of the CLA you may find here:
+When you open your first pull-request to ClickHouse repo, a bot will invite you to accept ClickHouse Individual CLA (Contributor License Agreement). It is a simple few click process. For subsequent pull-requests the bot will check if you have already signed it and won't bother you again.
-1) https://yandex.ru/legal/cla/?lang=en (in English) and
-2) https://yandex.ru/legal/cla/?lang=ru (in Russian).
 
-By adopting the CLA, you state the following:
+Optionally, to make contributions even more tight legally, your employer as a legal entity may want to sign a ClickHouse Corporate CLA with ClickHouse, Inc. If you're interested to do so, contact us at [legal@clickhouse.com](mailto:legal@clickhouse.com).
 
-* You obviously wish and are willingly licensing your contributions to us for our open source projects under the terms of the CLA,
+## Technical Info
-* You have read the terms and conditions of the CLA and agree with them in full,
-* You are legally able to provide and license your contributions as stated,
-* We may use your contributions for our open source projects and for any other our project too,
-* We rely on your assurances concerning the rights of third parties in relation to your contributions.
 
-If you agree with these principles, please read and adopt our CLA. By providing us your contributions, you hereby declare that you have already read and adopt our CLA, and we may freely merge your contributions with our corresponding open source project and use it in further in accordance with terms and conditions of the CLA.
+We have a [developer's guide](https://clickhouse.com/docs/en/development/developer_instruction/) for writing code for ClickHouse. Besides this guide, you can find [Overview of ClickHouse Architecture](https://clickhouse.com/docs/en/development/architecture/) and instructions on how to build ClickHouse in different environments.
 
-If you have already adopted terms and conditions of the CLA, you are able to provide your contributes. When you submit your pull request, please add the following information into it:
+If you want to contribute to documentation, read the [Contributing to ClickHouse Documentation](docs/README.md) guide.
-
-```
-I hereby agree to the terms of the CLA available at: [link].
-```
-
-Replace the bracketed text as follows:
-* [link] is the link at the current version of the CLA (you may add here a link https://yandex.ru/legal/cla/?lang=en (in English) or a link https://yandex.ru/legal/cla/?lang=ru (in Russian).
-
-It is enough to provide us such notification once.
-
-As an alternative, you can provide DCO instead of CLA. You can find the text of DCO here: https://developercertificate.org/
-It is enough to read and copy it verbatim to your pull request.
-
-If you don't agree with the CLA and don't want to provide DCO, you still can open a pull request to provide your contributions.

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2016-2021 Yandex LLC
+Copyright 2016-2022 ClickHouse, Inc.
 
                                  Apache License
                            Version 2.0, January 2004
@@ -188,7 +188,7 @@ Copyright 2016-2021 Yandex LLC
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2016-2021 Yandex LLC
+   Copyright 2016-2022 ClickHouse, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

PreLoad.cmake

@@ -18,3 +18,25 @@ if (NOT DEFINED ENV{CLION_IDE} AND NOT DEFINED ENV{XCODE_IDE})
         set(CMAKE_GENERATOR "Ninja" CACHE INTERNAL "" FORCE)
     endif ()
 endif()
+
+
+# Default toolchain - this is needed to avoid dependency on OS files.
+execute_process(COMMAND uname -s OUTPUT_VARIABLE OS)
+execute_process(COMMAND uname -m OUTPUT_VARIABLE ARCH)
+
+if (OS MATCHES "Linux"
+    AND NOT DEFINED CMAKE_TOOLCHAIN_FILE
+    AND NOT DISABLE_HERMETIC_BUILD
+    AND ($ENV{CC} MATCHES ".*clang.*" OR CMAKE_C_COMPILER MATCHES ".*clang.*"))
+
+    if (ARCH MATCHES "amd64|x86_64")
+        set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-x86_64.cmake" CACHE INTERNAL "" FORCE)
+    elseif (ARCH MATCHES "^(aarch64.*|AARCH64.*|arm64.*|ARM64.*)")
+        set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-aarch64.cmake" CACHE INTERNAL "" FORCE)
+    elseif (ARCH MATCHES "^(ppc64le.*|PPC64LE.*)")
+        set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-ppc64le.cmake" CACHE INTERNAL "" FORCE)
+    else ()
+        message (FATAL_ERROR "Unsupported architecture: ${ARCH}")
+    endif ()
+
+endif()

README.md

@@ -1,15 +1,15 @@
-[![ClickHouse — open source distributed column-oriented DBMS](https://github.com/ClickHouse/ClickHouse/raw/master/website/images/logo-400x240.png)](https://clickhouse.tech)
+[![ClickHouse — open source distributed column-oriented DBMS](https://github.com/ClickHouse/ClickHouse/raw/master/website/images/logo-400x240.png)](https://clickhouse.com)
 
-ClickHouse® is an open-source column-oriented database management system that allows generating analytical data reports in real time.
+ClickHouse® is an open-source column-oriented database management system that allows generating analytical data reports in real-time.
 
 ## Useful Links
 
-* [Official website](https://clickhouse.tech/) has quick high-level overview of ClickHouse on main page.
+* [Official website](https://clickhouse.com/) has a quick high-level overview of ClickHouse on the main page.
-* [Tutorial](https://clickhouse.tech/docs/en/getting_started/tutorial/) shows how to set up and query small ClickHouse cluster.
+* [Tutorial](https://clickhouse.com/docs/en/getting_started/tutorial/) shows how to set up and query a small ClickHouse cluster.
-* [Documentation](https://clickhouse.tech/docs/en/) provides more in-depth information.
+* [Documentation](https://clickhouse.com/docs/en/) provides more in-depth information.
 * [YouTube channel](https://www.youtube.com/c/ClickHouseDB) has a lot of content about ClickHouse in video format.
-* [Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-qfort0u8-TWqK4wIP0YSdoDE0btKa1w) and [Telegram](https://telegram.me/clickhouse_en) allow to chat with ClickHouse users in real-time.
+* [Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-rxm3rdrk-lIUmhLC3V8WTaL0TGxsOmg) and [Telegram](https://telegram.me/clickhouse_en) allow chatting with ClickHouse users in real-time.
-* [Blog](https://clickhouse.yandex/blog/en/) contains various ClickHouse-related articles, as well as announcements and reports about events.
+* [Blog](https://clickhouse.com/blog/en/) contains various ClickHouse-related articles, as well as announcements and reports about events.
-* [Code Browser](https://clickhouse.tech/codebrowser/html_report/ClickHouse/index.html) with syntax highlight and navigation.
+* [Code Browser (Woboq)](https://clickhouse.com/codebrowser/ClickHouse/index.html) with syntax highlight and navigation.
-* [Contacts](https://clickhouse.tech/#contacts) can help to get your questions answered if there are any.
+* [Code Browser (github.dev)](https://github.dev/ClickHouse/ClickHouse) with syntax highlight, powered by github.dev.
-* You can also [fill this form](https://clickhouse.tech/#meet) to meet Yandex ClickHouse team in person.
+* [Contacts](https://clickhouse.com/company/#contact) can help to get your questions answered if there are any.

SECURITY.md

@@ -1,28 +1,58 @@
 # Security Policy
 
-## Supported Versions
+## Security Announcements
+Security fixes will be announced by posting them in the [security changelog](https://clickhouse.com/docs/en/whats-new/security-changelog/).
 
-The following versions of ClickHouse server are
+## Scope and Supported Versions
-currently being supported with security updates:
+
+The following versions of ClickHouse server are currently being supported with security updates:
 
 | Version | Supported          |
 | ------- | ------------------ |
 | 1.x   | :x: |
 | 18.x   | :x:                |
 | 19.x   | :x:                |
-| 20.1   | :x: |
+| 20.x   | :x: |
-| 20.3   | :white_check_mark: |
+| 21.1   | :x: |
-| 20.4   | :x: |
+| 21.2   | :x: |
-| 20.5   | :x: |
+| 21.3   | :x: |
-| 20.6   | :x: |
+| 21.4   | :x: |
-| 20.7   | :x: |
+| 21.5   | :x: |
-| 20.8   | :white_check_mark: |
+| 21.6   | :x: |
-| 20.9   | :x: |
+| 21.7   | :x: |
-| 20.10  | :x: |
+| 21.8   | ✅ |
-| 20.11  | :white_check_mark: |
+| 21.9   | :x: |
-| 20.12  | :white_check_mark: |
+| 21.10   | :x: |
-| 21.1   | :white_check_mark: |
+| 21.11   | :x: |
+| 21.12   | :x: |
+| 22.1   | ✅ |
+| 22.2   | ✅ |
+| 22.3   | ✅ |
 
 ## Reporting a Vulnerability
 
-To report a potential vulnerability in ClickHouse please send the details about it to [clickhouse-feedback@yandex-team.com](mailto:clickhouse-feedback@yandex-team.com).
+We're extremely grateful for security researchers and users that report vulnerabilities to the ClickHouse Open Source Community. All reports are thoroughly investigated by developers.
+
+To report a potential vulnerability in ClickHouse please send the details about it to [security@clickhouse.com](mailto:security@clickhouse.com).
+
+### When Should I Report a Vulnerability?
+
+- You think you discovered a potential security vulnerability in ClickHouse
+- You are unsure how a vulnerability affects ClickHouse
+
+### When Should I NOT Report a Vulnerability?
+
+- You need help tuning ClickHouse components for security
+- You need help applying security related updates
+- Your issue is not security related
+
+## Security Vulnerability Response
+
+Each report is acknowledged and analyzed by ClickHouse maintainers within 5 working days.
+As the security issue moves from triage, to identified fix, to release planning we will keep the reporter updated.
+
+## Public Disclosure Timing
+
+A public disclosure date is negotiated by the ClickHouse maintainers and the bug submitter. We prefer to fully disclose the bug as soon as possible once a user mitigation is available. It is reasonable to delay disclosure when the bug or the fix is not yet fully understood, the solution is not well-tested, or for vendor coordination. The timeframe for disclosure is from immediate (especially if it's already publicly known) to 90 days. For a vulnerability with a straightforward mitigation, we expect report date to disclosure date to be on the order of 7 days. 
+
+

base/CMakeLists.txt

@@ -2,14 +2,10 @@ if (USE_CLANG_TIDY)
     set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}")
 endif ()
 
-add_subdirectory (common)
+add_subdirectory (base)
 add_subdirectory (daemon)
 add_subdirectory (loggers)
 add_subdirectory (pcg-random)
 add_subdirectory (widechar_width)
 add_subdirectory (readpassphrase)
 add_subdirectory (bridge)
-
-if (USE_MYSQL)
-    add_subdirectory (mysqlxx)
-endif ()

base/base/BorrowedObjectPool.h

@@ -6,8 +6,8 @@
 #include <mutex>
 #include <condition_variable>
 
-#include <common/defines.h>
+#include <base/defines.h>
-#include <common/MoveOrCopyIfThrow.h>
+#include <base/MoveOrCopyIfThrow.h>
 
 /** Pool for limited size objects that cannot be used from different threads simultaneously.
   * The main use case is to have fixed size of objects that can be reused in difference threads during their lifetime
@@ -88,10 +88,12 @@ public:
     /// Return object into pool. Client must return same object that was borrowed.
     inline void returnObject(T && object_to_return)
     {
-        std::unique_lock<std::mutex> lck(objects_mutex);
+        {
+            std::lock_guard<std::mutex> lock(objects_mutex);
 
-        objects.emplace_back(std::move(object_to_return));
+            objects.emplace_back(std::move(object_to_return));
-        --borrowed_objects_size;
+            --borrowed_objects_size;
+        }
 
         condition_variable.notify_one();
     }

base/base/CMakeLists.txt

@@ -0,0 +1,68 @@
+set (SRCS
+    argsToConfig.cpp
+    coverage.cpp
+    demangle.cpp
+    getAvailableMemoryAmount.cpp
+    getFQDNOrHostName.cpp
+    getMemoryAmount.cpp
+    getPageSize.cpp
+    getThreadId.cpp
+    JSON.cpp
+    LineReader.cpp
+    mremap.cpp
+    phdr_cache.cpp
+    preciseExp10.cpp
+    setTerminalEcho.cpp
+    shift10.cpp
+    sleep.cpp
+    terminalColors.cpp
+    errnoToString.cpp
+    StringRef.cpp
+    safeExit.cpp
+)
+
+if (ENABLE_REPLXX)
+    list (APPEND SRCS ReplxxLineReader.cpp)
+endif ()
+
+if (USE_DEBUG_HELPERS)
+    get_target_property(MAGIC_ENUM_INCLUDE_DIR ch_contrib::magic_enum INTERFACE_INCLUDE_DIRECTORIES)
+    # CMake generator expression will do insane quoting when it encounters special character like quotes, spaces, etc.
+    # Prefixing "SHELL:" will force it to use the original text.
+    set (INCLUDE_DEBUG_HELPERS "SHELL:-I\"${MAGIC_ENUM_INCLUDE_DIR}\" -include \"${ClickHouse_SOURCE_DIR}/base/base/iostream_debug_helpers.h\"")
+    # Use generator expression as we don't want to pollute CMAKE_CXX_FLAGS, which will interfere with CMake check system.
+    add_compile_options($<$<COMPILE_LANGUAGE:CXX>:${INCLUDE_DEBUG_HELPERS}>)
+endif ()
+
+add_library (common ${SRCS})
+
+if (WITH_COVERAGE)
+    target_compile_definitions(common PUBLIC WITH_COVERAGE=1)
+else ()
+    target_compile_definitions(common PUBLIC WITH_COVERAGE=0)
+endif ()
+
+target_include_directories(common PUBLIC .. "${CMAKE_CURRENT_BINARY_DIR}/..")
+
+if (OS_DARWIN AND NOT USE_STATIC_LIBRARIES)
+    target_link_libraries(common PUBLIC -Wl,-U,_inside_main)
+endif()
+
+target_link_libraries (common
+    PUBLIC
+        ch_contrib::cityhash
+        boost::headers_only
+        boost::system
+        Poco::Net
+        Poco::Net::SSL
+        Poco::Util
+        Poco::Foundation
+        ch_contrib::replxx
+        ch_contrib::cctz
+        ch_contrib::fmt
+        ch_contrib::magic_enum
+)
+
+if (ENABLE_TESTS)
+    add_subdirectory (tests)
+endif ()

base/base/CachedFn.h

@@ -0,0 +1,75 @@
+#pragma once
+
+#include <map>
+#include <tuple>
+#include <mutex>
+#include "FnTraits.h"
+
+/**
+ * Caching proxy for a functor that decays to a pointer-to-function.
+ * Saves pairs (func args, func result on args).
+ * Cache size is unlimited. Cache items are evicted only on manual drop.
+ * Invocation/update is O(log(saved cache values)).
+ *
+ * See Common/tests/cached_fn.cpp for examples.
+  */
+template <auto * Func>
+struct CachedFn
+{
+private:
+    using Traits = FnTraits<decltype(Func)>;
+    using DecayedArgs = TypeListMap<std::decay_t, typename Traits::Args>;
+    using Key = TypeListChangeRoot<std::tuple, DecayedArgs>;
+    using Result = typename Traits::Ret;
+
+    std::map<Key, Result> cache; // Can't use hashmap as tuples are unhashable by default
+    mutable std::mutex mutex;
+
+public:
+    template <class ...Args>
+    Result operator()(Args && ...args)
+    {
+        Key key{std::forward<Args>(args)...};
+
+        {
+            std::lock_guard lock(mutex);
+
+            if (auto it = cache.find(key); it != cache.end())
+                return it->second;
+        }
+
+        Result res = std::apply(Func, key);
+
+        {
+            std::lock_guard lock(mutex);
+            cache.emplace(std::move(key), res);
+        }
+
+        return res;
+    }
+
+    template <class ...Args>
+    void update(Args && ...args)
+    {
+        Key key{std::forward<Args>(args)...};
+        Result res = std::apply(Func, key);
+
+        {
+            std::lock_guard lock(mutex);
+            // TODO Can't use emplace(std::move(key), ..), causes test_host_ip_change errors.
+            cache[key] = std::move(res);
+        }
+    }
+
+    size_t size() const
+    {
+        std::lock_guard lock(mutex);
+        return cache.size();
+    }
+
+    void drop()
+    {
+        std::lock_guard lock(mutex);
+        cache.clear();
+    }
+};

base/base/DayNum.h

@@ -1,7 +1,7 @@
 #pragma once
 
-#include <common/types.h>
+#include <base/types.h>
-#include <common/strong_typedef.h>
+#include <base/strong_typedef.h>
 
 /** Represents number of days since 1970-01-01.
   * See DateLUTImpl for usage examples.

base/base/Decimal.h

@@ -0,0 +1,157 @@
+#pragma once
+#include <base/extended_types.h>
+
+#if !defined(NO_SANITIZE_UNDEFINED)
+#if defined(__clang__)
+    #define NO_SANITIZE_UNDEFINED __attribute__((__no_sanitize__("undefined")))
+#else
+    #define NO_SANITIZE_UNDEFINED
+#endif
+#endif
+
+namespace DB
+{
+template <class> struct Decimal;
+class DateTime64;
+
+using Decimal32 = Decimal<Int32>;
+using Decimal64 = Decimal<Int64>;
+using Decimal128 = Decimal<Int128>;
+using Decimal256 = Decimal<Int256>;
+
+template <class T>
+concept is_decimal =
+    std::is_same_v<T, Decimal32>
+    || std::is_same_v<T, Decimal64>
+    || std::is_same_v<T, Decimal128>
+    || std::is_same_v<T, Decimal256>
+    || std::is_same_v<T, DateTime64>;
+
+template <class T>
+concept is_over_big_int =
+    std::is_same_v<T, Int128>
+    || std::is_same_v<T, UInt128>
+    || std::is_same_v<T, Int256>
+    || std::is_same_v<T, UInt256>
+    || std::is_same_v<T, Decimal128>
+    || std::is_same_v<T, Decimal256>;
+
+template <class T> struct NativeTypeT { using Type = T; };
+template <is_decimal T> struct NativeTypeT<T> { using Type = typename T::NativeType; };
+template <class T> using NativeType = typename NativeTypeT<T>::Type;
+
+/// Own FieldType for Decimal.
+/// It is only a "storage" for decimal.
+/// To perform operations, you also have to provide a scale (number of digits after point).
+template <typename T>
+struct Decimal
+{
+    using NativeType = T;
+
+    constexpr Decimal() = default;
+    constexpr Decimal(Decimal<T> &&) = default;
+    constexpr Decimal(const Decimal<T> &) = default;
+
+    constexpr Decimal(const T & value_): value(value_) {}
+
+    template <typename U>
+    constexpr Decimal(const Decimal<U> & x): value(x.value) {}
+
+    constexpr Decimal<T> & operator = (Decimal<T> &&) = default;
+    constexpr Decimal<T> & operator = (const Decimal<T> &) = default;
+
+    constexpr operator T () const { return value; }
+
+    template <typename U>
+    constexpr U convertTo() const
+    {
+        if constexpr (is_decimal<U>)
+            return convertTo<typename U::NativeType>();
+        else
+            return static_cast<U>(value);
+    }
+
+    const Decimal<T> & operator += (const T & x) { value += x; return *this; }
+    const Decimal<T> & operator -= (const T & x) { value -= x; return *this; }
+    const Decimal<T> & operator *= (const T & x) { value *= x; return *this; }
+    const Decimal<T> & operator /= (const T & x) { value /= x; return *this; }
+    const Decimal<T> & operator %= (const T & x) { value %= x; return *this; }
+
+    template <typename U> const Decimal<T> & operator += (const Decimal<U> & x) { value += x.value; return *this; }
+    template <typename U> const Decimal<T> & operator -= (const Decimal<U> & x) { value -= x.value; return *this; }
+    template <typename U> const Decimal<T> & operator *= (const Decimal<U> & x) { value *= x.value; return *this; }
+    template <typename U> const Decimal<T> & operator /= (const Decimal<U> & x) { value /= x.value; return *this; }
+    template <typename U> const Decimal<T> & operator %= (const Decimal<U> & x) { value %= x.value; return *this; }
+
+    /// This is to avoid UB for sumWithOverflow()
+    void NO_SANITIZE_UNDEFINED addOverflow(const T & x) { value += x; }
+
+    T value;
+};
+
+template <typename T> inline bool operator< (const Decimal<T> & x, const Decimal<T> & y) { return x.value < y.value; }
+template <typename T> inline bool operator> (const Decimal<T> & x, const Decimal<T> & y) { return x.value > y.value; }
+template <typename T> inline bool operator<= (const Decimal<T> & x, const Decimal<T> & y) { return x.value <= y.value; }
+template <typename T> inline bool operator>= (const Decimal<T> & x, const Decimal<T> & y) { return x.value >= y.value; }
+template <typename T> inline bool operator== (const Decimal<T> & x, const Decimal<T> & y) { return x.value == y.value; }
+template <typename T> inline bool operator!= (const Decimal<T> & x, const Decimal<T> & y) { return x.value != y.value; }
+
+template <typename T> inline Decimal<T> operator+ (const Decimal<T> & x, const Decimal<T> & y) { return x.value + y.value; }
+template <typename T> inline Decimal<T> operator- (const Decimal<T> & x, const Decimal<T> & y) { return x.value - y.value; }
+template <typename T> inline Decimal<T> operator* (const Decimal<T> & x, const Decimal<T> & y) { return x.value * y.value; }
+template <typename T> inline Decimal<T> operator/ (const Decimal<T> & x, const Decimal<T> & y) { return x.value / y.value; }
+template <typename T> inline Decimal<T> operator- (const Decimal<T> & x) { return -x.value; }
+
+/// Distinguishable type to allow function resolution/deduction based on value type,
+/// but also relatively easy to convert to/from Decimal64.
+class DateTime64 : public Decimal64
+{
+public:
+    using Base = Decimal64;
+    using Base::Base;
+    using NativeType = Base::NativeType;
+
+    constexpr DateTime64(const Base & v): Base(v) {}
+};
+}
+
+constexpr DB::UInt64 max_uint_mask = std::numeric_limits<DB::UInt64>::max();
+
+namespace std
+{
+    template <typename T>
+    struct hash<DB::Decimal<T>>
+    {
+        size_t operator()(const DB::Decimal<T> & x) const { return hash<T>()(x.value); }
+    };
+
+    template <>
+    struct hash<DB::Decimal128>
+    {
+        size_t operator()(const DB::Decimal128 & x) const
+        {
+            return std::hash<DB::Int64>()(x.value >> 64)
+                ^ std::hash<DB::Int64>()(x.value & max_uint_mask);
+        }
+    };
+
+    template <>
+    struct hash<DB::DateTime64>
+    {
+        size_t operator()(const DB::DateTime64 & x) const
+        {
+            return std::hash<DB::DateTime64::NativeType>()(x);
+        }
+    };
+
+    template <>
+    struct hash<DB::Decimal256>
+    {
+        size_t operator()(const DB::Decimal256 & x) const
+        {
+            // FIXME temp solution
+            return std::hash<DB::Int64>()(static_cast<DB::Int64>(x.value >> 64 & max_uint_mask))
+                ^ std::hash<DB::Int64>()(static_cast<DB::Int64>(x.value & max_uint_mask));
+        }
+    };
+}

base/base/DecomposedFloat.h

@@ -3,7 +3,7 @@
 #include <cstdint>
 #include <cstddef>
 #include <cstring>
-#include <common/extended_types.h>
+#include <base/extended_types.h>
 
 
 /// Allows to check the internals of IEEE-754 floating point number.
@@ -91,10 +91,12 @@ struct DecomposedFloat
 
 
     /// Compare float with integer of arbitrary width (both signed and unsigned are supported). Assuming two's complement arithmetic.
+    /// This function is generic, big integers (128, 256 bit) are supported as well.
     /// Infinities are compared correctly. NaNs are treat similarly to infinities, so they can be less than all numbers.
     /// (note that we need total order)
+    /// Returns -1, 0 or 1.
     template <typename Int>
-    int compare(Int rhs)
+    int compare(Int rhs) const
     {
         if (rhs == 0)
             return sign();
@@ -137,10 +139,11 @@ struct DecomposedFloat
         if (normalized_exponent() >= static_cast<int16_t>(8 * sizeof(Int) - is_signed_v<Int>))
             return is_negative() ? -1 : 1;
 
-        using UInt = make_unsigned_t<Int>;
+        using UInt = std::conditional_t<(sizeof(Int) > sizeof(typename Traits::UInt)), make_unsigned_t<Int>, typename Traits::UInt>;
         UInt uint_rhs = rhs < 0 ? -rhs : rhs;
 
         /// Smaller octave: abs(rhs) < abs(float)
+        /// FYI, TIL: octave is also called "binade", https://en.wikipedia.org/wiki/Binade
         if (uint_rhs < (static_cast<UInt>(1) << normalized_exponent()))
             return is_negative() ? -1 : 1;
 
@@ -154,11 +157,11 @@ struct DecomposedFloat
 
         bool large_and_always_integer = normalized_exponent() >= static_cast<int16_t>(Traits::mantissa_bits);
 
-        typename Traits::UInt a = large_and_always_integer
+        UInt a = large_and_always_integer
-            ? mantissa() << (normalized_exponent() - Traits::mantissa_bits)
+            ? static_cast<UInt>(mantissa()) << (normalized_exponent() - Traits::mantissa_bits)
-            : mantissa() >> (Traits::mantissa_bits - normalized_exponent());
+            : static_cast<UInt>(mantissa()) >> (Traits::mantissa_bits - normalized_exponent());
 
-        typename Traits::UInt b = uint_rhs - (static_cast<UInt>(1) << normalized_exponent());
+        UInt b = uint_rhs - (static_cast<UInt>(1) << normalized_exponent());
 
         if (a < b)
             return is_negative() ? 1 : -1;
@@ -175,37 +178,37 @@ struct DecomposedFloat
 
 
     template <typename Int>
-    bool equals(Int rhs)
+    bool equals(Int rhs) const
     {
         return compare(rhs) == 0;
     }
 
     template <typename Int>
-    bool notEquals(Int rhs)
+    bool notEquals(Int rhs) const
     {
         return compare(rhs) != 0;
     }
 
     template <typename Int>
-    bool less(Int rhs)
+    bool less(Int rhs) const
     {
         return compare(rhs) < 0;
     }
 
     template <typename Int>
-    bool greater(Int rhs)
+    bool greater(Int rhs) const
     {
         return compare(rhs) > 0;
     }
 
     template <typename Int>
-    bool lessOrEquals(Int rhs)
+    bool lessOrEquals(Int rhs) const
     {
         return compare(rhs) <= 0;
     }
 
     template <typename Int>
-    bool greaterOrEquals(Int rhs)
+    bool greaterOrEquals(Int rhs) const
     {
         return compare(rhs) >= 0;
     }

base/base/EnumReflection.h

@@ -0,0 +1,38 @@
+#pragma once
+
+#include <magic_enum.hpp>
+#include <fmt/format.h>
+
+template <class T> concept is_enum = std::is_enum_v<T>;
+
+namespace detail
+{
+template <is_enum E, class F, size_t ...I>
+constexpr void static_for(F && f, std::index_sequence<I...>)
+{
+    (std::forward<F>(f)(std::integral_constant<E, magic_enum::enum_value<E>(I)>()) , ...);
+}
+}
+
+/**
+ * Iterate over enum values in compile-time (compile-time switch/case, loop unrolling).
+ *
+ * @example static_for<E>([](auto enum_value) { return template_func<enum_value>(); }
+ * ^ enum_value can be used as a template parameter
+ */
+template <is_enum E, class F>
+constexpr void static_for(F && f)
+{
+    constexpr size_t count = magic_enum::enum_count<E>();
+    detail::static_for<E>(std::forward<F>(f), std::make_index_sequence<count>());
+}
+
+/// Enable printing enum values as strings via fmt + magic_enum
+template <is_enum T>
+struct fmt::formatter<T> : fmt::formatter<std::string_view>
+{
+    constexpr auto format(T value, auto& format_context)
+    {
+        return formatter<string_view>::format(magic_enum::enum_name(value), format_context);
+    }
+};

base/base/ErrorHandlers.h

@@ -1,7 +1,7 @@
 #pragma once
 
 #include <Poco/ErrorHandler.h>
-#include <common/logger_useful.h>
+#include <base/logger_useful.h>
 #include <Common/Exception.h>
 
 

base/base/FnTraits.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#include "TypeList.h"
+
+namespace detail
+{
+template <class T>
+struct FnTraits { template <class> static constexpr bool value = false; };
+
+template <class R, class ...A>
+struct FnTraits<R(A...)>
+{
+    template <class F>
+    static constexpr bool value = std::is_invocable_r_v<R, F, A...>;
+
+    using Ret = R;
+    using Args = TypeList<A...>;
+};
+
+template <class R, class ...A>
+struct FnTraits<R(*)(A...)> : FnTraits<R(A...)> {};
+}
+
+template <class T> using FnTraits = detail::FnTraits<T>;
+
+/**
+ * A less-typing alias for std::is_invokable_r_v.
+ * @example void foo(Fn<bool(int, char)> auto && functor)
+ */
+template <class F, class FS>
+concept Fn = FnTraits<FS>::template value<F>;
+
+template <auto Value>
+using Constant = std::integral_constant<decltype(Value), Value>;

base/base/FunctorToStaticMethodAdaptor.h

@@ -0,0 +1,41 @@
+#include <functional>
+
+/** Adapt functor to static method where functor passed as context.
+  * Main use case to convert lambda into function that can be passed into JIT code.
+  */
+template <typename Functor>
+class FunctorToStaticMethodAdaptor : public FunctorToStaticMethodAdaptor<decltype(&Functor::operator())>
+{
+};
+
+template <typename R, typename C, typename ...Args>
+class FunctorToStaticMethodAdaptor<R (C::*)(Args...) const>
+{
+public:
+    static R call(C * ptr, Args &&... arguments)
+    {
+        return std::invoke(&C::operator(), ptr, std::forward<Args>(arguments)...);
+    }
+
+    static R unsafeCall(char * ptr, Args &&... arguments)
+    {
+        C * ptr_typed = reinterpret_cast<C*>(ptr);
+        return std::invoke(&C::operator(), ptr_typed, std::forward<Args>(arguments)...);
+    }
+};
+
+template <typename R, typename C, typename ...Args>
+class FunctorToStaticMethodAdaptor<R (C::*)(Args...)>
+{
+public:
+    static R call(C * ptr, Args &&... arguments)
+    {
+        return std::invoke(&C::operator(), ptr, std::forward<Args>(arguments)...);
+    }
+
+    static R unsafeCall(char * ptr, Args &&... arguments)
+    {
+        C * ptr_typed = static_cast<C*>(ptr);
+        return std::invoke(&C::operator(), ptr_typed, std::forward<Args>(arguments)...);
+    }
+};

base/base/JSON.cpp

@@ -3,9 +3,9 @@
 
 #include <Poco/UTF8Encoding.h>
 #include <Poco/NumberParser.h>
-#include <common/JSON.h>
+#include <base/JSON.h>
-#include <common/find_symbols.h>
+#include <base/find_symbols.h>
-#include <common/preciseExp10.h>
+#include <base/preciseExp10.h>
 
 #include <iostream>
 

base/base/JSON.h

@@ -2,8 +2,8 @@
 
 #include <typeinfo>
 #include <Poco/Exception.h>
-#include <common/StringRef.h>
+#include <base/StringRef.h>
-#include <common/types.h>
+#include <base/types.h>
 
 
 /** Очень простой класс для чтения JSON (или его кусочков).

base/base/LineReader.cpp

@@ -0,0 +1,204 @@
+#include <base/LineReader.h>
+
+#include <iostream>
+#include <string_view>
+#include <algorithm>
+
+#include <cassert>
+#include <string.h>
+#include <unistd.h>
+#include <sys/select.h>
+#include <sys/time.h>
+#include <sys/types.h>
+
+
+#ifdef HAS_RESERVED_IDENTIFIER
+#pragma clang diagnostic ignored "-Wreserved-identifier"
+#endif
+
+namespace
+{
+
+/// Trim ending whitespace inplace
+void trim(String & s)
+{
+    s.erase(std::find_if(s.rbegin(), s.rend(), [](int ch) { return !std::isspace(ch); }).base(), s.end());
+}
+
+/// Check if multi-line query is inserted from the paste buffer.
+/// Allows delaying the start of query execution until the entirety of query is inserted.
+bool hasInputData()
+{
+    timeval timeout = {0, 0};
+    fd_set fds{};
+    FD_ZERO(&fds);
+    FD_SET(STDIN_FILENO, &fds);
+    return select(1, &fds, nullptr, nullptr, &timeout) == 1;
+}
+
+struct NoCaseCompare
+{
+    bool operator()(const std::string & str1, const std::string & str2)
+    {
+        return std::lexicographical_compare(begin(str1), end(str1), begin(str2), end(str2), [](const char c1, const char c2)
+        {
+            return std::tolower(c1) < std::tolower(c2);
+        });
+    }
+};
+
+using Words = std::vector<std::string>;
+template <class Compare>
+void addNewWords(Words & to, const Words & from, Compare comp)
+{
+    size_t old_size = to.size();
+    size_t new_size = old_size + from.size();
+
+    to.reserve(new_size);
+    to.insert(to.end(), from.begin(), from.end());
+    auto middle = to.begin() + old_size;
+    std::inplace_merge(to.begin(), middle, to.end(), comp);
+
+    auto last_unique = std::unique(to.begin(), to.end());
+    to.erase(last_unique, to.end());
+}
+
+}
+
+replxx::Replxx::completions_t LineReader::Suggest::getCompletions(const String & prefix, size_t prefix_length)
+{
+    std::string_view last_word;
+
+    auto last_word_pos = prefix.find_last_of(word_break_characters);
+    if (std::string::npos == last_word_pos)
+        last_word = prefix;
+    else
+        last_word = std::string_view(prefix).substr(last_word_pos + 1, std::string::npos);
+    /// last_word can be empty.
+
+    std::pair<Words::const_iterator, Words::const_iterator> range;
+
+    std::lock_guard lock(mutex);
+
+    /// Only perform case sensitive completion when the prefix string contains any uppercase characters
+    if (std::none_of(prefix.begin(), prefix.end(), [](char32_t x) { return iswupper(static_cast<wint_t>(x)); }))
+        range = std::equal_range(
+            words_no_case.begin(), words_no_case.end(), last_word, [prefix_length](std::string_view s, std::string_view prefix_searched)
+            {
+                return strncasecmp(s.data(), prefix_searched.data(), prefix_length) < 0;
+            });
+    else
+        range = std::equal_range(words.begin(), words.end(), last_word, [prefix_length](std::string_view s, std::string_view prefix_searched)
+        {
+            return strncmp(s.data(), prefix_searched.data(), prefix_length) < 0;
+        });
+
+    return replxx::Replxx::completions_t(range.first, range.second);
+}
+
+void LineReader::Suggest::addWords(Words && new_words)
+{
+    Words new_words_no_case = new_words;
+    if (!new_words.empty())
+    {
+        std::sort(new_words.begin(), new_words.end());
+        std::sort(new_words_no_case.begin(), new_words_no_case.end(), NoCaseCompare{});
+    }
+
+    {
+        std::lock_guard lock(mutex);
+        addNewWords(words, new_words, std::less<std::string>{});
+        addNewWords(words_no_case, new_words_no_case, NoCaseCompare{});
+
+        assert(std::is_sorted(words.begin(), words.end()));
+        assert(std::is_sorted(words_no_case.begin(), words_no_case.end(), NoCaseCompare{}));
+    }
+}
+
+LineReader::LineReader(const String & history_file_path_, bool multiline_, Patterns extenders_, Patterns delimiters_)
+    : history_file_path(history_file_path_), multiline(multiline_), extenders(std::move(extenders_)), delimiters(std::move(delimiters_))
+{
+    /// FIXME: check extender != delimiter
+}
+
+String LineReader::readLine(const String & first_prompt, const String & second_prompt)
+{
+    String line;
+    bool need_next_line = false;
+
+    while (auto status = readOneLine(need_next_line ? second_prompt : first_prompt))
+    {
+        if (status == RESET_LINE)
+        {
+            line.clear();
+            need_next_line = false;
+            continue;
+        }
+
+        if (input.empty())
+        {
+            if (!line.empty() && !multiline && !hasInputData())
+                break;
+            else
+                continue;
+        }
+
+        const char * has_extender = nullptr;
+        for (const auto * extender : extenders)
+        {
+            if (input.ends_with(extender))
+            {
+                has_extender = extender;
+                break;
+            }
+        }
+
+        const char * has_delimiter = nullptr;
+        for (const auto * delimiter : delimiters)
+        {
+            if (input.ends_with(delimiter))
+            {
+                has_delimiter = delimiter;
+                break;
+            }
+        }
+
+        need_next_line = has_extender || (multiline && !has_delimiter) || hasInputData();
+
+        if (has_extender)
+        {
+            input.resize(input.size() - strlen(has_extender));
+            trim(input);
+            if (input.empty())
+                continue;
+        }
+
+        line += (line.empty() ? "" : "\n") + input;
+
+        if (!need_next_line)
+            break;
+    }
+
+    if (!line.empty() && line != prev_line)
+    {
+        addToHistory(line);
+        prev_line = line;
+    }
+
+    return line;
+}
+
+LineReader::InputStatus LineReader::readOneLine(const String & prompt)
+{
+    input.clear();
+
+    {
+        std::cout << prompt;
+        std::getline(std::cin, input);
+        if (!std::cin.good())
+            return ABORT;
+    }
+
+    trim(input);
+    return INPUT_LINE;
+}

base/base/LineReader.h

@@ -1,10 +1,12 @@
 #pragma once
 
-#include <common/types.h>
+#include <mutex>
-
 #include <atomic>
 #include <vector>
 #include <optional>
+#include <replxx.hxx>
+
+#include <base/types.h>
 
 class LineReader
 {
@@ -12,14 +14,16 @@ public:
     struct Suggest
     {
         using Words = std::vector<std::string>;
-        using WordsRange = std::pair<Words::const_iterator, Words::const_iterator>;
 
+        /// Get vector for the matched range of words if any.
+        replxx::Replxx::completions_t getCompletions(const String & prefix, size_t prefix_length);
+        void addWords(Words && new_words);
+
+    private:
         Words words;
         Words words_no_case;
-        std::atomic<bool> ready{false};
 
-        /// Get iterators for the matched range of words if any.
+        std::mutex mutex;
-        std::optional<WordsRange> getCompletions(const String & prefix, size_t prefix_length) const;
     };
 
     using Patterns = std::vector<const char *>;
@@ -53,7 +57,6 @@ protected:
 
     String input;
 
-private:
     bool multiline;
 
     Patterns extenders;

base/base/MoveOrCopyIfThrow.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include <common/types.h>
+#include <base/types.h>
 
 namespace detail
 {

base/base/ReplxxLineReader.cpp

@@ -0,0 +1,381 @@
+#include <base/ReplxxLineReader.h>
+#include <base/errnoToString.h>
+
+#include <chrono>
+#include <cerrno>
+#include <cstring>
+#include <unistd.h>
+#include <functional>
+#include <sys/file.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <csignal>
+#include <dlfcn.h>
+#include <fcntl.h>
+#include <fstream>
+#include <fmt/format.h>
+
+
+namespace
+{
+
+/// Trim ending whitespace inplace
+void trim(String & s)
+{
+    s.erase(std::find_if(s.rbegin(), s.rend(), [](unsigned char ch) { return !std::isspace(ch); }).base(), s.end());
+}
+
+std::string getEditor()
+{
+    const char * editor = std::getenv("EDITOR");
+
+    if (!editor || !*editor)
+        editor = "vim";
+
+    return editor;
+}
+
+/// Copied from replxx::src/util.cxx::now_ms_str() under the terms of 3-clause BSD license of Replxx.
+/// Copyright (c) 2017-2018, Marcin Konarski (amok at codestation.org)
+/// Copyright (c) 2010, Salvatore Sanfilippo (antirez at gmail dot com)
+/// Copyright (c) 2010, Pieter Noordhuis (pcnoordhuis at gmail dot com)
+std::string replxx_now_ms_str()
+{
+    std::chrono::milliseconds ms(std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()));
+    time_t t = ms.count() / 1000;
+    tm broken;
+    if (!localtime_r(&t, &broken))
+    {
+        return std::string();
+    }
+
+    static int const BUFF_SIZE(32);
+    char str[BUFF_SIZE];
+    strftime(str, BUFF_SIZE, "%Y-%m-%d %H:%M:%S.", &broken);
+    snprintf(str + sizeof("YYYY-mm-dd HH:MM:SS"), 5, "%03d", static_cast<int>(ms.count() % 1000));
+    return str;
+}
+
+/// Convert from readline to replxx format.
+///
+/// replxx requires each history line to prepended with time line:
+///
+///     ### YYYY-MM-DD HH:MM:SS.SSS
+///     select 1
+///
+/// And w/o those service lines it will load all lines from history file as
+/// one history line for suggestion. And if there are lots of lines in file it
+/// will take lots of time (getline() + tons of reallocations).
+///
+/// NOTE: this code uses std::ifstream/std::ofstream like original replxx code.
+void convertHistoryFile(const std::string & path, replxx::Replxx & rx)
+{
+    std::ifstream in(path);
+    if (!in)
+    {
+        rx.print("Cannot open %s reading (for conversion): %s\n",
+            path.c_str(), errnoToString(errno).c_str());
+        return;
+    }
+
+    std::string line;
+    if (getline(in, line).bad())
+    {
+        rx.print("Cannot read from %s (for conversion): %s\n",
+            path.c_str(), errnoToString(errno).c_str());
+        return;
+    }
+
+    /// This is the marker of the date, no need to convert.
+    static char const REPLXX_TIMESTAMP_PATTERN[] = "### dddd-dd-dd dd:dd:dd.ddd";
+    if (line.empty() || (line.starts_with("### ") && line.size() == strlen(REPLXX_TIMESTAMP_PATTERN)))
+    {
+        return;
+    }
+
+    std::vector<std::string> lines;
+    in.seekg(0);
+    while (getline(in, line).good())
+    {
+        lines.push_back(line);
+    }
+    in.close();
+
+    size_t lines_size = lines.size();
+    std::sort(lines.begin(), lines.end());
+    lines.erase(std::unique(lines.begin(), lines.end()), lines.end());
+    rx.print("The history file (%s) is in old format. %zu lines, %zu unique lines.\n",
+        path.c_str(), lines_size, lines.size());
+
+    std::ofstream out(path);
+    if (!out)
+    {
+        rx.print("Cannot open %s for writing (for conversion): %s\n",
+            path.c_str(), errnoToString(errno).c_str());
+        return;
+    }
+
+    const std::string & timestamp = replxx_now_ms_str();
+    for (const auto & out_line : lines)
+    {
+        out << "### " << timestamp << "\n" << out_line << std::endl;
+    }
+    out.close();
+}
+
+}
+
+static bool replxx_last_is_delimiter = false;
+void ReplxxLineReader::setLastIsDelimiter(bool flag)
+{
+    replxx_last_is_delimiter = flag;
+}
+
+ReplxxLineReader::ReplxxLineReader(
+    Suggest & suggest,
+    const String & history_file_path_,
+    bool multiline_,
+    Patterns extenders_,
+    Patterns delimiters_,
+    replxx::Replxx::highlighter_callback_t highlighter_)
+    : LineReader(history_file_path_, multiline_, std::move(extenders_), std::move(delimiters_)), highlighter(std::move(highlighter_))
+    , editor(getEditor())
+{
+    using namespace std::placeholders;
+    using Replxx = replxx::Replxx;
+
+    if (!history_file_path.empty())
+    {
+        history_file_fd = open(history_file_path.c_str(), O_RDWR);
+        if (history_file_fd < 0)
+        {
+            rx.print("Open of history file failed: %s\n", errnoToString(errno).c_str());
+        }
+        else
+        {
+            convertHistoryFile(history_file_path, rx);
+
+            if (flock(history_file_fd, LOCK_SH))
+            {
+                rx.print("Shared lock of history file failed: %s\n", errnoToString(errno).c_str());
+            }
+            else
+            {
+                if (!rx.history_load(history_file_path))
+                {
+                    rx.print("Loading history failed: %s\n", errnoToString(errno).c_str());
+                }
+
+                if (flock(history_file_fd, LOCK_UN))
+                {
+                    rx.print("Unlock of history file failed: %s\n", errnoToString(errno).c_str());
+                }
+            }
+        }
+    }
+
+    rx.install_window_change_handler();
+
+    auto callback = [&suggest] (const String & context, size_t context_size)
+    {
+        return suggest.getCompletions(context, context_size);
+    };
+
+    rx.set_completion_callback(callback);
+    rx.set_complete_on_empty(false);
+    rx.set_word_break_characters(word_break_characters);
+    rx.set_ignore_case(true);
+
+    if (highlighter)
+        rx.set_highlighter_callback(highlighter);
+
+    /// By default C-p/C-n binded to COMPLETE_NEXT/COMPLETE_PREV,
+    /// bind C-p/C-n to history-previous/history-next like readline.
+    rx.bind_key(Replxx::KEY::control('N'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::HISTORY_NEXT, code); });
+    rx.bind_key(Replxx::KEY::control('P'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::HISTORY_PREVIOUS, code); });
+
+    auto commit_action = [this](char32_t code)
+    {
+        /// If we allow multiline and there is already something in the input, start a newline.
+        /// NOTE: Lexer is only available if we use highlighter.
+        if (highlighter && multiline && !replxx_last_is_delimiter)
+            return rx.invoke(Replxx::ACTION::NEW_LINE, code);
+        replxx_last_is_delimiter = false;
+        return rx.invoke(Replxx::ACTION::COMMIT_LINE, code);
+    };
+    /// bind C-j to ENTER action.
+    rx.bind_key(Replxx::KEY::control('J'), commit_action);
+    rx.bind_key(Replxx::KEY::ENTER, commit_action);
+
+    /// By default COMPLETE_NEXT/COMPLETE_PREV was binded to C-p/C-n, re-bind
+    /// to M-P/M-N (that was used for HISTORY_COMMON_PREFIX_SEARCH before, but
+    /// it also binded to M-p/M-n).
+    rx.bind_key(Replxx::KEY::meta('N'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::COMPLETE_NEXT, code); });
+    rx.bind_key(Replxx::KEY::meta('P'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::COMPLETE_PREVIOUS, code); });
+    /// By default M-BACKSPACE is KILL_TO_WHITESPACE_ON_LEFT, while in readline it is backward-kill-word
+    rx.bind_key(Replxx::KEY::meta(Replxx::KEY::BACKSPACE), [this](char32_t code) { return rx.invoke(Replxx::ACTION::KILL_TO_BEGINING_OF_WORD, code); });
+    /// By default C-w is KILL_TO_BEGINING_OF_WORD, while in readline it is unix-word-rubout
+    rx.bind_key(Replxx::KEY::control('W'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::KILL_TO_WHITESPACE_ON_LEFT, code); });
+
+    rx.bind_key(Replxx::KEY::meta('E'), [this](char32_t) { openEditor(); return Replxx::ACTION_RESULT::CONTINUE; });
+}
+
+ReplxxLineReader::~ReplxxLineReader()
+{
+    if (close(history_file_fd))
+        rx.print("Close of history file failed: %s\n", errnoToString(errno).c_str());
+}
+
+LineReader::InputStatus ReplxxLineReader::readOneLine(const String & prompt)
+{
+    input.clear();
+
+    const char* cinput = rx.input(prompt);
+    if (cinput == nullptr)
+        return (errno != EAGAIN) ? ABORT : RESET_LINE;
+    input = cinput;
+
+    trim(input);
+    return INPUT_LINE;
+}
+
+void ReplxxLineReader::addToHistory(const String & line)
+{
+    // locking history file to prevent from inconsistent concurrent changes
+    //
+    // replxx::Replxx::history_save() already has lockf(),
+    // but replxx::Replxx::history_load() does not
+    // and that is why flock() is added here.
+    bool locked = false;
+    if (flock(history_file_fd, LOCK_EX))
+        rx.print("Lock of history file failed: %s\n", errnoToString(errno).c_str());
+    else
+        locked = true;
+
+    rx.history_add(line);
+
+    // flush changes to the disk
+    if (!rx.history_save(history_file_path))
+        rx.print("Saving history failed: %s\n", errnoToString(errno).c_str());
+
+    if (locked && 0 != flock(history_file_fd, LOCK_UN))
+        rx.print("Unlock of history file failed: %s\n", errnoToString(errno).c_str());
+}
+
+/// See comments in ShellCommand::executeImpl()
+/// (for the vfork via dlsym())
+int ReplxxLineReader::executeEditor(const std::string & path)
+{
+    std::vector<char> argv0(editor.data(), editor.data() + editor.size() + 1);
+    std::vector<char> argv1(path.data(), path.data() + path.size() + 1);
+    char * const argv[] = {argv0.data(), argv1.data(), nullptr};
+
+    static void * real_vfork = dlsym(RTLD_DEFAULT, "vfork");
+    if (!real_vfork)
+    {
+        rx.print("Cannot find symbol vfork in myself: %s\n", errnoToString(errno).c_str());
+        return -1;
+    }
+
+    pid_t pid = reinterpret_cast<pid_t (*)()>(real_vfork)();
+
+    if (-1 == pid)
+    {
+        rx.print("Cannot vfork: %s\n", errnoToString(errno).c_str());
+        return -1;
+    }
+
+    /// Child
+    if (0 == pid)
+    {
+        sigset_t mask;
+        sigemptyset(&mask);
+        sigprocmask(0, nullptr, &mask);
+        sigprocmask(SIG_UNBLOCK, &mask, nullptr);
+
+        execvp(editor.c_str(), argv);
+        rx.print("Cannot execute %s: %s\n", editor.c_str(), errnoToString(errno).c_str());
+        _exit(-1);
+    }
+
+    int status = 0;
+    do
+    {
+        int exited_pid = waitpid(pid, &status, 0);
+        if (exited_pid == -1)
+        {
+            if (errno == EINTR)
+                continue;
+
+            rx.print("Cannot waitpid: %s\n", errnoToString(errno).c_str());
+            return -1;
+        }
+        else
+            break;
+    } while (true);
+    return status;
+}
+
+void ReplxxLineReader::openEditor()
+{
+    char filename[] = "clickhouse_replxx_XXXXXX.sql";
+    int fd = ::mkstemps(filename, 4);
+    if (-1 == fd)
+    {
+        rx.print("Cannot create temporary file to edit query: %s\n", errnoToString(errno).c_str());
+        return;
+    }
+
+    replxx::Replxx::State state(rx.get_state());
+
+    size_t bytes_written = 0;
+    const char * begin = state.text();
+    size_t offset = strlen(state.text());
+    while (bytes_written != offset)
+    {
+        ssize_t res = ::write(fd, begin + bytes_written, offset - bytes_written);
+        if ((-1 == res || 0 == res) && errno != EINTR)
+        {
+            rx.print("Cannot write to temporary query file %s: %s\n", filename, errnoToString(errno).c_str());
+            break;
+        }
+        bytes_written += res;
+    }
+
+    if (0 != ::close(fd))
+    {
+        rx.print("Cannot close temporary query file %s: %s\n", filename, errnoToString(errno).c_str());
+        return;
+    }
+
+    if (0 == executeEditor(filename))
+    {
+        try
+        {
+            std::ifstream t(filename);
+            std::string str;
+            t.seekg(0, std::ios::end);
+            str.reserve(t.tellg());
+            t.seekg(0, std::ios::beg);
+            str.assign((std::istreambuf_iterator<char>(t)), std::istreambuf_iterator<char>());
+            rx.set_state(replxx::Replxx::State(str.c_str(), str.size()));
+        }
+        catch (...)
+        {
+            rx.print("Cannot read from temporary query file %s: %s\n", filename, errnoToString(errno).c_str());
+            return;
+        }
+    }
+
+    if (bracketed_paste_enabled)
+        enableBracketedPaste();
+
+    if (0 != ::unlink(filename))
+        rx.print("Cannot remove temporary query file %s: %s\n", filename, errnoToString(errno).c_str());
+}
+
+void ReplxxLineReader::enableBracketedPaste()
+{
+    bracketed_paste_enabled = true;
+    rx.enable_bracketed_paste();
+};

base/base/ReplxxLineReader.h

@@ -0,0 +1,39 @@
+#pragma once
+
+#include "LineReader.h"
+
+#include <replxx.hxx>
+
+
+class ReplxxLineReader : public LineReader
+{
+public:
+    ReplxxLineReader(
+        Suggest & suggest,
+        const String & history_file_path,
+        bool multiline,
+        Patterns extenders_,
+        Patterns delimiters_,
+        replxx::Replxx::highlighter_callback_t highlighter_);
+    ~ReplxxLineReader() override;
+
+    void enableBracketedPaste() override;
+
+    /// If highlight is on, we will set a flag to denote whether the last token is a delimiter.
+    /// This is useful to determine the behavior of <ENTER> key when multiline is enabled.
+    static void setLastIsDelimiter(bool flag);
+private:
+    InputStatus readOneLine(const String & prompt) override;
+    void addToHistory(const String & line) override;
+    int executeEditor(const std::string & path);
+    void openEditor();
+
+    replxx::Replxx rx;
+    replxx::Replxx::highlighter_callback_t highlighter;
+
+    // used to call flock() to synchronize multiple clients using same history file
+    int history_file_fd = -1;
+    bool bracketed_paste_enabled = false;
+
+    std::string editor;
+};

base/base/StringRef.h

@@ -7,8 +7,8 @@
 #include <functional>
 #include <iosfwd>
 
-#include <common/types.h>
+#include <base/types.h>
-#include <common/unaligned.h>
+#include <base/unaligned.h>
 
 #include <city.h>
 
@@ -19,6 +19,12 @@
 #if defined(__SSE4_2__)
     #include <smmintrin.h>
     #include <nmmintrin.h>
+    #define CRC_INT _mm_crc32_u64
+#endif
+
+#if defined(__aarch64__) && defined(__ARM_FEATURE_CRC32)
+    #include <arm_acle.h>
+    #define CRC_INT __crc32cd
 #endif
 
 
@@ -40,15 +46,17 @@ struct StringRef
 
     constexpr StringRef(const char * data_, size_t size_) : data(data_), size(size_) {}
 
-    StringRef(const std::string & s) : data(s.data()), size(s.size()) {}
+    StringRef(const std::string & s) : data(s.data()), size(s.size()) {} /// NOLINT
     constexpr explicit StringRef(std::string_view s) : data(s.data()), size(s.size()) {}
-    constexpr StringRef(const char * data_) : StringRef(std::string_view{data_}) {}
+    constexpr StringRef(const char * data_) : StringRef(std::string_view{data_}) {} /// NOLINT
     constexpr StringRef() = default;
 
     std::string toString() const { return std::string(data, size); }
 
     explicit operator std::string() const { return toString(); }
-    constexpr explicit operator std::string_view() const { return {data, size}; }
+    std::string_view toView() const { return std::string_view(data, size); }
+
+    constexpr explicit operator std::string_view() const { return std::string_view(data, size); }
 };
 
 /// Here constexpr doesn't implicate inline, see https://www.viva64.com/en/w/v1043/
@@ -96,6 +104,39 @@ inline bool compareSSE2x4(const char * p1, const char * p2)
 
 inline bool memequalSSE2Wide(const char * p1, const char * p2, size_t size)
 {
+    /** The order of branches and the trick with overlapping comparisons
+      * are the same as in memcpy implementation.
+      * See the comments in base/glibc-compatibility/memcpy/memcpy.h
+      */
+
+    if (size <= 16)
+    {
+        if (size >= 8)
+        {
+            /// Chunks of 8..16 bytes.
+            return unalignedLoad<uint64_t>(p1) == unalignedLoad<uint64_t>(p2)
+                && unalignedLoad<uint64_t>(p1 + size - 8) == unalignedLoad<uint64_t>(p2 + size - 8);
+        }
+        else if (size >= 4)
+        {
+            /// Chunks of 4..7 bytes.
+            return unalignedLoad<uint32_t>(p1) == unalignedLoad<uint32_t>(p2)
+                && unalignedLoad<uint32_t>(p1 + size - 4) == unalignedLoad<uint32_t>(p2 + size - 4);
+        }
+        else if (size >= 2)
+        {
+            /// Chunks of 2..3 bytes.
+            return unalignedLoad<uint16_t>(p1) == unalignedLoad<uint16_t>(p2)
+                && unalignedLoad<uint16_t>(p1 + size - 2) == unalignedLoad<uint16_t>(p2 + size - 2);
+        }
+        else if (size >= 1)
+        {
+            /// A single byte.
+            return *p1 == *p2;
+        }
+        return true;
+    }
+
     while (size >= 64)
     {
         if (compareSSE2x4(p1, p2))
@@ -108,39 +149,14 @@ inline bool memequalSSE2Wide(const char * p1, const char * p2, size_t size)
             return false;
     }
 
-    switch ((size % 64) / 16)
+    switch (size / 16)
     {
         case 3: if (!compareSSE2(p1 + 32, p2 + 32)) return false; [[fallthrough]];
         case 2: if (!compareSSE2(p1 + 16, p2 + 16)) return false; [[fallthrough]];
-        case 1: if (!compareSSE2(p1     , p2     )) return false; [[fallthrough]];
+        case 1: if (!compareSSE2(p1, p2)) return false;
-        case 0: break;
     }
 
-    p1 += (size % 64) / 16 * 16;
+    return compareSSE2(p1 + size - 16, p2 + size - 16);
-    p2 += (size % 64) / 16 * 16;
-
-    switch (size % 16)
-    {
-        case 15: if (p1[14] != p2[14]) return false; [[fallthrough]];
-        case 14: if (p1[13] != p2[13]) return false; [[fallthrough]];
-        case 13: if (p1[12] != p2[12]) return false; [[fallthrough]];
-        case 12: if (unalignedLoad<uint32_t>(p1 + 8) == unalignedLoad<uint32_t>(p2 + 8)) goto l8; else return false;
-        case 11: if (p1[10] != p2[10]) return false; [[fallthrough]];
-        case 10: if (p1[9] != p2[9]) return false; [[fallthrough]];
-        case 9:  if (p1[8] != p2[8]) return false;
-        l8: [[fallthrough]];
-        case 8:  return unalignedLoad<uint64_t>(p1) == unalignedLoad<uint64_t>(p2);
-        case 7:  if (p1[6] != p2[6]) return false; [[fallthrough]];
-        case 6:  if (p1[5] != p2[5]) return false; [[fallthrough]];
-        case 5:  if (p1[4] != p2[4]) return false; [[fallthrough]];
-        case 4:  return unalignedLoad<uint32_t>(p1) == unalignedLoad<uint32_t>(p2);
-        case 3:  if (p1[2] != p2[2]) return false; [[fallthrough]];
-        case 2:  return unalignedLoad<uint16_t>(p1) == unalignedLoad<uint16_t>(p2);
-        case 1:  if (p1[0] != p2[0]) return false; [[fallthrough]];
-        case 0:  break;
-    }
-
-    return true;
 }
 
 #endif
@@ -195,7 +211,7 @@ struct StringRefHash64
     }
 };
 
-#if defined(__SSE4_2__)
+#if defined(CRC_INT)
 
 /// Parts are taken from CityHash.
 
@@ -271,13 +287,13 @@ struct CRC32Hash
         do
         {
             UInt64 word = unalignedLoad<UInt64>(pos);
-            res = _mm_crc32_u64(res, word);
+            res = CRC_INT(res, word);
 
             pos += 8;
         } while (pos + 8 < end);
 
         UInt64 word = unalignedLoad<UInt64>(end - 8);    /// I'm not sure if this is normal.
-        res = _mm_crc32_u64(res, word);
+        res = CRC_INT(res, word);
 
         return res;
     }

base/base/TypeList.h

@@ -0,0 +1,44 @@
+#pragma once
+
+#include <cstddef>
+#include <type_traits>
+#include <utility>
+#include "defines.h"
+#include "TypePair.h"
+
+/// General-purpose typelist. Easy on compilation times as it does not use recursion.
+template <typename ...Args>
+struct TypeList { static constexpr size_t size = sizeof...(Args); };
+
+namespace TypeListUtils /// In some contexts it's more handy to use functions instead of aliases
+{
+    template <typename ...LArgs, typename ...RArgs>
+    constexpr TypeList<LArgs..., RArgs...> concat(TypeList<LArgs...>, TypeList<RArgs...>) { return {}; }
+
+    template <typename T, typename ...Args>
+    constexpr TypeList<T, Args...> prepend(TypeList<Args...>) { return {}; }
+
+    template <typename T, typename ...Args>
+    constexpr TypeList<Args..., T> append(TypeList<Args...>) { return {}; }
+
+    template <template <typename> typename F, typename ...Args>
+    constexpr TypeList<F<Args>...> map(TypeList<Args...>) { return {}; }
+
+    template <template <typename...> typename Root, typename ...Args>
+    constexpr Root<Args...> changeRoot(TypeList<Args...>) { return {}; }
+
+    template <typename F, typename ...Args>
+    constexpr void forEach(TypeList<Args...>, F && f) { (std::forward<F>(f)(Id<Args>{}), ...); }
+}
+
+template <typename TypeListLeft, typename TypeListRight>
+using TypeListConcat = decltype(TypeListUtils::concat(TypeListLeft{}, TypeListRight{}));
+
+template <typename T, typename List> using TypeListPrepend = decltype(TypeListUtils::prepend<T>(List{}));
+template <typename T, typename List> using TypeListAppend = decltype(TypeListUtils::append<T>(List{}));
+
+template <template <typename> typename F, typename List>
+using TypeListMap = decltype(TypeListUtils::map<F>(List{}));
+
+template <template <typename...> typename Root, typename List>
+using TypeListChangeRoot = decltype(TypeListUtils::changeRoot<Root>(List{}));

base/base/TypeLists.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include "TypeList.h"
+#include "extended_types.h"
+#include "Decimal.h"
+#include "UUID.h"
+
+namespace DB
+{
+
+using TypeListNativeInt = TypeList<UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64>;
+using TypeListFloat = TypeList<Float32, Float64>;
+using TypeListNativeNumber = TypeListConcat<TypeListNativeInt, TypeListFloat>;
+using TypeListWideInt = TypeList<UInt128, Int128, UInt256, Int256>;
+using TypeListInt = TypeListConcat<TypeListNativeInt, TypeListWideInt>;
+using TypeListIntAndFloat = TypeListConcat<TypeListInt, TypeListFloat>;
+using TypeListDecimal = TypeList<Decimal32, Decimal64, Decimal128, Decimal256>;
+using TypeListNumber = TypeListConcat<TypeListIntAndFloat, TypeListDecimal>;
+using TypeListNumberWithUUID = TypeListAppend<UUID, TypeListNumber>;
+
+}

base/base/TypeName.h

@@ -0,0 +1,51 @@
+#pragma once
+
+#include "Decimal.h"
+#include "UUID.h"
+
+namespace DB
+{
+struct Array;
+struct Tuple;
+struct Map;
+struct AggregateFunctionStateData;
+
+/**
+ * Obtain type string representation from real type if possible.
+ * @example TypeName<UInt8> == "UInt8"
+ */
+template <class T> constexpr inline std::string_view TypeName;
+
+#define TN_MAP(_A) \
+    template <> constexpr inline std::string_view TypeName<_A> = #_A;
+
+TN_MAP(UInt8)
+TN_MAP(UInt16)
+TN_MAP(UInt32)
+TN_MAP(UInt64)
+TN_MAP(UInt128)
+TN_MAP(UInt256)
+TN_MAP(Int8)
+TN_MAP(Int16)
+TN_MAP(Int32)
+TN_MAP(Int64)
+TN_MAP(Int128)
+TN_MAP(Int256)
+TN_MAP(Float32)
+TN_MAP(Float64)
+TN_MAP(String)
+TN_MAP(UUID)
+TN_MAP(Decimal32)
+TN_MAP(Decimal64)
+TN_MAP(Decimal128)
+TN_MAP(Decimal256)
+TN_MAP(DateTime64)
+TN_MAP(Array)
+TN_MAP(Tuple)
+TN_MAP(Map)
+
+/// Special case
+template <> constexpr inline std::string_view TypeName<AggregateFunctionStateData> = "AggregateFunctionState";
+
+#undef TN_MAP
+}

base/base/TypePair.h

@@ -0,0 +1,4 @@
+#pragma once
+
+template <typename T, typename V> struct TypePair {};
+template <typename T> struct Id {};

base/base/UUID.h

@@ -0,0 +1,9 @@
+#pragma once
+
+#include <base/strong_typedef.h>
+#include <base/extended_types.h>
+
+namespace DB
+{
+    using UUID = StrongTypedef<UInt128, struct UUIDTag>;
+}

base/base/arithmeticOverflow.h

@@ -1,7 +1,7 @@
 #pragma once
 
-#include <common/extended_types.h>
+#include <base/extended_types.h>
-#include <common/defines.h>
+#include <base/defines.h>
 
 
 namespace common
@@ -145,6 +145,19 @@ namespace common
         return __builtin_mul_overflow(x, y, &res);
     }
 
+    template <typename T, typename U, typename R>
+    inline bool mulOverflow(T x, U y, R & res)
+    {
+        // not built in type, wide integer
+        if constexpr (is_big_int_v<T>  || is_big_int_v<R> || is_big_int_v<U>)
+        {
+            res = mulIgnoreOverflow<R>(x, y);
+            return false;
+        }
+        else
+            return __builtin_mul_overflow(x, y, &res);
+    }
+
     template <>
     inline bool mulOverflow(int x, int y, int & res)
     {

base/base/arraySize.h

@@ -0,0 +1,7 @@
+#pragma once
+
+#include <cstdlib>
+
+/** \brief Returns number of elements in an automatic array. */
+template <typename T, std::size_t N>
+constexpr size_t arraySize(const T (&)[N]) noexcept { return N; }

base/base/bit_cast.h

@@ -0,0 +1,27 @@
+#pragma once
+
+#include <string.h>
+#include <algorithm>
+#include <type_traits>
+
+
+/** \brief Returns value `from` converted to type `To` while retaining bit representation.
+  *    `To` and `From` must satisfy `CopyConstructible`.
+  */
+template <typename To, typename From>
+std::decay_t<To> bit_cast(const From & from)
+{
+    To res {};
+    memcpy(static_cast<void*>(&res), &from, std::min(sizeof(res), sizeof(from)));
+    return res;
+}
+
+/** \brief Returns value `from` converted to type `To` while retaining bit representation.
+  *    `To` and `From` must satisfy `CopyConstructible`.
+  */
+template <typename To, typename From>
+std::decay_t<To> safe_bit_cast(const From & from)
+{
+    static_assert(sizeof(To) == sizeof(From), "bit cast on types of different width");
+    return bit_cast<To, From>(from);
+}

base/base/chrono_io.h

@@ -0,0 +1,46 @@
+#pragma once
+
+#include <chrono>
+#include <string>
+#include <sstream>
+#include <cctz/time_zone.h>
+
+
+inline std::string to_string(const std::time_t & time)
+{
+    return cctz::format("%Y-%m-%d %H:%M:%S", std::chrono::system_clock::from_time_t(time), cctz::local_time_zone());
+}
+
+template <typename Clock, typename Duration = typename Clock::duration>
+std::string to_string(const std::chrono::time_point<Clock, Duration> & tp)
+{
+    // Don't use DateLUT because it shows weird characters for
+    // TimePoint::max(). I wish we could use C++20 format, but it's not
+    // there yet.
+    // return DateLUT::instance().timeToString(std::chrono::system_clock::to_time_t(tp));
+
+    auto in_time_t = std::chrono::system_clock::to_time_t(tp);
+    return to_string(in_time_t);
+}
+
+template <typename Rep, typename Period = std::ratio<1>>
+std::string to_string(const std::chrono::duration<Rep, Period> & duration)
+{
+    auto seconds_as_int = std::chrono::duration_cast<std::chrono::seconds>(duration);
+    if (seconds_as_int == duration)
+        return std::to_string(seconds_as_int.count()) + "s";
+    auto seconds_as_double = std::chrono::duration_cast<std::chrono::duration<double>>(duration);
+    return std::to_string(seconds_as_double.count()) + "s";
+}
+
+template <typename Clock, typename Duration = typename Clock::duration>
+std::ostream & operator<<(std::ostream & o, const std::chrono::time_point<Clock, Duration> & tp)
+{
+    return o << to_string(tp);
+}
+
+template <typename Rep, typename Period = std::ratio<1>>
+std::ostream & operator<<(std::ostream & o, const std::chrono::duration<Rep, Period> & duration)
+{
+    return o << to_string(duration);
+}

base/base/defines.h

@@ -8,41 +8,28 @@
 /// NOTE:
 /// - __has_feature cannot be simply undefined,
 ///   since this will be broken if some C++ header will be included after
-///   including <common/defines.h>
+///   including <base/defines.h>
 /// - it should not have fallback to 0,
 ///   since this may create false-positive detection (common problem)
 #if defined(__clang__) && defined(__has_feature)
 #    define ch_has_feature __has_feature
 #endif
 
-#if defined(_MSC_VER)
+#if !defined(likely)
-#   if !defined(likely)
+#    define likely(x)   (__builtin_expect(!!(x), 1))
-#      define likely(x)   (x)
+#endif
-#   endif
+#if !defined(unlikely)
-#   if !defined(unlikely)
+#    define unlikely(x) (__builtin_expect(!!(x), 0))
-#      define unlikely(x) (x)
-#   endif
-#else
-#   if !defined(likely)
-#       define likely(x)   (__builtin_expect(!!(x), 1))
-#   endif
-#   if !defined(unlikely)
-#       define unlikely(x) (__builtin_expect(!!(x), 0))
-#   endif
 #endif
 
-#if defined(_MSC_VER)
+// more aliases: https://mailman.videolan.org/pipermail/x264-devel/2014-May/010660.html
-#    define ALWAYS_INLINE __forceinline
-#    define NO_INLINE static __declspec(noinline)
-#    define MAY_ALIAS
-#else
-#    define ALWAYS_INLINE __attribute__((__always_inline__))
-#    define NO_INLINE __attribute__((__noinline__))
-#    define MAY_ALIAS __attribute__((__may_alias__))
-#endif
 
-#if !defined(__x86_64__) && !defined(__aarch64__) && !defined(__PPC__)
+#define ALWAYS_INLINE __attribute__((__always_inline__))
-#    error "The only supported platforms are x86_64 and AArch64, PowerPC (work in progress)"
+#define NO_INLINE __attribute__((__noinline__))
+#define MAY_ALIAS __attribute__((__may_alias__))
+
+#if !defined(__x86_64__) && !defined(__aarch64__) && !defined(__PPC__) && !(defined(__riscv) && (__riscv_xlen == 64))
+#    error "The only supported platforms are x86_64 and AArch64, PowerPC (work in progress) and RISC-V 64 (experimental)"
 #endif
 
 /// Check for presence of address sanitizer
@@ -96,10 +83,6 @@
 #    define BOOST_USE_UCONTEXT 1
 #endif
 
-#if defined(ARCADIA_BUILD) && defined(BOOST_USE_UCONTEXT)
-#    undef BOOST_USE_UCONTEXT
-#endif
-
 /// TODO: Strange enough, there is no way to detect UB sanitizer.
 
 /// Explicitly allow undefined behaviour for certain functions. Use it as a function attribute.
@@ -117,6 +100,11 @@
 #    define ALWAYS_INLINE_NO_SANITIZE_UNDEFINED ALWAYS_INLINE
 #endif
 
+#if !__has_include(<sanitizer/asan_interface.h>) || !defined(ADDRESS_SANITIZER)
+#   define ASAN_UNPOISON_MEMORY_REGION(a, b)
+#   define ASAN_POISON_MEMORY_REGION(a, b)
+#endif
+
 /// A template function for suppressing warnings about unused variables or function results.
 template <typename... Args>
 constexpr void UNUSED(Args &&... args [[maybe_unused]])

base/base/demangle.cpp

@@ -0,0 +1,26 @@
+#include <base/demangle.h>
+
+#include <stdlib.h>
+#include <cxxabi.h>
+
+static DemangleResult tryDemangle(const char * name, int & status)
+{
+    return DemangleResult(abi::__cxa_demangle(name, nullptr, nullptr, &status));
+}
+
+DemangleResult tryDemangle(const char * name)
+{
+    int status = 0;
+    return tryDemangle(name, status);
+}
+
+std::string demangle(const char * name, int & status)
+{
+    auto result = tryDemangle(name, status);
+    if (result)
+    {
+        return std::string(result.get());
+    }
+
+    return name;
+}

base/base/extended_types.h

@@ -2,8 +2,8 @@
 
 #include <type_traits>
 
-#include <common/types.h>
+#include <base/types.h>
-#include <common/wide_integer.h>
+#include <base/wide_integer.h>
 
 
 using Int128 = wide::integer<128, signed>;
@@ -41,22 +41,14 @@ template <> struct is_unsigned<UInt256> { static constexpr bool value = true; };
 template <typename T>
 inline constexpr bool is_unsigned_v = is_unsigned<T>::value;
 
+template <class T> concept is_integer =
+    std::is_integral_v<T>
+    || std::is_same_v<T, Int128>
+    || std::is_same_v<T, UInt128>
+    || std::is_same_v<T, Int256>
+    || std::is_same_v<T, UInt256>;
 
-/// TODO: is_integral includes char, char8_t and wchar_t.
+template <class T> concept is_floating_point = std::is_floating_point_v<T>;
-template <typename T>
-struct is_integer
-{
-    static constexpr bool value = std::is_integral_v<T>;
-};
-
-template <> struct is_integer<Int128> { static constexpr bool value = true; };
-template <> struct is_integer<UInt128> { static constexpr bool value = true; };
-template <> struct is_integer<Int256> { static constexpr bool value = true; };
-template <> struct is_integer<UInt256> { static constexpr bool value = true; };
-
-template <typename T>
-inline constexpr bool is_integer_v = is_integer<T>::value;
-
 
 template <typename T>
 struct is_arithmetic

base/base/find_symbols.h

@@ -36,18 +36,7 @@
 
 namespace detail
 {
-
+template <char ...chars> constexpr bool is_in(char x) { return ((x == chars) || ...); }
-template <char s0>
-inline bool is_in(char x)
-{
-    return x == s0;
-}
-
-template <char s0, char s1, char... tail>
-inline bool is_in(char x)
-{
-    return x == s0 || is_in<s1, tail...>(x);
-}
 
 #if defined(__SSE2__)
 template <char s0>
@@ -67,16 +56,10 @@ inline __m128i mm_is_in(__m128i bytes)
 #endif
 
 template <bool positive>
-bool maybe_negate(bool x)
+constexpr bool maybe_negate(bool x) { return x == positive; }
-{
-    if constexpr (positive)
-        return x;
-    else
-        return !x;
-}
 
 template <bool positive>
-uint16_t maybe_negate(uint16_t x)
+constexpr uint16_t maybe_negate(uint16_t x)
 {
     if constexpr (positive)
         return x;
@@ -149,12 +132,13 @@ template <bool positive, ReturnMode return_mode, size_t num_chars,
     char c05 = 0, char c06 = 0, char c07 = 0, char c08 = 0,
     char c09 = 0, char c10 = 0, char c11 = 0, char c12 = 0,
     char c13 = 0, char c14 = 0, char c15 = 0, char c16 = 0>
-inline const char * find_first_symbols_sse42_impl(const char * const begin, const char * const end)
+inline const char * find_first_symbols_sse42(const char * const begin, const char * const end)
 {
     const char * pos = begin;
 
 #if defined(__SSE4_2__)
-#define MODE (_SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT)
+    constexpr int mode = _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT;
+
     __m128i set = _mm_setr_epi8(c01, c02, c03, c04, c05, c06, c07, c08, c09, c10, c11, c12, c13, c14, c15, c16);
 
     for (; pos + 15 < end; pos += 16)
@@ -163,16 +147,15 @@ inline const char * find_first_symbols_sse42_impl(const char * const begin, cons
 
         if constexpr (positive)
         {
-            if (_mm_cmpestrc(set, num_chars, bytes, 16, MODE))
+            if (_mm_cmpestrc(set, num_chars, bytes, 16, mode))
-                return pos + _mm_cmpestri(set, num_chars, bytes, 16, MODE);
+                return pos + _mm_cmpestri(set, num_chars, bytes, 16, mode);
         }
         else
         {
-            if (_mm_cmpestrc(set, num_chars, bytes, 16, MODE | _SIDD_NEGATIVE_POLARITY))
+            if (_mm_cmpestrc(set, num_chars, bytes, 16, mode | _SIDD_NEGATIVE_POLARITY))
-                return pos + _mm_cmpestri(set, num_chars, bytes, 16, MODE | _SIDD_NEGATIVE_POLARITY);
+                return pos + _mm_cmpestri(set, num_chars, bytes, 16, mode | _SIDD_NEGATIVE_POLARITY);
         }
     }
-#undef MODE
 #endif
 
     for (; pos < end; ++pos)
@@ -197,20 +180,15 @@ inline const char * find_first_symbols_sse42_impl(const char * const begin, cons
 }
 
 
-template <bool positive, ReturnMode return_mode, char... symbols>
-inline const char * find_first_symbols_sse42(const char * begin, const char * end)
-{
-    return find_first_symbols_sse42_impl<positive, return_mode, sizeof...(symbols), symbols...>(begin, end);
-}
-
 /// NOTE No SSE 4.2 implementation for find_last_symbols_or_null. Not worth to do.
 
 template <bool positive, ReturnMode return_mode, char... symbols>
 inline const char * find_first_symbols_dispatch(const char * begin, const char * end)
+    requires(0 <= sizeof...(symbols) && sizeof...(symbols) <= 16)
 {
 #if defined(__SSE4_2__)
     if (sizeof...(symbols) >= 5)
-        return find_first_symbols_sse42<positive, return_mode, symbols...>(begin, end);
+        return find_first_symbols_sse42<positive, return_mode, sizeof...(symbols), symbols...>(begin, end);
     else
 #endif
         return find_first_symbols_sse2<positive, return_mode, symbols...>(begin, end);

base/base/getAvailableMemoryAmount.cpp

@@ -0,0 +1,44 @@
+#include <stdexcept>
+#include <fstream>
+#include <base/getAvailableMemoryAmount.h>
+#include <base/getPageSize.h>
+
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/param.h>
+#if defined(BSD)
+#include <sys/sysctl.h>
+#include <sys/vmmeter.h>
+#endif
+
+
+uint64_t getAvailableMemoryAmountOrZero()
+{
+#if defined(_SC_AVPHYS_PAGES) // linux
+    return getPageSize() * sysconf(_SC_AVPHYS_PAGES);
+#elif defined(__FreeBSD__)
+    struct vmtotal vmt;
+    size_t vmt_size = sizeof(vmt);
+    if (sysctlbyname("vm.vmtotal", &vmt, &vmt_size, NULL, 0) == 0)
+        return getPageSize() * vmt.t_avm;
+    else
+        return 0;
+#else // darwin
+    unsigned int usermem;
+    size_t len = sizeof(usermem);
+    static int mib[2] = { CTL_HW, HW_USERMEM };
+    if (sysctl(mib, 2, &usermem, &len, nullptr, 0) == 0 && len == sizeof(usermem))
+        return usermem;
+    else
+        return 0;
+#endif
+}
+
+
+uint64_t getAvailableMemoryAmount()
+{
+    auto res = getAvailableMemoryAmountOrZero();
+    if (!res)
+        throw std::runtime_error("Cannot determine available memory amount");
+    return res;
+}

base/base/getAvailableMemoryAmount.h

@@ -0,0 +1,12 @@
+#pragma once
+
+#include <cstdint>
+
+/** Returns the size of currently available physical memory (RAM) in bytes.
+  * Returns 0 on unsupported platform or if it cannot determine the size of physical memory.
+  */
+uint64_t getAvailableMemoryAmountOrZero();
+
+/** Throws exception if it cannot determine the size of physical memory.
+  */
+uint64_t getAvailableMemoryAmount();

base/base/getFQDNOrHostName.cpp

@@ -1,5 +1,5 @@
 #include <Poco/Net/DNS.h>
-#include <common/getFQDNOrHostName.h>
+#include <base/getFQDNOrHostName.h>
 
 
 namespace

base/base/getMemoryAmount.cpp

@@ -0,0 +1,52 @@
+#include <stdexcept>
+#include <fstream>
+#include <base/getMemoryAmount.h>
+#include <base/getPageSize.h>
+
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/param.h>
+#if defined(BSD)
+#include <sys/sysctl.h>
+#endif
+
+
+/** Returns the size of physical memory (RAM) in bytes.
+  * Returns 0 on unsupported platform
+  */
+uint64_t getMemoryAmountOrZero()
+{
+    int64_t num_pages = sysconf(_SC_PHYS_PAGES);
+    if (num_pages <= 0)
+        return 0;
+
+    int64_t page_size = getPageSize();
+    if (page_size <= 0)
+        return 0;
+
+    uint64_t memory_amount = num_pages * page_size;
+
+#if defined(OS_LINUX)
+    // Try to lookup at the Cgroup limit
+    std::ifstream cgroup_limit("/sys/fs/cgroup/memory/memory.limit_in_bytes");
+    if (cgroup_limit.is_open())
+    {
+        uint64_t memory_limit = 0; // in case of read error
+        cgroup_limit >> memory_limit;
+        if (memory_limit > 0 && memory_limit < memory_amount)
+            memory_amount = memory_limit;
+    }
+#endif
+
+    return memory_amount;
+
+}
+
+
+uint64_t getMemoryAmount()
+{
+    auto res = getMemoryAmountOrZero();
+    if (!res)
+        throw std::runtime_error("Cannot determine memory amount");
+    return res;
+}

base/base/getPageSize.cpp

@@ -0,0 +1,11 @@
+#include <base/getPageSize.h>
+#include <unistd.h>
+#include <cstdlib>
+
+Int64 getPageSize()
+{
+    Int64 page_size = sysconf(_SC_PAGESIZE);
+    if (page_size < 0)
+        abort();
+    return page_size;
+}

base/base/getPageSize.h

@@ -0,0 +1,7 @@
+#pragma once
+
+#include <base/types.h>
+
+
+/// Get memory page size
+Int64 getPageSize();

base/base/getThreadId.cpp

@@ -1,4 +1,4 @@
-#include <common/getThreadId.h>
+#include <base/getThreadId.h>
 
 #if defined(OS_ANDROID)
     #include <sys/types.h>

base/base/insertAtEnd.h

@@ -0,0 +1,52 @@
+#pragma once
+
+#include <vector>
+
+/// Appends a specified vector with elements of another vector.
+template <typename T>
+void insertAtEnd(std::vector<T> & dest, const std::vector<T> & src)
+{
+    if (src.empty())
+        return;
+    dest.reserve(dest.size() + src.size());
+    dest.insert(dest.end(), src.begin(), src.end());
+}
+
+template <typename T>
+void insertAtEnd(std::vector<T> & dest, std::vector<T> && src)
+{
+    if (src.empty())
+        return;
+    if (dest.empty())
+    {
+        dest.swap(src);
+        return;
+    }
+    dest.reserve(dest.size() + src.size());
+    dest.insert(dest.end(), std::make_move_iterator(src.begin()), std::make_move_iterator(src.end()));
+    src.clear();
+}
+
+template <typename Container>
+void insertAtEnd(Container & dest, const Container & src)
+{
+    if (src.empty())
+        return;
+
+    dest.insert(dest.end(), src.begin(), src.end());
+}
+
+template <typename Container>
+void insertAtEnd(Container & dest, Container && src)
+{
+    if (src.empty())
+        return;
+    if (dest.empty())
+    {
+        dest.swap(src);
+        return;
+    }
+
+    dest.insert(dest.end(), std::make_move_iterator(src.begin()), std::make_move_iterator(src.end()));
+    src.clear();
+}

base/base/iostream_debug_helpers.h

@@ -6,6 +6,7 @@
 #include <tuple>
 #include <iomanip>
 #include <iostream>
+#include <magic_enum.hpp>
 
 /** Usage:
   *
@@ -61,6 +62,13 @@ std::enable_if_t<priority == 2, Out> & dumpImpl(Out & out, T && x, std::decay_t<
 }
 
 
+template <int priority, typename Out, typename T>
+std::enable_if_t<priority == 3 && std::is_enum_v<std::decay_t<T>>, Out> &
+dumpImpl(Out & out, T && x)
+{
+    return out << magic_enum::enum_name(x);
+}
+
 /// string and const char * - output not as container or pointer.
 
 template <int priority, typename Out, typename T>
@@ -131,15 +139,26 @@ Out & dumpValue(Out & out, T && x)
 template <typename Out, typename T>
 Out & dump(Out & out, const char * name, T && x)
 {
+    // Dumping string literal, printing name and demangled type is irrelevant.
+    if constexpr (std::is_same_v<const char *, std::decay_t<std::remove_reference_t<T>>>)
+    {
+         const auto name_len = strlen(name);
+         const auto value_len = strlen(x);
+         // `name` is the same as quoted `x`
+         if (name_len > 2 && value_len > 0 && name[0] == '"' && name[name_len - 1] == '"'
+                 && strncmp(name + 1, x, std::min(value_len, name_len) - 1) == 0)
+             return out << x;
+    }
+
     out << demangle(typeid(x).name()) << " " << name << " = ";
-    return dumpValue(out, x);
+    return dumpValue(out, x) << "; ";
 }
 
 #ifdef __clang__
 #pragma clang diagnostic ignored "-Wgnu-zero-variadic-macro-arguments"
 #endif
 
-#define DUMPVAR(VAR) ::dump(std::cerr, #VAR, (VAR)); std::cerr << "; ";
+#define DUMPVAR(VAR) ::dump(std::cerr, #VAR, (VAR));
 #define DUMPHEAD std::cerr << __FILE__ << ':' << __LINE__ << " [ " << getThreadId() << " ] ";
 #define DUMPTAIL std::cerr << '\n';
 

base/base/itoa.h

@@ -30,7 +30,7 @@
 #include <cstddef>
 #include <cstring>
 #include <type_traits>
-#include <common/extended_types.h>
+#include <base/extended_types.h>
 
 
 namespace impl

base/base/logger_useful.h

@@ -12,6 +12,8 @@ namespace
 {
     template <typename... Ts> constexpr size_t numArgs(Ts &&...) { return sizeof...(Ts); }
     template <typename T, typename... Ts> constexpr auto firstArg(T && x, Ts &&...) { return std::forward<T>(x); }
+    /// For implicit conversion of fmt::basic_runtime<> to char* for std::string ctor
+    template <typename T, typename... Ts> constexpr auto firstArg(fmt::basic_runtime<T> && data, Ts &&...) { return data.str.data(); }
 }
 
 
@@ -42,18 +44,10 @@ namespace
 } while (false)
 
 
+#define LOG_TEST(logger, ...)    LOG_IMPL(logger, DB::LogsLevel::test, Poco::Message::PRIO_TEST, __VA_ARGS__)
 #define LOG_TRACE(logger, ...)   LOG_IMPL(logger, DB::LogsLevel::trace, Poco::Message::PRIO_TRACE, __VA_ARGS__)
 #define LOG_DEBUG(logger, ...)   LOG_IMPL(logger, DB::LogsLevel::debug, Poco::Message::PRIO_DEBUG, __VA_ARGS__)
 #define LOG_INFO(logger, ...)    LOG_IMPL(logger, DB::LogsLevel::information, Poco::Message::PRIO_INFORMATION, __VA_ARGS__)
 #define LOG_WARNING(logger, ...) LOG_IMPL(logger, DB::LogsLevel::warning, Poco::Message::PRIO_WARNING, __VA_ARGS__)
 #define LOG_ERROR(logger, ...)   LOG_IMPL(logger, DB::LogsLevel::error, Poco::Message::PRIO_ERROR, __VA_ARGS__)
 #define LOG_FATAL(logger, ...)   LOG_IMPL(logger, DB::LogsLevel::error, Poco::Message::PRIO_FATAL, __VA_ARGS__)
-
-
-/// Compatibility for external projects.
-#if defined(ARCADIA_BUILD)
-    using Poco::Logger;
-    using Poco::Message;
-    using DB::LogsLevel;
-    using DB::CurrentThread;
-#endif

base/base/map.h

@@ -0,0 +1,52 @@
+#pragma once
+
+#include <type_traits>
+#include <boost/iterator/transform_iterator.hpp>
+
+namespace collections
+{
+
+/// \brief Strip type off top level reference and cv-qualifiers thus allowing storage in containers
+template <typename T>
+using unqualified_t = std::remove_cv_t<std::remove_reference_t<T>>;
+
+/** \brief Returns collection of the same container-type as the input collection,
+  *    with each element transformed by the application of `mapper`.
+  */
+template <template <typename...> class Collection, typename... Params, typename Mapper>
+auto map(const Collection<Params...> & collection, Mapper && mapper)
+{
+    using value_type = unqualified_t<decltype(mapper(*std::begin(collection)))>;
+
+    return Collection<value_type>(
+        boost::make_transform_iterator(std::begin(collection), std::forward<Mapper>(mapper)),
+        boost::make_transform_iterator(std::end(collection), std::forward<Mapper>(mapper)));
+}
+
+/** \brief Returns collection of specified container-type,
+  *    with each element transformed by the application of `mapper`.
+  *    Allows conversion between different container-types, e.g. std::vector to std::list
+  */
+template <template <typename...> class ResultCollection, typename Collection, typename Mapper>
+auto map(const Collection & collection, Mapper && mapper)
+{
+    using value_type = unqualified_t<decltype(mapper(*std::begin(collection)))>;
+
+    return ResultCollection<value_type>(
+        boost::make_transform_iterator(std::begin(collection), std::forward<Mapper>(mapper)),
+        boost::make_transform_iterator(std::end(collection), std::forward<Mapper>(mapper)));
+}
+
+/** \brief Returns collection of specified type,
+  *    with each element transformed by the application of `mapper`.
+  *    Allows leveraging implicit conversion between the result of applying `mapper` and R::value_type.
+  */
+template <typename ResultCollection, typename Collection, typename Mapper>
+auto map(const Collection & collection, Mapper && mapper)
+{
+    return ResultCollection(
+        boost::make_transform_iterator(std::begin(collection), std::forward<Mapper>(mapper)),
+        boost::make_transform_iterator(std::end(collection), std::forward<Mapper>(mapper)));
+}
+
+}

base/base/mremap.cpp

@@ -1,4 +1,4 @@
-#include <common/mremap.h>
+#include <base/mremap.h>
 
 #include <cstddef>
 #include <cstdlib>
@@ -19,22 +19,14 @@ void * mremap_fallback(
         return MAP_FAILED;
     }
 
-#if defined(_MSC_VER)
-    void * new_address = ::operator new(new_size);
-#else
     void * new_address = mmap(nullptr, new_size, mmap_prot, mmap_flags, mmap_fd, mmap_offset);
     if (MAP_FAILED == new_address)
         return MAP_FAILED;
-#endif
 
     memcpy(new_address, old_address, old_size);
 
-#if defined(_MSC_VER)
-    delete old_address;
-#else
     if (munmap(old_address, old_size))
         abort();
-#endif
 
     return new_address;
 }

base/base/mremap.h

@@ -2,9 +2,7 @@
 
 #include <cstddef>
 #include <sys/types.h>
-#if !defined(_MSC_VER)
 #include <sys/mman.h>
-#endif
 
 
 #ifdef MREMAP_MAYMOVE

base/base/phdr_cache.cpp

@@ -1,8 +1,12 @@
+#ifdef HAS_RESERVED_IDENTIFIER
+#pragma clang diagnostic ignored "-Wreserved-identifier"
+#endif
+
 /// This code was based on the code by Fedor Korotkiy (prime@yandex-team.ru) for YT product in Yandex.
 
-#include <common/defines.h>
+#include <base/defines.h>
 
-#if defined(__linux__) && !defined(THREAD_SANITIZER)
+#if defined(__linux__) && !defined(THREAD_SANITIZER) && !defined(USE_MUSL)
     #define USE_PHDR_CACHE 1
 #endif
 
@@ -119,6 +123,12 @@ bool hasPHDRCache()
 #else
 
 void updatePHDRCache() {}
-bool hasPHDRCache() { return false; }
+
+#if defined(USE_MUSL)
+    /// With statically linked with musl, dl_iterate_phdr is immutable.
+    bool hasPHDRCache() { return true; }
+#else
+    bool hasPHDRCache() { return false; }
+#endif
 
 #endif