Merge branch 'master' into mysql-macos

.clang-format

@@ -21,7 +21,6 @@ ConstructorInitializerAllOnOneLineOrOnePerLine: true
 ExperimentalAutoDetectBinPacking: true
 UseTab: Never
 TabWidth: 4
-IndentWidth: 4
 Standard: Cpp11
 PointerAlignment: Middle
 MaxEmptyLinesToKeep: 2
@@ -75,11 +74,12 @@ ConstructorInitializerIndentWidth: 4
 ContinuationIndentWidth: 4
 DerivePointerAlignment: false
 DisableFormat: false
+IndentRequiresClause: false
 IndentWidth: 4
 IndentWrappedFunctionNames: false
 MacroBlockBegin: ''
 MacroBlockEnd: ''
-NamespaceIndentation: Inner
+NamespaceIndentation: None
 ObjCBlockIndentWidth: 4
 ObjCSpaceAfterProperty: true
 ObjCSpaceBeforeProtocolList: true
@@ -89,6 +89,7 @@ PenaltyBreakFirstLessLess: 120
 PenaltyBreakString: 1000
 PenaltyExcessCharacter: 1000000
 PenaltyReturnTypeOnItsOwnLine: 60
+RemoveBracesLLVM: true
 SpaceAfterCStyleCast: false
 SpaceBeforeAssignmentOperators: true
 SpaceBeforeParens: ControlStatements

.clang-tidy

@@ -5,6 +5,9 @@
 # a) the new check is not controversial (this includes many checks in readability-* and google-*) or
 # b) too noisy (checks with > 100 new warnings are considered noisy, this includes e.g. cppcoreguidelines-*).
 
+# TODO: Once clang(-tidy) 17 is the minimum, we can convert this list to YAML
+# See https://releases.llvm.org/17.0.1/tools/clang/tools/extra/docs/ReleaseNotes.html#improvements-to-clang-tidy
+
 # TODO Let clang-tidy check headers in further directories
 #      --> HeaderFilterRegex: '^.*/(src|base|programs|utils)/.*(h|hpp)$'
 HeaderFilterRegex: '^.*/(base)/.*(h|hpp)$'
@@ -23,45 +26,21 @@ Checks: '*,
     -bugprone-implicit-widening-of-multiplication-result,
     -bugprone-narrowing-conversions,
     -bugprone-not-null-terminated-result,
+    -bugprone-reserved-identifier, # useful but too slow, TODO retry when https://reviews.llvm.org/rG1c282052624f9d0bd273bde0b47b30c96699c6c7 is merged
     -bugprone-unchecked-optional-access,
 
     -cert-dcl16-c,
+    -cert-dcl37-c,
+    -cert-dcl51-cpp,
     -cert-err58-cpp,
     -cert-msc32-c,
     -cert-msc51-cpp,
     -cert-oop54-cpp,
     -cert-oop57-cpp,
 
-    -clang-analyzer-optin.performance.Padding,
-    -clang-analyzer-optin.portability.UnixAPI,
-    -clang-analyzer-security.insecureAPI.bzero,
-    -clang-analyzer-security.insecureAPI.strcpy,
+    -clang-analyzer-unix.Malloc,
 
-    -cppcoreguidelines-avoid-c-arrays,
-    -cppcoreguidelines-avoid-goto,
-    -cppcoreguidelines-avoid-magic-numbers,
-    -cppcoreguidelines-avoid-non-const-global-variables,
-    -cppcoreguidelines-explicit-virtual-functions,
-    -cppcoreguidelines-init-variables,
-    -cppcoreguidelines-interfaces-global-init,
-    -cppcoreguidelines-macro-usage,
-    -cppcoreguidelines-narrowing-conversions,
-    -cppcoreguidelines-no-malloc,
-    -cppcoreguidelines-non-private-member-variables-in-classes,
-    -cppcoreguidelines-owning-memory,
-    -cppcoreguidelines-prefer-member-initializer,
-    -cppcoreguidelines-pro-bounds-array-to-pointer-decay,
-    -cppcoreguidelines-pro-bounds-constant-array-index,
-    -cppcoreguidelines-pro-bounds-pointer-arithmetic,
-    -cppcoreguidelines-pro-type-const-cast,
-    -cppcoreguidelines-pro-type-cstyle-cast,
-    -cppcoreguidelines-pro-type-member-init,
-    -cppcoreguidelines-pro-type-reinterpret-cast,
-    -cppcoreguidelines-pro-type-static-cast-downcast,
-    -cppcoreguidelines-pro-type-union-access,
-    -cppcoreguidelines-pro-type-vararg,
-    -cppcoreguidelines-slicing,
-    -cppcoreguidelines-special-member-functions,
+    -cppcoreguidelines-*, # impractical in a codebase as large as ClickHouse, also slow
 
     -darwin-*,
 
@@ -73,7 +52,6 @@ Checks: '*,
     -google-readability-function-size,
     -google-readability-namespace-comments,
     -google-readability-todo,
-    -google-upgrade-googletest-case,
 
     -hicpp-avoid-c-arrays,
     -hicpp-avoid-goto,
@@ -103,8 +81,11 @@ Checks: '*,
     -openmp-*,
 
     -misc-const-correctness,
+    -misc-include-cleaner, # useful but far too many occurrences
     -misc-no-recursion,
     -misc-non-private-member-variables-in-classes,
+    -misc-confusable-identifiers, # useful but slooow
+    -misc-use-anonymous-namespace,
 
     -modernize-avoid-c-arrays,
     -modernize-concat-nested-namespaces,
@@ -120,15 +101,19 @@ Checks: '*,
 
     -performance-inefficient-string-concatenation,
     -performance-no-int-to-ptr,
+    -performance-avoid-endl,
     -performance-unnecessary-value-param,
 
     -portability-simd-intrinsics,
 
+    -readability-avoid-unconditional-preprocessor-if,
     -readability-braces-around-statements,
+    -readability-convert-member-functions-to-static,
     -readability-else-after-return,
     -readability-function-cognitive-complexity,
     -readability-function-size,
     -readability-identifier-length,
+    -readability-identifier-naming, # useful but too slow
     -readability-implicit-bool-conversion,
     -readability-isolate-declaration,
     -readability-magic-numbers,
@@ -140,72 +125,39 @@ Checks: '*,
     -readability-uppercase-literal-suffix,
     -readability-use-anyofallof,
 
-    -zirkon-*,
-
-    -misc-*, # temporarily disabled due to being too slow
-    # also disable checks in other categories which are aliases of checks in misc-*:
-    # https://releases.llvm.org/15.0.0/tools/clang/tools/extra/docs/clang-tidy/checks/list.html
-    -cert-dcl54-cpp,                                            # alias of misc-new-delete-overloads
-    -hicpp-new-delete-operators,                                # alias of misc-new-delete-overloads
-    -cert-fio38-c,                                              # alias of misc-non-copyable-objects
-    -cert-dcl03-c,                                              # alias of misc-static-assert
-    -hicpp-static-assert,                                       # alias of misc-static-assert
-    -cert-err09-cpp,                                            # alias of misc-throw-by-value-catch-by-reference
-    -cert-err61-cpp,                                            # alias of misc-throw-by-value-catch-by-reference
-    -cppcoreguidelines-c-copy-assignment-signature,             # alias of misc-unconventional-assign-operator
-    -cppcoreguidelines-non-private-member-variables-in-classes, # alias of misc-non-private-member-variables-in-classes
+    -zircon-*,
 '
 
 WarningsAsErrors: '*'
 
-# TODO: use dictionary syntax for CheckOptions when minimum clang-tidy level rose to 15
-#           some-check.SomeOption: 'some value'
-#       instead of
-#           - key:             some-check.SomeOption
-#             value:           'some value'
+ExtraArgs:
+# clang-tidy 17 started to complain (for unknown reasons) that various pragmas are unknown ("clang-diagnostic-unknown-pragmas").
+# This is technically a compiler error, not a clang-tidy error. We could litter the code base with more pragmas that suppress
+# this error but it is better to pass the following flag to the compiler:
+- '-Wno-unknown-pragmas'
+- '-Wno-unused-command-line-argument' # similar issue
+
 CheckOptions:
-  - key: readability-identifier-naming.ClassCase
-    value: CamelCase
-  - key: readability-identifier-naming.EnumCase
-    value: CamelCase
-  - key: readability-identifier-naming.LocalVariableCase
-    value: lower_case
-  - key: readability-identifier-naming.StaticConstantCase
-    value: aNy_CasE
-  - key: readability-identifier-naming.MemberCase
-    value: lower_case
-  - key: readability-identifier-naming.PrivateMemberPrefix
-    value: ''
-  - key: readability-identifier-naming.ProtectedMemberPrefix
-    value: ''
-  - key: readability-identifier-naming.PublicMemberCase
-    value: lower_case
-  - key: readability-identifier-naming.MethodCase
-    value: camelBack
-  - key: readability-identifier-naming.PrivateMethodPrefix
-    value: ''
-  - key: readability-identifier-naming.ProtectedMethodPrefix
-    value: ''
-  - key: readability-identifier-naming.ParameterPackCase
-    value: lower_case
-  - key: readability-identifier-naming.StructCase
-    value: CamelCase
-  - key: readability-identifier-naming.TemplateTemplateParameterCase
-    value: CamelCase
-  - key: readability-identifier-naming.TemplateUsingCase
-    value: lower_case
-  - key: readability-identifier-naming.TypeTemplateParameterCase
-    value: CamelCase
-  - key: readability-identifier-naming.TypedefCase
-    value: CamelCase
-  - key: readability-identifier-naming.UnionCase
-    value: CamelCase
-  - key: readability-identifier-naming.UsingCase
-    value: CamelCase
-  - key: modernize-loop-convert.UseCxx20ReverseRanges
-    value: false
-  - key: performance-move-const-arg.CheckTriviallyCopyableMove
-    value: false
-    # Workaround clang-tidy bug: https://github.com/llvm/llvm-project/issues/46097
-  - key: readability-identifier-naming.TypeTemplateParameterIgnoredRegexp
-    value: expr-type
+  readability-identifier-naming.ClassCase: CamelCase
+  readability-identifier-naming.EnumCase: CamelCase
+  readability-identifier-naming.LocalVariableCase: lower_case
+  readability-identifier-naming.StaticConstantCase: aNy_CasE
+  readability-identifier-naming.MemberCase: lower_case
+  readability-identifier-naming.PrivateMemberPrefix: ''
+  readability-identifier-naming.ProtectedMemberPrefix: ''
+  readability-identifier-naming.PublicMemberCase: lower_case
+  readability-identifier-naming.MethodCase: camelBack
+  readability-identifier-naming.PrivateMethodPrefix: ''
+  readability-identifier-naming.ProtectedMethodPrefix: ''
+  readability-identifier-naming.ParameterPackCase: lower_case
+  readability-identifier-naming.StructCase: CamelCase
+  readability-identifier-naming.TemplateTemplateParameterCase: CamelCase
+  readability-identifier-naming.TemplateParameterCase: lower_case
+  readability-identifier-naming.TypeTemplateParameterCase: CamelCase
+  readability-identifier-naming.TypedefCase: CamelCase
+  readability-identifier-naming.UnionCase: CamelCase
+  modernize-loop-convert.UseCxx20ReverseRanges: false
+  performance-move-const-arg.CheckTriviallyCopyableMove: false
+  # Workaround clang-tidy bug: https://github.com/llvm/llvm-project/issues/46097
+  readability-identifier-naming.TypeTemplateParameterIgnoredRegexp: expr-type
+  cppcoreguidelines-avoid-do-while.IgnoreMacros: true

.clangd

@@ -0,0 +1,16 @@
+Diagnostics:
+    # clangd does parse .clang-tidy, but some checks are too slow to run in
+    # clang-tidy build, so let's enable them explicitly for clangd at least.
+    ClangTidy:
+        # The following checks had been disabled due to slowliness with C++23,
+        # for more details see [1].
+        #
+        #   [1]: https://github.com/llvm/llvm-project/issues/61418
+        #
+        # But the code base had been written in a style that had been checked
+        # by this check, so at least, let's enable it for clangd.
+        Add: [
+            # configured in .clang-tidy
+            readability-identifier-naming,
+            bugprone-reserved-identifier,
+        ]

.github/ISSUE_TEMPLATE/85_bug-report.md

@@ -7,6 +7,8 @@ assignees: ''
 
 ---
 
+> Please make sure that the version you're using is still supported (you can find the list [here](https://github.com/ClickHouse/ClickHouse/blob/master/SECURITY.md#scope-and-supported-versions)).
+
 > You have to provide the following information whenever possible.
 
 **Describe what's wrong**
@@ -21,8 +23,7 @@ assignees: ''
 
 **Enable crash reporting**
 
-> If possible, change "enabled" to true in "send_crash_reports" section in `config.xml`:
-
+> Change "enabled" to true in "send_crash_reports" section in `config.xml`:
 ```
 <send_crash_reports>
         <!-- Changing <enabled> to true allows sending crash reports to -->

.github/PULL_REQUEST_TEMPLATE.md

@@ -2,16 +2,16 @@
 A technical comment, you are free to remove or leave it as it is when PR is created
 The following categories are used in the next scripts, update them accordingly
 utils/changelog/changelog.py
-tests/ci/run_check.py
+tests/ci/cancel_and_rerun_workflow_lambda/app.py
 -->
 ### Changelog category (leave one):
 - New Feature
 - Improvement
-- Bug Fix (user-visible misbehavior in official stable or prestable release)
 - Performance Improvement
 - Backward Incompatible Change
 - Build/Testing/Packaging Improvement
 - Documentation (changelog entry is not required)
+- Bug Fix (user-visible misbehavior in an official stable release)
 - Not for changelog (changelog entry is not required)
 
 

.github/actions/clean/action.yml

@@ -0,0 +1,11 @@
+name: Clean runner
+description: Clean the runner's temp path on ending
+runs:
+  using: "composite"
+  steps:
+    - name: Clean
+      shell: bash
+      run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "${{runner.temp}}"

.github/actions/common_setup/action.yml

@@ -0,0 +1,30 @@
+name: Common setup
+description: Setup necessary environments
+inputs:
+  job_type:
+    description: the name to use in the TEMP_PATH and REPO_COPY
+    default: common
+    type: string
+  nested_job:
+    description: the fuse for unintended use inside of the reusable callable jobs
+    default: true
+    type: boolean
+runs:
+  using: "composite"
+  steps:
+    - name: Setup and check ENV
+      shell: bash
+      run: |
+          echo "Setup the common ENV variables"
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/${{inputs.job_type}}
+          EOF
+          if [ -z "${{env.GITHUB_JOB_OVERRIDDEN}}" ] && [ "true" == "${{inputs.nested_job}}" ]; then
+            echo "The GITHUB_JOB_OVERRIDDEN ENV is unset, and must be set for the nested jobs"
+            exit 1
+          fi
+    - name: Setup $TEMP_PATH
+      shell: bash
+      run: |
+          # to remove every leftovers
+          sudo rm -fr "$TEMP_PATH" && mkdir -p "$TEMP_PATH"

.github/workflows/auto_release.yml

@@ -0,0 +1,45 @@
+name: AutoRelease
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+
+concurrency:
+  group: auto-release
+on: # yamllint disable-line rule:truthy
+  # schedule:
+  #   - cron: '0 10-16 * * 1-5'
+  workflow_dispatch:
+
+jobs:
+  CherryPick:
+    runs-on: [self-hosted, style-checker-aarch64]
+    steps:
+      - name: Set envs
+        # https://docs.github.com/en/actions/learn-github-actions/workflow-commands-for-github-actions#multiline-strings
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/cherry_pick
+          ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
+          ${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
+          RCSK
+          REPO_OWNER=ClickHouse
+          REPO_NAME=ClickHouse
+          REPO_TEAM=core
+          EOF
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+          token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
+          fetch-depth: 0
+      - name: Auto-release
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 auto_release.py --release-after-days=3
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH"

.github/workflows/backport_branches.yml

@@ -1,3 +1,4 @@
+# yamllint disable rule:comments-indentation
 name: BackportPR
 
 env:
@@ -9,739 +10,269 @@ on: # yamllint disable-line rule:truthy
     branches:
       - 'backport/**'
 jobs:
-  PythonUnitTests:
+  RunConfig:
     runs-on: [self-hosted, style-checker]
+    outputs:
+      data: ${{ steps.runconfig.outputs.CI_DATA }}
     steps:
       - name: Check out repository code
         uses: ClickHouse/checkout@v1
         with:
-          clear-repository: true
+          clear-repository: true # to ensure correct digests
+          fetch-depth: 0 # to get version
+          filter: tree:0
+      - name: Labels check
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 run_check.py
       - name: Python unit tests
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 -m unittest discover -s . -p '*_test.py'
-  DockerHubPushAarch64:
-    runs-on: [self-hosted, style-checker-aarch64]
-    steps:
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Images check
+          echo "Testing the main ci directory"
+          python3 -m unittest discover -s . -p 'test_*.py'
+          for dir in *_lambda/; do
+            echo "Testing $dir"
+            python3 -m unittest discover -s "$dir" -p 'test_*.py'
+          done
+      - name: PrepareRunConfig
+        id: runconfig
         run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 docker_images_check.py --suffix aarch64
-      - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: changed_images_aarch64
-          path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json
-  DockerHubPushAmd64:
-    runs-on: [self-hosted, style-checker]
-    steps:
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Images check
+            echo "::group::configure CI run"
+            python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --outfile ${{ runner.temp }}/ci_run_data.json
+            echo "::endgroup::"
+
+            echo "::group::CI run configure results"
+            python3 -m json.tool ${{ runner.temp }}/ci_run_data.json
+            echo "::endgroup::"
+
+            {
+              echo 'CI_DATA<<EOF'
+              cat  ${{ runner.temp }}/ci_run_data.json
+              echo 'EOF'
+            } >> "$GITHUB_OUTPUT"
+      - name: Re-create GH statuses for skipped jobs if any
         run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 docker_images_check.py --suffix amd64
-      - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: changed_images_amd64
-          path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json
-  DockerHubPush:
-    needs: [DockerHubPushAmd64, DockerHubPushAarch64]
-    runs-on: [self-hosted, style-checker]
-    steps:
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Download changed aarch64 images
-        uses: actions/download-artifact@v3
-        with:
-          name: changed_images_aarch64
-          path: ${{ runner.temp }}
-      - name: Download changed amd64 images
-        uses: actions/download-artifact@v3
-        with:
-          name: changed_images_amd64
-          path: ${{ runner.temp }}
-      - name: Images check
-        run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64
-      - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: changed_images
-          path: ${{ runner.temp }}/changed_images.json
-  CompatibilityCheck:
-    needs: [BuilderDebRelease]
-    runs-on: [self-hosted, style-checker]
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/compatibility_check
-          REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse
-          REPORTS_PATH=${{runner.temp}}/reports_dir
-          EOF
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Download json reports
-        uses: actions/download-artifact@v3
-        with:
-          path: ${{ env.REPORTS_PATH }}
-      - name: CompatibilityCheck
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH"
+            python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ runner.temp }}/ci_run_data.json --update-gh-statuses
+  BuildDockers:
+    needs: [RunConfig]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_docker.yml
+    with:
+      data: ${{ needs.RunConfig.outputs.data }}
+  CompatibilityCheckX86:
+    needs: [RunConfig, BuilderDebRelease]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: Compatibility check (amd64)
+      runner_type: style-checker
+      data: ${{ needs.RunConfig.outputs.data }}
+      run_command: |
+        python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions
+  CompatibilityCheckAarch64:
+    needs: [RunConfig, BuilderDebAarch64]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: Compatibility check (aarch64)
+      runner_type: style-checker
+      data: ${{ needs.RunConfig.outputs.data }}
+      run_command: |
+        python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc
 #########################################################################################
 #################################### ORDINARY BUILDS ####################################
 #########################################################################################
   BuilderDebRelease:
-    needs: [DockerHubPush]
-    runs-on: [self-hosted, builder]
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/build_check
-          IMAGES_PATH=${{runner.temp}}/images_path
-          REPO_COPY=${{runner.temp}}/build_check/ClickHouse
-          CACHES_PATH=${{runner.temp}}/../ccaches
-          BUILD_NAME=package_release
-          EOF
-      - name: Download changed images
-        uses: actions/download-artifact@v3
-        with:
-          name: changed_images
-          path: ${{ env.IMAGES_PATH }}
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-          submodules: true
-          fetch-depth: 0 # For a proper version and performance artifacts
-      - name: Build
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
-      - name: Upload build URLs to artifacts
-        if: ${{ success() || failure() }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ env.BUILD_URLS }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
+    needs: [RunConfig, BuildDockers]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_build.yml
+    with:
+      build_name: package_release
+      checkout_depth: 0
+      data: ${{ needs.RunConfig.outputs.data }}
   BuilderDebAarch64:
-    needs: [DockerHubPush]
-    runs-on: [self-hosted, builder]
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/build_check
-          IMAGES_PATH=${{runner.temp}}/images_path
-          REPO_COPY=${{runner.temp}}/build_check/ClickHouse
-          CACHES_PATH=${{runner.temp}}/../ccaches
-          BUILD_NAME=package_aarch64
-          EOF
-      - name: Download changed images
-        uses: actions/download-artifact@v3
-        with:
-          name: changed_images
-          path: ${{ env.IMAGES_PATH }}
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-          submodules: true
-          fetch-depth: 0 # For a proper version and performance artifacts
-      - name: Build
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
-      - name: Upload build URLs to artifacts
-        if: ${{ success() || failure() }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ env.BUILD_URLS }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
+    needs: [RunConfig, BuildDockers]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_build.yml
+    with:
+      build_name: package_aarch64
+      checkout_depth: 0
+      data: ${{ needs.RunConfig.outputs.data }}
   BuilderDebAsan:
-    needs: [DockerHubPush]
-    runs-on: [self-hosted, builder]
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/build_check
-          IMAGES_PATH=${{runner.temp}}/images_path
-          REPO_COPY=${{runner.temp}}/build_check/ClickHouse
-          CACHES_PATH=${{runner.temp}}/../ccaches
-          BUILD_NAME=package_asan
-          EOF
-      - name: Download changed images
-        uses: actions/download-artifact@v3
-        with:
-          name: changed_images
-          path: ${{ env.IMAGES_PATH }}
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-          submodules: true
-      - name: Build
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
-      - name: Upload build URLs to artifacts
-        if: ${{ success() || failure() }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ env.BUILD_URLS }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
+    needs: [RunConfig, BuildDockers]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_build.yml
+    with:
+      build_name: package_asan
+      data: ${{ needs.RunConfig.outputs.data }}
   BuilderDebTsan:
-    needs: [DockerHubPush]
-    runs-on: [self-hosted, builder]
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/build_check
-          IMAGES_PATH=${{runner.temp}}/images_path
-          REPO_COPY=${{runner.temp}}/build_check/ClickHouse
-          CACHES_PATH=${{runner.temp}}/../ccaches
-          BUILD_NAME=package_tsan
-          EOF
-      - name: Download changed images
-        uses: actions/download-artifact@v3
-        with:
-          name: changed_images
-          path: ${{ env.IMAGES_PATH }}
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-          submodules: true
-      - name: Build
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
-      - name: Upload build URLs to artifacts
-        if: ${{ success() || failure() }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ env.BUILD_URLS }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
+    needs: [RunConfig, BuildDockers]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_build.yml
+    with:
+      build_name: package_tsan
+      data: ${{ needs.RunConfig.outputs.data }}
   BuilderDebDebug:
-    needs: [DockerHubPush]
-    runs-on: [self-hosted, builder]
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/build_check
-          IMAGES_PATH=${{runner.temp}}/images_path
-          REPO_COPY=${{runner.temp}}/build_check/ClickHouse
-          CACHES_PATH=${{runner.temp}}/../ccaches
-          BUILD_NAME=package_debug
-          EOF
-      - name: Download changed images
-        uses: actions/download-artifact@v3
-        with:
-          name: changed_images
-          path: ${{ env.IMAGES_PATH }}
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-          submodules: true
-      - name: Build
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
-      - name: Upload build URLs to artifacts
-        if: ${{ success() || failure() }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ env.BUILD_URLS }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
+    needs: [RunConfig, BuildDockers]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_build.yml
+    with:
+      build_name: package_debug
+      data: ${{ needs.RunConfig.outputs.data }}
   BuilderBinDarwin:
-    needs: [DockerHubPush]
-    runs-on: [self-hosted, builder]
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/build_check
-          IMAGES_PATH=${{runner.temp}}/images_path
-          REPO_COPY=${{runner.temp}}/build_check/ClickHouse
-          CACHES_PATH=${{runner.temp}}/../ccaches
-          BUILD_NAME=binary_darwin
-          EOF
-      - name: Download changed images
-        uses: actions/download-artifact@v3
-        with:
-          name: changed_images
-          path: ${{ env.IMAGES_PATH }}
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-          submodules: true
-          fetch-depth: 0 # otherwise we will have no info about contributors
-      - name: Build
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
-      - name: Upload build URLs to artifacts
-        if: ${{ success() || failure() }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ env.BUILD_URLS }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
+    needs: [RunConfig, BuildDockers]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_build.yml
+    with:
+      build_name: binary_darwin
+      data: ${{ needs.RunConfig.outputs.data }}
+      checkout_depth: 0
   BuilderBinDarwinAarch64:
-    needs: [DockerHubPush]
-    runs-on: [self-hosted, builder]
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/build_check
-          IMAGES_PATH=${{runner.temp}}/images_path
-          REPO_COPY=${{runner.temp}}/build_check/ClickHouse
-          CACHES_PATH=${{runner.temp}}/../ccaches
-          BUILD_NAME=binary_darwin_aarch64
-          EOF
-      - name: Download changed images
-        uses: actions/download-artifact@v3
-        with:
-          name: changed_images
-          path: ${{ env.IMAGES_PATH }}
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-          submodules: true
-          fetch-depth: 0 # otherwise we will have no info about contributors
-      - name: Build
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
-      - name: Upload build URLs to artifacts
-        if: ${{ success() || failure() }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ env.BUILD_URLS }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
+    needs: [RunConfig, BuildDockers]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_build.yml
+    with:
+      build_name: binary_darwin_aarch64
+      data: ${{ needs.RunConfig.outputs.data }}
+      checkout_depth: 0
 ############################################################################################
 ##################################### Docker images  #######################################
 ############################################################################################
   DockerServerImages:
-    needs:
-      - BuilderDebRelease
-      - BuilderDebAarch64
-    runs-on: [self-hosted, style-checker]
-    steps:
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-          fetch-depth: 0  # It MUST BE THE SAME for all dependencies and the job itself
-      - name: Check docker clickhouse/clickhouse-server building
-        run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 docker_server.py --release-type head --no-push
-          python3 docker_server.py --release-type head --no-push --no-ubuntu \
-            --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH"
+    needs: [RunConfig, BuilderDebRelease, BuilderDebAarch64]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: Docker server and keeper images
+      runner_type: style-checker
+      data: ${{ needs.RunConfig.outputs.data }}
+      checkout_depth: 0  # It MUST BE THE SAME for all dependencies and the job itself
+      run_command: |
+        cd "$GITHUB_WORKSPACE/tests/ci"
+        python3 docker_server.py --release-type head --no-push \
+          --image-repo clickhouse/clickhouse-server --image-path docker/server --allow-build-reuse
+        python3 docker_server.py --release-type head --no-push \
+          --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper --allow-build-reuse
 ############################################################################################
 ##################################### BUILD REPORTER #######################################
 ############################################################################################
   BuilderReport:
+    # run report check for failed builds to indicate the CI error
+    if: ${{ !cancelled() }}
     needs:
-      - BuilderDebRelease
+      - RunConfig
       - BuilderDebAarch64
       - BuilderDebAsan
-      - BuilderDebTsan
       - BuilderDebDebug
-    runs-on: [self-hosted, style-checker]
-    if: ${{ success() || failure() }}
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          CHECK_NAME=ClickHouse build check
-          REPORTS_PATH=${{runner.temp}}/reports_dir
-          TEMP_PATH=${{runner.temp}}/report_check
-          NEEDS_DATA_PATH=${{runner.temp}}/needs.json
-          EOF
-      - name: Download json reports
-        uses: actions/download-artifact@v3
-        with:
-          path: ${{ env.REPORTS_PATH }}
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Report Builder
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cat > "$NEEDS_DATA_PATH" << 'EOF'
-          ${{ toJSON(needs) }}
-          EOF
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 build_report_check.py "$CHECK_NAME"
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH"
+      - BuilderDebRelease
+      - BuilderDebTsan
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: ClickHouse build check
+      runner_type: style-checker
+      data: ${{ needs.RunConfig.outputs.data }}
+      additional_envs: |
+        NEEDS_DATA<<NDENV
+        ${{ toJSON(needs) }}
+        NDENV
+      run_command: |
+        python3 build_report_check.py "$CHECK_NAME"
   BuilderSpecialReport:
+    # run report check for failed builds to indicate the CI error
+    if: ${{ !cancelled() }}
     needs:
+      - RunConfig
       - BuilderBinDarwin
       - BuilderBinDarwinAarch64
-    runs-on: [self-hosted, style-checker]
-    if: ${{ success() || failure() }}
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/report_check
-          REPORTS_PATH=${{runner.temp}}/reports_dir
-          CHECK_NAME=ClickHouse special build check
-          NEEDS_DATA_PATH=${{runner.temp}}/needs.json
-          EOF
-      - name: Download json reports
-        uses: actions/download-artifact@v3
-        with:
-          path: ${{ env.REPORTS_PATH }}
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Report Builder
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cat > "$NEEDS_DATA_PATH" << 'EOF'
-          ${{ toJSON(needs) }}
-          EOF
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 build_report_check.py "$CHECK_NAME"
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH"
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: ClickHouse special build check
+      runner_type: style-checker
+      data: ${{ needs.RunConfig.outputs.data }}
+      additional_envs: |
+        NEEDS_DATA<<NDENV
+        ${{ toJSON(needs) }}
+        NDENV
+      run_command: |
+        python3 build_report_check.py "$CHECK_NAME"
 ############################################################################################
 #################################### INSTALL PACKAGES ######################################
 ############################################################################################
   InstallPackagesTestRelease:
-    needs: [BuilderDebRelease]
-    runs-on: [self-hosted, style-checker]
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/test_install
-          REPORTS_PATH=${{runner.temp}}/reports_dir
-          CHECK_NAME=Install packages (amd64)
-          REPO_COPY=${{runner.temp}}/test_install/ClickHouse
-          EOF
-      - name: Download json reports
-        uses: actions/download-artifact@v3
-        with:
-          path: ${{ env.REPORTS_PATH }}
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Test packages installation
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci"
-          python3 install_check.py "$CHECK_NAME"
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH"
+    needs: [RunConfig, BuilderDebRelease]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: Install packages (amd64)
+      runner_type: style-checker
+      data: ${{ needs.RunConfig.outputs.data }}
+      run_command: |
+        python3 install_check.py "$CHECK_NAME"
   InstallPackagesTestAarch64:
-    needs: [BuilderDebRelease]
-    runs-on: [self-hosted, style-checker-aarch64]
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/test_install
-          REPORTS_PATH=${{runner.temp}}/reports_dir
-          CHECK_NAME=Install packages (arm64)
-          REPO_COPY=${{runner.temp}}/test_install/ClickHouse
-          EOF
-      - name: Download json reports
-        uses: actions/download-artifact@v3
-        with:
-          path: ${{ env.REPORTS_PATH }}
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Test packages installation
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci"
-          python3 install_check.py "$CHECK_NAME"
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH"
+    needs: [RunConfig, BuilderDebAarch64]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: Install packages (arm64)
+      runner_type: style-checker-aarch64
+      data: ${{ needs.RunConfig.outputs.data }}
+      run_command: |
+        python3 install_check.py "$CHECK_NAME"
 ##############################################################################################
 ########################### FUNCTIONAl STATELESS TESTS #######################################
 ##############################################################################################
   FunctionalStatelessTestAsan:
-    needs: [BuilderDebAsan]
-    runs-on: [self-hosted, func-tester]
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/stateless_debug
-          REPORTS_PATH=${{runner.temp}}/reports_dir
-          CHECK_NAME=Stateless tests (asan)
-          REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
-          KILL_TIMEOUT=10800
-          EOF
-      - name: Download json reports
-        uses: actions/download-artifact@v3
-        with:
-          path: ${{ env.REPORTS_PATH }}
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Functional test
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci"
-          python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH"
+    needs: [RunConfig, BuilderDebAsan]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: Stateless tests (asan)
+      runner_type: func-tester
+      data: ${{ needs.RunConfig.outputs.data }}
 ##############################################################################################
 ############################ FUNCTIONAl STATEFUL TESTS #######################################
 ##############################################################################################
   FunctionalStatefulTestDebug:
-    needs: [BuilderDebDebug]
-    runs-on: [self-hosted, func-tester]
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/stateful_debug
-          REPORTS_PATH=${{runner.temp}}/reports_dir
-          CHECK_NAME=Stateful tests (debug)
-          REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse
-          KILL_TIMEOUT=3600
-          EOF
-      - name: Download json reports
-        uses: actions/download-artifact@v3
-        with:
-          path: ${{ env.REPORTS_PATH }}
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Functional test
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci"
-          python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH"
+    needs: [RunConfig, BuilderDebDebug]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: Stateful tests (debug)
+      runner_type: func-tester
+      data: ${{ needs.RunConfig.outputs.data }}
 ##############################################################################################
 ######################################### STRESS TESTS #######################################
 ##############################################################################################
   StressTestTsan:
-    needs: [BuilderDebTsan]
-    # func testers have 16 cores + 128 GB memory
-    # while stress testers have 36 cores + 72 memory
-    # It would be better to have something like 32 + 128,
-    # but such servers almost unavailable as spot instances.
-    runs-on: [self-hosted, func-tester]
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/stress_thread
-          REPORTS_PATH=${{runner.temp}}/reports_dir
-          CHECK_NAME=Stress test (tsan)
-          REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse
-          EOF
-      - name: Download json reports
-        uses: actions/download-artifact@v3
-        with:
-          path: ${{ env.REPORTS_PATH }}
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Stress test
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci"
-          python3 stress_check.py "$CHECK_NAME"
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH"
+    needs: [RunConfig, BuilderDebTsan]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: Stress test (tsan)
+      runner_type: stress-tester
+      data: ${{ needs.RunConfig.outputs.data }}
 #############################################################################################
 ############################# INTEGRATION TESTS #############################################
 #############################################################################################
   IntegrationTestsRelease:
-    needs: [BuilderDebRelease]
-    runs-on: [self-hosted, stress-tester]
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/integration_tests_release
-          REPORTS_PATH=${{runner.temp}}/reports_dir
-          CHECK_NAME=Integration tests (release)
-          REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse
-          EOF
-      - name: Download json reports
-        uses: actions/download-artifact@v3
-        with:
-          path: ${{ env.REPORTS_PATH }}
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Integration test
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci"
-          python3 integration_test_check.py "$CHECK_NAME"
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH"
+    needs: [RunConfig, BuilderDebRelease]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: Integration tests (release)
+      runner_type: stress-tester
+      data: ${{ needs.RunConfig.outputs.data }}
   FinishCheck:
+    if: ${{ !failure() && !cancelled() }}
     needs:
-      - DockerHubPush
-      - DockerServerImages
       - BuilderReport
       - BuilderSpecialReport
       - FunctionalStatelessTestAsan
       - FunctionalStatefulTestDebug
       - StressTestTsan
       - IntegrationTestsRelease
-      - CompatibilityCheck
+      - CompatibilityCheckX86
+      - CompatibilityCheckAarch64
     runs-on: [self-hosted, style-checker]
     steps:
       - name: Check out repository code

.github/workflows/cherry_pick.yml

@@ -35,7 +35,6 @@ jobs:
           fetch-depth: 0
       - name: Cherry pick
         run: |
-          sudo pip install GitPython
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 cherry_pick.py
       - name: Cleanup

.github/workflows/debug.yml

@@ -2,7 +2,7 @@
 name: Debug
 
 'on':
-  [push, pull_request, release, workflow_dispatch, workflow_call]
+  [push, pull_request, pull_request_review, release, workflow_dispatch, workflow_call]
 
 jobs:
   DebugInfo:

.github/workflows/docs_check.yml

@@ -1,172 +0,0 @@
-name: DocsCheck
-
-env:
-  # Force the stdout and stderr streams to be unbuffered
-  PYTHONUNBUFFERED: 1
-
-on:  # yamllint disable-line rule:truthy
-  pull_request:
-    types:
-      - synchronize
-      - reopened
-      - opened
-    branches:
-      - master
-    paths:
-      - 'docker/docs/**'
-      - 'docs/**'
-      - 'website/**'
-      - 'utils/check-style/aspell-ignore/**'
-jobs:
-  CheckLabels:
-    runs-on: [self-hosted, style-checker]
-    steps:
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Labels check
-        run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 run_check.py
-  DockerHubPushAarch64:
-    needs: CheckLabels
-    runs-on: [self-hosted, style-checker-aarch64]
-    steps:
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Images check
-        run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 docker_images_check.py --suffix aarch64
-      - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: changed_images_aarch64
-          path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json
-  DockerHubPushAmd64:
-    needs: CheckLabels
-    runs-on: [self-hosted, style-checker]
-    steps:
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Images check
-        run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 docker_images_check.py --suffix amd64
-      - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: changed_images_amd64
-          path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json
-  DockerHubPush:
-    needs: [DockerHubPushAmd64, DockerHubPushAarch64]
-    runs-on: [self-hosted, style-checker]
-    steps:
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Download changed aarch64 images
-        uses: actions/download-artifact@v3
-        with:
-          name: changed_images_aarch64
-          path: ${{ runner.temp }}
-      - name: Download changed amd64 images
-        uses: actions/download-artifact@v3
-        with:
-          name: changed_images_amd64
-          path: ${{ runner.temp }}
-      - name: Images check
-        run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64
-      - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: changed_images
-          path: ${{ runner.temp }}/changed_images.json
-  StyleCheck:
-    needs: DockerHubPush
-    runs-on: [self-hosted, style-checker]
-    if: ${{ success() || failure() }}
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{ runner.temp }}/style_check
-          ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
-          ${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
-          RCSK
-          EOF
-      - name: Download changed images
-        # even if artifact does not exist, e.g. on `do not test` label or failed Docker job
-        continue-on-error: true
-        uses: actions/download-artifact@v3
-        with:
-          name: changed_images
-          path: ${{ env.TEMP_PATH }}
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Style Check
-        run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 style_check.py
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH"
-  DocsCheck:
-    needs: DockerHubPush
-    runs-on: [self-hosted, func-tester-aarch64]
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/docs_check
-          REPO_COPY=${{runner.temp}}/docs_check/ClickHouse
-          EOF
-      - name: Download changed images
-        uses: actions/download-artifact@v3
-        with:
-          name: changed_images
-          path: ${{ env.TEMP_PATH }}
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Docs Check
-        run: |
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci"
-          python3 docs_check.py
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH"
-  FinishCheck:
-    needs:
-      - StyleCheck
-      - DockerHubPush
-      - DocsCheck
-    runs-on: [self-hosted, style-checker]
-    steps:
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Finish label
-        run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 finish_check.py
-          python3 merge_pr.py --check-approved

.github/workflows/docs_release.yml

@@ -1,118 +0,0 @@
-name: DocsReleaseChecks
-
-env:
-  # Force the stdout and stderr streams to be unbuffered
-  PYTHONUNBUFFERED: 1
-
-concurrency:
-  group: master-release
-  cancel-in-progress: true
-'on':
-  push:
-    branches:
-      - master
-    paths:
-      - '.github/**'
-      - 'docker/docs/release/**'
-      - 'docs/**'
-      - 'utils/list-versions/version_date.tsv'
-      - 'website/**'
-      - 'utils/check-style/aspell-ignore/**'
-  workflow_dispatch:
-jobs:
-  DockerHubPushAarch64:
-    runs-on: [self-hosted, style-checker-aarch64]
-    steps:
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Images check
-        run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 docker_images_check.py --suffix aarch64
-      - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: changed_images_aarch64
-          path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json
-  DockerHubPushAmd64:
-    runs-on: [self-hosted, style-checker]
-    steps:
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Images check
-        run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 docker_images_check.py --suffix amd64
-      - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: changed_images_amd64
-          path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json
-  DockerHubPush:
-    needs: [DockerHubPushAmd64, DockerHubPushAarch64]
-    runs-on: [self-hosted, style-checker]
-    steps:
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Download changed aarch64 images
-        uses: actions/download-artifact@v3
-        with:
-          name: changed_images_aarch64
-          path: ${{ runner.temp }}
-      - name: Download changed amd64 images
-        uses: actions/download-artifact@v3
-        with:
-          name: changed_images_amd64
-          path: ${{ runner.temp }}
-      - name: Images check
-        run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64
-      - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: changed_images
-          path: ${{ runner.temp }}/changed_images.json
-  DocsRelease:
-    needs: DockerHubPush
-    runs-on: [self-hosted, func-tester]
-    steps:
-      - name: Set envs
-        # https://docs.github.com/en/actions/learn-github-actions/workflow-commands-for-github-actions#multiline-strings
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/docs_release
-          REPO_COPY=${{runner.temp}}/docs_release/ClickHouse
-          CLOUDFLARE_TOKEN=${{secrets.CLOUDFLARE}}
-          ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
-          ${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
-          RCSK
-          EOF
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Download changed images
-        uses: actions/download-artifact@v3
-        with:
-          name: changed_images
-          path: ${{ env.TEMP_PATH }}
-      - name: Docs Release
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci"
-          python3 docs_release.py
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH"

.github/workflows/jepsen.yml

@@ -11,58 +11,17 @@ on: # yamllint disable-line rule:truthy
   workflow_call:
 jobs:
   KeeperJepsenRelease:
-    runs-on: [self-hosted, style-checker]
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/keeper_jepsen
-          REPO_COPY=${{runner.temp}}/keeper_jepsen/ClickHouse
-          EOF
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-          fetch-depth: 0
-      - name: Jepsen Test
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci"
-          python3 jepsen_check.py keeper
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH"
+    uses: ./.github/workflows/reusable_simple_job.yml
+    with:
+      test_name: Jepsen keeper check
+      runner_type: style-checker
+      run_command: |
+        python3 jepsen_check.py keeper
   # ServerJepsenRelease:
-  #   runs-on: [self-hosted, style-checker]
-  #   if: ${{ always() }}
-  #   needs: [KeeperJepsenRelease]
-  #   steps:
-  #     - name: Set envs
-  #       run: |
-  #         cat >> "$GITHUB_ENV" << 'EOF'
-  #         TEMP_PATH=${{runner.temp}}/server_jepsen
-  #         REPO_COPY=${{runner.temp}}/server_jepsen/ClickHouse
-  #         EOF
-  #     - name: Check out repository code
-  #       uses: ClickHouse/checkout@v1
-  #       with:
-  #         clear-repository: true
-  #         fetch-depth: 0
-  #     - name: Jepsen Test
-  #       run: |
-  #         sudo rm -fr "$TEMP_PATH"
-  #         mkdir -p "$TEMP_PATH"
-  #         cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-  #         cd "$REPO_COPY/tests/ci"
-  #         python3 jepsen_check.py server
-  #     - name: Cleanup
-  #       if: always()
-  #       run: |
-  #         docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-  #         docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-  #         sudo rm -fr "$TEMP_PATH"
+  #   uses: ./.github/workflows/reusable_simple_job.yml
+  #   with:
+  #     test_name: Jepsen server check
+  #     runner_type: style-checker
+  #     run_command: |
+  #       cd "$REPO_COPY/tests/ci"
+  #       python3 jepsen_check.py server

.github/workflows/libfuzzer.yml

@@ -0,0 +1,33 @@
+name: libFuzzer
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+
+on: # yamllint disable-line rule:truthy
+  #  schedule:
+  #    - cron: '0 0 2 31 1' # never for now
+  workflow_call:
+    inputs:
+      data:
+        description: json ci data
+        type: string
+        required: true
+
+jobs:
+  BuilderFuzzers:
+    uses: ./.github/workflows/reusable_build.yml
+    with:
+      build_name: fuzzers
+      data: ${{ inputs.data }}
+  libFuzzerTest:
+    needs: [BuilderFuzzers]
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: libFuzzer tests
+      runner_type: func-tester
+      data: ${{ inputs.data }}
+      additional_envs: |
+        KILL_TIMEOUT=10800
+      run_command: |
+        python3 libfuzzer_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"

.github/workflows/nightly.yml

@@ -13,124 +13,53 @@ jobs:
   Debug:
     # The task for having a preserved ENV and event.json for later investigation
     uses: ./.github/workflows/debug.yml
-  DockerHubPushAarch64:
-    runs-on: [self-hosted, style-checker-aarch64]
-    steps:
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Images check
-        run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 docker_images_check.py --suffix aarch64 --all
-      - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: changed_images_aarch64
-          path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json
-  DockerHubPushAmd64:
+  RunConfig:
     runs-on: [self-hosted, style-checker]
+    outputs:
+      data: ${{ steps.runconfig.outputs.CI_DATA }}
     steps:
       - name: Check out repository code
         uses: ClickHouse/checkout@v1
         with:
-          clear-repository: true
-      - name: Images check
+          clear-repository: true # to ensure correct digests
+          fetch-depth: 0 # to get version
+          filter: tree:0
+      - name: PrepareRunConfig
+        id: runconfig
         run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 docker_images_check.py --suffix amd64 --all
-      - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: changed_images_amd64
-          path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json
-  DockerHubPush:
-    needs: [DockerHubPushAmd64, DockerHubPushAarch64]
-    runs-on: [self-hosted, style-checker]
-    steps:
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Download changed aarch64 images
-        uses: actions/download-artifact@v3
-        with:
-          name: changed_images_aarch64
-          path: ${{ runner.temp }}
-      - name: Download changed amd64 images
-        uses: actions/download-artifact@v3
-        with:
-          name: changed_images_amd64
-          path: ${{ runner.temp }}
-      - name: Images check
-        run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64
-      - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: changed_images
-          path: ${{ runner.temp }}/changed_images.json
-  BuilderCoverity:
-    needs: DockerHubPush
-    runs-on: [self-hosted, builder]
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          BUILD_NAME=coverity
-          CACHES_PATH=${{runner.temp}}/../ccaches
-          IMAGES_PATH=${{runner.temp}}/images_path
-          REPO_COPY=${{runner.temp}}/build_check/ClickHouse
-          TEMP_PATH=${{runner.temp}}/build_check
-          EOF
-          echo "COVERITY_TOKEN=${{ secrets.COVERITY_TOKEN }}" >> "$GITHUB_ENV"
-      - name: Download changed images
-        uses: actions/download-artifact@v3
-        with:
-          name: changed_images
-          path: ${{ env.IMAGES_PATH }}
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-          submodules: true
-      - name: Build
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
-      - name: Upload Coverity Analysis
-        if: ${{ success() || failure() }}
-        run: |
-          curl --form token="${COVERITY_TOKEN}" \
-            --form email='security+coverity@clickhouse.com' \
-            --form file="@$TEMP_PATH/$BUILD_NAME/coverity-scan.tar.zst" \
-            --form version="${GITHUB_REF#refs/heads/}-${GITHUB_SHA::6}" \
-            --form description="Nighly Scan: $(date +'%Y-%m-%dT%H:%M:%S')" \
-            https://scan.coverity.com/builds?project=ClickHouse%2FClickHouse
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
+            echo "::group::configure CI run"
+            python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --skip-jobs --rebuild-all-docker --outfile ${{ runner.temp }}/ci_run_data.json
+            echo "::endgroup::"
+
+            echo "::group::CI run configure results"
+            python3 -m json.tool ${{ runner.temp }}/ci_run_data.json
+            echo "::endgroup::"
+            {
+              echo 'CI_DATA<<EOF'
+              cat  ${{ runner.temp }}/ci_run_data.json
+              echo 'EOF'
+            } >> "$GITHUB_OUTPUT"
+  BuildDockers:
+    needs: [RunConfig]
+    uses: ./.github/workflows/reusable_docker.yml
+    with:
+      data: "${{ needs.RunConfig.outputs.data }}"
+      set_latest: true
   SonarCloud:
     runs-on: [self-hosted, builder]
     env:
-      SONAR_SCANNER_VERSION: 4.7.0.2747
+      SONAR_SCANNER_VERSION: 4.8.0.2856
       SONAR_SERVER_URL: "https://sonarcloud.io"
       BUILD_WRAPPER_OUT_DIR: build_wrapper_output_directory # Directory where build-wrapper output will be placed
-      CC: clang-15
-      CXX: clang++-15
+      CC: clang-17
+      CXX: clang++-17
     steps:
       - name: Check out repository code
         uses: ClickHouse/checkout@v1
         with:
           clear-repository: true
           fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
+          filter: tree:0
           submodules: true
       - name: Set up JDK 11
         uses: actions/setup-java@v1
@@ -154,7 +83,7 @@ jobs:
       - name: Set Up Build Tools
         run: |
           sudo apt-get update
-          sudo apt-get install -yq git cmake ccache python3 ninja-build
+          sudo apt-get install -yq git cmake ccache ninja-build python3 yasm nasm
           sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
       - name: Run build-wrapper
         run: |
@@ -173,4 +102,5 @@ jobs:
             --define sonar.cfamily.build-wrapper-output="${{ env.BUILD_WRAPPER_OUT_DIR }}" \
             --define sonar.projectKey="ClickHouse_ClickHouse" \
             --define sonar.organization="clickhouse-java" \
+            --define sonar.cfamily.cpp23.enabled=true \
             --define sonar.exclusions="**/*.java,**/*.ts,**/*.js,**/*.css,**/*.sql"

.github/workflows/pull_request_approved.yml

@@ -0,0 +1,23 @@
+name: PullRequestApprovedCI
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+
+on:  # yamllint disable-line rule:truthy
+  pull_request_review:
+    types:
+      - submitted
+
+jobs:
+  MergeOnApproval:
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+      - name: Merge approved PR
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 merge_pr.py --check-approved

.github/workflows/release.yml

@@ -7,15 +7,28 @@ on: # yamllint disable-line rule:truthy
   release:
     types:
     - published
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Release tag'
+        required: true
+        type: string
 
 jobs:
   ReleasePublish:
     runs-on: [self-hosted, style-checker]
     steps:
+    - name: Set tag from input
+      if: github.event_name == 'workflow_dispatch'
+      run: |
+        echo "GITHUB_TAG=${{ github.event.inputs.tag }}" >> "$GITHUB_ENV"
+    - name: Set tag from REF
+      if: github.event_name == 'release'
+      run: |
+        echo "GITHUB_TAG=${GITHUB_REF#refs/tags/}" >> "$GITHUB_ENV"
     - name: Deploy packages and assets
       run: |
-        GITHUB_TAG="${GITHUB_REF#refs/tags/}"
-        curl --silent --data '' \
+        curl --silent --data '' --no-buffer \
           '${{ secrets.PACKAGES_RELEASE_URL }}/release/'"${GITHUB_TAG}"'?binary=binary_darwin&binary=binary_darwin_aarch64&sync=true'
   ############################################################################################
   ##################################### Docker images  #######################################
@@ -23,16 +36,27 @@ jobs:
   DockerServerImages:
     runs-on: [self-hosted, style-checker]
     steps:
+    - name: Set tag from input
+      if: github.event_name == 'workflow_dispatch'
+      run: |
+        echo "GITHUB_TAG=${{ github.event.inputs.tag }}" >> "$GITHUB_ENV"
+    - name: Set tag from REF
+      if: github.event_name == 'release'
+      run: |
+        echo "GITHUB_TAG=${GITHUB_REF#refs/tags/}" >> "$GITHUB_ENV"
     - name: Check out repository code
       uses: ClickHouse/checkout@v1
       with:
         clear-repository: true
         fetch-depth: 0  # otherwise we will have no version info
+        filter: tree:0
+        ref: ${{ env.GITHUB_TAG }}
     - name: Check docker clickhouse/clickhouse-server building
       run: |
         cd "$GITHUB_WORKSPACE/tests/ci"
-        python3 docker_server.py --release-type auto --version "${{ github.ref }}"
-        python3 docker_server.py --release-type auto --version "${{ github.ref }}" --no-ubuntu \
+        python3 docker_server.py --release-type auto --version "$GITHUB_TAG" \
+          --image-repo clickhouse/clickhouse-server --image-path docker/server
+        python3 docker_server.py --release-type auto --version "$GITHUB_TAG" \
           --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper
     - name: Cleanup
       if: always()

.github/workflows/reusable_build.yml

@@ -0,0 +1,88 @@
+### For the pure soul wishes to move it to another place
+# https://github.com/orgs/community/discussions/9050
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+
+name: Build ClickHouse
+'on':
+  workflow_call:
+    inputs:
+      build_name:
+        description: the value of build type from tests/ci/ci_config.py
+        required: true
+        type: string
+      checkout_depth:
+        description: the value of the git shallow checkout
+        required: false
+        type: number
+        default: 1
+      runner_type:
+        description: the label of runner to use
+        default: builder
+        type: string
+      data:
+        description: json ci data
+        type: string
+        required: true
+      additional_envs:
+        description: additional ENV variables to setup the job
+        type: string
+
+jobs:
+  Build:
+    name: Build-${{inputs.build_name}}
+    if: contains(fromJson(inputs.data).jobs_data.jobs_to_do, inputs.build_name)
+    env:
+      GITHUB_JOB_OVERRIDDEN: Build-${{inputs.build_name}}
+    runs-on: [self-hosted, '${{inputs.runner_type}}']
+    steps:
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+          ref: ${{ fromJson(inputs.data).git_ref }}
+          submodules: true
+          fetch-depth: ${{inputs.checkout_depth}}
+          filter: tree:0
+      - name: Set build envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          ${{inputs.additional_envs}}
+          DOCKER_TAG<<DOCKER_JSON
+          ${{ toJson(fromJson(inputs.data).docker_data.images) }}
+          DOCKER_JSON
+          EOF
+          python3 "$GITHUB_WORKSPACE"/tests/ci/ci_config.py --build-name "${{inputs.build_name}}" >> "$GITHUB_ENV"
+      - name: Apply sparse checkout for contrib # in order to check that it doesn't break build
+        # This step is done in GITHUB_WORKSPACE,
+        # because it's broken in REPO_COPY for some reason
+        if: ${{ env.BUILD_SPARSE_CHECKOUT == 'true' }}
+        run: |
+          rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed'
+          git -C "$GITHUB_WORKSPACE" checkout .  && echo 'restored'
+          "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK'
+          du -hs "$GITHUB_WORKSPACE/contrib" ||:
+          find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||:
+      - name: Common setup
+        uses: ./.github/actions/common_setup
+        with:
+          job_type: build_check
+      - name: Pre
+        run: |
+          python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --pre --job-name '${{inputs.build_name}}'
+      - name: Build
+        run: |
+          python3 "$GITHUB_WORKSPACE/tests/ci/build_check.py" "$BUILD_NAME"
+      - name: Post
+        # it still be build report to upload for failed build job
+        if: always()
+        run: |
+          python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --post --job-name '${{inputs.build_name}}'
+      - name: Mark as done
+        run: |
+          python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --mark-success --job-name '${{inputs.build_name}}'
+      - name: Clean
+        if: always()
+        uses: ./.github/actions/clean

.github/workflows/reusable_docker.yml

@@ -0,0 +1,68 @@
+name: Build docker images
+'on':
+  workflow_call:
+    inputs:
+      data:
+        description: json with ci data from todo job
+        required: true
+        type: string
+      set_latest:
+        description: set latest tag for resulting multiarch manifest
+        required: false
+        type: boolean
+        default: false
+jobs:
+  DockerBuildAarch64:
+    runs-on: [self-hosted, style-checker-aarch64]
+    if: |
+      !failure() && !cancelled() && toJson(fromJson(inputs.data).docker_data.missing_aarch64) != '[]'
+    steps:
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          ref: ${{ fromJson(inputs.data).git_ref }}
+      - name: Build images
+        run: |
+          python3 "${GITHUB_WORKSPACE}/tests/ci/docker_images_check.py" \
+            --suffix aarch64 \
+            --image-tags '${{ toJson(fromJson(inputs.data).docker_data.images) }}' \
+            --missing-images '${{ toJson(fromJson(inputs.data).docker_data.missing_aarch64) }}'
+  DockerBuildAmd64:
+    runs-on: [self-hosted, style-checker]
+    if: |
+      !failure() && !cancelled() && toJson(fromJson(inputs.data).docker_data.missing_amd64) != '[]'
+    steps:
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          ref: ${{ fromJson(inputs.data).git_ref }}
+      - name: Build images
+        run: |
+          python3 "${GITHUB_WORKSPACE}/tests/ci/docker_images_check.py" \
+            --suffix amd64 \
+            --image-tags '${{ toJson(fromJson(inputs.data).docker_data.images) }}' \
+            --missing-images '${{ toJson(fromJson(inputs.data).docker_data.missing_amd64) }}'
+  DockerMultiArchManifest:
+    needs: [DockerBuildAmd64, DockerBuildAarch64]
+    runs-on: [self-hosted, style-checker]
+    if: |
+      !failure() && !cancelled() && toJson(fromJson(inputs.data).docker_data.missing_multi) != '[]'
+    steps:
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          ref: ${{ fromJson(inputs.data).git_ref }}
+      - name: Build images
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          if [ "${{ inputs.set_latest }}" == "true" ]; then
+            echo "latest tag will be set for resulting manifests"
+            python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 \
+              --image-tags '${{ toJson(fromJson(inputs.data).docker_data.images) }}' \
+              --missing-images '${{ toJson(fromJson(inputs.data).docker_data.missing_multi) }}' \
+              --set-latest
+          else
+            python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 \
+              --image-tags '${{ toJson(fromJson(inputs.data).docker_data.images) }}' \
+              --missing-images '${{ toJson(fromJson(inputs.data).docker_data.missing_multi) }}'
+          fi

.github/workflows/reusable_simple_job.yml

@@ -0,0 +1,90 @@
+### For the pure soul wishes to move it to another place
+# https://github.com/orgs/community/discussions/9050
+
+name: Simple job
+'on':
+  workflow_call:
+    inputs:
+      test_name:
+        description: the value of test type from tests/ci/ci_config.py, ends up as $CHECK_NAME ENV
+        required: true
+        type: string
+      runner_type:
+        description: the label of runner to use
+        required: true
+        type: string
+      run_command:
+        description: the command to launch the check
+        default: ""
+        required: false
+        type: string
+      checkout_depth:
+        description: the value of the git shallow checkout
+        required: false
+        type: number
+        default: 1
+      submodules:
+        description: if the submodules should be checked out
+        required: false
+        type: boolean
+        default: false
+      additional_envs:
+        description: additional ENV variables to setup the job
+        type: string
+      working-directory:
+        description: sets custom working directory
+        type: string
+        default: ""
+      git_ref:
+        description: commit to use, merge commit for pr or head
+        required: false
+        type: string
+        default: ${{ github.event.after }} # no merge commit
+    secrets:
+      secret_envs:
+        description: if given, it's passed to the environments
+        required: false
+
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+  CHECK_NAME: ${{inputs.test_name}}
+
+jobs:
+  Test:
+    runs-on: [self-hosted, '${{inputs.runner_type}}']
+    name: ${{inputs.test_name}}
+    env:
+      GITHUB_JOB_OVERRIDDEN: ${{inputs.test_name}}
+    steps:
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+          ref: ${{ inputs.git_ref }}
+          submodules: ${{inputs.submodules}}
+          fetch-depth: ${{inputs.checkout_depth}}
+          filter: tree:0
+      - name: Set build envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          CHECK_NAME=${{ inputs.test_name }}
+          ${{inputs.additional_envs}}
+          ${{secrets.secret_envs}}
+          EOF
+      - name: Common setup
+        uses: ./.github/actions/common_setup
+        with:
+          job_type: test
+      - name: Run
+        run: |
+          if [ -n '${{ inputs.working-directory }}' ]; then
+            cd "${{ inputs.working-directory }}"
+          else
+            cd "$GITHUB_WORKSPACE/tests/ci"
+          fi
+          ${{ inputs.run_command }}
+      - name: Clean
+        if: always()
+        uses: ./.github/actions/clean

.github/workflows/reusable_test.yml

@@ -0,0 +1,119 @@
+### For the pure soul wishes to move it to another place
+# https://github.com/orgs/community/discussions/9050
+
+name: Testing workflow
+'on':
+  workflow_call:
+    inputs:
+      test_name:
+        description: the value of test type from tests/ci/ci_config.py, ends up as $CHECK_NAME ENV
+        required: true
+        type: string
+      runner_type:
+        description: the label of runner to use
+        required: true
+        type: string
+      run_command:
+        description: the command to launch the check
+        default: ""
+        required: false
+        type: string
+      checkout_depth:
+        description: the value of the git shallow checkout
+        required: false
+        type: number
+        default: 1
+      submodules:
+        description: if the submodules should be checked out
+        required: false
+        type: boolean
+        default: false
+      additional_envs:
+        description: additional ENV variables to setup the job
+        type: string
+      data:
+        description: ci data
+        type: string
+        required: true
+      working-directory:
+        description: sets custom working directory
+        type: string
+        default: ""
+    secrets:
+      secret_envs:
+        description: if given, it's passed to the environments
+        required: false
+
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+  CHECK_NAME: ${{inputs.test_name}}
+
+jobs:
+  Test:
+    runs-on: [self-hosted, '${{inputs.runner_type}}']
+    if: ${{ !failure() && !cancelled() && contains(fromJson(inputs.data).jobs_data.jobs_to_do, inputs.test_name) }}
+    name: ${{inputs.test_name}}${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].num_batches > 1 && format('-{0}',matrix.batch) || '' }}
+    env:
+      GITHUB_JOB_OVERRIDDEN: ${{inputs.test_name}}${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].num_batches > 1 && format('-{0}',matrix.batch) || '' }}
+    strategy:
+      fail-fast: false  # we always wait for entire matrix
+      matrix:
+        batch: ${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].batches }}
+    steps:
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+          ref: ${{ fromJson(inputs.data).git_ref }}
+          submodules: ${{inputs.submodules}}
+          fetch-depth: ${{inputs.checkout_depth}}
+          filter: tree:0
+      - name: Set build envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          CHECK_NAME=${{ inputs.test_name }}
+          ${{inputs.additional_envs}}
+          ${{secrets.secret_envs}}
+          DOCKER_TAG<<DOCKER_JSON
+          ${{ toJson(fromJson(inputs.data).docker_data.images) }}
+          DOCKER_JSON
+          EOF
+      - name: Common setup
+        uses: ./.github/actions/common_setup
+        with:
+          job_type: test
+      - name: Setup batch
+        if: ${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].num_batches > 1 }}
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          RUN_BY_HASH_NUM=${{matrix.batch}}
+          RUN_BY_HASH_TOTAL=${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].num_batches }}
+          EOF
+      - name: Pre run
+        run: |
+          python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --pre --job-name '${{inputs.test_name}}'
+      - name: Run
+        run: |
+          if [ -n "${{ inputs.working-directory }}" ]; then
+            cd "${{ inputs.working-directory }}"
+          else
+            cd "$GITHUB_WORKSPACE/tests/ci"
+          fi
+          if [ -n "$(echo '${{ inputs.run_command }}' | tr -d '\n')" ]; then
+            echo "Running command from workflow input"
+            ${{ inputs.run_command }}
+          else
+            echo "Running command from job config"
+            python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --run --job-name '${{inputs.test_name}}'
+          fi
+      - name: Post run
+        run: |
+          python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --post --job-name '${{inputs.test_name}}'
+      - name: Mark as done
+        run: |
+          python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --mark-success --job-name '${{inputs.test_name}}' --batch ${{matrix.batch}}
+      - name: Clean
+        if: always()
+        uses: ./.github/actions/clean

.github/workflows/tags_stable.yml

@@ -38,6 +38,7 @@ jobs:
       with:
         ref: master
         fetch-depth: 0
+        filter: tree:0
     - name: Update versions, docker version, changelog, security
       env:
         GITHUB_TOKEN: ${{ secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN }}

.github/workflows/woboq.yml

@@ -1,40 +0,0 @@
-name: WoboqBuilder
-env:
-  # Force the stdout and stderr streams to be unbuffered
-  PYTHONUNBUFFERED: 1
-
-concurrency:
-  group: woboq
-on: # yamllint disable-line rule:truthy
-  schedule:
-    - cron: '0 */18 * * *'
-  workflow_dispatch:
-jobs:
-  # don't use dockerhub push because this image updates so rarely
-  WoboqCodebrowser:
-    runs-on: [self-hosted, style-checker]
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/codebrowser
-          REPO_COPY=${{runner.temp}}/codebrowser/ClickHouse
-          IMAGES_PATH=${{runner.temp}}/images_path
-          EOF
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-          submodules: 'true'
-      - name: Codebrowser
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci" && python3 codebrowser_check.py
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH"

.gitignore

@@ -69,6 +69,7 @@ cmake-build-*
 *.pyc
 __pycache__
 *.pytest_cache
+.mypy_cache
 
 test.cpp
 CPackConfig.cmake
@@ -129,7 +130,6 @@ website/package-lock.json
 /.ccls-cache
 
 # clangd cache
-/.clangd
 /.cache
 
 /compile_commands.json
@@ -161,8 +161,11 @@ website/package-lock.json
 tests/queries/0_stateless/test_*
 tests/queries/0_stateless/*.binary
 tests/queries/0_stateless/*.generated-expect
+tests/queries/0_stateless/*.expect.history
+tests/integration/**/_gen
 
 # rust
 /rust/**/target
 # It is autogenerated from *.in
 /rust/**/.cargo/config.toml
+/rust/**/vendor

.gitmessage

@@ -0,0 +1,10 @@
+
+
+## To avoid merge commit in CI run (add a leading space to apply):
+#no-merge-commit
+
+## Running specified job (add a leading space to apply):
+#job_<JOB NAME>
+#job_stateless_tests_release
+#job_package_debug
+#job_integration_tests_asan

.gitmodules

@@ -1,9 +1,12 @@
+# Please do not use 'branch = ...' tags with submodule entries. Such tags make updating submodules a
+# little bit more convenient but they do *not* specify the tracked submodule branch. Thus, they are
+# more confusing than useful.
 [submodule "contrib/zstd"]
 	path = contrib/zstd
 	url = https://github.com/facebook/zstd
 [submodule "contrib/lz4"]
 	path = contrib/lz4
-	url = https://github.com/lz4/lz4
+	url = https://github.com/ClickHouse/lz4
 [submodule "contrib/librdkafka"]
 	path = contrib/librdkafka
 	url = https://github.com/ClickHouse/librdkafka
@@ -13,13 +16,12 @@
 [submodule "contrib/zlib-ng"]
 	path = contrib/zlib-ng
 	url = https://github.com/ClickHouse/zlib-ng
-	branch = clickhouse-2.0.x
 [submodule "contrib/googletest"]
 	path = contrib/googletest
 	url = https://github.com/google/googletest
 [submodule "contrib/capnproto"]
 	path = contrib/capnproto
-	url = https://github.com/capnproto/capnproto
+	url = https://github.com/ClickHouse/capnproto
 [submodule "contrib/double-conversion"]
 	path = contrib/double-conversion
 	url = https://github.com/google/double-conversion
@@ -35,20 +37,15 @@
 [submodule "contrib/unixodbc"]
 	path = contrib/unixodbc
 	url = https://github.com/ClickHouse/UnixODBC
-[submodule "contrib/protobuf"]
-	path = contrib/protobuf
-	url = https://github.com/ClickHouse/protobuf
-	branch = v3.13.0.1
+[submodule "contrib/google-protobuf"]
+	path = contrib/google-protobuf
+	url = https://github.com/ClickHouse/google-protobuf.git
 [submodule "contrib/boost"]
 	path = contrib/boost
 	url = https://github.com/ClickHouse/boost
-[submodule "contrib/base64"]
-	path = contrib/base64
-	url = https://github.com/ClickHouse/Turbo-Base64
 [submodule "contrib/arrow"]
 	path = contrib/arrow
 	url = https://github.com/ClickHouse/arrow
-	branch = blessed/release-6.0.1
 [submodule "contrib/thrift"]
 	path = contrib/thrift
 	url = https://github.com/apache/thrift
@@ -94,7 +91,6 @@
 [submodule "contrib/grpc"]
 	path = contrib/grpc
 	url = https://github.com/ClickHouse/grpc
-	branch = v1.33.2
 [submodule "contrib/aws"]
 	path = contrib/aws
 	url = https://github.com/ClickHouse/aws-sdk-cpp
@@ -141,11 +137,9 @@
 [submodule "contrib/cassandra"]
 	path = contrib/cassandra
 	url = https://github.com/ClickHouse/cpp-driver
-	branch = clickhouse
 [submodule "contrib/libuv"]
 	path = contrib/libuv
 	url = https://github.com/ClickHouse/libuv
-	branch = clickhouse
 [submodule "contrib/fmtlib"]
 	path = contrib/fmtlib
 	url = https://github.com/fmtlib/fmt
@@ -158,11 +152,9 @@
 [submodule "contrib/cyrus-sasl"]
 	path = contrib/cyrus-sasl
 	url = https://github.com/ClickHouse/cyrus-sasl
-	branch = cyrus-sasl-2.1
 [submodule "contrib/croaring"]
 	path = contrib/croaring
 	url = https://github.com/RoaringBitmap/CRoaring
-	branch = v0.2.66
 [submodule "contrib/miniselect"]
 	path = contrib/miniselect
 	url = https://github.com/danlark1/miniselect
@@ -175,7 +167,6 @@
 [submodule "contrib/abseil-cpp"]
 	path = contrib/abseil-cpp
 	url = https://github.com/abseil/abseil-cpp
-	branch = lts_2021_11_02
 [submodule "contrib/dragonbox"]
 	path = contrib/dragonbox
 	url = https://github.com/ClickHouse/dragonbox
@@ -188,7 +179,6 @@
 [submodule "contrib/boringssl"]
 	path = contrib/boringssl
 	url = https://github.com/ClickHouse/boringssl
-	branch = unknown_branch_from_artur
 [submodule "contrib/NuRaft"]
 	path = contrib/NuRaft
 	url = https://github.com/ClickHouse/NuRaft
@@ -197,7 +187,7 @@
 	url = https://github.com/ClickHouse/nanodbc
 [submodule "contrib/datasketches-cpp"]
 	path = contrib/datasketches-cpp
-	url = https://github.com/ClickHouse/datasketches-cpp
+	url = https://github.com/apache/datasketches-cpp
 [submodule "contrib/yaml-cpp"]
 	path = contrib/yaml-cpp
 	url = https://github.com/ClickHouse/yaml-cpp
@@ -249,7 +239,6 @@
 [submodule "contrib/annoy"]
 	path = contrib/annoy
 	url = https://github.com/ClickHouse/annoy
-	branch = ClickHouse-master
 [submodule "contrib/qpl"]
 	path = contrib/qpl
 	url = https://github.com/intel/qpl
@@ -259,24 +248,21 @@
 [submodule "contrib/wyhash"]
 	path = contrib/wyhash
 	url = https://github.com/wangyi-fudan/wyhash
-[submodule "contrib/hashidsxx"]
-	path = contrib/hashidsxx
-	url = https://github.com/schoentoon/hashidsxx
 [submodule "contrib/nats-io"]
 	path = contrib/nats-io
 	url = https://github.com/ClickHouse/nats.c
 [submodule "contrib/vectorscan"]
 	path = contrib/vectorscan
-	url = https://github.com/VectorCamp/vectorscan
-[submodule "contrib/c-ares"]
-	path = contrib/c-ares
-	url = https://github.com/ClickHouse/c-ares
+	url = https://github.com/VectorCamp/vectorscan.git
 [submodule "contrib/llvm-project"]
 	path = contrib/llvm-project
 	url = https://github.com/ClickHouse/llvm-project
 [submodule "contrib/corrosion"]
 	path = contrib/corrosion
 	url = https://github.com/corrosion-rs/corrosion
+[submodule "contrib/libssh"]
+	path = contrib/libssh
+	url = https://github.com/ClickHouse/libssh.git
 [submodule "contrib/morton-nd"]
 	path = contrib/morton-nd
 	url = https://github.com/morton-nd/morton-nd
@@ -289,13 +275,18 @@
 [submodule "contrib/openssl"]
 	path = contrib/openssl
 	url = https://github.com/openssl/openssl
-	branch = openssl-3.0
 [submodule "contrib/google-benchmark"]
 	path = contrib/google-benchmark
 	url = https://github.com/google/benchmark
 [submodule "contrib/libdivide"]
 	path = contrib/libdivide
 	url = https://github.com/ridiculousfish/libdivide
+[submodule "contrib/libbcrypt"]
+	path = contrib/libbcrypt
+	url = https://github.com/rg3/libbcrypt.git
+[submodule "contrib/ulid-c"]
+	path = contrib/ulid-c
+	url = https://github.com/ClickHouse/ulid-c.git
 [submodule "contrib/aws-crt-cpp"]
 	path = contrib/aws-crt-cpp
 	url = https://github.com/ClickHouse/aws-crt-cpp
@@ -332,3 +323,40 @@
 [submodule "contrib/liburing"]
 	path = contrib/liburing
 	url = https://github.com/axboe/liburing
+[submodule "contrib/libarchive"]
+	path = contrib/libarchive
+	url = https://github.com/libarchive/libarchive.git
+	ignore = dirty
+[submodule "contrib/libfiu"]
+	path = contrib/libfiu
+	url = https://github.com/ClickHouse/libfiu.git
+[submodule "contrib/isa-l"]
+	path = contrib/isa-l
+	url = https://github.com/ClickHouse/isa-l.git
+[submodule "contrib/c-ares"]
+	path = contrib/c-ares
+	url = https://github.com/c-ares/c-ares.git
+[submodule "contrib/incbin"]
+	path = contrib/incbin
+	url = https://github.com/graphitemaster/incbin.git
+[submodule "contrib/usearch"]
+	path = contrib/usearch
+	url = https://github.com/unum-cloud/usearch.git
+[submodule "contrib/SimSIMD"]
+	path = contrib/SimSIMD
+	url = https://github.com/ashvardanian/SimSIMD.git
+[submodule "contrib/FP16"]
+	path = contrib/FP16
+	url = https://github.com/Maratyszcza/FP16.git
+[submodule "contrib/robin-map"]
+	path = contrib/robin-map
+	url = https://github.com/Tessil/robin-map.git
+[submodule "contrib/aklomp-base64"]
+	path = contrib/aklomp-base64
+	url = https://github.com/aklomp/base64.git
+[submodule "contrib/pocketfft"]
+	path = contrib/pocketfft
+	url = https://github.com/mreineck/pocketfft.git
+[submodule "contrib/sqids-cpp"]
+	path = contrib/sqids-cpp
+	url = https://github.com/sqids/sqids-cpp.git

.yamllint

@@ -6,8 +6,10 @@ rules:
         level: warning
         indent-sequences: consistent
     line-length:
-        # there are some bash -c "", so this is OK
-        max: 300
+        # there are:
+        # - bash -c "", so this is OK
+        # - yaml in tests
+        max: 1000
         level: warning
     comments:
         min-spaces-from-content: 1

CMakeLists.txt

@@ -19,9 +19,13 @@ include (cmake/tools.cmake)
 include (cmake/ccache.cmake)
 include (cmake/clang_tidy.cmake)
 include (cmake/git.cmake)
+include (cmake/utils.cmake)
+
+# This is needed to set up the CMAKE_INSTALL_BINDIR variable.
+include (GNUInstallDirs)
 
 # Ignore export() since we don't use it,
-# but it gets broken with a global targets via link_libraries()
+# but it gets broken with global targets via link_libraries()
 macro (export)
 endmacro ()
 
@@ -57,8 +61,8 @@ if (ENABLE_CHECK_HEAVY_BUILDS)
     # set CPU time limit to 1000 seconds
     set (RLIMIT_CPU 1000)
 
-    # gcc10/gcc10/clang -fsanitize=memory is too heavy
-    if (SANITIZE STREQUAL "memory" OR COMPILER_GCC)
+    # -fsanitize=memory is too heavy
+    if (SANITIZE STREQUAL "memory")
        set (RLIMIT_DATA 10000000000) # 10G
     endif()
 
@@ -87,8 +91,6 @@ if (ENABLE_FUZZING)
     set (ENABLE_CLICKHOUSE_ODBC_BRIDGE OFF)
     set (ENABLE_LIBRARIES 0)
     set (ENABLE_SSL 1)
-    set (USE_UNWIND ON)
-    set (ENABLE_EMBEDDED_COMPILER 0)
     set (ENABLE_EXAMPLES 0)
     set (ENABLE_UTILS 0)
     set (ENABLE_THINLTO 0)
@@ -108,7 +110,11 @@ endif()
 # - sanitize.cmake
 add_library(global-libs INTERFACE)
 
-include (cmake/fuzzer.cmake)
+# We don't want to instrument everything with fuzzer, but only specific targets (see below),
+# also, since we build our own llvm, we specifically don't want to instrument
+# libFuzzer library itself - it would result in infinite recursion
+#include (cmake/fuzzer.cmake)
+
 include (cmake/sanitize.cmake)
 
 option(ENABLE_COLORED_BUILD "Enable colors in compiler output" ON)
@@ -121,6 +127,7 @@ if (ENABLE_COLORED_BUILD AND CMAKE_GENERATOR STREQUAL "Ninja")
     set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdiagnostics-color=always")
     # ... such manually setting of flags can be removed once CMake supports a variable to
     # activate colors in *all* build systems: https://gitlab.kitware.com/cmake/cmake/-/issues/15502
+    # --> available since CMake 3.24: https://stackoverflow.com/a/73349744
 endif ()
 
 include (cmake/check_flags.cmake)
@@ -134,24 +141,15 @@ if (COMPILER_CLANG)
         set(COMPILER_FLAGS "${COMPILER_FLAGS} -gdwarf-aranges")
     endif ()
 
-    if (HAS_USE_CTOR_HOMING)
-        # For more info see https://blog.llvm.org/posts/2021-04-05-constructor-homing-for-debug-info/
-        if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO")
-            set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xclang -fuse-ctor-homing")
-            set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Xclang -fuse-ctor-homing")
-        endif()
+    # See https://blog.llvm.org/posts/2021-04-05-constructor-homing-for-debug-info/
+    if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO")
+        set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xclang -fuse-ctor-homing")
+        set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Xclang -fuse-ctor-homing")
     endif()
 
     no_warning(enum-constexpr-conversion) # breaks Protobuf in clang-16
 endif ()
 
-# If compiler has support for -Wreserved-identifier. It is difficult to detect by clang version,
-# because there are two different branches of clang: clang and AppleClang.
-# (AppleClang is not supported by ClickHouse, but some developers have misfortune to use it).
-if (HAS_RESERVED_IDENTIFIER)
-    add_compile_definitions (HAS_RESERVED_IDENTIFIER)
-endif ()
-
 option(ENABLE_TESTS "Provide unit_test_dbms target with Google.Test unit tests" ON)
 option(ENABLE_EXAMPLES "Build all example programs in 'examples' subdirectories" OFF)
 option(ENABLE_BENCHMARKS "Build all benchmark programs in 'benchmarks' subdirectories" OFF)
@@ -163,8 +161,14 @@ elseif(GLIBC_COMPATIBILITY)
     message (${RECONFIGURE_MESSAGE_LEVEL} "Glibc compatibility cannot be enabled in current configuration")
 endif ()
 
-# Make sure the final executable has symbols exported
-set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -rdynamic")
+if (OS_LINUX)
+    # We should not export dynamic symbols, because:
+    # - The main clickhouse binary does not use dlopen,
+    #   and whatever is poisoning it by LD_PRELOAD should not link to our symbols.
+    # - The clickhouse-odbc-bridge and clickhouse-library-bridge binaries
+    #   should not expose their symbols to ODBC drivers and libraries.
+    set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-export-dynamic -Wl,--gc-sections")
+endif ()
 
 if (OS_DARWIN)
     # The `-all_load` flag forces loading of all symbols from all libraries,
@@ -178,38 +182,18 @@ else ()
     set(NO_WHOLE_ARCHIVE --no-whole-archive)
 endif ()
 
-option(ENABLE_CURL_BUILD "Enable curl, azure, sentry build on by default except MacOS." ON)
-if (OS_DARWIN)
-    # Disable the curl, azure, senry build on MacOS
-    set (ENABLE_CURL_BUILD OFF)
-endif ()
-
-# Ignored if `lld` is used
-option(ADD_GDB_INDEX_FOR_GOLD "Add .gdb-index to resulting binaries for gold linker.")
-
 if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE")
     # Can be lld or ld-lld or lld-13 or /path/to/lld.
-    if (LINKER_NAME MATCHES "lld" AND OS_LINUX)
+    if (LINKER_NAME MATCHES "lld")
         set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gdb-index")
-        set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--gdb-index")
         message (STATUS "Adding .gdb-index via --gdb-index linker option.")
-    # we use another tool for gdb-index, because gold linker removes section .debug_aranges, which used inside clickhouse stacktraces
-    # http://sourceware-org.1504.n7.nabble.com/gold-No-debug-aranges-section-when-linking-with-gdb-index-td540965.html#a556932
-    elseif (LINKER_NAME MATCHES "gold$" AND ADD_GDB_INDEX_FOR_GOLD)
-        find_program (GDB_ADD_INDEX_EXE NAMES "gdb-add-index" DOC "Path to gdb-add-index executable")
-        if (NOT GDB_ADD_INDEX_EXE)
-            set (USE_GDB_ADD_INDEX 0)
-            message (WARNING "Cannot add gdb index to binaries, because gold linker is used, but gdb-add-index executable not found.")
-        else()
-            set (USE_GDB_ADD_INDEX 1)
-            message (STATUS "gdb-add-index found: ${GDB_ADD_INDEX_EXE}")
-        endif()
     endif ()
 endif()
 
-if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE"
-    OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO"
-    OR CMAKE_BUILD_TYPE_UC STREQUAL "MINSIZEREL")
+if (NOT (SANITIZE_COVERAGE OR WITH_COVERAGE)
+    AND (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE"
+        OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO"
+        OR CMAKE_BUILD_TYPE_UC STREQUAL "MINSIZEREL"))
     set (OMIT_HEAVY_DEBUG_SYMBOLS_DEFAULT ON)
 else()
     set (OMIT_HEAVY_DEBUG_SYMBOLS_DEFAULT OFF)
@@ -221,9 +205,6 @@ option(OMIT_HEAVY_DEBUG_SYMBOLS
     "Do not generate debugger info for heavy modules (ClickHouse functions and dictionaries, some contrib)"
     ${OMIT_HEAVY_DEBUG_SYMBOLS_DEFAULT})
 
-if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
-    set(USE_DEBUG_HELPERS ON)
-endif()
 option(USE_DEBUG_HELPERS "Enable debug helpers" ${USE_DEBUG_HELPERS})
 
 option(BUILD_STANDALONE_KEEPER "Build keeper as small standalone binary" OFF)
@@ -235,7 +216,7 @@ endif ()
 
 # Create BuildID when using lld. For other linkers it is created by default.
 # (NOTE: LINKER_NAME can be either path or name, and in different variants)
-if (LINKER_NAME MATCHES "lld" AND OS_LINUX)
+if (LINKER_NAME MATCHES "lld")
     # SHA1 is not cryptographically secure but it is the best what lld is offering.
     set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--build-id=sha1")
 endif ()
@@ -288,79 +269,64 @@ endif ()
 option (ENABLE_BUILD_PATH_MAPPING "Enable remapping of file source paths in debug info, predefined preprocessor macros, and __builtin_FILE(). It's used to generate reproducible builds. See https://reproducible-builds.org/docs/build-path" ${ENABLE_BUILD_PATH_MAPPING_DEFAULT})
 
 if (ENABLE_BUILD_PATH_MAPPING)
-    set (COMPILER_FLAGS "${COMPILER_FLAGS} -ffile-prefix-map=${CMAKE_SOURCE_DIR}=.")
-    set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} -ffile-prefix-map=${CMAKE_SOURCE_DIR}=.")
+    set (COMPILER_FLAGS "${COMPILER_FLAGS} -ffile-prefix-map=${PROJECT_SOURCE_DIR}=.")
+    set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} -ffile-prefix-map=${PROJECT_SOURCE_DIR}=.")
 endif ()
 
 option (ENABLE_BUILD_PROFILING "Enable profiling of build time" OFF)
 if (ENABLE_BUILD_PROFILING)
      if (COMPILER_CLANG)
         set (COMPILER_FLAGS "${COMPILER_FLAGS} -ftime-trace")
+
+        if (LINKER_NAME MATCHES "lld")
+            set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--time-trace")
+            set (CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -Wl,--time-trace")
+        endif ()
      else ()
         message (${RECONFIGURE_MESSAGE_LEVEL} "Build profiling is only available with CLang")
      endif ()
 endif ()
 
-set (CMAKE_CXX_STANDARD 20)
-set (CMAKE_CXX_EXTENSIONS ON) # Same as gnu++2a (ON) vs c++2a (OFF): https://cmake.org/cmake/help/latest/prop_tgt/CXX_EXTENSIONS.html
+set (CMAKE_CXX_STANDARD 23)
+set (CMAKE_CXX_EXTENSIONS OFF)
 set (CMAKE_CXX_STANDARD_REQUIRED ON)
 
 set (CMAKE_C_STANDARD 11)
-set (CMAKE_C_EXTENSIONS ON)
+set (CMAKE_C_EXTENSIONS ON) # required by most contribs written in C
 set (CMAKE_C_STANDARD_REQUIRED ON)
 
-if (COMPILER_GCC OR COMPILER_CLANG)
+if (COMPILER_CLANG)
     # Enable C++14 sized global deallocation functions. It should be enabled by setting -std=c++14 but I'm not sure.
+    # See https://reviews.llvm.org/D112921
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsized-deallocation")
-endif ()
 
-# falign-functions=32 prevents from random performance regressions with the code change. Thus, providing more stable
-# benchmarks.
-if (COMPILER_GCC OR COMPILER_CLANG)
+    # falign-functions=32 prevents from random performance regressions with the code change. Thus, providing more stable
+    # benchmarks.
     set(COMPILER_FLAGS "${COMPILER_FLAGS} -falign-functions=32")
-endif ()
 
-if (ARCH_AMD64)
-    # align branches within a 32-Byte boundary to avoid the potential performance loss when code layout change,
-    # which makes benchmark results more stable.
-    set(BRANCHES_WITHIN_32B_BOUNDARIES "-mbranches-within-32B-boundaries")
-    if (COMPILER_GCC)
-        # gcc is in assembler, need to add "-Wa," prefix
-        set(BRANCHES_WITHIN_32B_BOUNDARIES "-Wa,${BRANCHES_WITHIN_32B_BOUNDARIES}")
-    endif()
-
-    include(CheckCXXCompilerFlag)
-    check_cxx_compiler_flag("${BRANCHES_WITHIN_32B_BOUNDARIES}" HAS_BRANCHES_WITHIN_32B_BOUNDARIES)
-    if (HAS_BRANCHES_WITHIN_32B_BOUNDARIES)
+    if (ARCH_AMD64)
+        # align branches within a 32-Byte boundary to avoid the potential performance loss when code layout change,
+        # which makes benchmark results more stable.
+        set(BRANCHES_WITHIN_32B_BOUNDARIES "-mbranches-within-32B-boundaries")
         set(COMPILER_FLAGS "${COMPILER_FLAGS} ${BRANCHES_WITHIN_32B_BOUNDARIES}")
     endif()
-endif()
-
-if (COMPILER_GCC)
-    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fcoroutines")
 endif ()
 
-# Compiler-specific coverage flags e.g. -fcoverage-mapping for gcc
-option(WITH_COVERAGE "Profile the resulting binary/binaries" OFF)
-
-if (WITH_COVERAGE AND COMPILER_CLANG)
-    set(COMPILER_FLAGS "${COMPILER_FLAGS} -fprofile-instr-generate -fcoverage-mapping")
-    # If we want to disable coverage for specific translation units
-    set(WITHOUT_COVERAGE "-fno-profile-instr-generate -fno-coverage-mapping")
-endif()
-
-if (WITH_COVERAGE AND COMPILER_GCC)
-    set(COMPILER_FLAGS "${COMPILER_FLAGS} -fprofile-arcs -ftest-coverage")
-    set(COVERAGE_OPTION "-lgcov")
-    set(WITHOUT_COVERAGE "-fno-profile-arcs -fno-test-coverage")
-endif()
-
 set (COMPILER_FLAGS "${COMPILER_FLAGS}")
 
 # Our built-in unwinder only supports DWARF version up to 4.
-set (DEBUG_INFO_FLAGS "-g -gdwarf-4")
+set (DEBUG_INFO_FLAGS "-g")
 
-set (CMAKE_CXX_FLAGS                     "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS}")
+# Disable omit frame pointer compiler optimization using -fno-omit-frame-pointer
+option(DISABLE_OMIT_FRAME_POINTER "Disable omit frame pointer compiler optimization" OFF)
+
+if (DISABLE_OMIT_FRAME_POINTER)
+    set (CMAKE_CXX_FLAGS_ADD "${CMAKE_CXX_FLAGS_ADD} -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer")
+    set (CMAKE_C_FLAGS_ADD "${CMAKE_C_FLAGS_ADD} -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer")
+    set (CMAKE_ASM_FLAGS_ADD "${CMAKE_ASM_FLAGS_ADD} -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer")
+endif()
+
+set (CMAKE_CXX_FLAGS                     "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS} ${CMAKE_CXX_FLAGS_ADD}")
 set (CMAKE_CXX_FLAGS_RELWITHDEBINFO      "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_CXX_FLAGS_ADD}")
 set (CMAKE_CXX_FLAGS_DEBUG               "${CMAKE_CXX_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} ${CMAKE_CXX_FLAGS_ADD}")
 
@@ -382,20 +348,15 @@ if (COMPILER_CLANG)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-absolute-paths")
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdiagnostics-absolute-paths")
 
-    if (NOT ENABLE_TESTS AND NOT SANITIZE)
+    if (NOT ENABLE_TESTS AND NOT SANITIZE AND OS_LINUX)
         # https://clang.llvm.org/docs/ThinLTO.html
-        # Applies to clang only.
+        # Applies to clang and linux only.
         # Disabled when building with tests or sanitizers.
         option(ENABLE_THINLTO "Clang-specific link time optimization" ON)
     endif()
 
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fstrict-vtable-pointers")
 
-    # Set new experimental pass manager, it's a performance, build time and binary size win.
-    # Can be removed after https://reviews.llvm.org/D66490 merged and released to at least two versions of clang.
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fexperimental-new-pass-manager")
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fexperimental-new-pass-manager")
-
     # We cannot afford to use LTO when compiling unit tests, and it's not enough
     # to only supply -fno-lto at the final linking stage. So we disable it
     # completely.
@@ -434,15 +395,22 @@ else()
 endif ()
 
 option (ENABLE_GWP_ASAN "Enable Gwp-Asan" ON)
-if (NOT OS_LINUX AND NOT OS_ANDROID)
+# We use mmap for allocations more heavily in debug builds,
+# but GWP-ASan also wants to use mmap frequently,
+# and due to a large number of memory mappings,
+# it does not work together well.
+if ((NOT OS_LINUX AND NOT OS_ANDROID) OR (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG"))
     set(ENABLE_GWP_ASAN OFF)
 endif ()
 
+option (ENABLE_FIU "Enable Fiu" ON)
+
 option(WERROR "Enable -Werror compiler option" ON)
 
 if (WERROR)
     # Don't pollute CMAKE_CXX_FLAGS with -Werror as it will break some CMake checks.
     # Instead, adopt modern cmake usage requirement.
+    # TODO: Set CMAKE_COMPILE_WARNING_AS_ERROR (cmake 3.24)
     target_compile_options(global-group INTERFACE "-Werror")
 endif ()
 
@@ -457,8 +425,11 @@ endif ()
 
 set (CMAKE_POSTFIX_VARIABLE "CMAKE_${CMAKE_BUILD_TYPE_UC}_POSTFIX")
 
-set (CMAKE_POSITION_INDEPENDENT_CODE OFF)
-if (OS_LINUX AND NOT ARCH_AARCH64)
+if (NOT SANITIZE)
+    set (CMAKE_POSITION_INDEPENDENT_CODE OFF)
+endif()
+
+if (OS_LINUX AND NOT (ARCH_AARCH64 OR ARCH_S390X) AND NOT SANITIZE)
     # Slightly more efficient code can be generated
     # It's disabled for ARM because otherwise ClickHouse cannot run on Android.
     set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-pie")
@@ -476,7 +447,12 @@ enable_testing() # Enable for tests without binary
 
 option(ENABLE_OPENSSL "This option performs a build with OpenSSL. NOTE! This option is insecure and should never be used. By default, ClickHouse uses and only supports BoringSSL" OFF)
 
-option(ENABLE_OPENSSL_DYNAMIC "This option removes SSL from ClickHouse and will link to the OpenSSL version supplied by OS." OFF)
+if (ARCH_S390X)
+    set(ENABLE_OPENSSL_DYNAMIC_DEFAULT ON)
+else ()
+    set(ENABLE_OPENSSL_DYNAMIC_DEFAULT OFF)
+endif ()
+option(ENABLE_OPENSSL_DYNAMIC "This option removes SSL from ClickHouse and will link to the OpenSSL version supplied by OS." ${ENABLE_OPENSSL_DYNAMIC_DEFAULT})
 
 # when installing to /usr - place configs to /etc but for /usr/local place to /usr/local/etc
 if (CMAKE_INSTALL_PREFIX STREQUAL "/usr")
@@ -487,14 +463,6 @@ endif ()
 
 message (STATUS "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE}")
 
-include (GNUInstallDirs)
-
-# When testing for memory leaks with Valgrind, don't link tcmalloc or jemalloc.
-
-if (TARGET global-group)
-    install (EXPORT global DESTINATION cmake)
-endif ()
-
 add_subdirectory (contrib EXCLUDE_FROM_ALL)
 
 if (NOT ENABLE_JEMALLOC)
@@ -556,14 +524,68 @@ include (cmake/print_flags.cmake)
 
 if (ENABLE_RUST)
     add_subdirectory (rust)
+
+    # With LTO Rust adds few symbols with global visiblity, the most common is
+    # rust_eh_personality. And this leads to linking errors because multiple
+    # Rust libraries contains the same symbol.
+    #
+    # If it was shared library, that we could use version script for linker to
+    # hide this symbols, but libraries are static.
+    #
+    # we could in theory compile everything to one library but this will be a
+    # mess
+    #
+    # But this should be OK since CI has lots of other builds that are done
+    # without LTO and it will find multiple definitions if there will be any.
+    #
+    # More information about this behaviour in Rust can be found here
+    # - https://github.com/rust-lang/rust/issues/44322
+    # - https://alanwu.space/post/symbol-hygiene/
+    if (ENABLE_THINLTO)
+        set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--allow-multiple-definition")
+    endif()
 endif()
 
+if (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" AND NOT SANITIZE AND OS_LINUX AND (ARCH_AMD64 OR ARCH_AARCH64))
+    set(CHECK_LARGE_OBJECT_SIZES_DEFAULT ON)
+else ()
+    set(CHECK_LARGE_OBJECT_SIZES_DEFAULT OFF)
+endif ()
+option(CHECK_LARGE_OBJECT_SIZES "Check that there are no large object files after build." ${CHECK_LARGE_OBJECT_SIZES_DEFAULT})
+
 add_subdirectory (base)
 add_subdirectory (src)
 add_subdirectory (programs)
-add_subdirectory (tests)
 add_subdirectory (utils)
 
+if (FUZZER)
+    # Bundle fuzzers target
+    add_custom_target(fuzzers)
+    # Instrument all targets fuzzer and link with libfuzzer
+    get_all_targets(all_targets)
+    foreach(target ${all_targets})
+        if (NOT(target STREQUAL "_fuzzer" OR target STREQUAL "_fuzzer_no_main"))
+            get_target_property(target_type ${target} TYPE)
+            if (NOT(target_type STREQUAL "INTERFACE_LIBRARY" OR target_type STREQUAL "UTILITY"))
+                target_compile_options(${target} PRIVATE "-fsanitize=fuzzer-no-link")
+            endif()
+            # clickhouse fuzzer isn't working correctly
+            # initial PR https://github.com/ClickHouse/ClickHouse/pull/27526
+            #if (target MATCHES ".+_fuzzer" OR target STREQUAL "clickhouse")
+            if (target_type STREQUAL "EXECUTABLE" AND target MATCHES ".+_fuzzer")
+                message(STATUS "${target} instrumented with fuzzer")
+                target_link_libraries(${target} PUBLIC ch_contrib::fuzzer)
+                # Add to fuzzers bundle
+                add_dependencies(fuzzers ${target})
+                get_target_filename(${target} target_bin_name)
+                get_target_property(target_bin_dir ${target} BINARY_DIR)
+                add_custom_command(TARGET fuzzers POST_BUILD COMMAND mv "${target_bin_dir}/${target_bin_name}" "${CMAKE_CURRENT_BINARY_DIR}/programs/" VERBATIM)
+            endif()
+        endif()
+    endforeach()
+    add_custom_command(TARGET fuzzers POST_BUILD COMMAND SRC=${CMAKE_SOURCE_DIR} BIN=${CMAKE_BINARY_DIR} OUT=${CMAKE_BINARY_DIR}/programs ${CMAKE_SOURCE_DIR}/tests/fuzz/build.sh VERBATIM)
+endif()
+
 include (cmake/sanitize_targets.cmake)
 
 # Build native targets if necessary
@@ -576,7 +598,7 @@ if (NATIVE_BUILD_TARGETS
 )
     message (STATUS "Building native targets...")
 
-    set (NATIVE_BUILD_DIR "${CMAKE_BINARY_DIR}/native")
+    set (NATIVE_BUILD_DIR "${PROJECT_BINARY_DIR}/native")
 
     execute_process(
         COMMAND ${CMAKE_COMMAND} -E make_directory "${NATIVE_BUILD_DIR}"
@@ -586,11 +608,11 @@ if (NATIVE_BUILD_TARGETS
         COMMAND ${CMAKE_COMMAND}
             "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}"
             "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}"
-            "-DENABLE_CCACHE=${ENABLE_CCACHE}"
+            "-DCOMPILER_CACHE=${COMPILER_CACHE}"
             # Avoid overriding .cargo/config.toml with native toolchain.
             "-DENABLE_RUST=OFF"
             "-DENABLE_CLICKHOUSE_SELF_EXTRACTING=${ENABLE_CLICKHOUSE_SELF_EXTRACTING}"
-        ${CMAKE_SOURCE_DIR}
+        ${PROJECT_SOURCE_DIR}
         WORKING_DIRECTORY "${NATIVE_BUILD_DIR}"
         COMMAND_ECHO STDOUT)
 

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2016-2023 ClickHouse, Inc.
+Copyright 2016-2024 ClickHouse, Inc.
 
                                  Apache License
                            Version 2.0, January 2004
@@ -188,7 +188,7 @@ Copyright 2016-2023 ClickHouse, Inc.
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2016-2023 ClickHouse, Inc.
+   Copyright 2016-2024 ClickHouse, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

PreLoad.cmake

@@ -19,8 +19,8 @@ endif()
 if (NOT "$ENV{CFLAGS}" STREQUAL ""
     OR NOT "$ENV{CXXFLAGS}" STREQUAL ""
     OR NOT "$ENV{LDFLAGS}" STREQUAL ""
-    OR CMAKE_C_FLAGS OR CMAKE_CXX_FLAGS OR CMAKE_EXE_LINKER_FLAGS OR CMAKE_SHARED_LINKER_FLAGS OR CMAKE_MODULE_LINKER_FLAGS
-    OR CMAKE_C_FLAGS_INIT OR CMAKE_CXX_FLAGS_INIT OR CMAKE_EXE_LINKER_FLAGS_INIT OR CMAKE_SHARED_LINKER_FLAGS_INIT OR CMAKE_MODULE_LINKER_FLAGS_INIT)
+    OR CMAKE_C_FLAGS OR CMAKE_CXX_FLAGS OR CMAKE_EXE_LINKER_FLAGS OR CMAKE_MODULE_LINKER_FLAGS
+    OR CMAKE_C_FLAGS_INIT OR CMAKE_CXX_FLAGS_INIT OR CMAKE_EXE_LINKER_FLAGS_INIT OR CMAKE_MODULE_LINKER_FLAGS_INIT)
 
     # if $ENV
     message("CFLAGS: $ENV{CFLAGS}")
@@ -36,7 +36,6 @@ if (NOT "$ENV{CFLAGS}" STREQUAL ""
     message("CMAKE_C_FLAGS_INIT: ${CMAKE_C_FLAGS_INIT}")
     message("CMAKE_CXX_FLAGS_INIT: ${CMAKE_CXX_FLAGS_INIT}")
     message("CMAKE_EXE_LINKER_FLAGS_INIT: ${CMAKE_EXE_LINKER_FLAGS_INIT}")
-    message("CMAKE_SHARED_LINKER_FLAGS_INIT: ${CMAKE_SHARED_LINKER_FLAGS_INIT}")
     message("CMAKE_MODULE_LINKER_FLAGS_INIT: ${CMAKE_MODULE_LINKER_FLAGS_INIT}")
 
     message(FATAL_ERROR "
@@ -79,12 +78,23 @@ if (OS MATCHES "Linux"
     AND ("$ENV{CC}" MATCHES ".*clang.*" OR CMAKE_C_COMPILER MATCHES ".*clang.*"))
 
     if (ARCH MATCHES "amd64|x86_64")
+        # NOTE: right now musl is not ready, since unwind is too slow with it
+        #
+        # FWIW the following had been tried:
+        # - update musl
+        # - compile musl with debug
+        # - compile musl with debug and -fasynchronous-unwind-tables
+        #
+        # But none of this changes anything so far.
         set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-x86_64.cmake" CACHE INTERNAL "")
     elseif (ARCH MATCHES "^(aarch64.*|AARCH64.*|arm64.*|ARM64.*)")
         set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-aarch64.cmake" CACHE INTERNAL "")
     elseif (ARCH MATCHES "^(ppc64le.*|PPC64LE.*)")
         set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-ppc64le.cmake" CACHE INTERNAL "")
+    elseif (ARCH MATCHES "^(s390x.*|S390X.*)")
+        set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-s390x.cmake" CACHE INTERNAL "")
     else ()
         message (FATAL_ERROR "Unsupported architecture: ${ARCH}")
     endif ()
+
 endif()

README.md

@@ -1,6 +1,22 @@
-[![ClickHouse — open source distributed column-oriented DBMS](https://github.com/ClickHouse/clickhouse-presentations/raw/master/images/logo-400x240.png)](https://clickhouse.com)
+<div align=center>
 
-ClickHouse® is an open-source column-oriented database management system that allows generating analytical data reports in real-time.
+[![Website](https://img.shields.io/website?up_message=AVAILABLE&down_message=DOWN&url=https%3A%2F%2Fclickhouse.com&style=for-the-badge)](https://clickhouse.com)
+[![Apache 2.0 License](https://img.shields.io/badge/license-Apache%202.0-blueviolet?style=for-the-badge)](https://www.apache.org/licenses/LICENSE-2.0)
+
+<picture align=center>
+    <source media="(prefers-color-scheme: dark)" srcset="https://github.com/ClickHouse/clickhouse-docs/assets/9611008/4ef9c104-2d3f-4646-b186-507358d2fe28">
+    <source media="(prefers-color-scheme: light)" srcset="https://github.com/ClickHouse/clickhouse-docs/assets/9611008/b001dc7b-5a45-4dcd-9275-e03beb7f9177">
+    <img alt="The ClickHouse company logo." src="https://github.com/ClickHouse/clickhouse-docs/assets/9611008/b001dc7b-5a45-4dcd-9275-e03beb7f9177">
+</picture>
+
+<h4>ClickHouse® is an open-source column-oriented database management system that allows generating analytical data reports in real-time.</h4>
+
+</div>
+
+## How To Install (Linux, macOS, FreeBSD)
+```
+curl https://clickhouse.com/ | sh
+```
 
 ## Useful Links
 
@@ -9,19 +25,26 @@ ClickHouse® is an open-source column-oriented database management system that a
 * [Tutorial](https://clickhouse.com/docs/en/getting_started/tutorial/) shows how to set up and query a small ClickHouse cluster.
 * [Documentation](https://clickhouse.com/docs/en/) provides more in-depth information.
 * [YouTube channel](https://www.youtube.com/c/ClickHouseDB) has a lot of content about ClickHouse in video format.
-* [Slack](https://clickhousedb.slack.com/) and [Telegram](https://telegram.me/clickhouse_en) allow chatting with ClickHouse users in real-time.
+* [Slack](https://clickhouse.com/slack) and [Telegram](https://telegram.me/clickhouse_en) allow chatting with ClickHouse users in real-time.
 * [Blog](https://clickhouse.com/blog/) contains various ClickHouse-related articles, as well as announcements and reports about events.
-* [Code Browser (Woboq)](https://clickhouse.com/codebrowser/ClickHouse/index.html) with syntax highlight and navigation.
-* [Code Browser (github.dev)](https://github.dev/ClickHouse/ClickHouse) with syntax highlight, powered by github.dev.
+* [Code Browser (github.dev)](https://github.dev/ClickHouse/ClickHouse) with syntax highlighting, powered by github.dev.
+* [Static Analysis (SonarCloud)](https://sonarcloud.io/project/issues?resolved=false&id=ClickHouse_ClickHouse) proposes C++ quality improvements.
 * [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any.
 
 ## Upcoming Events
-* [**ClickHouse Workshop**](https://clickhouse.com/company/events/2023-02-15-clickhouse-workshop?utm_source=github&utm_medium=social&utm_campaign=workshop) - Feb 15 & 16 - In this 2-day (3 hrs per day) free training, topics range from introductory content to a deep dive on interacting with and understanding your data. There will be both live training and hands-on labs.
-* [**v23.2 Release Webinar**](https://clickhouse.com/company/events/v23-2-release-webinar?utm_source=github&utm_medium=social&utm_campaign=release-webinar-2023-02) - Feb 23 - 23.2 is rapidly approaching. Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release.
-* [**ClickHouse Meetup in Amsterdam**](https://www.meetup.com/clickhouse-netherlands-user-group/events/291485868/) - Mar 9 - The first ClickHouse Amsterdam Meetup of 2023 is here! 🎉 Join us for short lightning talks and long discussions. Food, drinks & good times on us.
-* [**ClickHouse Meetup in SF Bay Area**](https://www.meetup.com/clickhouse-silicon-valley-meetup-group/events/291490121/) - Mar 14 - A night to meet with ClickHouse team in the San Francisco area! Food and drink are a given...but networking is the primary focus.
-* [**ClickHouse Meetup in Austin**](https://www.meetup.com/clickhouse-austin-user-group/events/291486654/) - Mar 16 - The first ClickHouse Meetup in Austin is happening soon! Interested in speaking, let us know!
+
+Keep an eye out for upcoming meetups around the world. Somewhere else you want us to be? Please feel free to reach out to tyler <at> clickhouse <dot> com.
 
 ## Recent Recordings
-* **FOSDEM 2023**: In the "Fast and Streaming Data" room Alexey gave a talk entitled "Building Analytical Apps With ClickHouse" that looks at the landscape of data tools, an interesting data set, and how you can interact with data quickly. Check out the recording on **[YouTube](https://www.youtube.com/watch?v=JlcI2Vfz_uk)**.
-* **Recording available**: [**v23.1 Release Webinar**](https://www.youtube.com/watch?v=zYSZXBnTMSE) 23.1 is the ClickHouse New Year release. Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release. Inverted indices, query cache, and so -- very -- much more.
+* **Recent Meetup Videos**: [Meetup Playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3iNDUzpY1S3L_iV4nARda_U) Whenever possible recordings of the ClickHouse Community Meetups are edited and presented as individual talks. Current featuring "Modern SQL in 2023", "Fast, Concurrent, and Consistent Asynchronous INSERTS in ClickHouse", and "Full-Text Indices: Design and Experiments"
+* **Recording available**: [**v23.10 Release Webinar**](https://www.youtube.com/watch?v=PGQS6uPb970) All the features of 23.10, one convenient video! Watch it now!
+* **All release webinar recordings**: [YouTube playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3jAlSy1JxyP8zluvXaN3nxU)
+
+  
+ ## Interested in joining ClickHouse and making it your full-time job? 
+  
+We are a globally diverse and distributed team, united behind a common goal of creating industry-leading, real-time analytics. Here, you will have an opportunity to solve some of the most cutting-edge technical challenges and have direct ownership of your work and vision. If you are a contributor by nature, a thinker and a doer - we’ll definitely click!
+  
+Check out our **current openings** here: https://clickhouse.com/company/careers 
+  
+Can't find what you are looking for, but want to let us know you are interested in joining ClickHouse? Email careers@clickhouse.com! 

SECURITY.md

@@ -13,19 +13,19 @@ The following versions of ClickHouse server are currently being supported with s
 
 | Version | Supported |
 |:-|:-|
-| 23.1 | ✔️ |
-| 22.12 | ✔️ |
-| 22.11 | ✔️ |
-| 22.10 | ❌ |
-| 22.9 | ❌ |
-| 22.8 | ✔️ |
-| 22.7 | ❌ |
-| 22.6 | ❌ |
-| 22.5 | ❌ |
-| 22.4 | ❌ |
-| 22.3 | ✔️ |
-| 22.2 | ❌ |
-| 22.1 | ❌ |
+| 23.12 | ✔️ |
+| 23.11 | ✔️ |
+| 23.10 | ✔️ |
+| 23.9 | ❌ |
+| 23.8 | ✔️ |
+| 23.7 | ❌ |
+| 23.6 | ❌ |
+| 23.5 | ❌ |
+| 23.4 | ❌ |
+| 23.3 | ✔️ |
+| 23.2 | ❌ |
+| 23.1 | ❌ |
+| 22.* | ❌ |
 | 21.* | ❌ |
 | 20.* | ❌ |
 | 19.* | ❌ |

base/base/CMakeLists.txt

@@ -1,7 +1,13 @@
+add_compile_options($<$<OR:$<COMPILE_LANGUAGE:C>,$<COMPILE_LANGUAGE:CXX>>:${COVERAGE_FLAGS}>)
+
 if (USE_CLANG_TIDY)
     set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}")
 endif ()
 
+# TODO: Remove this. We like to compile with C++23 (set by top-level CMakeLists) but Clang crashes with our libcxx
+# when instantiated from JSON.cpp. Try again when libcxx(abi) and Clang are upgraded to 16.
+set (CMAKE_CXX_STANDARD 20)
+
 set (SRCS
     argsToConfig.cpp
     coverage.cpp
@@ -41,6 +47,10 @@ else ()
     target_compile_definitions(common PUBLIC WITH_COVERAGE=0)
 endif ()
 
+if (TARGET ch_contrib::crc32_s390x)
+    target_link_libraries(common PUBLIC ch_contrib::crc32_s390x)
+endif()
+
 target_include_directories(common PUBLIC .. "${CMAKE_CURRENT_BINARY_DIR}/..")
 
 target_link_libraries (common

base/base/Decimal.h

@@ -1,5 +1,6 @@
 #pragma once
 #include <base/extended_types.h>
+#include <base/Decimal_fwd.h>
 
 #if !defined(NO_SANITIZE_UNDEFINED)
 #if defined(__clang__)
@@ -19,23 +20,6 @@ using Decimal64 = Decimal<Int64>;
 using Decimal128 = Decimal<Int128>;
 using Decimal256 = Decimal<Int256>;
 
-template <class T>
-concept is_decimal =
-    std::is_same_v<T, Decimal32>
-    || std::is_same_v<T, Decimal64>
-    || std::is_same_v<T, Decimal128>
-    || std::is_same_v<T, Decimal256>
-    || std::is_same_v<T, DateTime64>;
-
-template <class T>
-concept is_over_big_int =
-    std::is_same_v<T, Int128>
-    || std::is_same_v<T, UInt128>
-    || std::is_same_v<T, Int256>
-    || std::is_same_v<T, UInt256>
-    || std::is_same_v<T, Decimal128>
-    || std::is_same_v<T, Decimal256>;
-
 template <class T> struct NativeTypeT { using Type = T; };
 template <is_decimal T> struct NativeTypeT<T> { using Type = typename T::NativeType; };
 template <class T> using NativeType = typename NativeTypeT<T>::Type;

base/base/Decimal_fwd.h

@@ -0,0 +1,46 @@
+#pragma once
+
+#include <base/types.h>
+
+namespace wide
+{
+
+template <size_t Bits, typename Signed>
+class integer;
+
+}
+
+using Int128 = wide::integer<128, signed>;
+using UInt128 = wide::integer<128, unsigned>;
+using Int256 = wide::integer<256, signed>;
+using UInt256 = wide::integer<256, unsigned>;
+
+namespace DB
+{
+
+template <class> struct Decimal;
+
+using Decimal32 = Decimal<Int32>;
+using Decimal64 = Decimal<Int64>;
+using Decimal128 = Decimal<Int128>;
+using Decimal256 = Decimal<Int256>;
+
+class DateTime64;
+
+template <class T>
+concept is_decimal =
+    std::is_same_v<T, Decimal32>
+    || std::is_same_v<T, Decimal64>
+    || std::is_same_v<T, Decimal128>
+    || std::is_same_v<T, Decimal256>
+    || std::is_same_v<T, DateTime64>;
+
+template <class T>
+concept is_over_big_int =
+    std::is_same_v<T, Int128>
+    || std::is_same_v<T, UInt128>
+    || std::is_same_v<T, Int256>
+    || std::is_same_v<T, UInt256>
+    || std::is_same_v<T, Decimal128>
+    || std::is_same_v<T, Decimal256>;
+}

base/base/EnumReflection.h

@@ -3,6 +3,7 @@
 #include <magic_enum.hpp>
 #include <fmt/format.h>
 
+
 template <class T> concept is_enum = std::is_enum_v<T>;
 
 namespace detail

base/base/IPv4andIPv6.h

@@ -2,21 +2,23 @@
 
 #include <base/strong_typedef.h>
 #include <base/extended_types.h>
+#include <Common/formatIPv6.h>
 #include <Common/memcmpSmall.h>
 
 namespace DB
 {
 
-    using IPv4 = StrongTypedef<UInt32, struct IPv4Tag>;
+    struct IPv4 : StrongTypedef<UInt32, struct IPv4Tag>
+    {
+        using StrongTypedef::StrongTypedef;
+        using StrongTypedef::operator=;
+        constexpr explicit IPv4(UInt64 value): StrongTypedef(static_cast<UnderlyingType>(value)) {}
+    };
 
     struct IPv6 : StrongTypedef<UInt128, struct IPv6Tag>
     {
-        constexpr IPv6() = default;
-        constexpr explicit IPv6(const UInt128 & x) : StrongTypedef(x) {}
-        constexpr explicit IPv6(UInt128 && x) : StrongTypedef(std::move(x)) {}
-
-        IPv6 & operator=(const UInt128 & rhs) { StrongTypedef::operator=(rhs); return *this; }
-        IPv6 & operator=(UInt128 && rhs) { StrongTypedef::operator=(std::move(rhs)); return *this; }
+        using StrongTypedef::StrongTypedef;
+        using StrongTypedef::operator=;
 
         bool operator<(const IPv6 & rhs) const
         {
@@ -51,3 +53,25 @@ namespace DB
     };
 
 }
+
+namespace std
+{
+    /// For historical reasons we hash IPv6 as a FixedString(16)
+    template <>
+    struct hash<DB::IPv6>
+    {
+        size_t operator()(const DB::IPv6 & x) const
+        {
+            return std::hash<std::string_view>{}(std::string_view(reinterpret_cast<const char*>(&x.toUnderType()), IPV6_BINARY_LENGTH));
+        }
+    };
+
+    template <>
+    struct hash<DB::IPv4>
+    {
+        size_t operator()(const DB::IPv4 & x) const
+        {
+            return std::hash<DB::IPv4::UnderlyingType>()(x.toUnderType());
+        }
+    };
+}

base/base/JSON.cpp

@@ -7,8 +7,6 @@
 #include <base/find_symbols.h>
 #include <base/preciseExp10.h>
 
-#include <iostream>
-
 #define JSON_MAX_DEPTH 100
 
 
@@ -466,9 +464,8 @@ JSON::Pos JSON::searchField(const char * data, size_t size) const
     {
         if (!it->hasEscapes())
         {
-            if (static_cast<int>(size) + 2 > it->dataEnd() - it->data())
-                continue;
-            if (!strncmp(data, it->data() + 1, size))
+            const auto current_name = it->getRawName();
+            if (current_name.size() == size && 0 == memcmp(current_name.data(), data, size))
                 break;
         }
         else

base/base/StringRef.h

@@ -3,12 +3,15 @@
 #include <cassert>
 #include <stdexcept> // for std::logic_error
 #include <string>
+#include <type_traits>
 #include <vector>
 #include <functional>
 #include <iosfwd>
 
+#include <base/defines.h>
 #include <base/types.h>
 #include <base/unaligned.h>
+#include <base/simd.h>
 
 #include <city.h>
 
@@ -27,6 +30,15 @@
     #define CRC_INT __crc32cd
 #endif
 
+#if defined(__aarch64__) && defined(__ARM_NEON)
+    #include <arm_neon.h>
+    #pragma clang diagnostic ignored "-Wreserved-identifier"
+#endif
+
+#if defined(__s390x__)
+    #include <base/crc32c_s390x.h>
+    #define CRC_INT s390x_crc32c
+#endif
 
 /**
  * The std::string_view-like container to avoid creating strings to find substrings in the hash table.
@@ -72,14 +84,14 @@ using StringRefs = std::vector<StringRef>;
   * For more information, see hash_map_string_2.cpp
   */
 
-inline bool compareSSE2(const char * p1, const char * p2)
+inline bool compare8(const char * p1, const char * p2)
 {
     return 0xFFFF == _mm_movemask_epi8(_mm_cmpeq_epi8(
         _mm_loadu_si128(reinterpret_cast<const __m128i *>(p1)),
         _mm_loadu_si128(reinterpret_cast<const __m128i *>(p2))));
 }
 
-inline bool compareSSE2x4(const char * p1, const char * p2)
+inline bool compare64(const char * p1, const char * p2)
 {
     return 0xFFFF == _mm_movemask_epi8(
         _mm_and_si128(
@@ -99,7 +111,30 @@ inline bool compareSSE2x4(const char * p1, const char * p2)
                     _mm_loadu_si128(reinterpret_cast<const __m128i *>(p2) + 3)))));
 }
 
-inline bool memequalSSE2Wide(const char * p1, const char * p2, size_t size)
+#elif defined(__aarch64__) && defined(__ARM_NEON)
+
+inline bool compare8(const char * p1, const char * p2)
+{
+    uint64_t mask = getNibbleMask(vceqq_u8(
+            vld1q_u8(reinterpret_cast<const unsigned char *>(p1)), vld1q_u8(reinterpret_cast<const unsigned char *>(p2))));
+    return 0xFFFFFFFFFFFFFFFF == mask;
+}
+
+inline bool compare64(const char * p1, const char * p2)
+{
+    uint64_t mask = getNibbleMask(vandq_u8(
+        vandq_u8(vceqq_u8(vld1q_u8(reinterpret_cast<const unsigned char *>(p1)), vld1q_u8(reinterpret_cast<const unsigned char *>(p2))),
+            vceqq_u8(vld1q_u8(reinterpret_cast<const unsigned char *>(p1 + 16)), vld1q_u8(reinterpret_cast<const unsigned char *>(p2 + 16)))),
+        vandq_u8(vceqq_u8(vld1q_u8(reinterpret_cast<const unsigned char *>(p1 + 32)), vld1q_u8(reinterpret_cast<const unsigned char *>(p2 + 32))),
+            vceqq_u8(vld1q_u8(reinterpret_cast<const unsigned char *>(p1 + 48)), vld1q_u8(reinterpret_cast<const unsigned char *>(p2 + 48))))));
+    return 0xFFFFFFFFFFFFFFFF == mask;
+}
+
+#endif
+
+#if defined(__SSE2__) || (defined(__aarch64__) && defined(__ARM_NEON))
+
+inline bool memequalWide(const char * p1, const char * p2, size_t size)
 {
     /** The order of branches and the trick with overlapping comparisons
       * are the same as in memcpy implementation.
@@ -136,7 +171,7 @@ inline bool memequalSSE2Wide(const char * p1, const char * p2, size_t size)
 
     while (size >= 64)
     {
-        if (compareSSE2x4(p1, p2))
+        if (compare64(p1, p2))
         {
             p1 += 64;
             p2 += 64;
@@ -146,19 +181,18 @@ inline bool memequalSSE2Wide(const char * p1, const char * p2, size_t size)
             return false;
     }
 
-    switch (size / 16)
+    switch (size / 16) // NOLINT(bugprone-switch-missing-default-case)
     {
-        case 3: if (!compareSSE2(p1 + 32, p2 + 32)) return false; [[fallthrough]];
-        case 2: if (!compareSSE2(p1 + 16, p2 + 16)) return false; [[fallthrough]];
-        case 1: if (!compareSSE2(p1, p2)) return false;
+        case 3: if (!compare8(p1 + 32, p2 + 32)) return false; [[fallthrough]];
+        case 2: if (!compare8(p1 + 16, p2 + 16)) return false; [[fallthrough]];
+        case 1: if (!compare8(p1, p2)) return false;
     }
 
-    return compareSSE2(p1 + size - 16, p2 + size - 16);
+    return compare8(p1 + size - 16, p2 + size - 16);
 }
 
 #endif
 
-
 inline bool operator== (StringRef lhs, StringRef rhs)
 {
     if (lhs.size != rhs.size)
@@ -167,8 +201,8 @@ inline bool operator== (StringRef lhs, StringRef rhs)
     if (lhs.size == 0)
         return true;
 
-#if defined(__SSE2__)
-    return memequalSSE2Wide(lhs.data, rhs.data, lhs.size);
+#if defined(__SSE2__) || (defined(__aarch64__) && defined(__ARM_NEON))
+    return memequalWide(lhs.data, rhs.data, lhs.size);
 #else
     return 0 == memcmp(lhs.data, rhs.data, lhs.size);
 #endif
@@ -234,8 +268,8 @@ inline size_t hashLessThan8(const char * data, size_t size)
 
     if (size >= 4)
     {
-        UInt64 a = unalignedLoad<uint32_t>(data);
-        return hashLen16(size + (a << 3), unalignedLoad<uint32_t>(data + size - 4));
+        UInt64 a = unalignedLoadLittleEndian<uint32_t>(data);
+        return hashLen16(size + (a << 3), unalignedLoadLittleEndian<uint32_t>(data + size - 4));
     }
 
     if (size > 0)
@@ -255,8 +289,8 @@ inline size_t hashLessThan16(const char * data, size_t size)
 {
     if (size > 8)
     {
-        UInt64 a = unalignedLoad<UInt64>(data);
-        UInt64 b = unalignedLoad<UInt64>(data + size - 8);
+        UInt64 a = unalignedLoadLittleEndian<UInt64>(data);
+        UInt64 b = unalignedLoadLittleEndian<UInt64>(data + size - 8);
         return hashLen16(a, rotateByAtLeast1(b + size, static_cast<UInt8>(size))) ^ b;
     }
 
@@ -273,6 +307,8 @@ struct CRC32Hash
         if (size == 0)
             return 0;
 
+        chassert(pos);
+
         if (size < 8)
         {
             return static_cast<unsigned>(hashLessThan8(x.data, x.size));
@@ -283,13 +319,13 @@ struct CRC32Hash
 
         do
         {
-            UInt64 word = unalignedLoad<UInt64>(pos);
+            UInt64 word = unalignedLoadLittleEndian<UInt64>(pos);
             res = static_cast<unsigned>(CRC_INT(res, word));
 
             pos += 8;
         } while (pos + 8 < end);
 
-        UInt64 word = unalignedLoad<UInt64>(end - 8);    /// I'm not sure if this is normal.
+        UInt64 word = unalignedLoadLittleEndian<UInt64>(end - 8);    /// I'm not sure if this is normal.
         res = static_cast<unsigned>(CRC_INT(res, word));
 
         return res;
@@ -326,5 +362,16 @@ namespace ZeroTraits
     inline void set(StringRef & x) { x.size = 0; }
 }
 
+namespace PackedZeroTraits
+{
+    template <typename Second, template <typename, typename> class PackedPairNoInit>
+    inline bool check(const PackedPairNoInit<StringRef, Second> p)
+    { return 0 == p.key.size; }
+
+    template <typename Second, template <typename, typename> class PackedPairNoInit>
+    inline void set(PackedPairNoInit<StringRef, Second> & p)
+    { p.key.size = 0; }
+}
+
 
 std::ostream & operator<<(std::ostream & os, const StringRef & str);

base/base/TypeList.h

@@ -4,7 +4,6 @@
 #include <type_traits>
 #include <utility>
 #include "defines.h"
-#include "TypePair.h"
 
 /// General-purpose typelist. Easy on compilation times as it does not use recursion.
 template <typename ...Args>
@@ -28,7 +27,7 @@ namespace TypeListUtils /// In some contexts it's more handy to use functions in
     constexpr Root<Args...> changeRoot(TypeList<Args...>) { return {}; }
 
     template <typename F, typename ...Args>
-    constexpr void forEach(TypeList<Args...>, F && f) { (std::forward<F>(f)(Id<Args>{}), ...); }
+    constexpr void forEach(TypeList<Args...>, F && f) { (std::forward<F>(f)(TypeList<Args>{}), ...); }
 }
 
 template <typename TypeListLeft, typename TypeListRight>

base/base/TypePair.h

@@ -1,4 +0,0 @@
-#pragma once
-
-template <typename T, typename V> struct TypePair {};
-template <typename T> struct Id {};

base/base/argsToConfig.cpp

@@ -3,13 +3,29 @@
 #include <Poco/Util/LayeredConfiguration.h>
 #include <Poco/Util/MapConfiguration.h>
 
-
-void argsToConfig(const Poco::Util::Application::ArgVec & argv, Poco::Util::LayeredConfiguration & config, int priority)
+void argsToConfig(const Poco::Util::Application::ArgVec & argv,
+                  Poco::Util::LayeredConfiguration & config,
+                  int priority,
+                  const std::unordered_set<std::string>* alias_names)
 {
     /// Parsing all args and converting to config layer
     /// Test: -- --1=1 --1=2 --3 5 7 8 -9 10 -11=12 14= 15== --16==17 --=18 --19= --20 21 22 --23 --24 25 --26 -27 28 ---29=30 -- ----31 32 --33 3-4
     Poco::AutoPtr<Poco::Util::MapConfiguration> map_config = new Poco::Util::MapConfiguration;
     std::string key;
+
+    auto add_arg = [&map_config, &alias_names](const std::string & k, const std::string & v)
+    {
+        map_config->setString(k, v);
+
+        if (alias_names && !alias_names->contains(k))
+        {
+            std::string alias_key = k;
+            std::replace(alias_key.begin(), alias_key.end(), '-', '_');
+            if (alias_names->contains(alias_key))
+                map_config->setString(alias_key, v);
+        }
+    };
+
     for (const auto & arg : argv)
     {
         auto key_start = arg.find_first_not_of('-');
@@ -19,7 +35,7 @@ void argsToConfig(const Poco::Util::Application::ArgVec & argv, Poco::Util::Laye
         // old saved '--key', will set to some true value "1"
         if (!key.empty() && pos_minus != std::string::npos && pos_minus < key_start)
         {
-            map_config->setString(key, "1");
+            add_arg(key, "1");
             key = "";
         }
 
@@ -29,7 +45,7 @@ void argsToConfig(const Poco::Util::Application::ArgVec & argv, Poco::Util::Laye
             {
                 if (pos_minus == std::string::npos || pos_minus > key_start)
                 {
-                    map_config->setString(key, arg);
+                    add_arg(key, arg);
                 }
                 key = "";
             }
@@ -55,7 +71,7 @@ void argsToConfig(const Poco::Util::Application::ArgVec & argv, Poco::Util::Laye
         if (arg.size() > pos_eq)
             value = arg.substr(pos_eq + 1);
 
-        map_config->setString(key, value);
+        add_arg(key, value);
         key = "";
     }
 

base/base/argsToConfig.h

@@ -1,6 +1,8 @@
 #pragma once
 
 #include <Poco/Util/Application.h>
+#include <string>
+#include <unordered_set>
 
 namespace Poco::Util
 {
@@ -8,4 +10,7 @@ class LayeredConfiguration; // NOLINT(cppcoreguidelines-virtual-class-destructor
 }
 
 /// Import extra command line arguments to configuration. These are command line arguments after --.
-void argsToConfig(const Poco::Util::Application::ArgVec & argv, Poco::Util::LayeredConfiguration & config, int priority);
+void argsToConfig(const Poco::Util::Application::ArgVec & argv,
+                  Poco::Util::LayeredConfiguration & config,
+                  int priority,
+                  const std::unordered_set<std::string>* registered_alias_names = nullptr);

base/base/bit_cast.h

@@ -7,26 +7,33 @@
 
 /** Returns value `from` converted to type `To` while retaining bit representation.
   * `To` and `From` must satisfy `CopyConstructible`.
+  *
   * In contrast to std::bit_cast can cast types of different width.
+  *
+  * Note: for signed types of narrower size, the casted result is zero-extended
+  * instead of sign-extended as with regular static_cast.
+  * For example, -1 Int8 (represented as 0xFF) bit_casted to UInt64
+  * gives 255 (represented as 0x00000000000000FF) instead of 0xFFFFFFFFFFFFFFFF
   */
 template <typename To, typename From>
 std::decay_t<To> bit_cast(const From & from)
 {
-    /**
-     * Assume the source value is 0xAABBCCDD (i.e. sizeof(from) == 4).
-     * Its BE representation is 0xAABBCCDD, the LE representation is 0xDDCCBBAA.
-     * Further assume, sizeof(res) == 8 and that res is initially zeroed out.
-     * With LE, the result after bit_cast will be 0xDDCCBBAA00000000 --> input value == output value.
-     * With BE, the result after bit_cast will be 0x00000000AABBCCDD --> input value == output value.
-     */
+    /** Assume the source value is 0xAABBCCDD (i.e. sizeof(from) == 4).
+      * Its BE representation is 0xAABBCCDD, the LE representation is 0xDDCCBBAA.
+      * Further assume, sizeof(res) == 8 and that res is initially zeroed out.
+      * With LE, the result after bit_cast will be 0xDDCCBBAA00000000 --> input value == output value.
+      * With BE, the result after bit_cast will be 0x00000000AABBCCDD --> input value == output value.
+      */
     To res {};
     if constexpr (std::endian::native == std::endian::little)
-      memcpy(static_cast<void*>(&res), &from, std::min(sizeof(res), sizeof(from)));
+    {
+        memcpy(static_cast<void*>(&res), &from, std::min(sizeof(res), sizeof(from)));
+    }
     else
     {
-      uint32_t offset_to = (sizeof(res) > sizeof(from)) ? (sizeof(res) - sizeof(from)) : 0;
-      uint32_t offset_from = (sizeof(from) > sizeof(res)) ? (sizeof(from) - sizeof(res)) : 0;
-      memcpy(reinterpret_cast<char *>(&res) + offset_to, reinterpret_cast<const char *>(&from) + offset_from, std::min(sizeof(res), sizeof(from)));
+        uint32_t offset_to = (sizeof(res) > sizeof(from)) ? (sizeof(res) - sizeof(from)) : 0;
+        uint32_t offset_from = (sizeof(from) > sizeof(res)) ? (sizeof(from) - sizeof(res)) : 0;
+        memcpy(reinterpret_cast<char *>(&res) + offset_to, reinterpret_cast<const char *>(&from) + offset_from, std::min(sizeof(res), sizeof(from)));
     }
     return res;
 }

base/base/coverage.cpp

@@ -1,9 +1,15 @@
 #include "coverage.h"
 
+#pragma GCC diagnostic ignored "-Wreserved-identifier"
+
+
+/// WITH_COVERAGE enables the default implementation of code coverage,
+/// that dumps a map to the filesystem.
+
 #if WITH_COVERAGE
 
-#    include <mutex>
-#    include <unistd.h>
+#include <mutex>
+#include <unistd.h>
 
 
 #    if defined(__clang__)
@@ -29,3 +35,131 @@ void dumpCoverageReportIfPossible()
 
 #endif
 }
+
+
+/// SANITIZE_COVERAGE enables code instrumentation,
+/// but leaves the callbacks implementation to us,
+/// which we use to calculate coverage on a per-test basis
+/// and to write it to system tables.
+
+#if defined(SANITIZE_COVERAGE)
+
+namespace
+{
+    bool pc_guards_initialized = false;
+    bool pc_table_initialized = false;
+
+    uint32_t * guards_start = nullptr;
+    uint32_t * guards_end = nullptr;
+
+    uintptr_t * coverage_array = nullptr;
+    size_t coverage_array_size = 0;
+
+    uintptr_t * all_addresses_array = nullptr;
+    size_t all_addresses_array_size = 0;
+}
+
+extern "C"
+{
+
+/// This is called at least once for every DSO for initialization.
+/// But we will use it only for the main DSO.
+void __sanitizer_cov_trace_pc_guard_init(uint32_t * start, uint32_t * stop)
+{
+    if (pc_guards_initialized)
+        return;
+    pc_guards_initialized = true;
+
+    /// The function can be called multiple times, but we need to initialize only once.
+    if (start == stop || *start)
+        return;
+
+    guards_start = start;
+    guards_end = stop;
+    coverage_array_size = stop - start;
+
+    /// Note: we will leak this.
+    coverage_array = static_cast<uintptr_t*>(malloc(sizeof(uintptr_t) * coverage_array_size));
+
+    resetCoverage();
+}
+
+/// This is called at least once for every DSO for initialization
+/// and provides information about all instrumented addresses.
+void __sanitizer_cov_pcs_init(const uintptr_t * pcs_begin, const uintptr_t * pcs_end)
+{
+    if (pc_table_initialized)
+        return;
+    pc_table_initialized = true;
+
+    all_addresses_array = static_cast<uintptr_t*>(malloc(sizeof(uintptr_t) * coverage_array_size));
+    all_addresses_array_size = pcs_end - pcs_begin;
+
+    /// They are not a real pointers, but also contain a flag in the most significant bit,
+    /// in which we are not interested for now. Reset it.
+    for (size_t i = 0; i < all_addresses_array_size; ++i)
+        all_addresses_array[i] = pcs_begin[i] & 0x7FFFFFFFFFFFFFFFULL;
+}
+
+/// This is called at every basic block / edge, etc.
+void __sanitizer_cov_trace_pc_guard(uint32_t * guard)
+{
+    /// Duplicate the guard check.
+    if (!*guard)
+        return;
+    *guard = 0;
+
+    /// If you set *guard to 0 this code will not be called again for this edge.
+    /// Now we can get the PC and do whatever you want:
+    /// - store it somewhere or symbolize it and print right away.
+    /// The values of `*guard` are as you set them in
+    /// __sanitizer_cov_trace_pc_guard_init and so you can make them consecutive
+    /// and use them to dereference an array or a bit vector.
+    void * pc = __builtin_return_address(0);
+
+    coverage_array[guard - guards_start] = reinterpret_cast<uintptr_t>(pc);
+}
+
+}
+
+__attribute__((no_sanitize("coverage")))
+std::span<const uintptr_t> getCoverage()
+{
+    return {coverage_array, coverage_array_size};
+}
+
+__attribute__((no_sanitize("coverage")))
+std::span<const uintptr_t> getAllInstrumentedAddresses()
+{
+    return {all_addresses_array, all_addresses_array_size};
+}
+
+__attribute__((no_sanitize("coverage")))
+void resetCoverage()
+{
+    memset(coverage_array, 0, coverage_array_size * sizeof(*coverage_array));
+
+    /// The guard defines whether the __sanitizer_cov_trace_pc_guard should be called.
+    /// For example, you can unset it after first invocation to prevent excessive work.
+    /// Initially set all the guards to 1 to enable callbacks.
+    for (uint32_t * x = guards_start; x < guards_end; ++x)
+        *x = 1;
+}
+
+#else
+
+std::span<const uintptr_t> getCoverage()
+{
+    return {};
+}
+
+std::span<const uintptr_t> getAllInstrumentedAddresses()
+{
+    return {};
+}
+
+void resetCoverage()
+{
+}
+
+#endif

base/base/coverage.h

@@ -1,5 +1,8 @@
 #pragma once
 
+#include <span>
+#include <cstdint>
+
 /// Flush coverage report to file, depending on coverage system
 /// proposed by compiler (llvm for clang and gcov for gcc).
 ///
@@ -7,3 +10,16 @@
 /// Thread safe (use exclusive lock).
 /// Idempotent, may be called multiple times.
 void dumpCoverageReportIfPossible();
+
+/// This is effective if SANITIZE_COVERAGE is enabled at build time.
+/// Get accumulated unique program addresses of the instrumented parts of the code,
+/// seen so far after program startup or after previous reset.
+/// The returned span will be represented as a sparse map, containing mostly zeros, which you should filter away.
+std::span<const uintptr_t> getCoverage();
+
+/// Get all instrumented addresses that could be in the coverage.
+std::span<const uintptr_t> getAllInstrumentedAddresses();
+
+/// Reset the accumulated coverage.
+/// This is useful to compare coverage of different tests, including differential coverage.
+void resetCoverage();

base/base/crc32c_s390x.h

@@ -0,0 +1,26 @@
+#pragma once
+
+#include <crc32-s390x.h>
+
+inline uint32_t s390x_crc32c_u8(uint32_t crc, uint8_t v)
+{
+    return crc32c_le_vx(crc, reinterpret_cast<unsigned char *>(&v), sizeof(v));
+}
+
+inline uint32_t s390x_crc32c_u16(uint32_t crc, uint16_t v)
+{
+    v = __builtin_bswap16(v);
+    return crc32c_le_vx(crc, reinterpret_cast<unsigned char *>(&v), sizeof(v));
+}
+
+inline uint32_t s390x_crc32c_u32(uint32_t crc, uint32_t v)
+{
+    v = __builtin_bswap32(v);
+    return crc32c_le_vx(crc, reinterpret_cast<unsigned char *>(&v), sizeof(v));
+}
+
+inline uint64_t s390x_crc32c(uint64_t crc, uint64_t v)
+{
+    v = __builtin_bswap64(v);
+    return crc32c_le_vx(static_cast<uint32_t>(crc), reinterpret_cast<unsigned char *>(&v), sizeof(uint64_t));
+}

base/base/defines.h

@@ -28,8 +28,8 @@
 #define NO_INLINE __attribute__((__noinline__))
 #define MAY_ALIAS __attribute__((__may_alias__))
 
-#if !defined(__x86_64__) && !defined(__aarch64__) && !defined(__PPC__) && !(defined(__riscv) && (__riscv_xlen == 64))
-#    error "The only supported platforms are x86_64 and AArch64, PowerPC (work in progress) and RISC-V 64 (experimental)"
+#if !defined(__x86_64__) && !defined(__aarch64__) && !defined(__PPC__) && !defined(__s390x__) && !(defined(__riscv) && (__riscv_xlen == 64))
+#    error "The only supported platforms are x86_64 and AArch64, PowerPC (work in progress), s390x (work in progress) and RISC-V 64 (experimental)"
 #endif
 
 /// Check for presence of address sanitizer
@@ -73,18 +73,6 @@
 #    endif
 #endif
 
-#if defined(ADDRESS_SANITIZER)
-#    define BOOST_USE_ASAN 1
-#    define BOOST_USE_UCONTEXT 1
-#endif
-
-#if defined(THREAD_SANITIZER)
-#    define BOOST_USE_TSAN 1
-#    define BOOST_USE_UCONTEXT 1
-#endif
-
-/// TODO: Strange enough, there is no way to detect UB sanitizer.
-
 /// Explicitly allow undefined behaviour for certain functions. Use it as a function attribute.
 /// It is useful in case when compiler cannot see (and exploit) it, but UBSan can.
 /// Example: multiplication of signed integers with possibility of overflow when both sides are from user input.
@@ -127,14 +115,20 @@
 /// because SIGABRT is easier to debug than SIGTRAP (the second one makes gdb crazy)
 #if !defined(chassert)
     #if defined(ABORT_ON_LOGICAL_ERROR)
-        #define chassert(x) static_cast<bool>(x) ? void(0) : ::DB::abortOnFailedAssertion(#x)
+        // clang-format off
+        #include <base/types.h>
+        namespace DB
+        {
+            [[noreturn]] void abortOnFailedAssertion(const String & description);
+        }
+        #define chassert(x) do { static_cast<bool>(x) ? void(0) : ::DB::abortOnFailedAssertion(#x); } while (0)
         #define UNREACHABLE() abort()
+        // clang-format off
     #else
         /// Here sizeof() trick is used to suppress unused warning for result,
         /// since simple "(void)x" will evaluate the expression, while
         /// "sizeof(!(x))" will not.
-        #define NIL_EXPRESSION(x) (void)sizeof(!(x))
-        #define chassert(x) NIL_EXPRESSION(x)
+        #define chassert(x) (void)sizeof(!(x))
         #define UNREACHABLE() __builtin_unreachable()
     #endif
 #endif
@@ -155,6 +149,7 @@
 #    define TSA_ACQUIRE_SHARED(...) __attribute__((acquire_shared_capability(__VA_ARGS__)))          /// function acquires a shared capability, but does not release it
 #    define TSA_TRY_ACQUIRE_SHARED(...) __attribute__((try_acquire_shared_capability(__VA_ARGS__)))  /// function tries to acquire a shared capability and returns a boolean value indicating success or failure
 #    define TSA_RELEASE_SHARED(...) __attribute__((release_shared_capability(__VA_ARGS__)))          /// function releases the given shared capability
+#    define TSA_SCOPED_LOCKABLE __attribute__((scoped_lockable)) /// object of a class has scoped lockable capability
 
 /// Macros for suppressing TSA warnings for specific reads/writes (instead of suppressing it for the whole function)
 /// They use a lambda function to apply function attribute to a single statement. This enable us to suppress warnings locally instead of
@@ -182,6 +177,7 @@
 #    define TSA_ACQUIRE_SHARED(...)
 #    define TSA_TRY_ACQUIRE_SHARED(...)
 #    define TSA_RELEASE_SHARED(...)
+#    define TSA_SCOPED_LOCKABLE
 
 #    define TSA_SUPPRESS_WARNING_FOR_READ(x) (x)
 #    define TSA_SUPPRESS_WARNING_FOR_WRITE(x) (x)
@@ -190,6 +186,6 @@
 
 /// A template function for suppressing warnings about unused variables or function results.
 template <typename... Args>
-constexpr void UNUSED(Args &&... args [[maybe_unused]])
+constexpr void UNUSED(Args &&... args [[maybe_unused]]) // NOLINT(cppcoreguidelines-missing-std-forward)
 {
 }

base/base/find_symbols.h

@@ -2,6 +2,7 @@
 
 #include <cstdint>
 #include <string>
+#include <array>
 
 #if defined(__SSE2__)
     #include <emmintrin.h>
@@ -34,9 +35,51 @@
   * If no such characters, returns nullptr.
   */
 
+struct SearchSymbols
+{
+    static constexpr auto BUFFER_SIZE = 16;
+
+    SearchSymbols() = default;
+
+    explicit SearchSymbols(std::string in)
+        : str(std::move(in))
+    {
+#if defined(__SSE4_2__)
+        if (str.size() > BUFFER_SIZE)
+        {
+            throw std::runtime_error("SearchSymbols can contain at most " + std::to_string(BUFFER_SIZE) + " symbols and " + std::to_string(str.size()) + " was provided\n");
+        }
+
+        char tmp_safety_buffer[BUFFER_SIZE] = {0};
+
+        memcpy(tmp_safety_buffer, str.data(), str.size());
+
+        simd_vector = _mm_loadu_si128(reinterpret_cast<const __m128i *>(tmp_safety_buffer));
+#endif
+    }
+
+#if defined(__SSE4_2__)
+    __m128i simd_vector;
+#endif
+    std::string str;
+};
+
 namespace detail
 {
-template <char ...chars> constexpr bool is_in(char x) { return ((x == chars) || ...); }
+template <char ...chars> constexpr bool is_in(char x) { return ((x == chars) || ...); } // NOLINT(misc-redundant-expression)
+
+static bool is_in(char c, const char * symbols, size_t num_chars)
+{
+    for (size_t i = 0u; i < num_chars; ++i)
+    {
+        if (c == symbols[i])
+        {
+            return true;
+        }
+    }
+
+    return false;
+}
 
 #if defined(__SSE2__)
 template <char s0>
@@ -53,6 +96,43 @@ inline __m128i mm_is_in(__m128i bytes)
     __m128i eq = mm_is_in<s1, tail...>(bytes);
     return _mm_or_si128(eq0, eq);
 }
+
+inline __m128i mm_is_in(__m128i bytes, const char * symbols, size_t num_chars)
+{
+    __m128i accumulator = _mm_setzero_si128();
+    for (size_t i = 0; i < num_chars; ++i)
+    {
+        __m128i eq = _mm_cmpeq_epi8(bytes, _mm_set1_epi8(symbols[i]));
+        accumulator = _mm_or_si128(accumulator, eq);
+    }
+
+    return accumulator;
+}
+
+inline std::array<__m128i, 16u> mm_is_in_prepare(const char * symbols, size_t num_chars)
+{
+    std::array<__m128i, 16u> result {};
+
+    for (size_t i = 0; i < num_chars; ++i)
+    {
+        result[i] = _mm_set1_epi8(symbols[i]);
+    }
+
+    return result;
+}
+
+inline __m128i mm_is_in_execute(__m128i bytes, const std::array<__m128i, 16u> & needles)
+{
+    __m128i accumulator = _mm_setzero_si128();
+
+    for (const auto & needle : needles)
+    {
+        __m128i eq = _mm_cmpeq_epi8(bytes, needle);
+        accumulator = _mm_or_si128(accumulator, eq);
+    }
+
+    return accumulator;
+}
 #endif
 
 template <bool positive>
@@ -99,6 +179,32 @@ inline const char * find_first_symbols_sse2(const char * const begin, const char
     return return_mode == ReturnMode::End ? end : nullptr;
 }
 
+template <bool positive, ReturnMode return_mode>
+inline const char * find_first_symbols_sse2(const char * const begin, const char * const end, const char * symbols, size_t num_chars)
+{
+    const char * pos = begin;
+
+#if defined(__SSE2__)
+    const auto needles = mm_is_in_prepare(symbols, num_chars);
+    for (; pos + 15 < end; pos += 16)
+    {
+        __m128i bytes = _mm_loadu_si128(reinterpret_cast<const __m128i *>(pos));
+
+        __m128i eq = mm_is_in_execute(bytes, needles);
+
+        uint16_t bit_mask = maybe_negate<positive>(uint16_t(_mm_movemask_epi8(eq)));
+        if (bit_mask)
+            return pos + __builtin_ctz(bit_mask);
+    }
+#endif
+
+    for (; pos < end; ++pos)
+        if (maybe_negate<positive>(is_in(*pos, symbols, num_chars)))
+            return pos;
+
+    return return_mode == ReturnMode::End ? end : nullptr;
+}
+
 
 template <bool positive, ReturnMode return_mode, char... symbols>
 inline const char * find_last_symbols_sse2(const char * const begin, const char * const end)
@@ -159,26 +265,61 @@ inline const char * find_first_symbols_sse42(const char * const begin, const cha
 #endif
 
     for (; pos < end; ++pos)
-        if (   (num_chars >= 1 && maybe_negate<positive>(*pos == c01))
-            || (num_chars >= 2 && maybe_negate<positive>(*pos == c02))
-            || (num_chars >= 3 && maybe_negate<positive>(*pos == c03))
-            || (num_chars >= 4 && maybe_negate<positive>(*pos == c04))
-            || (num_chars >= 5 && maybe_negate<positive>(*pos == c05))
-            || (num_chars >= 6 && maybe_negate<positive>(*pos == c06))
-            || (num_chars >= 7 && maybe_negate<positive>(*pos == c07))
-            || (num_chars >= 8 && maybe_negate<positive>(*pos == c08))
-            || (num_chars >= 9 && maybe_negate<positive>(*pos == c09))
-            || (num_chars >= 10 && maybe_negate<positive>(*pos == c10))
-            || (num_chars >= 11 && maybe_negate<positive>(*pos == c11))
-            || (num_chars >= 12 && maybe_negate<positive>(*pos == c12))
-            || (num_chars >= 13 && maybe_negate<positive>(*pos == c13))
-            || (num_chars >= 14 && maybe_negate<positive>(*pos == c14))
-            || (num_chars >= 15 && maybe_negate<positive>(*pos == c15))
-            || (num_chars >= 16 && maybe_negate<positive>(*pos == c16)))
+        if (   (num_chars == 1 && maybe_negate<positive>(is_in<c01>(*pos)))
+            || (num_chars == 2 && maybe_negate<positive>(is_in<c01, c02>(*pos)))
+            || (num_chars == 3 && maybe_negate<positive>(is_in<c01, c02, c03>(*pos)))
+            || (num_chars == 4 && maybe_negate<positive>(is_in<c01, c02, c03, c04>(*pos)))
+            || (num_chars == 5 && maybe_negate<positive>(is_in<c01, c02, c03, c04, c05>(*pos)))
+            || (num_chars == 6 && maybe_negate<positive>(is_in<c01, c02, c03, c04, c05, c06>(*pos)))
+            || (num_chars == 7 && maybe_negate<positive>(is_in<c01, c02, c03, c04, c05, c06, c07>(*pos)))
+            || (num_chars == 8 && maybe_negate<positive>(is_in<c01, c02, c03, c04, c05, c06, c07, c08>(*pos)))
+            || (num_chars == 9 && maybe_negate<positive>(is_in<c01, c02, c03, c04, c05, c06, c07, c08, c09>(*pos)))
+            || (num_chars == 10 && maybe_negate<positive>(is_in<c01, c02, c03, c04, c05, c06, c07, c08, c09, c10>(*pos)))
+            || (num_chars == 11 && maybe_negate<positive>(is_in<c01, c02, c03, c04, c05, c06, c07, c08, c09, c10, c11>(*pos)))
+            || (num_chars == 12 && maybe_negate<positive>(is_in<c01, c02, c03, c04, c05, c06, c07, c08, c09, c10, c11, c12>(*pos)))
+            || (num_chars == 13 && maybe_negate<positive>(is_in<c01, c02, c03, c04, c05, c06, c07, c08, c09, c10, c11, c12, c13>(*pos)))
+            || (num_chars == 14 && maybe_negate<positive>(is_in<c01, c02, c03, c04, c05, c06, c07, c08, c09, c10, c11, c12, c13, c14>(*pos)))
+            || (num_chars == 15 && maybe_negate<positive>(is_in<c01, c02, c03, c04, c05, c06, c07, c08, c09, c10, c11, c12, c13, c14, c15>(*pos)))
+            || (num_chars == 16 && maybe_negate<positive>(is_in<c01, c02, c03, c04, c05, c06, c07, c08, c09, c10, c11, c12, c13, c14, c15, c16>(*pos))))
             return pos;
     return return_mode == ReturnMode::End ? end : nullptr;
 }
 
+template <bool positive, ReturnMode return_mode>
+inline const char * find_first_symbols_sse42(const char * const begin, const char * const end, const SearchSymbols & symbols)
+{
+    const char * pos = begin;
+
+    const auto num_chars = symbols.str.size();
+
+#if defined(__SSE4_2__)
+    constexpr int mode = _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT;
+
+    const __m128i set = symbols.simd_vector;
+
+    for (; pos + 15 < end; pos += 16)
+    {
+        __m128i bytes = _mm_loadu_si128(reinterpret_cast<const __m128i *>(pos));
+
+        if constexpr (positive)
+        {
+            if (_mm_cmpestrc(set, num_chars, bytes, 16, mode))
+                return pos + _mm_cmpestri(set, num_chars, bytes, 16, mode);
+        }
+        else
+        {
+            if (_mm_cmpestrc(set, num_chars, bytes, 16, mode | _SIDD_NEGATIVE_POLARITY))
+                return pos + _mm_cmpestri(set, num_chars, bytes, 16, mode | _SIDD_NEGATIVE_POLARITY);
+        }
+    }
+#endif
+
+    for (; pos < end; ++pos)
+        if (maybe_negate<positive>(is_in(*pos, symbols.str.data(), num_chars)))
+            return pos;
+
+    return return_mode == ReturnMode::End ? end : nullptr;
+}
 
 /// NOTE No SSE 4.2 implementation for find_last_symbols_or_null. Not worth to do.
 
@@ -194,6 +335,17 @@ inline const char * find_first_symbols_dispatch(const char * begin, const char *
         return find_first_symbols_sse2<positive, return_mode, symbols...>(begin, end);
 }
 
+template <bool positive, ReturnMode return_mode>
+inline const char * find_first_symbols_dispatch(const std::string_view haystack, const SearchSymbols & symbols)
+{
+#if defined(__SSE4_2__)
+    if (symbols.str.size() >= 5)
+        return find_first_symbols_sse42<positive, return_mode>(haystack.begin(), haystack.end(), symbols);
+    else
+#endif
+        return find_first_symbols_sse2<positive, return_mode>(haystack.begin(), haystack.end(), symbols.str.data(), symbols.str.size());
+}
+
 }
 
 
@@ -211,6 +363,11 @@ inline char * find_first_symbols(char * begin, char * end)
     return const_cast<char *>(detail::find_first_symbols_dispatch<true, detail::ReturnMode::End, symbols...>(begin, end));
 }
 
+inline const char * find_first_symbols(std::string_view haystack, const SearchSymbols & symbols)
+{
+    return detail::find_first_symbols_dispatch<true, detail::ReturnMode::End>(haystack, symbols);
+}
+
 template <char... symbols>
 inline const char * find_first_not_symbols(const char * begin, const char * end)
 {
@@ -223,6 +380,11 @@ inline char * find_first_not_symbols(char * begin, char * end)
     return const_cast<char *>(detail::find_first_symbols_dispatch<false, detail::ReturnMode::End, symbols...>(begin, end));
 }
 
+inline const char * find_first_not_symbols(std::string_view haystack, const SearchSymbols & symbols)
+{
+    return detail::find_first_symbols_dispatch<false, detail::ReturnMode::End>(haystack, symbols);
+}
+
 template <char... symbols>
 inline const char * find_first_symbols_or_null(const char * begin, const char * end)
 {
@@ -235,6 +397,11 @@ inline char * find_first_symbols_or_null(char * begin, char * end)
     return const_cast<char *>(detail::find_first_symbols_dispatch<true, detail::ReturnMode::Nullptr, symbols...>(begin, end));
 }
 
+inline const char * find_first_symbols_or_null(std::string_view haystack, const SearchSymbols & symbols)
+{
+    return detail::find_first_symbols_dispatch<true, detail::ReturnMode::Nullptr>(haystack, symbols);
+}
+
 template <char... symbols>
 inline const char * find_first_not_symbols_or_null(const char * begin, const char * end)
 {
@@ -247,6 +414,10 @@ inline char * find_first_not_symbols_or_null(char * begin, char * end)
     return const_cast<char *>(detail::find_first_symbols_dispatch<false, detail::ReturnMode::Nullptr, symbols...>(begin, end));
 }
 
+inline const char * find_first_not_symbols_or_null(std::string_view haystack, const SearchSymbols & symbols)
+{
+    return detail::find_first_symbols_dispatch<false, detail::ReturnMode::Nullptr>(haystack, symbols);
+}
 
 template <char... symbols>
 inline const char * find_last_symbols_or_null(const char * begin, const char * end)
@@ -277,7 +448,7 @@ inline char * find_last_not_symbols_or_null(char * begin, char * end)
 /// See https://github.com/boostorg/algorithm/issues/63
 /// And https://bugs.llvm.org/show_bug.cgi?id=41141
 template <char... symbols, typename To>
-inline void splitInto(To & to, const std::string & what, bool token_compress = false)
+inline To & splitInto(To & to, std::string_view what, bool token_compress = false)
 {
     const char * pos = what.data();
     const char * end = pos + what.size();
@@ -293,4 +464,6 @@ inline void splitInto(To & to, const std::string & what, bool token_compress = f
         else
             pos = delimiter_or_end;
     }
+
+    return to;
 }

base/base/getMemoryAmount.cpp

@@ -28,14 +28,28 @@ uint64_t getMemoryAmountOrZero()
 
 #if defined(OS_LINUX)
     // Try to lookup at the Cgroup limit
-    std::ifstream cgroup_limit("/sys/fs/cgroup/memory/memory.limit_in_bytes");
-    if (cgroup_limit.is_open())
+
+    // CGroups v2
+    std::ifstream cgroupv2_limit("/sys/fs/cgroup/memory.max");
+    if (cgroupv2_limit.is_open())
     {
-        uint64_t memory_limit = 0; // in case of read error
-        cgroup_limit >> memory_limit;
+        uint64_t memory_limit = 0;
+        cgroupv2_limit >> memory_limit;
         if (memory_limit > 0 && memory_limit < memory_amount)
             memory_amount = memory_limit;
     }
+    else
+    {
+        // CGroups v1
+        std::ifstream cgroup_limit("/sys/fs/cgroup/memory/memory.limit_in_bytes");
+        if (cgroup_limit.is_open())
+        {
+            uint64_t memory_limit = 0; // in case of read error
+            cgroup_limit >> memory_limit;
+            if (memory_limit > 0 && memory_limit < memory_amount)
+                memory_amount = memory_limit;
+        }
+    }
 #endif
 
     return memory_amount;

base/base/getThreadId.cpp

@@ -15,25 +15,34 @@
 
 
 static thread_local uint64_t current_tid = 0;
+
+static void setCurrentThreadId()
+{
+#if defined(OS_ANDROID)
+    current_tid = gettid();
+#elif defined(OS_LINUX)
+    current_tid = static_cast<uint64_t>(syscall(SYS_gettid)); /// This call is always successful. - man gettid
+#elif defined(OS_FREEBSD)
+    current_tid = pthread_getthreadid_np();
+#elif defined(OS_SUNOS)
+    // On Solaris-derived systems, this returns the ID of the LWP, analogous
+    // to a thread.
+    current_tid = static_cast<uint64_t>(pthread_self());
+#else
+    if (0 != pthread_threadid_np(nullptr, &current_tid))
+        throw std::logic_error("pthread_threadid_np returned error");
+#endif
+}
+
 uint64_t getThreadId()
 {
     if (!current_tid)
-    {
-#if defined(OS_ANDROID)
-        current_tid = gettid();
-#elif defined(OS_LINUX)
-        current_tid = static_cast<uint64_t>(syscall(SYS_gettid)); /// This call is always successful. - man gettid
-#elif defined(OS_FREEBSD)
-        current_tid = pthread_getthreadid_np();
-#elif defined(OS_SUNOS)
-        // On Solaris-derived systems, this returns the ID of the LWP, analogous
-        // to a thread.
-        current_tid = static_cast<uint64_t>(pthread_self());
-#else
-        if (0 != pthread_threadid_np(nullptr, &current_tid))
-            throw std::logic_error("pthread_threadid_np returned error");
-#endif
-    }
+        setCurrentThreadId();
 
     return current_tid;
 }
+
+void updateCurrentThreadIdAfterFork()
+{
+    setCurrentThreadId();
+}

base/base/getThreadId.h

@@ -3,3 +3,5 @@
 
 /// Obtain thread id from OS. The value is cached in thread local variable.
 uint64_t getThreadId();
+
+void updateCurrentThreadIdAfterFork();

base/base/hex.h

@@ -0,0 +1,301 @@
+#pragma once
+
+#include <bit>
+#include <cstring>
+#include "types.h"
+
+namespace CityHash_v1_0_2 { struct uint128; }
+
+namespace wide
+{
+    template <size_t Bits, typename Signed>
+    class integer;
+}
+
+namespace impl
+{
+    /// Maps 0..15 to 0..9A..F or 0..9a..f correspondingly.
+    constexpr inline std::string_view hex_digit_to_char_uppercase_table = "0123456789ABCDEF";
+    constexpr inline std::string_view hex_digit_to_char_lowercase_table = "0123456789abcdef";
+
+    /// Maps 0..255 to 00..FF or 00..ff correspondingly.
+    constexpr inline std::string_view hex_byte_to_char_uppercase_table = //
+        "000102030405060708090A0B0C0D0E0F"
+        "101112131415161718191A1B1C1D1E1F"
+        "202122232425262728292A2B2C2D2E2F"
+        "303132333435363738393A3B3C3D3E3F"
+        "404142434445464748494A4B4C4D4E4F"
+        "505152535455565758595A5B5C5D5E5F"
+        "606162636465666768696A6B6C6D6E6F"
+        "707172737475767778797A7B7C7D7E7F"
+        "808182838485868788898A8B8C8D8E8F"
+        "909192939495969798999A9B9C9D9E9F"
+        "A0A1A2A3A4A5A6A7A8A9AAABACADAEAF"
+        "B0B1B2B3B4B5B6B7B8B9BABBBCBDBEBF"
+        "C0C1C2C3C4C5C6C7C8C9CACBCCCDCECF"
+        "D0D1D2D3D4D5D6D7D8D9DADBDCDDDEDF"
+        "E0E1E2E3E4E5E6E7E8E9EAEBECEDEEEF"
+        "F0F1F2F3F4F5F6F7F8F9FAFBFCFDFEFF";
+
+    constexpr inline std::string_view hex_byte_to_char_lowercase_table = //
+        "000102030405060708090a0b0c0d0e0f"
+        "101112131415161718191a1b1c1d1e1f"
+        "202122232425262728292a2b2c2d2e2f"
+        "303132333435363738393a3b3c3d3e3f"
+        "404142434445464748494a4b4c4d4e4f"
+        "505152535455565758595a5b5c5d5e5f"
+        "606162636465666768696a6b6c6d6e6f"
+        "707172737475767778797a7b7c7d7e7f"
+        "808182838485868788898a8b8c8d8e8f"
+        "909192939495969798999a9b9c9d9e9f"
+        "a0a1a2a3a4a5a6a7a8a9aaabacadaeaf"
+        "b0b1b2b3b4b5b6b7b8b9babbbcbdbebf"
+        "c0c1c2c3c4c5c6c7c8c9cacbcccdcecf"
+        "d0d1d2d3d4d5d6d7d8d9dadbdcdddedf"
+        "e0e1e2e3e4e5e6e7e8e9eaebecedeeef"
+        "f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff";
+
+    /// Maps 0..255 to 00000000..11111111 correspondingly.
+    constexpr inline std::string_view bin_byte_to_char_table = //
+        "0000000000000001000000100000001100000100000001010000011000000111"
+        "0000100000001001000010100000101100001100000011010000111000001111"
+        "0001000000010001000100100001001100010100000101010001011000010111"
+        "0001100000011001000110100001101100011100000111010001111000011111"
+        "0010000000100001001000100010001100100100001001010010011000100111"
+        "0010100000101001001010100010101100101100001011010010111000101111"
+        "0011000000110001001100100011001100110100001101010011011000110111"
+        "0011100000111001001110100011101100111100001111010011111000111111"
+        "0100000001000001010000100100001101000100010001010100011001000111"
+        "0100100001001001010010100100101101001100010011010100111001001111"
+        "0101000001010001010100100101001101010100010101010101011001010111"
+        "0101100001011001010110100101101101011100010111010101111001011111"
+        "0110000001100001011000100110001101100100011001010110011001100111"
+        "0110100001101001011010100110101101101100011011010110111001101111"
+        "0111000001110001011100100111001101110100011101010111011001110111"
+        "0111100001111001011110100111101101111100011111010111111001111111"
+        "1000000010000001100000101000001110000100100001011000011010000111"
+        "1000100010001001100010101000101110001100100011011000111010001111"
+        "1001000010010001100100101001001110010100100101011001011010010111"
+        "1001100010011001100110101001101110011100100111011001111010011111"
+        "1010000010100001101000101010001110100100101001011010011010100111"
+        "1010100010101001101010101010101110101100101011011010111010101111"
+        "1011000010110001101100101011001110110100101101011011011010110111"
+        "1011100010111001101110101011101110111100101111011011111010111111"
+        "1100000011000001110000101100001111000100110001011100011011000111"
+        "1100100011001001110010101100101111001100110011011100111011001111"
+        "1101000011010001110100101101001111010100110101011101011011010111"
+        "1101100011011001110110101101101111011100110111011101111011011111"
+        "1110000011100001111000101110001111100100111001011110011011100111"
+        "1110100011101001111010101110101111101100111011011110111011101111"
+        "1111000011110001111100101111001111110100111101011111011011110111"
+        "1111100011111001111110101111101111111100111111011111111011111111";
+
+    /// Maps 0..9, A..F, a..f to 0..15. Other chars are mapped to implementation specific value.
+    constexpr inline std::string_view hex_char_to_digit_table
+        = {"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\xff\xff\xff\xff\xff\xff" //0-9
+        "\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" //A-Z
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" //a-z
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff",
+        256};
+
+    /// Converts a hex digit '0'..'f' or '0'..'F' to its value 0..15.
+    constexpr UInt8 unhexDigit(char c)
+    {
+        return hex_char_to_digit_table[static_cast<UInt8>(c)];
+    }
+
+    /// Converts an unsigned integer in the native endian to hexadecimal representation and back. Used as a base class for HexConversion<T>.
+    template <typename TUInt, typename = void>
+    struct HexConversionUInt
+    {
+        static const constexpr size_t num_hex_digits = sizeof(TUInt) * 2;
+
+        static void hex(TUInt uint_, char * out, std::string_view table)
+        {
+            union
+            {
+                TUInt value;
+                UInt8 uint8[sizeof(TUInt)];
+            };
+
+            value = uint_;
+
+            for (size_t i = 0; i < sizeof(TUInt); ++i)
+            {
+                if constexpr (std::endian::native == std::endian::little)
+                    memcpy(out + i * 2, &table[static_cast<size_t>(uint8[sizeof(TUInt) - 1 - i]) * 2], 2);
+                else
+                    memcpy(out + i * 2, &table[static_cast<size_t>(uint8[i]) * 2], 2);
+            }
+        }
+
+        static TUInt unhex(const char * data)
+        {
+            TUInt res;
+            if constexpr (sizeof(TUInt) == 1)
+            {
+                res = static_cast<UInt8>(unhexDigit(data[0])) * 0x10 + static_cast<UInt8>(unhexDigit(data[1]));
+            }
+            else if constexpr (sizeof(TUInt) == 2)
+            {
+                res = static_cast<UInt16>(unhexDigit(data[0])) * 0x1000 + static_cast<UInt16>(unhexDigit(data[1])) * 0x100
+                    + static_cast<UInt16>(unhexDigit(data[2])) * 0x10 + static_cast<UInt16>(unhexDigit(data[3]));
+            }
+            else if constexpr ((sizeof(TUInt) <= 8) || ((sizeof(TUInt) % 8) != 0))
+            {
+                res = 0;
+                for (size_t i = 0; i < sizeof(TUInt) * 2; ++i, ++data)
+                {
+                    res <<= 4;
+                    res += unhexDigit(*data);
+                }
+            }
+            else
+            {
+                res = 0;
+                for (size_t i = 0; i < sizeof(TUInt) / 8; ++i, data += 16)
+                {
+                    res <<= 64;
+                    res += HexConversionUInt<UInt64>::unhex(data);
+                }
+            }
+            return res;
+        }
+    };
+
+    /// Helper template class to convert a value of any supported type to hexadecimal representation and back.
+    template <typename T, typename SFINAE = void>
+    struct HexConversion;
+
+    template <typename TUInt>
+    struct HexConversion<TUInt, std::enable_if_t<std::is_integral_v<TUInt>>> : public HexConversionUInt<TUInt> {};
+
+    template <size_t Bits, typename Signed>
+    struct HexConversion<wide::integer<Bits, Signed>> : public HexConversionUInt<wide::integer<Bits, Signed>> {};
+
+    template <typename CityHashUInt128> /// Partial specialization here allows not to include <city.h> in this header.
+    struct HexConversion<CityHashUInt128, std::enable_if_t<std::is_same_v<CityHashUInt128, typename CityHash_v1_0_2::uint128>>>
+    {
+        static const constexpr size_t num_hex_digits = 32;
+
+        static void hex(const CityHashUInt128 & uint_, char * out, std::string_view table)
+        {
+            HexConversion<UInt64>::hex(uint_.high64, out, table);
+            HexConversion<UInt64>::hex(uint_.low64, out + 16, table);
+        }
+
+        static CityHashUInt128 unhex(const char * data)
+        {
+            CityHashUInt128 res;
+            res.high64 = HexConversion<UInt64>::unhex(data);
+            res.low64 = HexConversion<UInt64>::unhex(data + 16);
+            return res;
+        }
+    };
+}
+
+/// Produces a hexadecimal representation of an integer value with leading zeros (for checksums).
+/// The function supports native integer types, wide::integer, CityHash_v1_0_2::uint128.
+/// It can be used with signed types as well, however they are written as corresponding unsigned numbers
+/// using two's complement (i.e. for example "-1" is written as "0xFF", not as "-0x01").
+template <typename T>
+void writeHexUIntUppercase(const T & value, char * out)
+{
+    impl::HexConversion<T>::hex(value, out, impl::hex_byte_to_char_uppercase_table);
+}
+
+template <typename T>
+void writeHexUIntLowercase(const T & value, char * out)
+{
+    impl::HexConversion<T>::hex(value, out, impl::hex_byte_to_char_lowercase_table);
+}
+
+template <typename T>
+std::string getHexUIntUppercase(const T & value)
+{
+    std::string res(impl::HexConversion<T>::num_hex_digits, '\0');
+    writeHexUIntUppercase(value, res.data());
+    return res;
+}
+
+template <typename T>
+std::string getHexUIntLowercase(const T & value)
+{
+    std::string res(impl::HexConversion<T>::num_hex_digits, '\0');
+    writeHexUIntLowercase(value, res.data());
+    return res;
+}
+
+constexpr char hexDigitUppercase(unsigned char c)
+{
+    return impl::hex_digit_to_char_uppercase_table[c];
+}
+
+constexpr char hexDigitLowercase(unsigned char c)
+{
+    return impl::hex_digit_to_char_lowercase_table[c];
+}
+
+inline void writeHexByteUppercase(UInt8 byte, void * out)
+{
+    memcpy(out, &impl::hex_byte_to_char_uppercase_table[static_cast<size_t>(byte) * 2], 2);
+}
+
+inline void writeHexByteLowercase(UInt8 byte, void * out)
+{
+    memcpy(out, &impl::hex_byte_to_char_lowercase_table[static_cast<size_t>(byte) * 2], 2);
+}
+
+/// Converts a hex representation with leading zeros back to an integer value.
+/// The function supports native integer types, wide::integer, CityHash_v1_0_2::uint128.
+template <typename T>
+constexpr T unhexUInt(const char * data)
+{
+    return impl::HexConversion<T>::unhex(data);
+}
+
+/// Converts a hexadecimal digit '0'..'f' or '0'..'F' to UInt8.
+constexpr UInt8 unhex(char c)
+{
+    return impl::unhexDigit(c);
+}
+
+/// Converts two hexadecimal digits to UInt8.
+constexpr UInt8 unhex2(const char * data)
+{
+    return unhexUInt<UInt8>(data);
+}
+
+/// Converts four hexadecimal digits to UInt16.
+constexpr UInt16 unhex4(const char * data)
+{
+    return unhexUInt<UInt16>(data);
+}
+
+/// Produces a binary representation of a single byte.
+inline void writeBinByte(UInt8 byte, void * out)
+{
+    memcpy(out, &impl::bin_byte_to_char_table[static_cast<size_t>(byte) * 8], 8);
+}
+
+/// Converts byte array to a hex string. Useful for debug logging.
+inline std::string hexString(const void * data, size_t size)
+{
+    const char * p = reinterpret_cast<const char *>(data);
+    std::string s(size * 2, '\0');
+    for (size_t i = 0; i < size; ++i)
+        writeHexByteLowercase(p[i], s.data() + i * 2);
+    return s;
+}

base/base/interpolate.h

@@ -0,0 +1,18 @@
+#pragma once
+#include <cassert>
+#include <cmath>
+
+/** Linear interpolation in logarithmic coordinates.
+  * Exponential interpolation is related to linear interpolation
+  * exactly in same way as geometric mean is related to arithmetic mean.
+  */
+constexpr double interpolateExponential(double min, double max, double ratio)
+{
+    assert(min > 0 && ratio >= 0 && ratio <= 1);
+    return min * std::pow(max / min, ratio);
+}
+
+constexpr double interpolateLinear(double min, double max, double ratio)
+{
+    return std::lerp(min, max, ratio);
+}

base/base/iostream_debug_helpers.h

@@ -20,14 +20,14 @@ Out & dumpValue(Out &, T &&);
 
 /// Catch-all case.
 template <int priority, typename Out, typename T>
-std::enable_if_t<priority == -1, Out> & dumpImpl(Out & out, T &&)
+std::enable_if_t<priority == -1, Out> & dumpImpl(Out & out, T &&) // NOLINT(cppcoreguidelines-missing-std-forward)
 {
     return out << "{...}";
 }
 
 /// An object, that could be output with operator <<.
 template <int priority, typename Out, typename T>
-std::enable_if_t<priority == 0, Out> & dumpImpl(Out & out, T && x, std::decay_t<decltype(std::declval<Out &>() << std::declval<T>())> * = nullptr)
+std::enable_if_t<priority == 0, Out> & dumpImpl(Out & out, T && x, std::decay_t<decltype(std::declval<Out &>() << std::declval<T>())> * = nullptr) // NOLINT(cppcoreguidelines-missing-std-forward)
 {
     return out << x;
 }
@@ -37,7 +37,7 @@ template <int priority, typename Out, typename T>
 std::enable_if_t<priority == 1
     /// Protect from the case when operator * do effectively nothing (function pointer).
     && !std::is_same_v<std::decay_t<T>, std::decay_t<decltype(*std::declval<T>())>>
-    , Out> & dumpImpl(Out & out, T && x, std::decay_t<decltype(*std::declval<T>())> * = nullptr)
+    , Out> & dumpImpl(Out & out, T && x, std::decay_t<decltype(*std::declval<T>())> * = nullptr) // NOLINT(cppcoreguidelines-missing-std-forward)
 {
     if (!x)
         return out << "nullptr";
@@ -46,7 +46,7 @@ std::enable_if_t<priority == 1
 
 /// Container.
 template <int priority, typename Out, typename T>
-std::enable_if_t<priority == 2, Out> & dumpImpl(Out & out, T && x, std::decay_t<decltype(std::begin(std::declval<T>()))> * = nullptr)
+std::enable_if_t<priority == 2, Out> & dumpImpl(Out & out, T && x, std::decay_t<decltype(std::begin(std::declval<T>()))> * = nullptr) // NOLINT(cppcoreguidelines-missing-std-forward)
 {
     bool first = true;
     out << "{";
@@ -64,7 +64,7 @@ std::enable_if_t<priority == 2, Out> & dumpImpl(Out & out, T && x, std::decay_t<
 
 template <int priority, typename Out, typename T>
 std::enable_if_t<priority == 3 && std::is_enum_v<std::decay_t<T>>, Out> &
-dumpImpl(Out & out, T && x)
+dumpImpl(Out & out, T && x) // NOLINT(cppcoreguidelines-missing-std-forward)
 {
     return out << magic_enum::enum_name(x);
 }
@@ -73,7 +73,7 @@ dumpImpl(Out & out, T && x)
 
 template <int priority, typename Out, typename T>
 std::enable_if_t<priority == 3 && (std::is_same_v<std::decay_t<T>, std::string> || std::is_same_v<std::decay_t<T>, const char *>), Out> &
-dumpImpl(Out & out, T && x)
+dumpImpl(Out & out, T && x) // NOLINT(cppcoreguidelines-missing-std-forward)
 {
     return out << std::quoted(x);
 }
@@ -82,7 +82,7 @@ dumpImpl(Out & out, T && x)
 
 template <int priority, typename Out, typename T>
 std::enable_if_t<priority == 3 && std::is_same_v<std::decay_t<T>, unsigned char>, Out> &
-dumpImpl(Out & out, T && x)
+dumpImpl(Out & out, T && x) // NOLINT(cppcoreguidelines-missing-std-forward)
 {
     return out << int(x);
 }
@@ -90,7 +90,7 @@ dumpImpl(Out & out, T && x)
 
 /// Tuple, pair
 template <size_t N, typename Out, typename T>
-Out & dumpTupleImpl(Out & out, T && x)
+Out & dumpTupleImpl(Out & out, T && x) // NOLINT(cppcoreguidelines-missing-std-forward)
 {
     if constexpr (N == 0)
         out << "{";
@@ -108,14 +108,14 @@ Out & dumpTupleImpl(Out & out, T && x)
 }
 
 template <int priority, typename Out, typename T>
-std::enable_if_t<priority == 4, Out> & dumpImpl(Out & out, T && x, std::decay_t<decltype(std::get<0>(std::declval<T>()))> * = nullptr)
+std::enable_if_t<priority == 4, Out> & dumpImpl(Out & out, T && x, std::decay_t<decltype(std::get<0>(std::declval<T>()))> * = nullptr) // NOLINT(cppcoreguidelines-missing-std-forward)
 {
     return dumpTupleImpl<0>(out, x);
 }
 
 
 template <int priority, typename Out, typename T>
-Out & dumpDispatchPriorities(Out & out, T && x, std::decay_t<decltype(dumpImpl<priority>(std::declval<Out &>(), std::declval<T>()))> *)
+Out & dumpDispatchPriorities(Out & out, T && x, std::decay_t<decltype(dumpImpl<priority>(std::declval<Out &>(), std::declval<T>()))> *) // NOLINT(cppcoreguidelines-missing-std-forward)
 {
     return dumpImpl<priority>(out, x);
 }
@@ -124,21 +124,21 @@ Out & dumpDispatchPriorities(Out & out, T && x, std::decay_t<decltype(dumpImpl<p
 struct LowPriority { LowPriority(void *) {} };
 
 template <int priority, typename Out, typename T>
-Out & dumpDispatchPriorities(Out & out, T && x, LowPriority)
+Out & dumpDispatchPriorities(Out & out, T && x, LowPriority) // NOLINT(cppcoreguidelines-missing-std-forward)
 {
     return dumpDispatchPriorities<priority - 1>(out, x, nullptr);
 }
 
 
 template <typename Out, typename T>
-Out & dumpValue(Out & out, T && x)
+Out & dumpValue(Out & out, T && x) // NOLINT(cppcoreguidelines-missing-std-forward)
 {
     return dumpDispatchPriorities<5>(out, x, nullptr);
 }
 
 
 template <typename Out, typename T>
-Out & dump(Out & out, const char * name, T && x)
+Out & dump(Out & out, const char * name, T && x) // NOLINT(cppcoreguidelines-missing-std-forward)
 {
     // Dumping string literal, printing name and demangled type is irrelevant.
     if constexpr (std::is_same_v<const char *, std::decay_t<std::remove_reference_t<T>>>)

base/base/move_extend.h

@@ -0,0 +1,9 @@
+#pragma once
+
+/// Extend @p to by moving elements from @p from to @p to end
+/// @return @p to iterator to first of moved elements.
+template <class To, class From>
+typename To::iterator moveExtend(To & to, From && from)
+{
+    return to.insert(to.end(), std::make_move_iterator(from.begin()), std::make_move_iterator(from.end()));
+}

base/base/phdr_cache.cpp

@@ -1,6 +1,4 @@
-#ifdef HAS_RESERVED_IDENTIFIER
 #pragma clang diagnostic ignored "-Wreserved-identifier"
-#endif
 
 /// This code was based on the code by Fedor Korotkiy https://www.linkedin.com/in/fedor-korotkiy-659a1838/
 

base/base/scope_guard.h

@@ -9,9 +9,9 @@ class [[nodiscard]] BasicScopeGuard
 {
 public:
     constexpr BasicScopeGuard() = default;
-    constexpr BasicScopeGuard(BasicScopeGuard && src) : function{src.release()} {} // NOLINT(hicpp-noexcept-move, performance-noexcept-move-constructor)
+    constexpr BasicScopeGuard(BasicScopeGuard && src) : function{src.release()} {} // NOLINT(hicpp-noexcept-move, performance-noexcept-move-constructor, cppcoreguidelines-noexcept-move-operations)
 
-    constexpr BasicScopeGuard & operator=(BasicScopeGuard && src) // NOLINT(hicpp-noexcept-move, performance-noexcept-move-constructor)
+    constexpr BasicScopeGuard & operator=(BasicScopeGuard && src) // NOLINT(hicpp-noexcept-move, performance-noexcept-move-constructor, cppcoreguidelines-noexcept-move-operations)
     {
         if (this != &src)
         {
@@ -23,11 +23,11 @@ public:
 
     template <typename G>
     requires std::is_convertible_v<G, F>
-    constexpr BasicScopeGuard(BasicScopeGuard<G> && src) : function{src.release()} {} // NOLINT(google-explicit-constructor)
+    constexpr BasicScopeGuard(BasicScopeGuard<G> && src) : function{src.release()} {} // NOLINT(google-explicit-constructor, cppcoreguidelines-rvalue-reference-param-not-moved, cppcoreguidelines-noexcept-move-operations)
 
     template <typename G>
     requires std::is_convertible_v<G, F>
-    constexpr BasicScopeGuard & operator=(BasicScopeGuard<G> && src)
+    constexpr BasicScopeGuard & operator=(BasicScopeGuard<G> && src) // NOLINT(cppcoreguidelines-rvalue-reference-param-not-moved, cppcoreguidelines-noexcept-move-operations)
     {
         if (this != &src)
         {
@@ -43,7 +43,7 @@ public:
 
     template <typename G>
     requires std::is_convertible_v<G, F>
-    constexpr BasicScopeGuard(G && function_) : function{std::move(function_)} {} // NOLINT(google-explicit-constructor, bugprone-forwarding-reference-overload, bugprone-move-forwarding-reference)
+    constexpr BasicScopeGuard(G && function_) : function{std::move(function_)} {} // NOLINT(google-explicit-constructor, bugprone-forwarding-reference-overload, bugprone-move-forwarding-reference, cppcoreguidelines-missing-std-forward)
 
     ~BasicScopeGuard() { invoke(); }
 
@@ -70,7 +70,7 @@ public:
 
     template <typename G>
     requires std::is_convertible_v<G, F>
-    BasicScopeGuard<F> & join(BasicScopeGuard<G> && other)
+    BasicScopeGuard<F> & join(BasicScopeGuard<G> && other) // NOLINT(cppcoreguidelines-rvalue-reference-param-not-moved)
     {
         if (other.function)
         {

base/base/simd.h

@@ -0,0 +1,14 @@
+#pragma once
+
+#if defined(__aarch64__) && defined(__ARM_NEON)
+
+#    include <arm_neon.h>
+#      pragma clang diagnostic ignored "-Wreserved-identifier"
+
+/// Returns a 64 bit mask of nibbles (4 bits for each byte).
+inline uint64_t getNibbleMask(uint8x16_t res)
+{
+    return vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(res), 4)), 0);
+}
+
+#endif

base/base/sort.h

@@ -131,3 +131,29 @@ void sort(RandomIt first, RandomIt last)
     using comparator = std::less<value_type>;
     ::sort(first, last, comparator());
 }
+
+/** Try to fast sort elements for common sorting patterns:
+  * 1. If elements are already sorted.
+  * 2. If elements are already almost sorted.
+  * 3. If elements are already sorted in reverse order.
+  *
+  * Returns true if fast sort was performed or elements were already sorted, false otherwise.
+  */
+template <typename RandomIt, typename Compare>
+bool trySort(RandomIt first, RandomIt last, Compare compare)
+{
+#ifndef NDEBUG
+    ::shuffle(first, last);
+#endif
+
+    ComparatorWrapper<Compare> compare_wrapper = compare;
+    return ::pdqsort_try_sort(first, last, compare_wrapper);
+}
+
+template <typename RandomIt>
+bool trySort(RandomIt first, RandomIt last)
+{
+    using value_type = typename std::iterator_traits<RandomIt>::value_type;
+    using comparator = std::less<value_type>;
+    return ::trySort(first, last, comparator());
+}

base/base/strong_typedef.h

@@ -23,10 +23,10 @@ public:
     constexpr StrongTypedef(): t() {}
 
     constexpr StrongTypedef(const Self &) = default;
-    constexpr StrongTypedef(Self &&) noexcept(std::is_nothrow_move_constructible_v<T>) = default;
+    constexpr StrongTypedef(Self &&) noexcept(std::is_nothrow_move_constructible_v<T>) = default; // NOLINT(cppcoreguidelines-noexcept-move-operations, hicpp-noexcept-move, performance-noexcept-move-constructor)
 
     Self & operator=(const Self &) = default;
-    Self & operator=(Self &&) noexcept(std::is_nothrow_move_assignable_v<T>)= default;
+    Self & operator=(Self &&) noexcept(std::is_nothrow_move_assignable_v<T>)= default; // NOLINT(cppcoreguidelines-noexcept-move-operations, hicpp-noexcept-move, performance-noexcept-move-constructor)
 
     template <class Enable = typename std::is_copy_assignable<T>::type>
     Self & operator=(const T & rhs) { t = rhs; return *this;}
@@ -35,7 +35,7 @@ public:
     Self & operator=(T && rhs) { t = std::move(rhs); return *this;}
 
     // NOLINTBEGIN(google-explicit-constructor)
-    operator const T & () const { return t; }
+    constexpr operator const T & () const { return t; }
     operator T & () { return t; }
     // NOLINTEND(google-explicit-constructor)
 

base/base/types.h

@@ -13,11 +13,7 @@ using char8_t = unsigned char;
 #endif
 
 /// This is needed for more strict aliasing. https://godbolt.org/z/xpJBSb https://stackoverflow.com/a/57453713
-#if !defined(PVS_STUDIO) /// But PVS-Studio does not treat it correctly.
 using UInt8 = char8_t;
-#else
-using UInt8 = uint8_t;
-#endif
 
 using UInt16 = uint16_t;
 using UInt32 = uint32_t;

base/base/unaligned.h

@@ -5,44 +5,6 @@
 #include <bit>
 
 
-inline void reverseMemcpy(void * dst, const void * src, size_t size)
-{
-    uint8_t * uint_dst = reinterpret_cast<uint8_t *>(dst);
-    const uint8_t * uint_src = reinterpret_cast<const uint8_t *>(src);
-
-    uint_dst += size;
-    while (size)
-    {
-        --uint_dst;
-        *uint_dst = *uint_src;
-        ++uint_src;
-        --size;
-    }
-}
-
-template <typename T>
-inline T unalignedLoadLE(const void * address)
-{
-    T res {};
-    if constexpr (std::endian::native == std::endian::little)
-        memcpy(&res, address, sizeof(res));
-    else
-        reverseMemcpy(&res, address, sizeof(res));
-    return res;
-}
-
-
-template <typename T>
-inline void unalignedStoreLE(void * address,
-                           const typename std::enable_if<true, T>::type & src)
-{
-    static_assert(std::is_trivially_copyable_v<T>);
-    if constexpr (std::endian::native == std::endian::little)
-        memcpy(address, &src, sizeof(src));
-    else
-        reverseMemcpy(address, &src, sizeof(src));
-}
-
 template <typename T>
 inline T unalignedLoad(const void * address)
 {
@@ -62,3 +24,70 @@ inline void unalignedStore(void * address,
     static_assert(std::is_trivially_copyable_v<T>);
     memcpy(address, &src, sizeof(src));
 }
+
+
+inline void reverseMemcpy(void * dst, const void * src, size_t size)
+{
+    uint8_t * uint_dst = reinterpret_cast<uint8_t *>(dst);
+    const uint8_t * uint_src = reinterpret_cast<const uint8_t *>(src);
+
+    uint_dst += size;
+    while (size)
+    {
+        --uint_dst;
+        *uint_dst = *uint_src;
+        ++uint_src;
+        --size;
+    }
+}
+
+template <std::endian endian, typename T>
+inline T unalignedLoadEndian(const void * address)
+{
+    T res {};
+    if constexpr (std::endian::native == endian)
+        memcpy(&res, address, sizeof(res));
+    else
+        reverseMemcpy(&res, address, sizeof(res));
+    return res;
+}
+
+
+template <std::endian endian, typename T>
+inline void unalignedStoreEndian(void * address, T & src)
+{
+    static_assert(std::is_trivially_copyable_v<T>);
+    if constexpr (std::endian::native == endian)
+        memcpy(address, &src, sizeof(src));
+    else
+        reverseMemcpy(address, &src, sizeof(src));
+}
+
+
+template <typename T>
+inline T unalignedLoadLittleEndian(const void * address)
+{
+    return unalignedLoadEndian<std::endian::little, T>(address);
+}
+
+
+template <typename T>
+inline void unalignedStoreLittleEndian(void * address,
+    const typename std::enable_if<true, T>::type & src)
+{
+    unalignedStoreEndian<std::endian::little>(address, src);
+}
+
+template <typename T>
+inline T unalignedLoadBigEndian(const void * address)
+{
+    return unalignedLoadEndian<std::endian::big, T>(address);
+}
+
+
+template <typename T>
+inline void unalignedStoreBigEndian(void * address,
+    const typename std::enable_if<true, T>::type & src)
+{
+    unalignedStoreEndian<std::endian::big>(address, src);
+}

base/base/unit.h

@@ -5,10 +5,8 @@ constexpr size_t KiB = 1024;
 constexpr size_t MiB = 1024 * KiB;
 constexpr size_t GiB = 1024 * MiB;
 
-#ifdef HAS_RESERVED_IDENTIFIER
-#  pragma clang diagnostic push
-#  pragma clang diagnostic ignored "-Wreserved-identifier"
-#endif
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wreserved-identifier"
 
 // NOLINTBEGIN(google-runtime-int)
 constexpr size_t operator"" _KiB(unsigned long long val) { return val * KiB; }
@@ -16,6 +14,4 @@ constexpr size_t operator"" _MiB(unsigned long long val) { return val * MiB; }
 constexpr size_t operator"" _GiB(unsigned long long val) { return val * GiB; }
 // NOLINTEND(google-runtime-int)
 
-#ifdef HAS_RESERVED_IDENTIFIER
-#  pragma clang diagnostic pop
-#endif
+#pragma clang diagnostic pop

base/base/wide_integer_impl.h

@@ -12,7 +12,6 @@
 #include <tuple>
 #include <limits>
 
-#include <boost/multiprecision/cpp_bin_float.hpp>
 #include <boost/math/special_functions/fpclassify.hpp>
 
 // NOLINTBEGIN(*)
@@ -22,11 +21,14 @@
 #define CONSTEXPR_FROM_DOUBLE constexpr
 using FromDoubleIntermediateType = long double;
 #else
+#include <boost/multiprecision/cpp_bin_float.hpp>
 /// `wide_integer_from_builtin` can't be constexpr with non-literal `cpp_bin_float_double_extended`
 #define CONSTEXPR_FROM_DOUBLE
 using FromDoubleIntermediateType = boost::multiprecision::cpp_bin_float_double_extended;
 #endif
 
+namespace CityHash_v1_0_2 { struct uint128; }
+
 namespace wide
 {
 
@@ -63,7 +65,7 @@ class IsTupleLike
     static void check(...);
 
 public:
-    static constexpr const bool value = !std::is_void<decltype(check<T>(nullptr))>::value;
+    static constexpr const bool value = !std::is_void_v<decltype(check<T>(nullptr))>;
 };
 
 }
@@ -77,7 +79,7 @@ class numeric_limits<wide::integer<Bits, Signed>>
 {
 public:
     static constexpr bool is_specialized = true;
-    static constexpr bool is_signed = is_same<Signed, signed>::value;
+    static constexpr bool is_signed = is_same_v<Signed, signed>;
     static constexpr bool is_integer = true;
     static constexpr bool is_exact = true;
     static constexpr bool has_infinity = false;
@@ -89,7 +91,7 @@ public:
     static constexpr bool is_iec559 = false;
     static constexpr bool is_bounded = true;
     static constexpr bool is_modulo = true;
-    static constexpr int digits = Bits - (is_same<Signed, signed>::value ? 1 : 0);
+    static constexpr int digits = Bits - (is_same_v<Signed, signed> ? 1 : 0);
     static constexpr int digits10 = digits * 0.30103 /*std::log10(2)*/;
     static constexpr int max_digits10 = 0;
     static constexpr int radix = 2;
@@ -102,7 +104,7 @@ public:
 
     static constexpr wide::integer<Bits, Signed> min() noexcept
     {
-        if (is_same<Signed, signed>::value)
+        if constexpr (is_same_v<Signed, signed>)
         {
             using T = wide::integer<Bits, signed>;
             T res{};
@@ -116,7 +118,7 @@ public:
     {
         using T = wide::integer<Bits, Signed>;
         T res{};
-        res.items[T::_impl::big(0)] = is_same<Signed, signed>::value
+        res.items[T::_impl::big(0)] = is_same_v<Signed, signed>
             ? std::numeric_limits<typename wide::integer<Bits, Signed>::signed_base_type>::max()
             : std::numeric_limits<typename wide::integer<Bits, Signed>::base_type>::max();
         for (unsigned i = 1; i < wide::integer<Bits, Signed>::_impl::item_count; ++i)
@@ -155,13 +157,13 @@ struct common_type<wide::integer<Bits, Signed>, Arithmetic>
         std::is_floating_point_v<Arithmetic>,
         Arithmetic,
         std::conditional_t<
-            sizeof(Arithmetic) < Bits * sizeof(long),
+            sizeof(Arithmetic) * 8 < Bits,
             wide::integer<Bits, Signed>,
             std::conditional_t<
-                Bits * sizeof(long) < sizeof(Arithmetic),
+                Bits < sizeof(Arithmetic) * 8,
                 Arithmetic,
                 std::conditional_t<
-                    Bits * sizeof(long) == sizeof(Arithmetic) && (std::is_same_v<Signed, signed> || std::is_signed_v<Arithmetic>),
+                    Bits == sizeof(Arithmetic) * 8 && (std::is_same_v<Signed, signed> || std::is_signed_v<Arithmetic>),
                     Arithmetic,
                     wide::integer<Bits, Signed>>>>>;
 };
@@ -281,6 +283,17 @@ struct integer<Bits, Signed>::_impl
         }
     }
 
+    template <typename CityHashUInt128 = CityHash_v1_0_2::uint128>
+    constexpr static void wide_integer_from_cityhash_uint128(integer<Bits, Signed> & self, const CityHashUInt128 & value) noexcept
+    {
+        static_assert(sizeof(item_count) >= 2);
+
+        if constexpr (std::endian::native == std::endian::little)
+            wide_integer_from_tuple_like(self, std::make_pair(value.low64, value.high64));
+        else
+            wide_integer_from_tuple_like(self, std::make_pair(value.high64, value.low64));
+    }
+
     /**
      * N.B. t is constructed from double, so max(t) = max(double) ~ 2^310
      * the recursive call happens when t / 2^64 > 2^64, so there won't be more than 5 of them.
@@ -314,7 +327,14 @@ struct integer<Bits, Signed>::_impl
 
         const T alpha = t / static_cast<T>(max_int);
 
-        if (alpha <= static_cast<T>(max_int))
+        /** Here we have to use strict comparison.
+          * The max_int is 2^64 - 1.
+          * When casted to floating point type, it will be rounded to the closest representable number,
+          * which is 2^64.
+          * But 2^64 is not representable in uint64_t,
+          * so the maximum representable number will be strictly less.
+          */
+        if (alpha < static_cast<T>(max_int))
             self = static_cast<uint64_t>(alpha);
         else // max(double) / 2^64 will surely contain less than 52 precision bits, so speed up computations.
             set_multiplier<double>(self, static_cast<double>(alpha));
@@ -360,7 +380,7 @@ struct integer<Bits, Signed>::_impl
         constexpr const unsigned to_copy = min_bits / base_bits;
 
         for (unsigned i = 0; i < to_copy; ++i)
-            self.items[little(i)] = rhs.items[little(i)];
+            self.items[little(i)] = rhs.items[integer<Bits2, Signed2>::_impl::little(i)];
 
         if constexpr (Bits > Bits2)
         {
@@ -732,9 +752,10 @@ public:
             if (std::numeric_limits<T>::is_signed && (is_negative(lhs) != is_negative(rhs)))
                 return is_negative(rhs);
 
+            integer<Bits, Signed> t = rhs;
             for (unsigned i = 0; i < item_count; ++i)
             {
-                base_type rhs_item = get_item(rhs, big(i));
+                base_type rhs_item = get_item(t, big(i));
 
                 if (lhs.items[big(i)] != rhs_item)
                     return lhs.items[big(i)] > rhs_item;
@@ -757,9 +778,10 @@ public:
             if (std::numeric_limits<T>::is_signed && (is_negative(lhs) != is_negative(rhs)))
                 return is_negative(lhs);
 
+            integer<Bits, Signed> t = rhs;
             for (unsigned i = 0; i < item_count; ++i)
             {
-                base_type rhs_item = get_item(rhs, big(i));
+                base_type rhs_item = get_item(t, big(i));
 
                 if (lhs.items[big(i)] != rhs_item)
                     return lhs.items[big(i)] < rhs_item;
@@ -779,9 +801,10 @@ public:
     {
         if constexpr (should_keep_size<T>())
         {
+            integer<Bits, Signed> t = rhs;
             for (unsigned i = 0; i < item_count; ++i)
             {
-                base_type rhs_item = get_item(rhs, any(i));
+                base_type rhs_item = get_item(t, any(i));
 
                 if (lhs.items[any(i)] != rhs_item)
                     return false;
@@ -1026,6 +1049,8 @@ constexpr integer<Bits, Signed>::integer(T rhs) noexcept
         _impl::wide_integer_from_wide_integer(*this, rhs);
     else if  constexpr (IsTupleLike<T>::value)
         _impl::wide_integer_from_tuple_like(*this, rhs);
+    else if constexpr (std::is_same_v<std::remove_cvref_t<T>, CityHash_v1_0_2::uint128>)
+        _impl::wide_integer_from_cityhash_uint128(*this, rhs);
     else
         _impl::wide_integer_from_builtin(*this, rhs);
 }
@@ -1041,6 +1066,8 @@ constexpr integer<Bits, Signed>::integer(std::initializer_list<T> il) noexcept
             _impl::wide_integer_from_wide_integer(*this, *il.begin());
         else if  constexpr (IsTupleLike<T>::value)
             _impl::wide_integer_from_tuple_like(*this, *il.begin());
+        else if constexpr (std::is_same_v<std::remove_cvref_t<T>, CityHash_v1_0_2::uint128>)
+            _impl::wide_integer_from_cityhash_uint128(*this, *il.begin());
         else
             _impl::wide_integer_from_builtin(*this, *il.begin());
     }
@@ -1078,6 +1105,8 @@ constexpr integer<Bits, Signed> & integer<Bits, Signed>::operator=(T rhs) noexce
 {
     if  constexpr (IsTupleLike<T>::value)
         _impl::wide_integer_from_tuple_like(*this, rhs);
+    else if constexpr (std::is_same_v<std::remove_cvref_t<T>, CityHash_v1_0_2::uint128>)
+        _impl::wide_integer_from_cityhash_uint128(*this, rhs);
     else
         _impl::wide_integer_from_builtin(*this, rhs);
     return *this;
@@ -1239,7 +1268,7 @@ constexpr integer<Bits, Signed>::operator long double() const noexcept
     for (unsigned i = 0; i < _impl::item_count; ++i)
     {
         long double t = res;
-        res *= std::numeric_limits<base_type>::max();
+        res *= static_cast<long double>(std::numeric_limits<base_type>::max());
         res += t;
         res += tmp.items[_impl::big(i)];
     }

base/base/wide_integer_to_string.h

@@ -64,6 +64,6 @@ struct fmt::formatter<wide::integer<Bits, Signed>>
     template <typename FormatContext>
     auto format(const wide::integer<Bits, Signed> & value, FormatContext & ctx)
     {
-        return format_to(ctx.out(), "{}", to_string(value));
+        return fmt::format_to(ctx.out(), "{}", to_string(value));
     }
 };

base/glibc-compatibility/CMakeLists.txt

@@ -5,9 +5,6 @@ if (GLIBC_COMPATIBILITY)
     endif()
 
     enable_language(ASM)
-    include(CheckIncludeFile)
-
-    check_include_file("sys/random.h" HAVE_SYS_RANDOM_H)
 
     add_headers_and_sources(glibc_compatibility .)
     add_headers_and_sources(glibc_compatibility musl)
@@ -21,11 +18,6 @@ if (GLIBC_COMPATIBILITY)
         message (FATAL_ERROR "glibc_compatibility can only be used on x86_64 or aarch64.")
     endif ()
 
-    list(REMOVE_ITEM glibc_compatibility_sources musl/getentropy.c)
-    if(HAVE_SYS_RANDOM_H)
-        list(APPEND glibc_compatibility_sources musl/getentropy.c)
-    endif()
-
     # Need to omit frame pointers to match the performance of glibc
     set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fomit-frame-pointer")
 
@@ -43,12 +35,6 @@ if (GLIBC_COMPATIBILITY)
 
     target_link_libraries(global-libs INTERFACE glibc-compatibility ${MEMCPY_LIBRARY})
 
-    install(
-        TARGETS glibc-compatibility ${MEMCPY_LIBRARY}
-        EXPORT global
-        ARCHIVE DESTINATION lib
-    )
-
     message (STATUS "Some symbols from glibc will be replaced for compatibility")
 
 elseif (CLICKHOUSE_OFFICIAL_BUILD)

base/glibc-compatibility/glibc-compatibility.c

@@ -30,7 +30,6 @@ int __gai_sigqueue(int sig, const union sigval val, pid_t caller_pid)
 }
 
 
-#include <sys/select.h>
 #include <stdlib.h>
 #include <features.h>
 
@@ -195,7 +194,6 @@ long splice(int fd_in, off_t *off_in, int fd_out, off_t *off_out, size_t len, un
 #include <sys/stat.h>
 #include <stdint.h>
 
-#if !defined(__aarch64__)
 struct statx {
 	uint32_t stx_mask;
 	uint32_t stx_blksize;
@@ -226,7 +224,6 @@ int statx(int fd, const char *restrict path, int flag,
 {
 	return syscall(SYS_statx, fd, path, flag, mask, statxbuf);
 }
-#endif
 
 
 #include <syscall.h>
@@ -237,6 +234,17 @@ ssize_t getrandom(void *buf, size_t buflen, unsigned flags)
     return syscall(SYS_getrandom, buf, buflen, flags);
 }
 
+/* Structure for scatter/gather I/O.  */
+struct iovec
+{
+    void *iov_base;    /* Pointer to data.  */
+    size_t iov_len;    /* Length of data.  */
+};
+
+ssize_t preadv(int __fd, const struct iovec *__iovec, int __count, __off_t __offset)
+{
+    return syscall(SYS_preadv, __fd, __iovec, __count, (long)(__offset), (long)(__offset>>32));
+}
 
 #include <errno.h>
 #include <limits.h>

base/glibc-compatibility/memcpy/memcpy.cpp

@@ -1,5 +1,6 @@
 #include "memcpy.h"
 
+__attribute__((no_sanitize("coverage")))
 extern "C" void * memcpy(void * __restrict dst, const void * __restrict src, size_t size)
 {
     return inline_memcpy(dst, src, size);

base/glibc-compatibility/memcpy/memcpy.h

@@ -93,7 +93,7 @@
   * See https://habr.com/en/company/yandex/blog/457612/
   */
 
-
+__attribute__((no_sanitize("coverage")))
 static inline void * inline_memcpy(void * __restrict dst_, const void * __restrict src_, size_t size)
 {
     /// We will use pointer arithmetic, so char pointer will be used.

base/glibc-compatibility/musl/expf.c

@@ -0,0 +1,81 @@
+/* origin: FreeBSD /usr/src/lib/msun/src/e_expf.c */
+/*
+ * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com.
+ */
+/*
+ * ====================================================
+ * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved.
+ *
+ * Developed at SunPro, a Sun Microsystems, Inc. business.
+ * Permission to use, copy, modify, and distribute this
+ * software is freely granted, provided that this notice
+ * is preserved.
+ * ====================================================
+ */
+
+#include "libm.h"
+
+static const float
+    half[2] = {0.5,-0.5},
+    ln2hi   = 6.9314575195e-1f,  /* 0x3f317200 */
+    ln2lo   = 1.4286067653e-6f,  /* 0x35bfbe8e */
+    invln2  = 1.4426950216e+0f,  /* 0x3fb8aa3b */
+    /*
+ * Domain [-0.34568, 0.34568], range ~[-4.278e-9, 4.447e-9]:
+ * |x*(exp(x)+1)/(exp(x)-1) - p(x)| < 2**-27.74
+ */
+    P1 =  1.6666625440e-1f, /*  0xaaaa8f.0p-26 */
+    P2 = -2.7667332906e-3f; /* -0xb55215.0p-32 */
+
+float expf(float x)
+{
+    float_t hi, lo, c, xx, y;
+    int k, sign;
+    uint32_t hx;
+
+    GET_FLOAT_WORD(hx, x);
+    sign = hx >> 31;   /* sign bit of x */
+    hx &= 0x7fffffff;  /* high word of |x| */
+
+    /* special cases */
+    if (hx >= 0x42aeac50) {  /* if |x| >= -87.33655f or NaN */
+        if (hx >= 0x42b17218 && !sign) {  /* x >= 88.722839f */
+            /* overflow */
+            x *= 0x1p127f;
+            return x;
+        }
+        if (sign) {
+            /* underflow */
+            FORCE_EVAL(-0x1p-149f/x);
+            if (hx >= 0x42cff1b5)  /* x <= -103.972084f */
+                return 0;
+        }
+    }
+
+    /* argument reduction */
+    if (hx > 0x3eb17218) {  /* if |x| > 0.5 ln2 */
+        if (hx > 0x3f851592)  /* if |x| > 1.5 ln2 */
+            k = invln2*x + half[sign];
+        else
+            k = 1 - sign - sign;
+        hi = x - k*ln2hi;  /* k*ln2hi is exact here */
+        lo = k*ln2lo;
+        x = hi - lo;
+    } else if (hx > 0x39000000) {  /* |x| > 2**-14 */
+        k = 0;
+        hi = x;
+        lo = 0;
+    } else {
+        /* raise inexact */
+        FORCE_EVAL(0x1p127f + x);
+        return 1 + x;
+    }
+
+    /* x is now in primary range */
+    xx = x*x;
+    c = x - xx*(P1+xx*P2);
+    y = 1 + (x*c/(2-c) - lo + hi);
+    if (k == 0)
+        return y;
+    return scalbnf(y, k);
+}

base/glibc-compatibility/musl/fallocate.c

@@ -8,3 +8,8 @@ int fallocate(int fd, int mode, off_t base, off_t len)
 {
 	return syscall(SYS_fallocate, fd, mode, base, len);
 }
+
+int fallocate64(int fd, int mode, off_t base, off_t len)
+{
+	return fallocate(fd, mode, base, len);
+}

base/glibc-compatibility/musl/lgamma.c

@@ -78,9 +78,6 @@
  *
  */
 
-// Disable warnings by PVS-Studio
-//-V::GA
-
 static const double
 pi  =  3.14159265358979311600e+00, /* 0x400921FB, 0x54442D18 */
 a0  =  7.72156649015328655494e-02, /* 0x3FB3C467, 0xE37DB0C8 */

base/glibc-compatibility/musl/lgammal.c

@@ -85,9 +85,6 @@
  *
  */
 
-// Disable warnings by PVS-Studio
-//-V::GA
-
 #include <stdint.h>
 #include <math.h>
 #include "libm.h"

base/glibc-compatibility/musl/libm.h

@@ -155,7 +155,7 @@ static inline long double fp_barrierl(long double x)
 static inline void fp_force_evalf(float x)
 {
 	volatile float y;
-	y = x; //-V1001
+	y = x;
 }
 #endif
 
@@ -164,7 +164,7 @@ static inline void fp_force_evalf(float x)
 static inline void fp_force_eval(double x)
 {
 	volatile double y;
-	y = x; //-V1001
+	y = x;
 }
 #endif
 
@@ -173,7 +173,7 @@ static inline void fp_force_eval(double x)
 static inline void fp_force_evall(long double x)
 {
 	volatile long double y;
-	y = x; //-V1001
+	y = x;
 }
 #endif
 

base/glibc-compatibility/musl/logf.c

@@ -53,7 +53,7 @@ float logf(float x)
 	tmp = ix - OFF;
 	i = (tmp >> (23 - LOGF_TABLE_BITS)) % N;
 	k = (int32_t)tmp >> 23; /* arithmetic shift */
-	iz = ix - (tmp & 0x1ff << 23);
+	iz = ix - (tmp & 0xff800000);
 	invc = T[i].invc;
 	logc = T[i].logc;
 	z = (double_t)asfloat(iz);

base/glibc-compatibility/musl/powf.c

@@ -3,9 +3,6 @@
  * SPDX-License-Identifier: MIT
  */
 
-// Disable warnings by PVS-Studio
-//-V::GA
-
 #include <math.h>
 #include <stdint.h>
 #include "libm.h"

base/glibc-compatibility/musl/pwritev.c

@@ -9,3 +9,8 @@ ssize_t pwritev(int fd, const struct iovec *iov, int count, off_t ofs)
     /// There was cancellable syscall (syscall_cp), but I don't care.
 	return syscall(SYS_pwritev, fd, iov, count, (long)(ofs), (long)(ofs>>32));
 }
+
+ssize_t pwritev64(int fd, const struct iovec *iov, int count, off_t ofs)
+{
+	return pwritev(fd, iov, count, ofs);
+}

base/glibc-compatibility/musl/scalbnf.c

@@ -0,0 +1,31 @@
+#include <math.h>
+#include <stdint.h>
+
+float scalbnf(float x, int n)
+{
+    union {float f; uint32_t i;} u;
+    float_t y = x;
+
+    if (n > 127) {
+        y *= 0x1p127f;
+        n -= 127;
+        if (n > 127) {
+            y *= 0x1p127f;
+            n -= 127;
+            if (n > 127)
+                n = 127;
+        }
+    } else if (n < -126) {
+        y *= 0x1p-126f;
+        n += 126;
+        if (n < -126) {
+            y *= 0x1p-126f;
+            n += 126;
+            if (n < -126)
+                n = -126;
+        }
+    }
+    u.i = (uint32_t)(0x7f+n)<<23;
+    x = y * u.f;
+    return x;
+}

base/harmful/CMakeLists.txt

@@ -1,2 +1 @@
 add_library(harmful harmful.c)
-install(TARGETS harmful EXPORT global ARCHIVE DESTINATION lib)

base/harmful/harmful.c

@@ -31,7 +31,8 @@ TRAP(argp_state_help)
 TRAP(argp_usage)
 TRAP(asctime)
 TRAP(clearenv)
-TRAP(crypt)
+// Redefined at contrib/libbcrypt/crypt_blowfish/wrapper.c:186
+// TRAP(crypt)
 TRAP(ctime)
 TRAP(cuserid)
 TRAP(drand48)

base/pcg-random/pcg_extras.hpp

@@ -455,7 +455,7 @@ auto bounded_rand(RngType& rng, typename RngType::result_type upper_bound)
     typedef typename RngType::result_type rtype;
     rtype threshold = (RngType::max() - RngType::min() + rtype(1) - upper_bound)
                     % upper_bound;
-    for (;;) { //-V1044
+    for (;;) {
         rtype r = rng() - RngType::min();
         if (r >= threshold)
             return r % upper_bound;
@@ -463,7 +463,7 @@ auto bounded_rand(RngType& rng, typename RngType::result_type upper_bound)
 }
 
 template <typename Iter, typename RandType>
-void shuffle(Iter from, Iter to, RandType&& rng)
+void shuffle(Iter from, Iter to, RandType&& rng) // NOLINT(cppcoreguidelines-missing-std-forward)
 {
     typedef typename std::iterator_traits<Iter>::difference_type delta_t;
     typedef typename std::remove_reference<RandType>::type::result_type result_t;

base/pcg-random/pcg_random.hpp

@@ -930,7 +930,7 @@ struct rxs_m_xs_mixin {
         constexpr bitcount_t shift = bits - xtypebits;
         constexpr bitcount_t mask = (1 << opbits) - 1;
         bitcount_t rshift =
-            opbits ? bitcount_t(internal >> (bits - opbits)) & mask : 0; //-V547
+            opbits ? bitcount_t(internal >> (bits - opbits)) & mask : 0;
         internal ^= internal >> (opbits + rshift);
         internal *= mcg_multiplier<itype>::multiplier();
         xtype result = internal >> shift;
@@ -952,7 +952,7 @@ struct rxs_m_xs_mixin {
 
         internal *= mcg_unmultiplier<itype>::unmultiplier();
 
-        bitcount_t rshift = opbits ? (internal >> (bits - opbits)) & mask : 0; //-V547
+        bitcount_t rshift = opbits ? (internal >> (bits - opbits)) & mask : 0;
         internal = unxorshift(internal, bits, opbits + rshift);
 
         return internal;
@@ -977,7 +977,7 @@ struct rxs_m_mixin {
                                  :                    2;
         constexpr bitcount_t shift = bits - xtypebits;
         constexpr bitcount_t mask = (1 << opbits) - 1;
-        bitcount_t rshift = opbits ? (internal >> (bits - opbits)) & mask : 0; //-V547
+        bitcount_t rshift = opbits ? (internal >> (bits - opbits)) & mask : 0;
         internal ^= internal >> (opbits + rshift);
         internal *= mcg_multiplier<itype>::multiplier();
         xtype result = internal >> shift;
@@ -1368,7 +1368,7 @@ void extended<table_pow2,advance_pow2,baseclass,extvalclass,kdd>::selfinit()
     //      - any strange correlations would only be apparent if we
     //        were to backstep the generator so that the base generator
     //        was generating the same values again
-    result_type xdiff = baseclass::operator()() - baseclass::operator()(); //-V501
+    result_type xdiff = baseclass::operator()() - baseclass::operator()();
     for (size_t i = 0; i < table_size; ++i) {
         data_[i] = baseclass::operator()() ^ xdiff;
     }

base/poco/Crypto/CMakeLists.txt

@@ -21,6 +21,7 @@ if (ENABLE_SSL)
             -Wno-unreachable-code-return
             -Wno-unused-parameter
             -Wno-zero-as-null-pointer-constant
+            -Wno-used-but-marked-unused
     )
     target_include_directories (_poco_crypto SYSTEM PUBLIC "include")
     target_link_libraries (_poco_crypto PUBLIC Poco::Foundation OpenSSL::SSL OpenSSL::Crypto)

base/poco/Crypto/include/Poco/Crypto/Cipher.h

@@ -18,121 +18,124 @@
 #define Crypto_Cipher_INCLUDED
 
 
-#include "Poco/Crypto/Crypto.h"
-#include "Poco/RefCountedObject.h"
-#include "Poco/AutoPtr.h"
 #include <istream>
 #include <ostream>
 #include <vector>
+#include "Poco/AutoPtr.h"
+#include "Poco/Crypto/Crypto.h"
+#include "Poco/RefCountedObject.h"
 
 
-namespace Poco {
-namespace Crypto {
-
-
-class CryptoTransform;
-
-
-class Crypto_API Cipher: public Poco::RefCountedObject
-	/// Represents the abstract base class from which all implementations of
-	/// symmetric/asymmetric encryption algorithms must inherit.  Use the CipherFactory
-	/// class to obtain an instance of this class:
-	///
-	///     CipherFactory& factory = CipherFactory::defaultFactory();
-	///     // Creates a 256-bit AES cipher
-	///     Cipher* pCipher = factory.createCipher(CipherKey("aes-256"));
-	///     Cipher* pRSACipher = factory.createCipher(RSAKey(RSAKey::KL_1024, RSAKey::EXP_SMALL));
-	///
-	/// Check the different Key constructors on how to initialize/create
-	/// a key. The above example auto-generates random keys.
-	///
-	/// Note that you won't be able to decrypt data encrypted with a random key
-	/// once the Cipher is destroyed unless you persist the generated key and IV.
-	/// An example usage for random keys is to encrypt data saved in a temporary
-	/// file.
-	///
-	/// Once your key is set up, you can use the Cipher object to encrypt or
-	/// decrypt strings or, in conjunction with a CryptoInputStream or a
-	/// CryptoOutputStream, to encrypt streams of data.
-	///
-	/// Since encrypted strings will contain arbitrary binary data that will cause
-	/// problems in applications that are not binary-safe (eg., when sending
-	/// encrypted data in e-mails), the encryptString() and decryptString() can
-	/// encode (or decode, respectively) encrypted data using a "transport encoding".
-	/// Supported encodings are Base64 and BinHex.
-	///
-	/// The following example encrypts and decrypts a string utilizing Base64
-	/// encoding:
-	///
-	///     std::string plainText = "This is my secret information";
-	///     std::string encrypted = pCipher->encryptString(plainText, Cipher::ENC_BASE64);
-	///     std::string decrypted = pCipher->decryptString(encrypted, Cipher::ENC_BASE64);
-	///
-	/// In order to encrypt a stream of data (eg. to encrypt files), you can use
-	/// a CryptoStream:
-	///
-	///     // Create an output stream that will encrypt all data going through it
-	///     // and write pass it to the underlying file stream.
-	///     Poco::FileOutputStream sink("encrypted.dat");
-	///     CryptoOutputStream encryptor(sink, pCipher->createEncryptor());
-	///     
-	///     Poco::FileInputStream source("source.txt");
-	///     Poco::StreamCopier::copyStream(source, encryptor);
-	///     
-	///     // Always close output streams to flush all internal buffers
-	///     encryptor.close();
-	///     sink.close();
+namespace Poco
+{
+namespace Crypto
 {
-public:
-	typedef Poco::AutoPtr<Cipher> Ptr;
-	typedef std::vector<unsigned char> ByteVec;
-
-	enum Encoding
-		/// Transport encoding to use for encryptString() and decryptString().
-	{
-		ENC_NONE         = 0x00, /// Plain binary output
-		ENC_BASE64       = 0x01, /// Base64-encoded output
-		ENC_BINHEX       = 0x02, /// BinHex-encoded output
-		ENC_BASE64_NO_LF = 0x81, /// Base64-encoded output, no linefeeds
-		ENC_BINHEX_NO_LF = 0x82  /// BinHex-encoded output, no linefeeds
-		
-	};
-
-	virtual ~Cipher();
-		/// Destroys the Cipher.
-
-	virtual const std::string& name() const = 0;
-		/// Returns the name of the Cipher.
-
-	virtual CryptoTransform* createEncryptor() = 0;
-		/// Creates an encryptor object to be used with a CryptoStream.
-
-	virtual CryptoTransform* createDecryptor() = 0;
-		/// Creates a decryptor object to be used with a CryptoStream.
-
-	virtual std::string encryptString(const std::string& str, Encoding encoding = ENC_NONE);
-		/// Directly encrypt a string and encode it using the given encoding.
-
-	virtual std::string decryptString(const std::string& str, Encoding encoding = ENC_NONE);
-		/// Directly decrypt a string that is encoded with the given encoding.
-
-	virtual void encrypt(std::istream& source, std::ostream& sink, Encoding encoding = ENC_NONE);
-		/// Directly encrypts an input stream and encodes it using the given encoding.
-
-	virtual void decrypt(std::istream& source, std::ostream& sink, Encoding encoding = ENC_NONE);
-		/// Directly decrypt an input stream that is encoded with the given encoding.
-
-protected:
-	Cipher();
-		/// Creates a new Cipher object.
-
-private:
-	Cipher(const Cipher&);
-	Cipher& operator = (const Cipher&);
-};
 
 
-} } // namespace Poco::Crypto
+    class CryptoTransform;
+
+
+    class Crypto_API Cipher : public Poco::RefCountedObject
+    /// Represents the abstract base class from which all implementations of
+    /// symmetric/asymmetric encryption algorithms must inherit.  Use the CipherFactory
+    /// class to obtain an instance of this class:
+    ///
+    ///     CipherFactory& factory = CipherFactory::defaultFactory();
+    ///     // Creates a 256-bit AES cipher
+    ///     Cipher* pCipher = factory.createCipher(CipherKey("aes-256"));
+    ///     Cipher* pRSACipher = factory.createCipher(RSAKey(RSAKey::KL_1024, RSAKey::EXP_SMALL));
+    ///
+    /// Check the different Key constructors on how to initialize/create
+    /// a key. The above example auto-generates random keys.
+    ///
+    /// Note that you won't be able to decrypt data encrypted with a random key
+    /// once the Cipher is destroyed unless you persist the generated key and IV.
+    /// An example usage for random keys is to encrypt data saved in a temporary
+    /// file.
+    ///
+    /// Once your key is set up, you can use the Cipher object to encrypt or
+    /// decrypt strings or, in conjunction with a CryptoInputStream or a
+    /// CryptoOutputStream, to encrypt streams of data.
+    ///
+    /// Since encrypted strings will contain arbitrary binary data that will cause
+    /// problems in applications that are not binary-safe (eg., when sending
+    /// encrypted data in e-mails), the encryptString() and decryptString() can
+    /// encode (or decode, respectively) encrypted data using a "transport encoding".
+    /// Supported encodings are Base64 and BinHex.
+    ///
+    /// The following example encrypts and decrypts a string utilizing Base64
+    /// encoding:
+    ///
+    ///     std::string plainText = "This is my secret information";
+    ///     std::string encrypted = pCipher->encryptString(plainText, Cipher::ENC_BASE64);
+    ///     std::string decrypted = pCipher->decryptString(encrypted, Cipher::ENC_BASE64);
+    ///
+    /// In order to encrypt a stream of data (eg. to encrypt files), you can use
+    /// a CryptoStream:
+    ///
+    ///     // Create an output stream that will encrypt all data going through it
+    ///     // and write pass it to the underlying file stream.
+    ///     Poco::FileOutputStream sink("encrypted.dat");
+    ///     CryptoOutputStream encryptor(sink, pCipher->createEncryptor());
+    ///
+    ///     Poco::FileInputStream source("source.txt");
+    ///     Poco::StreamCopier::copyStream(source, encryptor);
+    ///
+    ///     // Always close output streams to flush all internal buffers
+    ///     encryptor.close();
+    ///     sink.close();
+    {
+    public:
+        typedef Poco::AutoPtr<Cipher> Ptr;
+        typedef std::vector<unsigned char> ByteVec;
+
+        enum Encoding
+        /// Transport encoding to use for encryptString() and decryptString().
+        {
+            ENC_NONE = 0x00, /// Plain binary output
+            ENC_BASE64 = 0x01, /// Base64-encoded output
+            ENC_BINHEX = 0x02, /// BinHex-encoded output
+            ENC_BASE64_NO_LF = 0x81, /// Base64-encoded output, no linefeeds
+            ENC_BINHEX_NO_LF = 0x82 /// BinHex-encoded output, no linefeeds
+
+        };
+
+        virtual ~Cipher();
+        /// Destroys the Cipher.
+
+        virtual const std::string & name() const = 0;
+        /// Returns the name of the Cipher.
+
+        virtual CryptoTransform * createEncryptor() = 0;
+        /// Creates an encryptor object to be used with a CryptoStream.
+
+        virtual CryptoTransform * createDecryptor() = 0;
+        /// Creates a decryptor object to be used with a CryptoStream.
+
+        virtual std::string encryptString(const std::string & str, Encoding encoding = ENC_NONE);
+        /// Directly encrypt a string and encode it using the given encoding.
+
+        virtual std::string decryptString(const std::string & str, Encoding encoding = ENC_NONE);
+        /// Directly decrypt a string that is encoded with the given encoding.
+
+        virtual void encrypt(std::istream & source, std::ostream & sink, Encoding encoding = ENC_NONE);
+        /// Directly encrypts an input stream and encodes it using the given encoding.
+
+        virtual void decrypt(std::istream & source, std::ostream & sink, Encoding encoding = ENC_NONE);
+        /// Directly decrypt an input stream that is encoded with the given encoding.
+
+    protected:
+        Cipher();
+        /// Creates a new Cipher object.
+
+    private:
+        Cipher(const Cipher &);
+        Cipher & operator=(const Cipher &);
+    };
+
+
+}
+} // namespace Poco::Crypto
 
 
 #endif // Crypto_Cipher_INCLUDED

base/poco/Crypto/include/Poco/Crypto/CipherFactory.h

@@ -21,55 +21,58 @@
 #include "Poco/Crypto/Crypto.h"
 
 
-namespace Poco {
-namespace Crypto {
-
-
-class Cipher;
-class CipherKey;
-class RSAKey;
-
-
-class Crypto_API CipherFactory
-	/// A factory for Cipher objects. See the Cipher class for examples on how to
-	/// use the CipherFactory.
+namespace Poco
+{
+namespace Crypto
 {
-public:
-	CipherFactory();
-		/// Creates a new CipherFactory object.
-
-	virtual ~CipherFactory();
-		/// Destroys the CipherFactory.
-
-	Cipher* createCipher(const CipherKey& key);
-		/// Creates a Cipher object for the given Cipher name. Valid cipher 
-		/// names depend on the OpenSSL version the library is linked with;  
-		/// see the output of
-		///
-		///     openssl enc --help
-		///
-		/// for a list of supported block and stream ciphers.
-		///
-		/// Common examples are:
-		///
-		///   * AES: "aes-128", "aes-256"
-		///   * DES: "des", "des3"
-		///   * Blowfish: "bf"
-
-	Cipher* createCipher(const RSAKey& key, RSAPaddingMode paddingMode = RSA_PADDING_PKCS1);
-		/// Creates a RSACipher using the given RSA key and padding mode
-		/// for public key encryption/private key decryption.
-	
-	static CipherFactory& defaultFactory();
-		/// Returns the default CipherFactory.
-
-private:
-	CipherFactory(const CipherFactory&);
-	CipherFactory& operator = (const CipherFactory&);
-};
 
 
-} } // namespace Poco::Crypto
+    class Cipher;
+    class CipherKey;
+    class RSAKey;
+
+
+    class Crypto_API CipherFactory
+    /// A factory for Cipher objects. See the Cipher class for examples on how to
+    /// use the CipherFactory.
+    {
+    public:
+        CipherFactory();
+        /// Creates a new CipherFactory object.
+
+        virtual ~CipherFactory();
+        /// Destroys the CipherFactory.
+
+        Cipher * createCipher(const CipherKey & key);
+        /// Creates a Cipher object for the given Cipher name. Valid cipher
+        /// names depend on the OpenSSL version the library is linked with;
+        /// see the output of
+        ///
+        ///     openssl enc --help
+        ///
+        /// for a list of supported block and stream ciphers.
+        ///
+        /// Common examples are:
+        ///
+        ///   * AES: "aes-128", "aes-256"
+        ///   * DES: "des", "des3"
+        ///   * Blowfish: "bf"
+
+        Cipher * createCipher(const RSAKey & key, RSAPaddingMode paddingMode = RSA_PADDING_PKCS1);
+        /// Creates a RSACipher using the given RSA key and padding mode
+        /// for public key encryption/private key decryption.
+
+        static CipherFactory & defaultFactory();
+        /// Returns the default CipherFactory.
+
+    private:
+        CipherFactory(const CipherFactory &);
+        CipherFactory & operator=(const CipherFactory &);
+    };
+
+
+}
+} // namespace Poco::Crypto
 
 
 #endif // Crypto_CipherFactory_INCLUDED

base/poco/Crypto/include/Poco/Crypto/CipherImpl.h

@@ -18,52 +18,55 @@
 #define Crypto_CipherImpl_INCLUDED
 
 
-#include "Poco/Crypto/Crypto.h"
+#include <openssl/evp.h>
 #include "Poco/Crypto/Cipher.h"
 #include "Poco/Crypto/CipherKey.h"
+#include "Poco/Crypto/Crypto.h"
 #include "Poco/Crypto/OpenSSLInitializer.h"
-#include <openssl/evp.h>
 
 
-namespace Poco {
-namespace Crypto {
-
-
-class CipherImpl: public Cipher
-	/// An implementation of the Cipher class for OpenSSL's crypto library.
+namespace Poco
 {
-public:
-	CipherImpl(const CipherKey& key);
-		/// Creates a new CipherImpl object for the given CipherKey.
-
-	virtual ~CipherImpl();
-		/// Destroys the CipherImpl.
-
-	const std::string& name() const;
-		/// Returns the name of the cipher.
-
-	CryptoTransform* createEncryptor();
-		/// Creates an encryptor object.
-
-	CryptoTransform* createDecryptor();
-		/// Creates a decryptor object.
-
-private:
-	CipherKey _key;
-	OpenSSLInitializer _openSSLInitializer;
-};
-
-
-//
-// Inlines
-//
-inline const std::string& CipherImpl::name() const
+namespace Crypto
 {
-	return _key.name();
+
+
+    class CipherImpl : public Cipher
+    /// An implementation of the Cipher class for OpenSSL's crypto library.
+    {
+    public:
+        CipherImpl(const CipherKey & key);
+        /// Creates a new CipherImpl object for the given CipherKey.
+
+        virtual ~CipherImpl();
+        /// Destroys the CipherImpl.
+
+        const std::string & name() const;
+        /// Returns the name of the cipher.
+
+        CryptoTransform * createEncryptor();
+        /// Creates an encryptor object.
+
+        CryptoTransform * createDecryptor();
+        /// Creates a decryptor object.
+
+    private:
+        CipherKey _key;
+        OpenSSLInitializer _openSSLInitializer;
+    };
+
+
+    //
+    // Inlines
+    //
+    inline const std::string & CipherImpl::name() const
+    {
+        return _key.name();
+    }
+
+
 }
-
-
-} } // namespace Poco::Crypto
+} // namespace Poco::Crypto
 
 
 #endif // Crypto_CipherImpl_INCLUDED

base/poco/Crypto/include/Poco/Crypto/CipherKey.h

@@ -18,184 +18,186 @@
 #define Crypto_CipherKey_INCLUDED
 
 
-#include "Poco/Crypto/Crypto.h"
 #include "Poco/Crypto/CipherKeyImpl.h"
+#include "Poco/Crypto/Crypto.h"
 
 
-namespace Poco {
-namespace Crypto {
-
-
-class Crypto_API CipherKey
-	/// CipherKey stores the key information for decryption/encryption of data.
-	/// To create a random key, using the following code:
-	///
-	///     CipherKey key("aes-256");
-	///
-	/// Note that you won't be able to decrypt data encrypted with a random key
-	/// once the Cipher is destroyed unless you persist the generated key and IV.
-	/// An example usage for random keys is to encrypt data saved in a temporary
-	/// file.
-	///
-	/// To create a key using a human-readable password
-	/// string, use the following code. We create a AES Cipher and 
-	/// use a salt value to make the key more robust:
-	///
-	///     std::string password = "secret";
-	///     std::string salt("asdff8723lasdf(**923412");
-	///     CipherKey key("aes-256", password, salt);
-	///
-	/// You may also control the digest and the number of iterations used to generate the key
-	/// by specifying the specific values. Here we create a key with the same data as before,
-	/// except that we use 100 iterations instead of DEFAULT_ITERATION_COUNT, and sha1 instead of
-	/// the default md5:
-	///
-	///     std::string password = "secret";
-	///     std::string salt("asdff8723lasdf(**923412");
-	///     std::string digest ("sha1");
-	///     CipherKey key("aes-256", password, salt, 100, digest);
-	///
+namespace Poco
 {
-public:
-	typedef CipherKeyImpl::Mode Mode;
-	typedef CipherKeyImpl::ByteVec ByteVec;
-
-	enum
-	{
-		DEFAULT_ITERATION_COUNT = 2000
-			/// Default iteration count to use with
-			/// generateKey(). RSA security recommends
-			/// an iteration count of at least 1000.
-	};
-
-	CipherKey(const std::string& name, 
-		const std::string& passphrase, 
-		const std::string& salt = "",
-		int iterationCount = DEFAULT_ITERATION_COUNT,
-		const std::string& digest = "md5");
-		/// Creates a new CipherKeyImpl object using the given
-		/// cipher name, passphrase, salt value, iteration count and digest.
-
-	CipherKey(const std::string& name, 
-		const ByteVec& key, 
-		const ByteVec& iv);
-		/// Creates a new CipherKeyImpl object using the given cipher
-		/// name, key and initialization vector (IV).
-		///
-		/// The size of the IV must match the cipher's expected
-		/// IV size (see ivSize()), except for GCM mode, which allows
-		/// a custom IV size.
-
-	CipherKey(const std::string& name);
-		/// Creates a new CipherKeyImpl object. Autoinitializes key and 
-		/// initialization vector.
-
-	~CipherKey();
-		/// Destroys the CipherKeyImpl.
-
-	const std::string& name() const;
-		/// Returns the name of the Cipher.
-
-	int keySize() const;
-		/// Returns the key size of the Cipher.
-
-	int blockSize() const;
-		/// Returns the block size of the Cipher.
-
-	int ivSize() const;
-		/// Returns the IV size of the Cipher.
-
-	Mode mode() const;
-		/// Returns the Cipher's mode of operation.
-	
-	const ByteVec& getKey() const;
-		/// Returns the key for the Cipher.
-
-	void setKey(const ByteVec& key);
-		/// Sets the key for the Cipher.
-
-	const ByteVec& getIV() const;
-		/// Returns the initialization vector (IV) for the Cipher.
-
-	void setIV(const ByteVec& iv);
-		/// Sets the initialization vector (IV) for the Cipher.
-		///
-		/// The size of the vector must match the cipher's expected
-		/// IV size (see ivSize()), except for GCM mode, which allows
-		/// a custom IV size.
-
-	CipherKeyImpl::Ptr impl();
-		/// Returns the impl object
-
-private:
-	CipherKeyImpl::Ptr _pImpl;
-};
-
-
-//
-// inlines
-//
-inline const std::string& CipherKey::name() const
+namespace Crypto
 {
-	return _pImpl->name();
+
+
+    class Crypto_API CipherKey
+    /// CipherKey stores the key information for decryption/encryption of data.
+    /// To create a random key, using the following code:
+    ///
+    ///     CipherKey key("aes-256");
+    ///
+    /// Note that you won't be able to decrypt data encrypted with a random key
+    /// once the Cipher is destroyed unless you persist the generated key and IV.
+    /// An example usage for random keys is to encrypt data saved in a temporary
+    /// file.
+    ///
+    /// To create a key using a human-readable password
+    /// string, use the following code. We create a AES Cipher and
+    /// use a salt value to make the key more robust:
+    ///
+    ///     std::string password = "secret";
+    ///     std::string salt("asdff8723lasdf(**923412");
+    ///     CipherKey key("aes-256", password, salt);
+    ///
+    /// You may also control the digest and the number of iterations used to generate the key
+    /// by specifying the specific values. Here we create a key with the same data as before,
+    /// except that we use 100 iterations instead of DEFAULT_ITERATION_COUNT, and sha1 instead of
+    /// the default md5:
+    ///
+    ///     std::string password = "secret";
+    ///     std::string salt("asdff8723lasdf(**923412");
+    ///     std::string digest ("sha1");
+    ///     CipherKey key("aes-256", password, salt, 100, digest);
+    ///
+    {
+    public:
+        typedef CipherKeyImpl::Mode Mode;
+        typedef CipherKeyImpl::ByteVec ByteVec;
+
+        enum
+        {
+            DEFAULT_ITERATION_COUNT = 2000
+            /// Default iteration count to use with
+            /// generateKey(). RSA security recommends
+            /// an iteration count of at least 1000.
+        };
+
+        CipherKey(
+            const std::string & name,
+            const std::string & passphrase,
+            const std::string & salt = "",
+            int iterationCount = DEFAULT_ITERATION_COUNT,
+            const std::string & digest = "md5");
+        /// Creates a new CipherKeyImpl object using the given
+        /// cipher name, passphrase, salt value, iteration count and digest.
+
+        CipherKey(const std::string & name, const ByteVec & key, const ByteVec & iv);
+        /// Creates a new CipherKeyImpl object using the given cipher
+        /// name, key and initialization vector (IV).
+        ///
+        /// The size of the IV must match the cipher's expected
+        /// IV size (see ivSize()), except for GCM mode, which allows
+        /// a custom IV size.
+
+        CipherKey(const std::string & name);
+        /// Creates a new CipherKeyImpl object. Autoinitializes key and
+        /// initialization vector.
+
+        ~CipherKey();
+        /// Destroys the CipherKeyImpl.
+
+        const std::string & name() const;
+        /// Returns the name of the Cipher.
+
+        int keySize() const;
+        /// Returns the key size of the Cipher.
+
+        int blockSize() const;
+        /// Returns the block size of the Cipher.
+
+        int ivSize() const;
+        /// Returns the IV size of the Cipher.
+
+        Mode mode() const;
+        /// Returns the Cipher's mode of operation.
+
+        const ByteVec & getKey() const;
+        /// Returns the key for the Cipher.
+
+        void setKey(const ByteVec & key);
+        /// Sets the key for the Cipher.
+
+        const ByteVec & getIV() const;
+        /// Returns the initialization vector (IV) for the Cipher.
+
+        void setIV(const ByteVec & iv);
+        /// Sets the initialization vector (IV) for the Cipher.
+        ///
+        /// The size of the vector must match the cipher's expected
+        /// IV size (see ivSize()), except for GCM mode, which allows
+        /// a custom IV size.
+
+        CipherKeyImpl::Ptr impl();
+        /// Returns the impl object
+
+    private:
+        CipherKeyImpl::Ptr _pImpl;
+    };
+
+
+    //
+    // inlines
+    //
+    inline const std::string & CipherKey::name() const
+    {
+        return _pImpl->name();
+    }
+
+
+    inline int CipherKey::keySize() const
+    {
+        return _pImpl->keySize();
+    }
+
+
+    inline int CipherKey::blockSize() const
+    {
+        return _pImpl->blockSize();
+    }
+
+
+    inline int CipherKey::ivSize() const
+    {
+        return _pImpl->ivSize();
+    }
+
+
+    inline CipherKey::Mode CipherKey::mode() const
+    {
+        return _pImpl->mode();
+    }
+
+
+    inline const CipherKey::ByteVec & CipherKey::getKey() const
+    {
+        return _pImpl->getKey();
+    }
+
+
+    inline void CipherKey::setKey(const CipherKey::ByteVec & key)
+    {
+        _pImpl->setKey(key);
+    }
+
+
+    inline const CipherKey::ByteVec & CipherKey::getIV() const
+    {
+        return _pImpl->getIV();
+    }
+
+
+    inline void CipherKey::setIV(const CipherKey::ByteVec & iv)
+    {
+        _pImpl->setIV(iv);
+    }
+
+
+    inline CipherKeyImpl::Ptr CipherKey::impl()
+    {
+        return _pImpl;
+    }
+
+
 }
-
-
-inline int CipherKey::keySize() const
-{
-	return _pImpl->keySize();
-}
-
-
-inline int CipherKey::blockSize() const
-{
-	return _pImpl->blockSize();
-}
-
-
-inline int CipherKey::ivSize() const
-{
-	return _pImpl->ivSize();
-}
-
-
-inline CipherKey::Mode CipherKey::mode() const
-{
-	return _pImpl->mode();
-}
-
-
-inline const CipherKey::ByteVec& CipherKey::getKey() const
-{
-	return _pImpl->getKey();
-}
-
-
-inline void CipherKey::setKey(const CipherKey::ByteVec& key)
-{
-	_pImpl->setKey(key);
-}
-
-
-inline const CipherKey::ByteVec& CipherKey::getIV() const
-{
-	return _pImpl->getIV();
-}
-
-
-inline void CipherKey::setIV(const CipherKey::ByteVec& iv)
-{
-	_pImpl->setIV(iv);
-}
-
-
-inline CipherKeyImpl::Ptr CipherKey::impl()
-{
-	return _pImpl;
-}
-
-
-} } // namespace Poco::Crypto
+} // namespace Poco::Crypto
 
 
 #endif // Crypto_CipherKey_INCLUDED

base/poco/Crypto/include/Poco/Crypto/CipherKeyImpl.h

@@ -18,151 +18,151 @@
 #define Crypto_CipherKeyImpl_INCLUDED
 
 
+#include <vector>
+#include "Poco/AutoPtr.h"
 #include "Poco/Crypto/Crypto.h"
 #include "Poco/Crypto/OpenSSLInitializer.h"
 #include "Poco/RefCountedObject.h"
-#include "Poco/AutoPtr.h"
-#include <vector>
 
 
 struct evp_cipher_st;
 typedef struct evp_cipher_st EVP_CIPHER;
 
 
-namespace Poco {
-namespace Crypto {
-
-
-class CipherKeyImpl: public RefCountedObject
-	/// An implementation of the CipherKey class for OpenSSL's crypto library.
+namespace Poco
 {
-public:
-	typedef std::vector<unsigned char> ByteVec;
-	typedef Poco::AutoPtr<CipherKeyImpl> Ptr;
-
-	enum Mode
-		/// Cipher mode of operation. This mode determines how multiple blocks
-		/// are connected; this is essential to improve security.
-	{
-		MODE_STREAM_CIPHER,	/// Stream cipher
-		MODE_ECB,			/// Electronic codebook (plain concatenation)
-		MODE_CBC,			/// Cipher block chaining (default)
-		MODE_CFB,			/// Cipher feedback
-		MODE_OFB,			/// Output feedback
-		MODE_CTR,           /// Counter mode
-		MODE_GCM,           /// Galois/Counter mode
-		MODE_CCM            /// Counter with CBC-MAC
-	};
-
-	CipherKeyImpl(const std::string& name,
-		const std::string& passphrase,
-		const std::string& salt,
-		int iterationCount,
-		const std::string& digest);
-		/// Creates a new CipherKeyImpl object, using
-		/// the given cipher name, passphrase, salt value
-		/// and iteration count.
-
-	CipherKeyImpl(const std::string& name,
-		const ByteVec& key,
-		const ByteVec& iv);
-		/// Creates a new CipherKeyImpl object, using the
-		/// given cipher name, key and initialization vector.
-
-	CipherKeyImpl(const std::string& name);
-		/// Creates a new CipherKeyImpl object. Autoinitializes key
-		/// and initialization vector.
-
-	virtual ~CipherKeyImpl();
-		/// Destroys the CipherKeyImpl.
-
-	const std::string& name() const;
-		/// Returns the name of the Cipher.
-
-	int keySize() const;
-		/// Returns the key size of the Cipher.
-
-	int blockSize() const;
-		/// Returns the block size of the Cipher.
-
-	int ivSize() const;
-		/// Returns the IV size of the Cipher.
-
-	Mode mode() const;
-		/// Returns the Cipher's mode of operation.
-
-	const ByteVec& getKey() const;
-		/// Returns the key for the Cipher.
-
-	void setKey(const ByteVec& key);
-		/// Sets the key for the Cipher.
-
-	const ByteVec& getIV() const;
-		/// Returns the initialization vector (IV) for the Cipher.
-
-	void setIV(const ByteVec& iv);
-		/// Sets the initialization vector (IV) for the Cipher.
-
-	const EVP_CIPHER* cipher();
-		/// Returns the cipher object
-
-private:
-	void generateKey(const std::string& passphrase,
-		const std::string& salt,
-		int iterationCount);
-	 	/// Generates key and IV from a password and optional salt string.
-
-	void generateKey();
-		/// Generates key and IV from random data.
-
-	void getRandomBytes(ByteVec& vec, std::size_t count);
-		/// Stores random bytes in vec.
-
-private:
-	const EVP_CIPHER*  _pCipher;
-	const EVP_MD*      _pDigest;
-	std::string	       _name;
-	ByteVec		       _key;
-	ByteVec		       _iv;
-	OpenSSLInitializer _openSSLInitializer;
-};
-
-
-//
-// Inlines
-//
-inline const std::string& CipherKeyImpl::name() const
+namespace Crypto
 {
-	return _name;
+
+
+    class CipherKeyImpl : public RefCountedObject
+    /// An implementation of the CipherKey class for OpenSSL's crypto library.
+    {
+    public:
+        typedef std::vector<unsigned char> ByteVec;
+        typedef Poco::AutoPtr<CipherKeyImpl> Ptr;
+
+        enum Mode
+        /// Cipher mode of operation. This mode determines how multiple blocks
+        /// are connected; this is essential to improve security.
+        {
+            MODE_STREAM_CIPHER, /// Stream cipher
+            MODE_ECB, /// Electronic codebook (plain concatenation)
+            MODE_CBC, /// Cipher block chaining (default)
+            MODE_CFB, /// Cipher feedback
+            MODE_OFB, /// Output feedback
+            MODE_CTR, /// Counter mode
+            MODE_GCM, /// Galois/Counter mode
+            MODE_CCM /// Counter with CBC-MAC
+        };
+
+        CipherKeyImpl(
+            const std::string & name,
+            const std::string & passphrase,
+            const std::string & salt,
+            int iterationCount,
+            const std::string & digest);
+        /// Creates a new CipherKeyImpl object, using
+        /// the given cipher name, passphrase, salt value
+        /// and iteration count.
+
+        CipherKeyImpl(const std::string & name, const ByteVec & key, const ByteVec & iv);
+        /// Creates a new CipherKeyImpl object, using the
+        /// given cipher name, key and initialization vector.
+
+        CipherKeyImpl(const std::string & name);
+        /// Creates a new CipherKeyImpl object. Autoinitializes key
+        /// and initialization vector.
+
+        virtual ~CipherKeyImpl();
+        /// Destroys the CipherKeyImpl.
+
+        const std::string & name() const;
+        /// Returns the name of the Cipher.
+
+        int keySize() const;
+        /// Returns the key size of the Cipher.
+
+        int blockSize() const;
+        /// Returns the block size of the Cipher.
+
+        int ivSize() const;
+        /// Returns the IV size of the Cipher.
+
+        Mode mode() const;
+        /// Returns the Cipher's mode of operation.
+
+        const ByteVec & getKey() const;
+        /// Returns the key for the Cipher.
+
+        void setKey(const ByteVec & key);
+        /// Sets the key for the Cipher.
+
+        const ByteVec & getIV() const;
+        /// Returns the initialization vector (IV) for the Cipher.
+
+        void setIV(const ByteVec & iv);
+        /// Sets the initialization vector (IV) for the Cipher.
+
+        const EVP_CIPHER * cipher();
+        /// Returns the cipher object
+
+    private:
+        void generateKey(const std::string & passphrase, const std::string & salt, int iterationCount);
+        /// Generates key and IV from a password and optional salt string.
+
+        void generateKey();
+        /// Generates key and IV from random data.
+
+        void getRandomBytes(ByteVec & vec, std::size_t count);
+        /// Stores random bytes in vec.
+
+    private:
+        const EVP_CIPHER * _pCipher;
+        const EVP_MD * _pDigest;
+        std::string _name;
+        ByteVec _key;
+        ByteVec _iv;
+        OpenSSLInitializer _openSSLInitializer;
+    };
+
+
+    //
+    // Inlines
+    //
+    inline const std::string & CipherKeyImpl::name() const
+    {
+        return _name;
+    }
+
+
+    inline const CipherKeyImpl::ByteVec & CipherKeyImpl::getKey() const
+    {
+        return _key;
+    }
+
+
+    inline void CipherKeyImpl::setKey(const ByteVec & key)
+    {
+        poco_assert(key.size() == static_cast<ByteVec::size_type>(keySize()));
+        _key = key;
+    }
+
+
+    inline const CipherKeyImpl::ByteVec & CipherKeyImpl::getIV() const
+    {
+        return _iv;
+    }
+
+
+    inline const EVP_CIPHER * CipherKeyImpl::cipher()
+    {
+        return _pCipher;
+    }
+
+
 }
-
-
-inline const CipherKeyImpl::ByteVec& CipherKeyImpl::getKey() const
-{
-	return _key;
-}
-
-
-inline void CipherKeyImpl::setKey(const ByteVec& key)
-{
-	poco_assert(key.size() == static_cast<ByteVec::size_type>(keySize()));
-	_key = key;
-}
-
-
-inline const CipherKeyImpl::ByteVec& CipherKeyImpl::getIV() const
-{
-	return _iv;
-}
-
-
-inline const EVP_CIPHER* CipherKeyImpl::cipher()
-{
-	return _pCipher;
-}
-
-
-} } // namespace Poco::Crypto
+} // namespace Poco::Crypto
 
 
 #endif // Crypto_CipherKeyImpl_INCLUDED

base/poco/Crypto/include/Poco/Crypto/Crypto.h

@@ -24,39 +24,37 @@
 #define POCO_EXTERNAL_OPENSSL_SLPRO 2
 
 
-#include "Poco/Foundation.h"
 #include <openssl/opensslv.h>
+#include "Poco/Foundation.h"
 
 
 #ifndef OPENSSL_VERSION_PREREQ
-	#if defined(OPENSSL_VERSION_MAJOR) && defined(OPENSSL_VERSION_MINOR)
-		#define OPENSSL_VERSION_PREREQ(maj, min) \
-			((OPENSSL_VERSION_MAJOR << 16) + OPENSSL_VERSION_MINOR >= ((maj) << 16) + (min))
-	#else
-		#define OPENSSL_VERSION_PREREQ(maj, min) \
-			(OPENSSL_VERSION_NUMBER >= (((maj) << 28) | ((min) << 20)))
-	#endif
+#    if defined(OPENSSL_VERSION_MAJOR) && defined(OPENSSL_VERSION_MINOR)
+#        define OPENSSL_VERSION_PREREQ(maj, min) ((OPENSSL_VERSION_MAJOR << 16) + OPENSSL_VERSION_MINOR >= ((maj) << 16) + (min))
+#    else
+#        define OPENSSL_VERSION_PREREQ(maj, min) (OPENSSL_VERSION_NUMBER >= (((maj) << 28) | ((min) << 20)))
+#    endif
 #endif
 
 
 enum RSAPaddingMode
-	/// The padding mode used for RSA public key encryption.
+/// The padding mode used for RSA public key encryption.
 {
-	RSA_PADDING_PKCS1,
-		/// PKCS #1 v1.5 padding. This currently is the most widely used mode. 
-		
-	RSA_PADDING_PKCS1_OAEP,
-		/// EME-OAEP as defined in PKCS #1 v2.0 with SHA-1, MGF1 and an empty 
-		/// encoding parameter. This mode is recommended for all new applications.
-		
-	RSA_PADDING_SSLV23,
-		/// PKCS #1 v1.5 padding with an SSL-specific modification that denotes 
-		/// that the server is SSL3 capable. 
-		
-	RSA_PADDING_NONE
-		/// Raw RSA encryption. This mode should only be used to implement cryptographically 
-		/// sound padding modes in the application code. Encrypting user data directly with RSA 
-		/// is insecure. 
+    RSA_PADDING_PKCS1,
+    /// PKCS #1 v1.5 padding. This currently is the most widely used mode.
+
+    RSA_PADDING_PKCS1_OAEP,
+    /// EME-OAEP as defined in PKCS #1 v2.0 with SHA-1, MGF1 and an empty
+    /// encoding parameter. This mode is recommended for all new applications.
+
+    RSA_PADDING_SSLV23,
+    /// PKCS #1 v1.5 padding with an SSL-specific modification that denotes
+    /// that the server is SSL3 capable.
+
+    RSA_PADDING_NONE
+    /// Raw RSA encryption. This mode should only be used to implement cryptographically
+    /// sound padding modes in the application code. Encrypting user data directly with RSA
+    /// is insecure.
 };
 
 
@@ -68,128 +66,51 @@ enum RSAPaddingMode
 // Crypto_API functions as being imported from a DLL, whereas this DLL sees symbols
 // defined with this macro as being exported.
 //
-#if defined(_WIN32)
-	#if defined(POCO_DLL)
-		#if defined(Crypto_EXPORTS)
-			#define Crypto_API __declspec(dllexport)
-		#else
-			#define Crypto_API __declspec(dllimport)
-		#endif
-	#endif
-#endif
 
 
 #if !defined(Crypto_API)
-	#if !defined(POCO_NO_GCC_API_ATTRIBUTE) && defined (__GNUC__) && (__GNUC__ >= 4)
-		#define Crypto_API __attribute__ ((visibility ("default")))
-	#else
-		#define Crypto_API
-	#endif
+#    if !defined(POCO_NO_GCC_API_ATTRIBUTE) && defined(__GNUC__) && (__GNUC__ >= 4)
+#        define Crypto_API __attribute__((visibility("default")))
+#    else
+#        define Crypto_API
+#    endif
 #endif
 
 
 //
 // Automatically link Crypto and OpenSSL libraries.
 //
-#if defined(_MSC_VER)
-	#if !defined(POCO_NO_AUTOMATIC_LIBS)
-		#if defined(POCO_INTERNAL_OPENSSL_MSVC_VER)
-			#if defined(POCO_EXTERNAL_OPENSSL)
-				#pragma message("External OpenSSL defined but internal headers used - possible mismatch!")
-			#endif // POCO_EXTERNAL_OPENSSL
-			#if !defined(_DEBUG)
-				#define POCO_DEBUG_SUFFIX ""
-				#if !defined (_DLL)
-					#define POCO_STATIC_SUFFIX "mt"
-				#else // _DLL
-					#define POCO_STATIC_SUFFIX ""
-				#endif
-			#else // _DEBUG
-				#define POCO_DEBUG_SUFFIX "d"
-				#if !defined (_DLL)
-					#define POCO_STATIC_SUFFIX "mt"
-				#else // _DLL
-					#define POCO_STATIC_SUFFIX ""
-				#endif
-			#endif
-			#pragma comment(lib, "libcrypto" POCO_STATIC_SUFFIX POCO_DEBUG_SUFFIX ".lib")
-			#pragma comment(lib, "libssl" POCO_STATIC_SUFFIX POCO_DEBUG_SUFFIX ".lib")
-			#if !defined(_WIN64) && !defined (_DLL) && \
-						(POCO_INTERNAL_OPENSSL_MSVC_VER == 120) && \
-						(POCO_MSVC_VERSION < POCO_INTERNAL_OPENSSL_MSVC_VER)
-				#pragma comment(lib, "libPreVS2013CRT" POCO_STATIC_SUFFIX POCO_DEBUG_SUFFIX ".lib")
-			#endif
-			#if !defined (_DLL) && (POCO_MSVS_VERSION >= 2015)
-				#pragma comment(lib, "legacy_stdio_definitions.lib")
-				#pragma comment(lib, "legacy_stdio_wide_specifiers.lib")
-			#endif
-		#elif defined(POCO_EXTERNAL_OPENSSL)
-			#if POCO_EXTERNAL_OPENSSL == POCO_EXTERNAL_OPENSSL_SLPRO
-				#if defined(POCO_DLL)
-					#if OPENSSL_VERSION_PREREQ(1,1)
-						#pragma comment(lib, "libcrypto.lib")
-						#pragma comment(lib, "libssl.lib")
-					#else
-						#pragma comment(lib, "libeay32.lib")
-						#pragma comment(lib, "ssleay32.lib")
-					#endif
-			  	#else
-					#if OPENSSL_VERSION_PREREQ(1,1)
-						#if defined(_WIN64)
-							#pragma comment(lib, "libcrypto64" POCO_LIB_SUFFIX)
-							#pragma comment(lib, "libssl64" POCO_LIB_SUFFIX)
-						#else
-							#pragma comment(lib, "libcrypto32" POCO_LIB_SUFFIX)
-							#pragma comment(lib, "libssl32" POCO_LIB_SUFFIX)
-						#endif
-					#else
-						#pragma comment(lib, "libeay32" POCO_LIB_SUFFIX)
-						#pragma comment(lib, "ssleay32" POCO_LIB_SUFFIX)
-					#endif
-				#endif
-			#elif POCO_EXTERNAL_OPENSSL == POCO_EXTERNAL_OPENSSL_DEFAULT
-				#if OPENSSL_VERSION_PREREQ(1,1)
-					#pragma comment(lib, "libcrypto.lib")
-					#pragma comment(lib, "libssl.lib")
-				#else
-					#pragma comment(lib, "libeay32.lib")
-					#pragma comment(lib, "ssleay32.lib")
-				#endif
-			#endif
-		#endif // POCO_INTERNAL_OPENSSL_MSVC_VER
-		#if !defined(Crypto_EXPORTS)
-			#pragma comment(lib, "PocoCrypto" POCO_LIB_SUFFIX)
-		#endif
-	#endif // POCO_NO_AUTOMATIC_LIBS
-#endif
 
 
-namespace Poco {
-namespace Crypto {
+namespace Poco
+{
+namespace Crypto
+{
 
 
-void Crypto_API initializeCrypto();
-	/// Initialize the Crypto library, as well as the underlying OpenSSL
-	/// libraries, by calling OpenSSLInitializer::initialize().
-	///
-	/// Should be called before using any class from the Crypto library.
-	/// The Crypto library will be initialized automatically, through
-	/// OpenSSLInitializer instances held by various Crypto classes
-	/// (Cipher, CipherKey, RSAKey, X509Certificate).
-	/// However, it is recommended to call initializeCrypto()
-	/// in any case at application startup.
-	///
-	/// Can be called multiple times; however, for every call to
-	/// initializeCrypto(), a matching call to uninitializeCrypto()
-	/// must be performed.
+    void Crypto_API initializeCrypto();
+    /// Initialize the Crypto library, as well as the underlying OpenSSL
+    /// libraries, by calling OpenSSLInitializer::initialize().
+    ///
+    /// Should be called before using any class from the Crypto library.
+    /// The Crypto library will be initialized automatically, through
+    /// OpenSSLInitializer instances held by various Crypto classes
+    /// (Cipher, CipherKey, RSAKey, X509Certificate).
+    /// However, it is recommended to call initializeCrypto()
+    /// in any case at application startup.
+    ///
+    /// Can be called multiple times; however, for every call to
+    /// initializeCrypto(), a matching call to uninitializeCrypto()
+    /// must be performed.
 
 
-void Crypto_API uninitializeCrypto();
-	/// Uninitializes the Crypto library by calling
-	/// OpenSSLInitializer::uninitialize().
+    void Crypto_API uninitializeCrypto();
+    /// Uninitializes the Crypto library by calling
+    /// OpenSSLInitializer::uninitialize().
 
 
-} } // namespace Poco::Crypto
+}
+} // namespace Poco::Crypto
 
 
 #endif // Crypto_Crypto_INCLUDED

base/poco/Crypto/include/Poco/Crypto/CryptoException.h

@@ -23,34 +23,37 @@
 #include "Poco/Exception.h"
 
 
-namespace Poco {
-namespace Crypto {
-
-
-POCO_DECLARE_EXCEPTION(Crypto_API, CryptoException, Poco::Exception)
-
-
-class Crypto_API OpenSSLException : public CryptoException
+namespace Poco
+{
+namespace Crypto
 {
-public:
-	OpenSSLException(int code = 0);
-	OpenSSLException(const std::string& msg, int code = 0);
-	OpenSSLException(const std::string& msg, const std::string& arg, int code = 0);
-	OpenSSLException(const std::string& msg, const Poco::Exception& exc, int code = 0);
-	OpenSSLException(const OpenSSLException& exc);
-	~OpenSSLException() throw();
-	OpenSSLException& operator = (const OpenSSLException& exc);
-	const char* name() const throw();
-	const char* className() const throw();
-	Poco::Exception* clone() const;
-	void rethrow() const;
-
-private:
-	void setExtMessage();
-};
 
 
-} } // namespace Poco::Crypto
+    POCO_DECLARE_EXCEPTION(Crypto_API, CryptoException, Poco::Exception)
+
+
+    class Crypto_API OpenSSLException : public CryptoException
+    {
+    public:
+        OpenSSLException(int code = 0);
+        OpenSSLException(const std::string & msg, int code = 0);
+        OpenSSLException(const std::string & msg, const std::string & arg, int code = 0);
+        OpenSSLException(const std::string & msg, const Poco::Exception & exc, int code = 0);
+        OpenSSLException(const OpenSSLException & exc);
+        ~OpenSSLException() throw();
+        OpenSSLException & operator=(const OpenSSLException & exc);
+        const char * name() const throw();
+        const char * className() const throw();
+        Poco::Exception * clone() const;
+        void rethrow() const;
+
+    private:
+        void setExtMessage();
+    };
+
+
+}
+} // namespace Poco::Crypto
 
 
 #endif // Crypto_CryptoException_INCLUDED

base/poco/Crypto/include/Poco/Crypto/CryptoStream.h

@@ -19,174 +19,177 @@
 #define Crypto_CryptoStream_INCLUDED
 
 
-#include "Poco/Crypto/Crypto.h"
-#include "Poco/BufferedStreamBuf.h"
-#include "Poco/Buffer.h"
 #include <iostream>
+#include "Poco/Buffer.h"
+#include "Poco/BufferedStreamBuf.h"
+#include "Poco/Crypto/Crypto.h"
 
 
-namespace Poco {
-namespace Crypto {
-
-
-class CryptoTransform;
-class Cipher;
-
-
-class Crypto_API CryptoStreamBuf: public Poco::BufferedStreamBuf
-	/// This stream buffer performs cryptographic transformation on the data
-	/// going through it.
+namespace Poco
 {
-public:
-	CryptoStreamBuf(std::istream& istr, CryptoTransform* pTransform, std::streamsize bufferSize = 8192);
-	CryptoStreamBuf(std::ostream& ostr, CryptoTransform* pTransform, std::streamsize bufferSize = 8192);
-
-	virtual ~CryptoStreamBuf();
-
-	void close();
-		/// Flushes all buffers and finishes the encryption.
-
-protected:
-	int readFromDevice(char* buffer, std::streamsize length);
-	int writeToDevice(const char* buffer, std::streamsize length);
-
-private:
-	CryptoTransform* _pTransform;
-	std::istream*	 _pIstr;
-	std::ostream*	 _pOstr;
-	bool			 _eof;
-
-	Poco::Buffer<unsigned char> _buffer;
-
-	CryptoStreamBuf(const CryptoStreamBuf&);
-	CryptoStreamBuf& operator = (const CryptoStreamBuf&);
-};
-
-
-class Crypto_API CryptoIOS: public virtual std::ios
-	/// The base class for CryptoInputStream and CryptoOutputStream.
-	///
-	/// This class is needed to ensure correct initialization order of the
-	/// stream buffer and base classes.
+namespace Crypto
 {
-public:
-	CryptoIOS(std::istream& istr, CryptoTransform* pTransform, std::streamsize bufferSize = 8192);
-	CryptoIOS(std::ostream& ostr, CryptoTransform* pTransform, std::streamsize bufferSize = 8192);
-	~CryptoIOS();
-	CryptoStreamBuf* rdbuf();
-
-protected:
-	CryptoStreamBuf _buf;
-};
 
 
-class Crypto_API CryptoInputStream: public CryptoIOS, public std::istream
-	/// This stream transforms all data passing through it using the given
-	/// CryptoTransform.
-	///
-	/// Use a CryptoTransform object provided by Cipher::createEncrytor() or
-	/// Cipher::createDecryptor() to create an encrypting or decrypting stream,
-	/// respectively.
-{
-public:
-	CryptoInputStream(std::istream& istr, CryptoTransform* pTransform, std::streamsize bufferSize = 8192);
-		/// Create a new CryptoInputStream object. The CryptoInputStream takes the
-		/// ownership of the given CryptoTransform object.
-
-	CryptoInputStream(std::istream& istr, Cipher& cipher, std::streamsize bufferSize = 8192);
-		/// Create a new encrypting CryptoInputStream object using the given cipher.
-
-	~CryptoInputStream();
-		/// Destroys the CryptoInputStream.
-};
+    class CryptoTransform;
+    class Cipher;
 
 
-class Crypto_API CryptoOutputStream: public CryptoIOS, public std::ostream
-	/// This stream transforms all data passing through it using the given
-	/// CryptoTransform.
-	///
-	/// Use a CryptoTransform object provided by Cipher::createEncrytor() or
-	/// Cipher::createDecryptor() to create an encrypting or decrypting stream,
-	/// respectively.
-	///
-	/// After all data has been passed through the stream, close() must be called
-	/// to ensure completion of cryptographic transformation.
-{
-public:
-	CryptoOutputStream(std::ostream& ostr, CryptoTransform* pTransform, std::streamsize bufferSize = 8192);
-		/// Create a new CryptoOutputStream object. The CryptoOutputStream takes the
-		/// ownership of the given CryptoTransform object.
+    class Crypto_API CryptoStreamBuf : public Poco::BufferedStreamBuf
+    /// This stream buffer performs cryptographic transformation on the data
+    /// going through it.
+    {
+    public:
+        CryptoStreamBuf(std::istream & istr, CryptoTransform * pTransform, std::streamsize bufferSize = 8192);
+        CryptoStreamBuf(std::ostream & ostr, CryptoTransform * pTransform, std::streamsize bufferSize = 8192);
 
-	CryptoOutputStream(std::ostream& ostr, Cipher& cipher, std::streamsize bufferSize = 8192);
-		/// Create a new decrypting CryptoOutputStream object using the given cipher.
+        virtual ~CryptoStreamBuf();
 
-	~CryptoOutputStream();
-		/// Destroys the CryptoOutputStream.
+        void close();
+        /// Flushes all buffers and finishes the encryption.
 
-	void close();
-		/// Flushes all buffers and finishes the encryption.
-};
+    protected:
+        int readFromDevice(char * buffer, std::streamsize length);
+        int writeToDevice(const char * buffer, std::streamsize length);
+
+    private:
+        CryptoTransform * _pTransform;
+        std::istream * _pIstr;
+        std::ostream * _pOstr;
+        bool _eof;
+
+        Poco::Buffer<unsigned char> _buffer;
+
+        CryptoStreamBuf(const CryptoStreamBuf &);
+        CryptoStreamBuf & operator=(const CryptoStreamBuf &);
+    };
 
 
-class Crypto_API DecryptingInputStream: public CryptoIOS, public std::istream
-	/// This stream decrypts all data passing through it using the given
-	/// Cipher.
-{
-public:
-	DecryptingInputStream(std::istream& istr, Cipher& cipher, std::streamsize bufferSize = 8192);
-		/// Create a new DecryptingInputStream object using the given cipher.
+    class Crypto_API CryptoIOS : public virtual std::ios
+    /// The base class for CryptoInputStream and CryptoOutputStream.
+    ///
+    /// This class is needed to ensure correct initialization order of the
+    /// stream buffer and base classes.
+    {
+    public:
+        CryptoIOS(std::istream & istr, CryptoTransform * pTransform, std::streamsize bufferSize = 8192);
+        CryptoIOS(std::ostream & ostr, CryptoTransform * pTransform, std::streamsize bufferSize = 8192);
+        ~CryptoIOS();
+        CryptoStreamBuf * rdbuf();
 
-	~DecryptingInputStream();
-		/// Destroys the DecryptingInputStream.
-};
+    protected:
+        CryptoStreamBuf _buf;
+    };
 
 
-class Crypto_API DecryptingOutputStream: public CryptoIOS, public std::ostream
-	/// This stream decrypts all data passing through it using the given
-	/// Cipher.
-{
-public:
-	DecryptingOutputStream(std::ostream& ostr, Cipher& cipher, std::streamsize bufferSize = 8192);
-		/// Create a new DecryptingOutputStream object using the given cipher.
+    class Crypto_API CryptoInputStream : public CryptoIOS, public std::istream
+    /// This stream transforms all data passing through it using the given
+    /// CryptoTransform.
+    ///
+    /// Use a CryptoTransform object provided by Cipher::createEncrytor() or
+    /// Cipher::createDecryptor() to create an encrypting or decrypting stream,
+    /// respectively.
+    {
+    public:
+        CryptoInputStream(std::istream & istr, CryptoTransform * pTransform, std::streamsize bufferSize = 8192);
+        /// Create a new CryptoInputStream object. The CryptoInputStream takes the
+        /// ownership of the given CryptoTransform object.
 
-	~DecryptingOutputStream();
-		/// Destroys the DecryptingOutputStream.
+        CryptoInputStream(std::istream & istr, Cipher & cipher, std::streamsize bufferSize = 8192);
+        /// Create a new encrypting CryptoInputStream object using the given cipher.
 
-	void close();
-		/// Flushes all buffers and finishes the decryption.
-};
+        ~CryptoInputStream();
+        /// Destroys the CryptoInputStream.
+    };
 
 
-class Crypto_API EncryptingInputStream: public CryptoIOS, public std::istream
-	/// This stream encrypts all data passing through it using the given
-	/// Cipher.
-{
-public:
-	EncryptingInputStream(std::istream& istr, Cipher& cipher, std::streamsize bufferSize = 8192);
-		/// Create a new EncryptingInputStream object using the given cipher.
+    class Crypto_API CryptoOutputStream : public CryptoIOS, public std::ostream
+    /// This stream transforms all data passing through it using the given
+    /// CryptoTransform.
+    ///
+    /// Use a CryptoTransform object provided by Cipher::createEncrytor() or
+    /// Cipher::createDecryptor() to create an encrypting or decrypting stream,
+    /// respectively.
+    ///
+    /// After all data has been passed through the stream, close() must be called
+    /// to ensure completion of cryptographic transformation.
+    {
+    public:
+        CryptoOutputStream(std::ostream & ostr, CryptoTransform * pTransform, std::streamsize bufferSize = 8192);
+        /// Create a new CryptoOutputStream object. The CryptoOutputStream takes the
+        /// ownership of the given CryptoTransform object.
 
-	~EncryptingInputStream();
-		/// Destroys the EncryptingInputStream.
-};
+        CryptoOutputStream(std::ostream & ostr, Cipher & cipher, std::streamsize bufferSize = 8192);
+        /// Create a new decrypting CryptoOutputStream object using the given cipher.
+
+        ~CryptoOutputStream();
+        /// Destroys the CryptoOutputStream.
+
+        void close();
+        /// Flushes all buffers and finishes the encryption.
+    };
 
 
-class Crypto_API EncryptingOutputStream: public CryptoIOS, public std::ostream
-	/// This stream encrypts all data passing through it using the given
-	/// Cipher.
-{
-public:
-	EncryptingOutputStream(std::ostream& ostr, Cipher& cipher, std::streamsize bufferSize = 8192);
-		/// Create a new EncryptingOutputStream object using the given cipher.
+    class Crypto_API DecryptingInputStream : public CryptoIOS, public std::istream
+    /// This stream decrypts all data passing through it using the given
+    /// Cipher.
+    {
+    public:
+        DecryptingInputStream(std::istream & istr, Cipher & cipher, std::streamsize bufferSize = 8192);
+        /// Create a new DecryptingInputStream object using the given cipher.
 
-	~EncryptingOutputStream();
-		/// Destroys the EncryptingOutputStream.
-
-	void close();
-		/// Flushes all buffers and finishes the encryption.
-};
+        ~DecryptingInputStream();
+        /// Destroys the DecryptingInputStream.
+    };
 
 
-} } // namespace Poco::Crypto
+    class Crypto_API DecryptingOutputStream : public CryptoIOS, public std::ostream
+    /// This stream decrypts all data passing through it using the given
+    /// Cipher.
+    {
+    public:
+        DecryptingOutputStream(std::ostream & ostr, Cipher & cipher, std::streamsize bufferSize = 8192);
+        /// Create a new DecryptingOutputStream object using the given cipher.
+
+        ~DecryptingOutputStream();
+        /// Destroys the DecryptingOutputStream.
+
+        void close();
+        /// Flushes all buffers and finishes the decryption.
+    };
+
+
+    class Crypto_API EncryptingInputStream : public CryptoIOS, public std::istream
+    /// This stream encrypts all data passing through it using the given
+    /// Cipher.
+    {
+    public:
+        EncryptingInputStream(std::istream & istr, Cipher & cipher, std::streamsize bufferSize = 8192);
+        /// Create a new EncryptingInputStream object using the given cipher.
+
+        ~EncryptingInputStream();
+        /// Destroys the EncryptingInputStream.
+    };
+
+
+    class Crypto_API EncryptingOutputStream : public CryptoIOS, public std::ostream
+    /// This stream encrypts all data passing through it using the given
+    /// Cipher.
+    {
+    public:
+        EncryptingOutputStream(std::ostream & ostr, Cipher & cipher, std::streamsize bufferSize = 8192);
+        /// Create a new EncryptingOutputStream object using the given cipher.
+
+        ~EncryptingOutputStream();
+        /// Destroys the EncryptingOutputStream.
+
+        void close();
+        /// Flushes all buffers and finishes the encryption.
+    };
+
+
+}
+} // namespace Poco::Crypto
 
 
 #endif // Crypto_CryptoStream_INCLUDED

base/poco/Crypto/include/Poco/Crypto/CryptoTransform.h

@@ -18,70 +18,71 @@
 #define Crypto_CryptoTransform_INCLUDED
 
 
-#include "Poco/Crypto/Crypto.h"
 #include <ios>
+#include "Poco/Crypto/Crypto.h"
 
 
-namespace Poco {
-namespace Crypto {
-
-
-class Crypto_API CryptoTransform
-	/// This interface represents the basic operations for cryptographic
-	/// transformations to be used with a CryptoInputStream or a
-	/// CryptoOutputStream.
-	///
-	/// Implementations of this class are returned by the Cipher class to
-	/// perform encryption or decryption of data.
+namespace Poco
+{
+namespace Crypto
 {
-public:
-	CryptoTransform();
-		/// Creates a new CryptoTransform object.
-
-	virtual ~CryptoTransform();
-		/// Destroys the CryptoTransform.
-
-	virtual std::size_t blockSize() const = 0;
-		/// Returns the block size for this CryptoTransform.
-
-	virtual int setPadding(int padding);
-		/// Enables or disables padding. By default encryption operations are padded using standard block
-		/// padding and the padding is checked and removed when decrypting. If the padding parameter is zero then
-		/// no padding is performed, the total amount of data encrypted or decrypted must then be a multiple of
-		/// the block size or an error will occur.
-
-	virtual std::string getTag(std::size_t tagSize = 16) = 0;
-		/// Returns the GCM tag after encrypting using GCM mode.
-		///
-		/// Must be called after finalize().
-
-	virtual void setTag(const std::string& tag) = 0;
-		/// Sets the GCM tag for authenticated decryption using GCM mode.
-		///
-		/// Must be set before finalize() is called, otherwise
-		/// decryption will fail.
-
-	virtual std::streamsize transform(
-		const unsigned char* input,
-		std::streamsize		 inputLength,
-		unsigned char*		 output,
-		std::streamsize		 outputLength) = 0;
-		/// Transforms a chunk of data. The inputLength is arbitrary and does not
-		/// need to be a multiple of the block size. The output buffer has a maximum
-		/// capacity of the given outputLength that must be at least
-		///   inputLength + blockSize() - 1
-		/// Returns the number of bytes written to the output buffer.
-
-	virtual std::streamsize finalize(unsigned char* output, std::streamsize length) = 0;
-		/// Finalizes the transformation. The output buffer must contain enough
-		/// space for at least two blocks, ie.
-		///   length >= 2*blockSize()
-		/// must be true.  Returns the number of bytes written to the output
-		/// buffer.
-};
 
 
-} } // namespace Poco::Crypto
+    class Crypto_API CryptoTransform
+    /// This interface represents the basic operations for cryptographic
+    /// transformations to be used with a CryptoInputStream or a
+    /// CryptoOutputStream.
+    ///
+    /// Implementations of this class are returned by the Cipher class to
+    /// perform encryption or decryption of data.
+    {
+    public:
+        CryptoTransform();
+        /// Creates a new CryptoTransform object.
+
+        virtual ~CryptoTransform();
+        /// Destroys the CryptoTransform.
+
+        virtual std::size_t blockSize() const = 0;
+        /// Returns the block size for this CryptoTransform.
+
+        virtual int setPadding(int padding);
+        /// Enables or disables padding. By default encryption operations are padded using standard block
+        /// padding and the padding is checked and removed when decrypting. If the padding parameter is zero then
+        /// no padding is performed, the total amount of data encrypted or decrypted must then be a multiple of
+        /// the block size or an error will occur.
+
+        virtual std::string getTag(std::size_t tagSize = 16) = 0;
+        /// Returns the GCM tag after encrypting using GCM mode.
+        ///
+        /// Must be called after finalize().
+
+        virtual void setTag(const std::string & tag) = 0;
+        /// Sets the GCM tag for authenticated decryption using GCM mode.
+        ///
+        /// Must be set before finalize() is called, otherwise
+        /// decryption will fail.
+
+        virtual std::streamsize
+        transform(const unsigned char * input, std::streamsize inputLength, unsigned char * output, std::streamsize outputLength)
+            = 0;
+        /// Transforms a chunk of data. The inputLength is arbitrary and does not
+        /// need to be a multiple of the block size. The output buffer has a maximum
+        /// capacity of the given outputLength that must be at least
+        ///   inputLength + blockSize() - 1
+        /// Returns the number of bytes written to the output buffer.
+
+        virtual std::streamsize finalize(unsigned char * output, std::streamsize length) = 0;
+        /// Finalizes the transformation. The output buffer must contain enough
+        /// space for at least two blocks, ie.
+        ///   length >= 2*blockSize()
+        /// must be true.  Returns the number of bytes written to the output
+        /// buffer.
+    };
+
+
+}
+} // namespace Poco::Crypto
 
 
 #endif // Crypto_CryptoTransform_INCLUDED

base/poco/Crypto/include/Poco/Crypto/DigestEngine.h

@@ -18,63 +18,66 @@
 #define Crypto_DigestEngine_INCLUDED
 
 
+#include <openssl/evp.h>
 #include "Poco/Crypto/Crypto.h"
 #include "Poco/Crypto/OpenSSLInitializer.h"
 #include "Poco/DigestEngine.h"
-#include <openssl/evp.h>
 
 
-namespace Poco {
-namespace Crypto {
-
-
-class Crypto_API DigestEngine: public Poco::DigestEngine
-	/// This class implements a Poco::DigestEngine for all
-	/// digest algorithms supported by OpenSSL.
+namespace Poco
 {
-public:
-	DigestEngine(const std::string& name);
-		/// Creates a DigestEngine using the digest with the given name
-		/// (e.g., "MD5", "SHA1", "SHA256", "SHA512", etc.).
-		/// See the OpenSSL documentation for a list of supported digest algorithms.
-		///
-		/// Throws a Poco::NotFoundException if no algorithm with the given name exists.
-
-	~DigestEngine();
-		/// Destroys the DigestEngine.
-
-	const std::string& algorithm() const;
-		/// Returns the name of the digest algorithm.
-
-	int nid() const;
-		/// Returns the NID (OpenSSL object identifier) of the digest algorithm.
-
-	// DigestEngine
-	std::size_t digestLength() const;
-	void reset();
-	const Poco::DigestEngine::Digest& digest();
-
-protected:
-	void updateImpl(const void* data, std::size_t length);
-
-private:
-	std::string _name;
-	EVP_MD_CTX* _pContext;
-	Poco::DigestEngine::Digest _digest;
-	OpenSSLInitializer _openSSLInitializer;
-};
-
-
-//
-// inlines
-//
-inline const std::string& DigestEngine::algorithm() const
+namespace Crypto
 {
-	return _name;
+
+
+    class Crypto_API DigestEngine : public Poco::DigestEngine
+    /// This class implements a Poco::DigestEngine for all
+    /// digest algorithms supported by OpenSSL.
+    {
+    public:
+        DigestEngine(const std::string & name);
+        /// Creates a DigestEngine using the digest with the given name
+        /// (e.g., "MD5", "SHA1", "SHA256", "SHA512", etc.).
+        /// See the OpenSSL documentation for a list of supported digest algorithms.
+        ///
+        /// Throws a Poco::NotFoundException if no algorithm with the given name exists.
+
+        ~DigestEngine();
+        /// Destroys the DigestEngine.
+
+        const std::string & algorithm() const;
+        /// Returns the name of the digest algorithm.
+
+        int nid() const;
+        /// Returns the NID (OpenSSL object identifier) of the digest algorithm.
+
+        // DigestEngine
+        std::size_t digestLength() const;
+        void reset();
+        const Poco::DigestEngine::Digest & digest();
+
+    protected:
+        void updateImpl(const void * data, std::size_t length);
+
+    private:
+        std::string _name;
+        EVP_MD_CTX * _pContext;
+        Poco::DigestEngine::Digest _digest;
+        OpenSSLInitializer _openSSLInitializer;
+    };
+
+
+    //
+    // inlines
+    //
+    inline const std::string & DigestEngine::algorithm() const
+    {
+        return _name;
+    }
+
+
 }
-
-
-} } // namespace Poco::Crypto
+} // namespace Poco::Crypto
 
 
 #endif // Crypto_DigestEngine_INCLUDED

base/poco/Crypto/include/Poco/Crypto/ECDSADigestEngine.h

@@ -19,83 +19,85 @@
 #define Crypto_ECDSADigestEngine_INCLUDED
 
 
-#include "Poco/Crypto/Crypto.h"
-#include "Poco/Crypto/ECKey.h"
-#include "Poco/DigestEngine.h"
-#include "Poco/Crypto/DigestEngine.h"
 #include <istream>
 #include <ostream>
+#include "Poco/Crypto/Crypto.h"
+#include "Poco/Crypto/DigestEngine.h"
+#include "Poco/Crypto/ECKey.h"
+#include "Poco/DigestEngine.h"
 
 
-namespace Poco {
-namespace Crypto {
-
-
-class Crypto_API ECDSADigestEngine: public Poco::DigestEngine
-	/// This class implements a Poco::DigestEngine that can be
-	/// used to compute a secure digital signature.
-	///
-	/// First another Poco::Crypto::DigestEngine is created and
-	/// used to compute a cryptographic hash of the data to be
-	/// signed. Then, the hash value is encrypted, using
-	/// the ECDSA private key.
-	///
-	/// To verify a signature, pass it to the verify() 
-	/// member function. It will decrypt the signature
-	/// using the ECDSA public key and compare the resulting
-	/// hash with the actual hash of the data.
+namespace Poco
+{
+namespace Crypto
 {
-public:
-
-	ECDSADigestEngine(const ECKey& key, const std::string &name);
-		/// Creates the ECDSADigestEngine with the given ECDSA key,
-		/// using the hash algorithm with the given name
-		/// (e.g., "SHA1", "SHA256", "SHA512", etc.).
-		/// See the OpenSSL documentation for a list of supported digest algorithms.
-		///
-		/// Throws a Poco::NotFoundException if no algorithm with the given name exists.
-
-	~ECDSADigestEngine();
-		/// Destroys the ECDSADigestEngine.
-
-	std::size_t digestLength() const;
-		/// Returns the length of the digest in bytes.
-
-	void reset();
-		/// Resets the engine so that a new
-		/// digest can be computed.
-		
-	const DigestEngine::Digest& digest();
-		/// Finishes the computation of the digest 
-		/// (the first time it's called) and
-		/// returns the message digest. 
-		///
-		/// Can be called multiple times.
-
-	const DigestEngine::Digest& signature();
-		/// Signs the digest using the ECDSADSA algorithm
-		/// and the private key (the first time it's
-		/// called) and returns the result.
-		///
-		/// Can be called multiple times.
-
-	bool verify(const DigestEngine::Digest& signature);
-		/// Verifies the data against the signature.
-		///
-		/// Returns true if the signature can be verified, false otherwise.
-
-protected:
-	void updateImpl(const void* data, std::size_t length);
-
-private:
-	ECKey _key;
-	Poco::Crypto::DigestEngine _engine;
-	Poco::DigestEngine::Digest _digest;
-	Poco::DigestEngine::Digest _signature;
-};
 
 
-} } // namespace Poco::Crypto
+    class Crypto_API ECDSADigestEngine : public Poco::DigestEngine
+    /// This class implements a Poco::DigestEngine that can be
+    /// used to compute a secure digital signature.
+    ///
+    /// First another Poco::Crypto::DigestEngine is created and
+    /// used to compute a cryptographic hash of the data to be
+    /// signed. Then, the hash value is encrypted, using
+    /// the ECDSA private key.
+    ///
+    /// To verify a signature, pass it to the verify()
+    /// member function. It will decrypt the signature
+    /// using the ECDSA public key and compare the resulting
+    /// hash with the actual hash of the data.
+    {
+    public:
+        ECDSADigestEngine(const ECKey & key, const std::string & name);
+        /// Creates the ECDSADigestEngine with the given ECDSA key,
+        /// using the hash algorithm with the given name
+        /// (e.g., "SHA1", "SHA256", "SHA512", etc.).
+        /// See the OpenSSL documentation for a list of supported digest algorithms.
+        ///
+        /// Throws a Poco::NotFoundException if no algorithm with the given name exists.
+
+        ~ECDSADigestEngine();
+        /// Destroys the ECDSADigestEngine.
+
+        std::size_t digestLength() const;
+        /// Returns the length of the digest in bytes.
+
+        void reset();
+        /// Resets the engine so that a new
+        /// digest can be computed.
+
+        const DigestEngine::Digest & digest();
+        /// Finishes the computation of the digest
+        /// (the first time it's called) and
+        /// returns the message digest.
+        ///
+        /// Can be called multiple times.
+
+        const DigestEngine::Digest & signature();
+        /// Signs the digest using the ECDSADSA algorithm
+        /// and the private key (the first time it's
+        /// called) and returns the result.
+        ///
+        /// Can be called multiple times.
+
+        bool verify(const DigestEngine::Digest & signature);
+        /// Verifies the data against the signature.
+        ///
+        /// Returns true if the signature can be verified, false otherwise.
+
+    protected:
+        void updateImpl(const void * data, std::size_t length);
+
+    private:
+        ECKey _key;
+        Poco::Crypto::DigestEngine _engine;
+        Poco::DigestEngine::Digest _digest;
+        Poco::DigestEngine::Digest _signature;
+    };
+
+
+}
+} // namespace Poco::Crypto
 
 
 #endif // Crypto_ECDSADigestEngine_INCLUDED