Merge branch 'master' into riscv-in-ci

.clang-format

@@ -21,7 +21,6 @@ ConstructorInitializerAllOnOneLineOrOnePerLine: true
 ExperimentalAutoDetectBinPacking: true
 UseTab: Never
 TabWidth: 4
-IndentWidth: 4
 Standard: Cpp11
 PointerAlignment: Middle
 MaxEmptyLinesToKeep: 2
@@ -75,6 +74,7 @@ ConstructorInitializerIndentWidth: 4
 ContinuationIndentWidth: 4
 DerivePointerAlignment: false
 DisableFormat: false
+IndentRequiresClause: false
 IndentWidth: 4
 IndentWrappedFunctionNames: false
 MacroBlockBegin: ''

.clang-tidy

@@ -1,6 +1,14 @@
-# Enable all checks + disale selected checks. Feel free to remove disabled checks from below list if
+# To run clang-tidy from CMake, build ClickHouse with -DENABLE_CLANG_TIDY=1. To show all warnings, it is
+# recommended to pass "-k0" to Ninja.
+
+# Enable all checks + disable selected checks. Feel free to remove disabled checks from below list if
 # a) the new check is not controversial (this includes many checks in readability-* and google-*) or
 # b) too noisy (checks with > 100 new warnings are considered noisy, this includes e.g. cppcoreguidelines-*).
+
+# TODO Let clang-tidy check headers in further directories
+#      --> HeaderFilterRegex: '^.*/(src|base|programs|utils)/.*(h|hpp)$'
+HeaderFilterRegex: '^.*/(base)/.*(h|hpp)$'
+
 Checks: '*,
     -abseil-*,
 
@@ -8,14 +16,19 @@ Checks: '*,
 
     -android-*,
 
+    -bugprone-assignment-in-if-condition,
     -bugprone-branch-clone,
     -bugprone-easily-swappable-parameters,
     -bugprone-exception-escape,
     -bugprone-implicit-widening-of-multiplication-result,
     -bugprone-narrowing-conversions,
     -bugprone-not-null-terminated-result,
+    -bugprone-reserved-identifier, # useful but too slow, TODO retry when https://reviews.llvm.org/rG1c282052624f9d0bd273bde0b47b30c96699c6c7 is merged
+    -bugprone-unchecked-optional-access,
 
     -cert-dcl16-c,
+    -cert-dcl37-c,
+    -cert-dcl51-cpp,
     -cert-err58-cpp,
     -cert-msc32-c,
     -cert-msc51-cpp,
@@ -24,11 +37,12 @@ Checks: '*,
 
     -clang-analyzer-optin.performance.Padding,
     -clang-analyzer-optin.portability.UnixAPI,
-
     -clang-analyzer-security.insecureAPI.bzero,
     -clang-analyzer-security.insecureAPI.strcpy,
 
     -cppcoreguidelines-avoid-c-arrays,
+    -cppcoreguidelines-avoid-const-or-ref-data-members,
+    -cppcoreguidelines-avoid-do-while,
     -cppcoreguidelines-avoid-goto,
     -cppcoreguidelines-avoid-magic-numbers,
     -cppcoreguidelines-avoid-non-const-global-variables,
@@ -54,8 +68,6 @@ Checks: '*,
     -cppcoreguidelines-slicing,
     -cppcoreguidelines-special-member-functions,
 
-    -concurrency-mt-unsafe,
-
     -darwin-*,
 
     -fuchsia-*,
@@ -95,11 +107,15 @@ Checks: '*,
 
     -openmp-*,
 
+    -misc-const-correctness,
     -misc-no-recursion,
     -misc-non-private-member-variables-in-classes,
+    -misc-confusable-identifiers, # useful but slooow
+    -misc-use-anonymous-namespace,
 
     -modernize-avoid-c-arrays,
     -modernize-concat-nested-namespaces,
+    -modernize-macro-to-enum,
     -modernize-pass-by-value,
     -modernize-return-braced-init-list,
     -modernize-use-auto,
@@ -116,68 +132,49 @@ Checks: '*,
     -portability-simd-intrinsics,
 
     -readability-braces-around-statements,
+    -readability-convert-member-functions-to-static,
     -readability-else-after-return,
     -readability-function-cognitive-complexity,
     -readability-function-size,
     -readability-identifier-length,
+    -readability-identifier-naming, # useful but too slow
     -readability-implicit-bool-conversion,
     -readability-isolate-declaration,
     -readability-magic-numbers,
     -readability-named-parameter,
     -readability-redundant-declaration,
+    -readability-simplify-boolean-expr,
     -readability-static-accessed-through-instance,
     -readability-suspicious-call-argument,
     -readability-uppercase-literal-suffix,
     -readability-use-anyofallof,
 
-    -zirkon-*,
+    -zircon-*,
 '
 
 WarningsAsErrors: '*'
 
 CheckOptions:
-  - key: readability-identifier-naming.ClassCase
-    value: CamelCase
-  - key: readability-identifier-naming.EnumCase
-    value: CamelCase
-  - key: readability-identifier-naming.LocalVariableCase
-    value: lower_case
-  - key: readability-identifier-naming.StaticConstantCase
-    value: aNy_CasE
-  - key: readability-identifier-naming.MemberCase
-    value: lower_case
-  - key: readability-identifier-naming.PrivateMemberPrefix
-    value: ''
-  - key: readability-identifier-naming.ProtectedMemberPrefix
-    value: ''
-  - key: readability-identifier-naming.PublicMemberCase
-    value: lower_case
-  - key: readability-identifier-naming.MethodCase
-    value: camelBack
-  - key: readability-identifier-naming.PrivateMethodPrefix
-    value: ''
-  - key: readability-identifier-naming.ProtectedMethodPrefix
-    value: ''
-  - key: readability-identifier-naming.ParameterPackCase
-    value: lower_case
-  - key: readability-identifier-naming.StructCase
-    value: CamelCase
-  - key: readability-identifier-naming.TemplateTemplateParameterCase
-    value: CamelCase
-  - key: readability-identifier-naming.TemplateUsingCase
-    value: lower_case
-  - key: readability-identifier-naming.TypeTemplateParameterCase
-    value: CamelCase
-  - key: readability-identifier-naming.TypedefCase
-    value: CamelCase
-  - key: readability-identifier-naming.UnionCase
-    value: CamelCase
-  - key: readability-identifier-naming.UsingCase
-    value: CamelCase
-  - key: modernize-loop-convert.UseCxx20ReverseRanges
-    value: false
-  - key: performance-move-const-arg.CheckTriviallyCopyableMove
-    value: false
-    # Workaround clang-tidy bug: https://github.com/llvm/llvm-project/issues/46097
-  - key: readability-identifier-naming.TypeTemplateParameterIgnoredRegexp
-    value: expr-type
+  readability-identifier-naming.ClassCase: CamelCase
+  readability-identifier-naming.EnumCase: CamelCase
+  readability-identifier-naming.LocalVariableCase: lower_case
+  readability-identifier-naming.StaticConstantCase: aNy_CasE
+  readability-identifier-naming.MemberCase: lower_case
+  readability-identifier-naming.PrivateMemberPrefix: ''
+  readability-identifier-naming.ProtectedMemberPrefix: ''
+  readability-identifier-naming.PublicMemberCase: lower_case
+  readability-identifier-naming.MethodCase: camelBack
+  readability-identifier-naming.PrivateMethodPrefix: ''
+  readability-identifier-naming.ProtectedMethodPrefix: ''
+  readability-identifier-naming.ParameterPackCase: lower_case
+  readability-identifier-naming.StructCase: CamelCase
+  readability-identifier-naming.TemplateTemplateParameterCase: CamelCase
+  readability-identifier-naming.TemplateParameterCase: lower_case
+  readability-identifier-naming.TypeTemplateParameterCase: CamelCase
+  readability-identifier-naming.TypedefCase: CamelCase
+  readability-identifier-naming.UnionCase: CamelCase
+  modernize-loop-convert.UseCxx20ReverseRanges: false
+  performance-move-const-arg.CheckTriviallyCopyableMove: false
+  # Workaround clang-tidy bug: https://github.com/llvm/llvm-project/issues/46097
+  readability-identifier-naming.TypeTemplateParameterIgnoredRegexp: expr-type
+  cppcoreguidelines-avoid-do-while.IgnoreMacros: true

.clangd

@@ -0,0 +1,16 @@
+Diagnostics:
+    # clangd does parse .clang-tidy, but some checks are too slow to run in
+    # clang-tidy build, so let's enable them explicitly for clangd at least.
+    ClangTidy:
+        # The following checks had been disabled due to slowliness with C++23,
+        # for more details see [1].
+        #
+        #   [1]: https://github.com/llvm/llvm-project/issues/61418
+        #
+        # But the code base had been written in a style that had been checked
+        # by this check, so at least, let's enable it for clangd.
+        Add: [
+            # configured in .clang-tidy
+            readability-identifier-naming,
+            bugprone-reserved-identifier,
+        ]

.exrc

@@ -0,0 +1 @@
+au BufRead,BufNewFile * set tabstop=4 softtabstop=0 expandtab shiftwidth=4 smarttab tags=tags,../tags

.git-blame-ignore-revs

@@ -0,0 +1,15 @@
+# This is a file that can be used by git-blame to ignore some revisions.
+# (git 2.23+, released in August 2019)
+#
+# Can be configured as follow:
+#
+#     $ git config blame.ignoreRevsFile .git-blame-ignore-revs
+#
+# For more information you can look at git-blame(1) man page.
+
+# Changed tabs to spaces in code [#CLICKHOUSE-3]
+137ad95929ee016cc6d3c03bccb5586941c163ff
+
+# dbms/ → src/
+# (though it is unlikely that you will see it in blame)
+06446b4f08a142d6f1bc30664c47ded88ab51782

.gitattributes

@@ -1,3 +1,4 @@
 contrib/* linguist-vendored
 *.h linguist-language=C++
 tests/queries/0_stateless/data_json/* binary
+tests/queries/0_stateless/*.reference -crlf

.github/ISSUE_TEMPLATE/10_question.md

@@ -7,6 +7,6 @@ assignees: ''
 
 ---
 
-> Make sure to check documentation https://clickhouse.com/docs/en/ first. If the question is concise and probably has a short answer, asking it in Telegram chat https://telegram.me/clickhouse_en is probably the fastest way to find the answer. For more complicated questions, consider asking them on StackOverflow with "clickhouse" tag https://stackoverflow.com/questions/tagged/clickhouse 
+> Make sure to check documentation https://clickhouse.com/docs/en/ first. If the question is concise and probably has a short answer, asking it in [community Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-1gh9ds7f4-PgDhJAaF8ad5RbWBAAjzFg) is probably the fastest way to find the answer. For more complicated questions, consider asking them on StackOverflow with "clickhouse" tag https://stackoverflow.com/questions/tagged/clickhouse 
 
 > If you still prefer GitHub issues, remove all this text and ask your question here.

.github/ISSUE_TEMPLATE/85_bug-report.md

@@ -13,6 +13,8 @@ assignees: ''
 
 > A clear and concise description of what works not as it is supposed to.
 
+> A link to reproducer in [https://fiddle.clickhouse.com/](https://fiddle.clickhouse.com/).
+
 **Does it reproduce on recent release?**
 
 [The list of releases](https://github.com/ClickHouse/ClickHouse/blob/master/utils/list-versions/version_date.tsv)

.github/ISSUE_TEMPLATE/96_installation-issues.md

@@ -0,0 +1,31 @@
+---
+name: Installation issue
+about: Issue with ClickHouse installation from https://clickhouse.com/docs/en/install/
+title: ''
+labels: comp-install
+assignees: ''
+
+---
+
+**I have tried the following solutions**: https://clickhouse.com/docs/en/faq/troubleshooting/#troubleshooting-installation-errors
+
+**Installation type**
+
+Packages, docker, single binary, curl?
+
+**Source of the ClickHouse**
+
+A link to the source. Or the command you've tried
+
+**Expected result**
+
+What you expected
+
+**The actual result**
+
+What you get
+
+**How to reproduce**
+
+* For Linux-based operating systems: provide a script for clear docker container from the official image
+* For anything else: steps to reproduce on as much as possible clear system

.github/PULL_REQUEST_TEMPLATE.md

@@ -2,21 +2,40 @@
 A technical comment, you are free to remove or leave it as it is when PR is created
 The following categories are used in the next scripts, update them accordingly
 utils/changelog/changelog.py
-tests/ci/run_check.py
+tests/ci/cancel_and_rerun_workflow_lambda/app.py
 -->
 ### Changelog category (leave one):
 - New Feature
 - Improvement
-- Bug Fix (user-visible misbehavior in official stable or prestable release)
 - Performance Improvement
 - Backward Incompatible Change
 - Build/Testing/Packaging Improvement
 - Documentation (changelog entry is not required)
+- Bug Fix (user-visible misbehavior in an official stable release)
 - Not for changelog (changelog entry is not required)
 
 
 ### Changelog entry (a user-readable short description of the changes that goes to CHANGELOG.md):
 ...
 
+### Documentation entry for user-facing changes
+
+- [ ] Documentation is written (mandatory for new features)
+
+<!---
+Directly edit documentation source files in the "docs" folder with the same pull-request as code changes
+
+or
+
+Add a user-readable short description of the changes that should be added to docs.clickhouse.com below.
+
+At a minimum, the following information should be added (but add more as needed).
+- Motivation: Why is this function, table engine, etc. useful to ClickHouse users?
+
+- Parameters: If the feature being added takes arguments, options or is influenced by settings, please list them below with a brief explanation.
+
+- Example use: A query or command.
+-->
+
 
 > Information about CI checks: https://clickhouse.com/docs/en/development/continuous-integration/

.github/workflows/backport_branches.yml

@@ -9,49 +9,62 @@ on: # yamllint disable-line rule:truthy
     branches:
       - 'backport/**'
 jobs:
+  CheckLabels:
+    runs-on: [self-hosted, style-checker]
+    # Run the first check always, even if the CI is cancelled
+    if: ${{ always() }}
+    steps:
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+      - name: Labels check
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 run_check.py
   PythonUnitTests:
     runs-on: [self-hosted, style-checker]
+    needs: CheckLabels
     steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Python unit tests
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 -m unittest discover -s . -p '*_test.py'
   DockerHubPushAarch64:
     runs-on: [self-hosted, style-checker-aarch64]
+    needs: CheckLabels
     steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Images check
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 docker_images_check.py --suffix aarch64
       - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: changed_images_aarch64
           path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json
   DockerHubPushAmd64:
     runs-on: [self-hosted, style-checker]
+    needs: CheckLabels
     steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Images check
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 docker_images_check.py --suffix amd64
       - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: changed_images_amd64
           path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json
@@ -59,18 +72,17 @@ jobs:
     needs: [DockerHubPushAmd64, DockerHubPushAarch64]
     runs-on: [self-hosted, style-checker]
     steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Download changed aarch64 images
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images_aarch64
           path: ${{ runner.temp }}
       - name: Download changed amd64 images
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images_amd64
           path: ${{ runner.temp }}
@@ -79,11 +91,11 @@ jobs:
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64
       - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: changed_images
           path: ${{ runner.temp }}/changed_images.json
-  CompatibilityCheck:
+  CompatibilityCheckX86:
     needs: [BuilderDebRelease]
     runs-on: [self-hosted, style-checker]
     steps:
@@ -94,28 +106,56 @@ jobs:
           REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse
           REPORTS_PATH=${{runner.temp}}/reports_dir
           EOF
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Download json reports
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           path: ${{ env.REPORTS_PATH }}
-      - name: CompatibilityCheck
+      - name: CompatibilityCheckX86
         run: |
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py
+          cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions
       - name: Cleanup
         if: always()
         run: |
-          # shellcheck disable=SC2046
-          docker kill $(docker ps -q) ||:
-          # shellcheck disable=SC2046
-          docker rm -f $(docker ps -a -q) ||:
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH"
+  CompatibilityCheckAarch64:
+    needs: [BuilderDebAarch64]
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/compatibility_check
+          REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          EOF
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+      - name: Download json reports
+        uses: actions/download-artifact@v3
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: CompatibilityCheckAarch64
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH"
 #########################################################################################
 #################################### ORDINARY BUILDS ####################################
@@ -134,38 +174,33 @@ jobs:
           BUILD_NAME=package_release
           EOF
       - name: Download changed images
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images
           path: ${{ env.IMAGES_PATH }}
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
         with:
+          clear-repository: true
+          submodules: true
           fetch-depth: 0 # For a proper version and performance artifacts
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
           cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
       - name: Upload build URLs to artifacts
         if: ${{ success() || failure() }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: ${{ env.BUILD_URLS }}
           path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
-          # shellcheck disable=SC2046
-          docker kill $(docker ps -q) ||:
-          # shellcheck disable=SC2046
-          docker rm -f $(docker ps -a -q) ||:
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
   BuilderDebAarch64:
     needs: [DockerHubPush]
@@ -181,38 +216,33 @@ jobs:
           BUILD_NAME=package_aarch64
           EOF
       - name: Download changed images
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images
           path: ${{ env.IMAGES_PATH }}
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
         with:
+          clear-repository: true
+          submodules: true
           fetch-depth: 0 # For a proper version and performance artifacts
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
           cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
       - name: Upload build URLs to artifacts
         if: ${{ success() || failure() }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: ${{ env.BUILD_URLS }}
           path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
-          # shellcheck disable=SC2046
-          docker kill $(docker ps -q) ||:
-          # shellcheck disable=SC2046
-          docker rm -f $(docker ps -a -q) ||:
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
   BuilderDebAsan:
     needs: [DockerHubPush]
@@ -228,36 +258,32 @@ jobs:
           BUILD_NAME=package_asan
           EOF
       - name: Download changed images
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images
           path: ${{ env.IMAGES_PATH }}
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+          submodules: true
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
           cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
       - name: Upload build URLs to artifacts
         if: ${{ success() || failure() }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: ${{ env.BUILD_URLS }}
           path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
-          # shellcheck disable=SC2046
-          docker kill $(docker ps -q) ||:
-          # shellcheck disable=SC2046
-          docker rm -f $(docker ps -a -q) ||:
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
   BuilderDebTsan:
     needs: [DockerHubPush]
@@ -273,36 +299,32 @@ jobs:
           BUILD_NAME=package_tsan
           EOF
       - name: Download changed images
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images
           path: ${{ env.IMAGES_PATH }}
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+          submodules: true
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
           cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
       - name: Upload build URLs to artifacts
         if: ${{ success() || failure() }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: ${{ env.BUILD_URLS }}
           path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
-          # shellcheck disable=SC2046
-          docker kill $(docker ps -q) ||:
-          # shellcheck disable=SC2046
-          docker rm -f $(docker ps -a -q) ||:
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
   BuilderDebDebug:
     needs: [DockerHubPush]
@@ -318,36 +340,123 @@ jobs:
           BUILD_NAME=package_debug
           EOF
       - name: Download changed images
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images
           path: ${{ env.IMAGES_PATH }}
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+          submodules: true
+      - name: Apply sparse checkout for contrib # in order to check that it doesn't break build
+        run: |
+          rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed'
+          git -C "$GITHUB_WORKSPACE" checkout .  && echo 'restored'
+          "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK'
+          du -hs "$GITHUB_WORKSPACE/contrib" ||:
+          find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||:
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
           cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
       - name: Upload build URLs to artifacts
         if: ${{ success() || failure() }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: ${{ env.BUILD_URLS }}
           path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
-          # shellcheck disable=SC2046
-          docker kill $(docker ps -q) ||:
-          # shellcheck disable=SC2046
-          docker rm -f $(docker ps -a -q) ||:
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
+  BuilderBinDarwin:
+    needs: [DockerHubPush]
+    runs-on: [self-hosted, builder]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/build_check
+          IMAGES_PATH=${{runner.temp}}/images_path
+          REPO_COPY=${{runner.temp}}/build_check/ClickHouse
+          CACHES_PATH=${{runner.temp}}/../ccaches
+          BUILD_NAME=binary_darwin
+          EOF
+      - name: Download changed images
+        uses: actions/download-artifact@v3
+        with:
+          name: changed_images
+          path: ${{ env.IMAGES_PATH }}
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+          submodules: true
+          fetch-depth: 0 # otherwise we will have no info about contributors
+      - name: Build
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
+      - name: Upload build URLs to artifacts
+        if: ${{ success() || failure() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
+  BuilderBinDarwinAarch64:
+    needs: [DockerHubPush]
+    runs-on: [self-hosted, builder]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/build_check
+          IMAGES_PATH=${{runner.temp}}/images_path
+          REPO_COPY=${{runner.temp}}/build_check/ClickHouse
+          CACHES_PATH=${{runner.temp}}/../ccaches
+          BUILD_NAME=binary_darwin_aarch64
+          EOF
+      - name: Download changed images
+        uses: actions/download-artifact@v3
+        with:
+          name: changed_images
+          path: ${{ env.IMAGES_PATH }}
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+          submodules: true
+          fetch-depth: 0 # otherwise we will have no info about contributors
+      - name: Build
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
+      - name: Upload build URLs to artifacts
+        if: ${{ success() || failure() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
 ############################################################################################
 ##################################### Docker images  #######################################
@@ -358,26 +467,23 @@ jobs:
       - BuilderDebAarch64
     runs-on: [self-hosted, style-checker]
     steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
         with:
+          clear-repository: true
           fetch-depth: 0  # It MUST BE THE SAME for all dependencies and the job itself
       - name: Check docker clickhouse/clickhouse-server building
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 docker_server.py --release-type head --no-push
-          python3 docker_server.py --release-type head --no-push --no-ubuntu \
+          python3 docker_server.py --release-type head --no-push \
+            --image-repo clickhouse/clickhouse-server --image-path docker/server
+          python3 docker_server.py --release-type head --no-push \
             --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper
       - name: Cleanup
         if: always()
         run: |
-          # shellcheck disable=SC2046
-          docker kill $(docker ps -q) ||:
-          # shellcheck disable=SC2046
-          docker rm -f $(docker ps -a -q) ||:
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH"
 ############################################################################################
 ##################################### BUILD REPORTER #######################################
@@ -390,6 +496,7 @@ jobs:
       - BuilderDebTsan
       - BuilderDebDebug
     runs-on: [self-hosted, style-checker]
+    if: ${{ success() || failure() }}
     steps:
       - name: Set envs
         run: |
@@ -400,14 +507,13 @@ jobs:
           NEEDS_DATA_PATH=${{runner.temp}}/needs.json
           EOF
       - name: Download json reports
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           path: ${{ env.REPORTS_PATH }}
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Report Builder
         run: |
           sudo rm -fr "$TEMP_PATH"
@@ -420,10 +526,115 @@ jobs:
       - name: Cleanup
         if: always()
         run: |
-          # shellcheck disable=SC2046
-          docker kill $(docker ps -q) ||:
-          # shellcheck disable=SC2046
-          docker rm -f $(docker ps -a -q) ||:
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH"
+  BuilderSpecialReport:
+    needs:
+      - BuilderBinDarwin
+      - BuilderBinDarwinAarch64
+    runs-on: [self-hosted, style-checker]
+    if: ${{ success() || failure() }}
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/report_check
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          CHECK_NAME=ClickHouse special build check
+          NEEDS_DATA_PATH=${{runner.temp}}/needs.json
+          EOF
+      - name: Download json reports
+        uses: actions/download-artifact@v3
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+      - name: Report Builder
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cat > "$NEEDS_DATA_PATH" << 'EOF'
+          ${{ toJSON(needs) }}
+          EOF
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 build_report_check.py "$CHECK_NAME"
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH"
+############################################################################################
+#################################### INSTALL PACKAGES ######################################
+############################################################################################
+  InstallPackagesTestRelease:
+    needs: [BuilderDebRelease]
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/test_install
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          CHECK_NAME=Install packages (amd64)
+          REPO_COPY=${{runner.temp}}/test_install/ClickHouse
+          EOF
+      - name: Download json reports
+        uses: actions/download-artifact@v3
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+      - name: Test packages installation
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci"
+          python3 install_check.py "$CHECK_NAME"
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH"
+  InstallPackagesTestAarch64:
+    needs: [BuilderDebAarch64]
+    runs-on: [self-hosted, style-checker-aarch64]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/test_install
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          CHECK_NAME=Install packages (arm64)
+          REPO_COPY=${{runner.temp}}/test_install/ClickHouse
+          EOF
+      - name: Download json reports
+        uses: actions/download-artifact@v3
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+      - name: Test packages installation
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci"
+          python3 install_check.py "$CHECK_NAME"
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH"
 ##############################################################################################
 ########################### FUNCTIONAl STATELESS TESTS #######################################
@@ -437,19 +648,18 @@ jobs:
           cat >> "$GITHUB_ENV" << 'EOF'
           TEMP_PATH=${{runner.temp}}/stateless_debug
           REPORTS_PATH=${{runner.temp}}/reports_dir
-          CHECK_NAME=Stateless tests (address)
+          CHECK_NAME=Stateless tests (asan)
           REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
           KILL_TIMEOUT=10800
           EOF
       - name: Download json reports
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           path: ${{ env.REPORTS_PATH }}
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Functional test
         run: |
           sudo rm -fr "$TEMP_PATH"
@@ -460,10 +670,8 @@ jobs:
       - name: Cleanup
         if: always()
         run: |
-          # shellcheck disable=SC2046
-          docker kill $(docker ps -q) ||:
-          # shellcheck disable=SC2046
-          docker rm -f $(docker ps -a -q) ||:
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH"
 ##############################################################################################
 ############################ FUNCTIONAl STATEFUL TESTS #######################################
@@ -482,14 +690,13 @@ jobs:
           KILL_TIMEOUT=3600
           EOF
       - name: Download json reports
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           path: ${{ env.REPORTS_PATH }}
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Functional test
         run: |
           sudo rm -fr "$TEMP_PATH"
@@ -500,10 +707,8 @@ jobs:
       - name: Cleanup
         if: always()
         run: |
-          # shellcheck disable=SC2046
-          docker kill $(docker ps -q) ||:
-          # shellcheck disable=SC2046
-          docker rm -f $(docker ps -a -q) ||:
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH"
 ##############################################################################################
 ######################################### STRESS TESTS #######################################
@@ -521,18 +726,17 @@ jobs:
           cat >> "$GITHUB_ENV" << 'EOF'
           TEMP_PATH=${{runner.temp}}/stress_thread
           REPORTS_PATH=${{runner.temp}}/reports_dir
-          CHECK_NAME=Stress test (thread)
+          CHECK_NAME=Stress test (tsan)
           REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse
           EOF
       - name: Download json reports
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           path: ${{ env.REPORTS_PATH }}
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Stress test
         run: |
           sudo rm -fr "$TEMP_PATH"
@@ -543,10 +747,8 @@ jobs:
       - name: Cleanup
         if: always()
         run: |
-          # shellcheck disable=SC2046
-          docker kill $(docker ps -q) ||:
-          # shellcheck disable=SC2046
-          docker rm -f $(docker ps -a -q) ||:
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH"
 #############################################################################################
 ############################# INTEGRATION TESTS #############################################
@@ -564,14 +766,13 @@ jobs:
           REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse
           EOF
       - name: Download json reports
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           path: ${{ env.REPORTS_PATH }}
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Integration test
         run: |
           sudo rm -fr "$TEMP_PATH"
@@ -582,29 +783,29 @@ jobs:
       - name: Cleanup
         if: always()
         run: |
-          # shellcheck disable=SC2046
-          docker kill $(docker ps -q) ||:
-          # shellcheck disable=SC2046
-          docker rm -f $(docker ps -a -q) ||:
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH"
   FinishCheck:
     needs:
       - DockerHubPush
       - DockerServerImages
       - BuilderReport
+      - BuilderSpecialReport
       - FunctionalStatelessTestAsan
       - FunctionalStatefulTestDebug
       - StressTestTsan
       - IntegrationTestsRelease
-      - CompatibilityCheck
+      - CompatibilityCheckX86
+      - CompatibilityCheckAarch64
     runs-on: [self-hosted, style-checker]
     steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Finish label
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 finish_check.py
+          python3 merge_pr.py

.github/workflows/cancel.yml

@@ -6,7 +6,7 @@ env:
 
 on: # yamllint disable-line rule:truthy
   workflow_run:
-    workflows: ["PullRequestCI", "ReleaseCI", "DocsCheck", "BackportPR"]
+    workflows: ["PullRequestCI", "ReleaseBranchCI", "DocsCheck", "BackportPR"]
     types:
       - requested
 jobs:

.github/workflows/cherry_pick.yml

@@ -28,20 +28,18 @@ jobs:
           REPO_TEAM=core
           EOF
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
         with:
+          clear-repository: true
           token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
           fetch-depth: 0
       - name: Cherry pick
         run: |
-          sudo pip install GitPython
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 cherry_pick.py
       - name: Cleanup
         if: always()
         run: |
-          # shellcheck disable=SC2046
-          docker kill $(docker ps -q) ||:
-          # shellcheck disable=SC2046
-          docker rm -f $(docker ps -a -q) ||:
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH"

.github/workflows/debug.yml

@@ -2,10 +2,10 @@
 name: Debug
 
 'on':
-  [push, pull_request, release, workflow_dispatch]
+  [push, pull_request, pull_request_review, release, workflow_dispatch, workflow_call]
 
 jobs:
   DebugInfo:
     runs-on: ubuntu-latest
     steps:
-    - uses: hmarr/debug-action@1201a20fc9d278ddddd5f0f46922d06513892491
+    - uses: hmarr/debug-action@a701ed95a46e6f2fb0df25e1a558c16356fae35a

.github/workflows/docs_check.yml

@@ -13,18 +13,20 @@ on:  # yamllint disable-line rule:truthy
     branches:
       - master
     paths:
+      - 'CHANGELOG.md'
+      - 'README.md'
+      - 'SECURITY.md'
       - 'docker/docs/**'
       - 'docs/**'
-      - 'website/**'
+      - 'utils/check-style/aspell-ignore/**'
 jobs:
   CheckLabels:
     runs-on: [self-hosted, style-checker]
     steps:
-      - name: Clear repository
-        run: |
-          sudo rm -rf "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Labels check
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
@@ -33,17 +35,16 @@ jobs:
     needs: CheckLabels
     runs-on: [self-hosted, style-checker-aarch64]
     steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Images check
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 docker_images_check.py --suffix aarch64
       - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: changed_images_aarch64
           path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json
@@ -51,17 +52,16 @@ jobs:
     needs: CheckLabels
     runs-on: [self-hosted, style-checker]
     steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Images check
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 docker_images_check.py --suffix amd64
       - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: changed_images_amd64
           path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json
@@ -69,18 +69,17 @@ jobs:
     needs: [DockerHubPushAmd64, DockerHubPushAarch64]
     runs-on: [self-hosted, style-checker]
     steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Download changed aarch64 images
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images_aarch64
           path: ${{ runner.temp }}
       - name: Download changed amd64 images
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images_amd64
           path: ${{ runner.temp }}
@@ -89,7 +88,7 @@ jobs:
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64
       - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: changed_images
           path: ${{ runner.temp }}/changed_images.json
@@ -109,15 +108,14 @@ jobs:
       - name: Download changed images
         # even if artifact does not exist, e.g. on `do not test` label or failed Docker job
         continue-on-error: true
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images
           path: ${{ env.TEMP_PATH }}
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Style Check
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
@@ -125,10 +123,8 @@ jobs:
       - name: Cleanup
         if: always()
         run: |
-          # shellcheck disable=SC2046
-          docker kill $(docker ps -q) ||:
-          # shellcheck disable=SC2046
-          docker rm -f $(docker ps -a -q) ||:
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH"
   DocsCheck:
     needs: DockerHubPush
@@ -141,15 +137,14 @@ jobs:
           REPO_COPY=${{runner.temp}}/docs_check/ClickHouse
           EOF
       - name: Download changed images
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images
           path: ${{ env.TEMP_PATH }}
-      - name: Clear repository
-        run: |
-          sudo rm -rf "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Docs Check
         run: |
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -158,10 +153,8 @@ jobs:
       - name: Cleanup
         if: always()
         run: |
-          # shellcheck disable=SC2046
-          docker kill $(docker ps -q) ||:
-          # shellcheck disable=SC2046
-          docker rm -f $(docker ps -a -q) ||:
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH"
   FinishCheck:
     needs:
@@ -170,12 +163,12 @@ jobs:
       - DocsCheck
     runs-on: [self-hosted, style-checker]
     steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Finish label
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 finish_check.py
+          python3 merge_pr.py --check-approved

.github/workflows/docs_release.yml

@@ -1,123 +0,0 @@
-name: DocsReleaseChecks
-
-env:
-  # Force the stdout and stderr streams to be unbuffered
-  PYTHONUNBUFFERED: 1
-
-concurrency:
-  group: master-release
-  cancel-in-progress: true
-'on':
-  push:
-    branches:
-      - master
-    paths:
-      - '.github/**'
-      - 'docker/docs/release/**'
-      - 'docs/**'
-      - 'utils/list-versions/version_date.tsv'
-      - 'website/**'
-  workflow_dispatch:
-jobs:
-  DockerHubPushAarch64:
-    runs-on: [self-hosted, style-checker-aarch64]
-    steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
-      - name: Check out repository code
-        uses: actions/checkout@v2
-      - name: Images check
-        run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 docker_images_check.py --suffix aarch64
-      - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: changed_images_aarch64
-          path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json
-  DockerHubPushAmd64:
-    runs-on: [self-hosted, style-checker]
-    steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
-      - name: Check out repository code
-        uses: actions/checkout@v2
-      - name: Images check
-        run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 docker_images_check.py --suffix amd64
-      - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: changed_images_amd64
-          path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json
-  DockerHubPush:
-    needs: [DockerHubPushAmd64, DockerHubPushAarch64]
-    runs-on: [self-hosted, style-checker]
-    steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
-      - name: Check out repository code
-        uses: actions/checkout@v2
-      - name: Download changed aarch64 images
-        uses: actions/download-artifact@v2
-        with:
-          name: changed_images_aarch64
-          path: ${{ runner.temp }}
-      - name: Download changed amd64 images
-        uses: actions/download-artifact@v2
-        with:
-          name: changed_images_amd64
-          path: ${{ runner.temp }}
-      - name: Images check
-        run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64
-      - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: changed_images
-          path: ${{ runner.temp }}/changed_images.json
-  DocsRelease:
-    needs: DockerHubPush
-    runs-on: [self-hosted, func-tester]
-    steps:
-      - name: Set envs
-        # https://docs.github.com/en/actions/learn-github-actions/workflow-commands-for-github-actions#multiline-strings
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/docs_release
-          REPO_COPY=${{runner.temp}}/docs_release/ClickHouse
-          CLOUDFLARE_TOKEN=${{secrets.CLOUDFLARE}}
-          ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
-          ${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
-          RCSK
-          EOF
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
-      - name: Check out repository code
-        uses: actions/checkout@v2
-      - name: Download changed images
-        uses: actions/download-artifact@v2
-        with:
-          name: changed_images
-          path: ${{ env.TEMP_PATH }}
-      - name: Docs Release
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci"
-          python3 docs_release.py
-      - name: Cleanup
-        if: always()
-        run: |
-          # shellcheck disable=SC2046
-          docker kill $(docker ps -q) ||:
-          # shellcheck disable=SC2046
-          docker rm -f $(docker ps -a -q) ||:
-          sudo rm -fr "$TEMP_PATH"

.github/workflows/jepsen.yml

@@ -19,12 +19,10 @@ jobs:
           TEMP_PATH=${{runner.temp}}/keeper_jepsen
           REPO_COPY=${{runner.temp}}/keeper_jepsen/ClickHouse
           EOF
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
         with:
+          clear-repository: true
           fetch-depth: 0
       - name: Jepsen Test
         run: |
@@ -32,12 +30,39 @@ jobs:
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
           cd "$REPO_COPY/tests/ci"
-          python3 keeper_jepsen_check.py
+          python3 jepsen_check.py keeper
       - name: Cleanup
         if: always()
         run: |
-          # shellcheck disable=SC2046
-          docker kill $(docker ps -q) ||:
-          # shellcheck disable=SC2046
-          docker rm -f $(docker ps -a -q) ||:
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH"
+  # ServerJepsenRelease:
+  #   runs-on: [self-hosted, style-checker]
+  #   if: ${{ always() }}
+  #   needs: [KeeperJepsenRelease]
+  #   steps:
+  #     - name: Set envs
+  #       run: |
+  #         cat >> "$GITHUB_ENV" << 'EOF'
+  #         TEMP_PATH=${{runner.temp}}/server_jepsen
+  #         REPO_COPY=${{runner.temp}}/server_jepsen/ClickHouse
+  #         EOF
+  #     - name: Check out repository code
+  #       uses: ClickHouse/checkout@v1
+  #       with:
+  #         clear-repository: true
+  #         fetch-depth: 0
+  #     - name: Jepsen Test
+  #       run: |
+  #         sudo rm -fr "$TEMP_PATH"
+  #         mkdir -p "$TEMP_PATH"
+  #         cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+  #         cd "$REPO_COPY/tests/ci"
+  #         python3 jepsen_check.py server
+  #     - name: Cleanup
+  #       if: always()
+  #       run: |
+  #         docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+  #         docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+  #         sudo rm -fr "$TEMP_PATH"

.github/workflows/nightly.yml

@@ -10,37 +10,38 @@ env:
   workflow_dispatch:
 
 jobs:
+  Debug:
+    # The task for having a preserved ENV and event.json for later investigation
+    uses: ./.github/workflows/debug.yml
   DockerHubPushAarch64:
     runs-on: [self-hosted, style-checker-aarch64]
     steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Images check
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 docker_images_check.py --suffix aarch64 --all
       - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: changed_images_aarch64
           path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json
   DockerHubPushAmd64:
     runs-on: [self-hosted, style-checker]
     steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Images check
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 docker_images_check.py --suffix amd64 --all
       - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: changed_images_amd64
           path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json
@@ -48,18 +49,17 @@ jobs:
     needs: [DockerHubPushAmd64, DockerHubPushAarch64]
     runs-on: [self-hosted, style-checker]
     steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Download changed aarch64 images
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images_aarch64
           path: ${{ runner.temp }}
       - name: Download changed amd64 images
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images_amd64
           path: ${{ runner.temp }}
@@ -68,10 +68,13 @@ jobs:
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64
       - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: changed_images
           path: ${{ runner.temp }}/changed_images.json
+  Codebrowser:
+    needs: [DockerHubPush]
+    uses: ./.github/workflows/woboq.yml
   BuilderCoverity:
     needs: DockerHubPush
     runs-on: [self-hosted, builder]
@@ -87,22 +90,17 @@ jobs:
           EOF
           echo "COVERITY_TOKEN=${{ secrets.COVERITY_TOKEN }}" >> "$GITHUB_ENV"
       - name: Download changed images
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images
           path: ${{ env.IMAGES_PATH }}
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        id: coverity-checkout
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
         with:
-          fetch-depth: 0 # otherwise we will have no info about contributors
+          clear-repository: true
+          submodules: true
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -112,15 +110,71 @@ jobs:
         run: |
           curl --form token="${COVERITY_TOKEN}" \
             --form email='security+coverity@clickhouse.com' \
-            --form file="@$TEMP_PATH/$BUILD_NAME/coverity-scan.tgz" \
+            --form file="@$TEMP_PATH/$BUILD_NAME/coverity-scan.tar.gz" \
             --form version="${GITHUB_REF#refs/heads/}-${GITHUB_SHA::6}" \
             --form description="Nighly Scan: $(date +'%Y-%m-%dT%H:%M:%S')" \
             https://scan.coverity.com/builds?project=ClickHouse%2FClickHouse
       - name: Cleanup
         if: always()
         run: |
-          # shellcheck disable=SC2046
-          docker kill $(docker ps -q) ||:
-          # shellcheck disable=SC2046
-          docker rm -f $(docker ps -a -q) ||:
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
+  SonarCloud:
+    runs-on: [self-hosted, builder]
+    env:
+      SONAR_SCANNER_VERSION: 4.8.0.2856
+      SONAR_SERVER_URL: "https://sonarcloud.io"
+      BUILD_WRAPPER_OUT_DIR: build_wrapper_output_directory # Directory where build-wrapper output will be placed
+      CC: clang-16
+      CXX: clang++-16
+    steps:
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+          fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
+          submodules: true
+      - name: Set up JDK 11
+        uses: actions/setup-java@v1
+        with:
+          java-version: 11
+      - name: Download and set up sonar-scanner
+        env:
+          SONAR_SCANNER_DOWNLOAD_URL: https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-${{ env.SONAR_SCANNER_VERSION }}-linux.zip
+        run: |
+          mkdir -p "$HOME/.sonar"
+          curl -sSLo "$HOME/.sonar/sonar-scanner.zip" "${{ env.SONAR_SCANNER_DOWNLOAD_URL }}"
+          unzip -o "$HOME/.sonar/sonar-scanner.zip" -d "$HOME/.sonar/"
+          echo "$HOME/.sonar/sonar-scanner-${{ env.SONAR_SCANNER_VERSION }}-linux/bin" >> "$GITHUB_PATH"
+      - name: Download and set up build-wrapper
+        env:
+          BUILD_WRAPPER_DOWNLOAD_URL: ${{ env.SONAR_SERVER_URL }}/static/cpp/build-wrapper-linux-x86.zip
+        run: |
+          curl -sSLo "$HOME/.sonar/build-wrapper-linux-x86.zip" "${{ env.BUILD_WRAPPER_DOWNLOAD_URL }}"
+          unzip -o "$HOME/.sonar/build-wrapper-linux-x86.zip" -d "$HOME/.sonar/"
+          echo "$HOME/.sonar/build-wrapper-linux-x86" >> "$GITHUB_PATH"
+      - name: Set Up Build Tools
+        run: |
+          sudo apt-get update
+          sudo apt-get install -yq git cmake ccache ninja-build python3 yasm nasm
+          sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
+      - name: Run build-wrapper
+        run: |
+          mkdir build
+          cd build
+          cmake ..
+          cd ..
+          build-wrapper-linux-x86-64 --out-dir ${{ env.BUILD_WRAPPER_OUT_DIR }} cmake --build build/
+      - name: Run sonar-scanner
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+        run: |
+          sonar-scanner \
+            --define sonar.host.url="${{ env.SONAR_SERVER_URL }}" \
+            --define sonar.cfamily.build-wrapper-output="${{ env.BUILD_WRAPPER_OUT_DIR }}" \
+            --define sonar.projectKey="ClickHouse_ClickHouse" \
+            --define sonar.organization="clickhouse-java" \
+            --define sonar.cfamily.cpp23.enabled=true \
+            --define sonar.exclusions="**/*.java,**/*.ts,**/*.js,**/*.css,**/*.sql"

.github/workflows/pull_request_approved.yml

@@ -0,0 +1,23 @@
+name: PullRequestApprovedCI
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+
+on:  # yamllint disable-line rule:truthy
+  pull_request_review:
+    types:
+      - submitted
+
+jobs:
+  MergeOnApproval:
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+      - name: Merge approved PR
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 merge_pr.py --check-approved

.github/workflows/release.yml

@@ -1,4 +1,4 @@
-name: ReleaseWorkflow
+name: PublishedReleaseCI
 # - Gets artifacts from S3
 # - Sends it to JFROG Artifactory
 # - Adds them to the release assets
@@ -7,62 +7,59 @@ on: # yamllint disable-line rule:truthy
   release:
     types:
     - published
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Release tag'
+        required: true
+        type: string
 
 jobs:
   ReleasePublish:
     runs-on: [self-hosted, style-checker]
     steps:
-    - name: Set envs
+    - name: Set tag from input
+      if: github.event_name == 'workflow_dispatch'
       run: |
-        cat >> "$GITHUB_ENV" << 'EOF'
-        JFROG_API_KEY=${{ secrets.JFROG_KEY_API_PACKAGES }}
-        TEMP_PATH=${{runner.temp}}/release_packages
-        REPO_COPY=${{runner.temp}}/release_packages/ClickHouse
-        EOF
-    - name: Check out repository code
-      uses: actions/checkout@v2
-      with:
-        # Always use the most recent script version
-        ref: master
-    - name: Download packages and push to Artifactory
+        echo "GITHUB_TAG=${{ github.event.inputs.tag }}" >> "$GITHUB_ENV"
+    - name: Set tag from REF
+      if: github.event_name == 'release'
       run: |
-        rm -rf "$TEMP_PATH" && mkdir -p "$TEMP_PATH"
-        cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-        cd "$REPO_COPY"
-        python3 ./tests/ci/push_to_artifactory.py --release "${{ github.ref }}" \
-          --commit '${{ github.sha }}' --all
-    - name: Upload packages to release assets
-      uses: svenstaro/upload-release-action@v2
-      with:
-        repo_token: ${{ secrets.GITHUB_TOKEN }}
-        file: ${{runner.temp}}/push_to_artifactory/*
-        overwrite: true
-        tag: ${{ github.ref }}
-        file_glob: true
+        echo "GITHUB_TAG=${GITHUB_REF#refs/tags/}" >> "$GITHUB_ENV"
+    - name: Deploy packages and assets
+      run: |
+        curl --silent --data '' --no-buffer \
+          '${{ secrets.PACKAGES_RELEASE_URL }}/release/'"${GITHUB_TAG}"'?binary=binary_darwin&binary=binary_darwin_aarch64&sync=true'
   ############################################################################################
   ##################################### Docker images  #######################################
   ############################################################################################
   DockerServerImages:
     runs-on: [self-hosted, style-checker]
     steps:
-    - name: Clear repository
+    - name: Set tag from input
+      if: github.event_name == 'workflow_dispatch'
       run: |
-        sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+        echo "GITHUB_TAG=${{ github.event.inputs.tag }}" >> "$GITHUB_ENV"
+    - name: Set tag from REF
+      if: github.event_name == 'release'
+      run: |
+        echo "GITHUB_TAG=${GITHUB_REF#refs/tags/}" >> "$GITHUB_ENV"
     - name: Check out repository code
-      uses: actions/checkout@v2
+      uses: ClickHouse/checkout@v1
       with:
+        clear-repository: true
         fetch-depth: 0  # otherwise we will have no version info
+        ref: ${{ env.GITHUB_TAG }}
     - name: Check docker clickhouse/clickhouse-server building
       run: |
         cd "$GITHUB_WORKSPACE/tests/ci"
-        python3 docker_server.py --release-type auto --version "${{ github.ref }}"
-        python3 docker_server.py --release-type auto --version "${{ github.ref }}" --no-ubuntu \
+        python3 docker_server.py --release-type auto --version "$GITHUB_TAG" \
+          --image-repo clickhouse/clickhouse-server --image-path docker/server
+        python3 docker_server.py --release-type auto --version "$GITHUB_TAG" \
           --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper
     - name: Cleanup
       if: always()
       run: |
-        # shellcheck disable=SC2046
-        docker kill $(docker ps -q) ||:
-        # shellcheck disable=SC2046
-        docker rm -f $(docker ps -a -q) ||:
+        docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+        docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
         sudo rm -fr "$TEMP_PATH"

.github/workflows/tags_stable.yml

@@ -34,15 +34,16 @@ jobs:
       run: |
         echo "GITHUB_TAG=${GITHUB_REF#refs/tags/}" >> "$GITHUB_ENV"
     - name: Check out repository code
-      uses: actions/checkout@v2
+      uses: ClickHouse/checkout@v1
       with:
         ref: master
         fetch-depth: 0
-    - name: Generate versions
+    - name: Update versions, docker version, changelog, security
       env:
         GITHUB_TOKEN: ${{ secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN }}
       run: |
         ./utils/list-versions/list-versions.sh > ./utils/list-versions/version_date.tsv
+        ./utils/list-versions/update-docker-version.sh
         GID=$(id -g "${UID}")
         docker run -u "${UID}:${GID}" -e PYTHONUNBUFFERED=1 \
             --volume="${GITHUB_WORKSPACE}:/ClickHouse" clickhouse/style-test \
@@ -50,6 +51,7 @@ jobs:
                 --gh-user-or-token="$GITHUB_TOKEN" --jobs=5 \
                 --output="/ClickHouse/docs/changelogs/${GITHUB_TAG}.md" "${GITHUB_TAG}"
         git add "./docs/changelogs/${GITHUB_TAG}.md"
+        python3 ./utils/security-generator/generate_security.py > SECURITY.md
         git diff HEAD
     - name: Create Pull Request
       uses: peter-evans/create-pull-request@v3
@@ -59,6 +61,7 @@ jobs:
         committer: "robot-clickhouse <robot-clickhouse@users.noreply.github.com>"
         commit-message: Update version_date.tsv and changelogs after ${{ env.GITHUB_TAG }}
         branch: auto/${{ env.GITHUB_TAG }}
+        assignees: ${{ github.event.sender.login }}  # assign the PR to the tag pusher
         delete-branch: true
         title: Update version_date.tsv and changelogs after ${{ env.GITHUB_TAG }}
         labels: do not test

.github/workflows/woboq.yml

@@ -6,13 +6,13 @@ env:
 concurrency:
   group: woboq
 on: # yamllint disable-line rule:truthy
-  schedule:
-    - cron: '0 */18 * * *'
   workflow_dispatch:
+  workflow_call:
 jobs:
   # don't use dockerhub push because this image updates so rarely
   WoboqCodebrowser:
     runs-on: [self-hosted, style-checker]
+    timeout-minutes: 420  # the task is pretty heavy, so there's an additional hour
     steps:
       - name: Set envs
         run: |
@@ -21,13 +21,15 @@ jobs:
           REPO_COPY=${{runner.temp}}/codebrowser/ClickHouse
           IMAGES_PATH=${{runner.temp}}/images_path
           EOF
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
         with:
+          clear-repository: true
           submodules: 'true'
+      - name: Download json reports
+        uses: actions/download-artifact@v3
+        with:
+          path: ${{ env.IMAGES_PATH }}
       - name: Codebrowser
         run: |
           sudo rm -fr "$TEMP_PATH"
@@ -37,8 +39,6 @@ jobs:
       - name: Cleanup
         if: always()
         run: |
-          # shellcheck disable=SC2046
-          docker kill $(docker ps -q) ||:
-          # shellcheck disable=SC2046
-          docker rm -f $(docker ps -a -q) ||:
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH"

.gitignore

@@ -17,6 +17,7 @@
 
 # logs
 *.log
+*.debuglog
 *.stderr
 *.stdout
 
@@ -58,6 +59,10 @@ cmake_install.cmake
 CTestTestfile.cmake
 *.a
 *.o
+*.so
+*.dll
+*.lib
+*.dylib
 cmake-build-*
 
 # Python cache
@@ -76,6 +81,7 @@ core
 vgcore*
 
 *.deb
+*.tar.zst
 *.build
 *.upload
 *.changes
@@ -123,7 +129,6 @@ website/package-lock.json
 /.ccls-cache
 
 # clangd cache
-/.clangd
 /.cache
 
 /compile_commands.json
@@ -148,4 +153,16 @@ website/package-lock.json
 /programs/server/data
 /programs/server/metadata
 /programs/server/store
+/programs/server/uuid
+/programs/server/coordination
 
+# temporary test files
+tests/queries/0_stateless/test_*
+tests/queries/0_stateless/*.binary
+tests/queries/0_stateless/*.generated-expect
+tests/queries/0_stateless/*.expect.history
+
+# rust
+/rust/**/target
+# It is autogenerated from *.in
+/rust/**/.cargo/config.toml

.gitmodules

@@ -1,97 +1,83 @@
-[submodule "contrib/poco"]
-	path = contrib/poco
-	url = https://github.com/ClickHouse/poco.git
-	branch = clickhouse
 [submodule "contrib/zstd"]
 	path = contrib/zstd
-	url = https://github.com/facebook/zstd.git
+	url = https://github.com/facebook/zstd
 [submodule "contrib/lz4"]
 	path = contrib/lz4
-	url = https://github.com/lz4/lz4.git
+	url = https://github.com/lz4/lz4
 [submodule "contrib/librdkafka"]
 	path = contrib/librdkafka
-	url = https://github.com/ClickHouse/librdkafka.git
+	url = https://github.com/ClickHouse/librdkafka
 [submodule "contrib/cctz"]
 	path = contrib/cctz
-	url = https://github.com/ClickHouse/cctz.git
+	url = https://github.com/ClickHouse/cctz
 [submodule "contrib/zlib-ng"]
 	path = contrib/zlib-ng
-	url = https://github.com/ClickHouse/zlib-ng.git
+	url = https://github.com/ClickHouse/zlib-ng
 	branch = clickhouse-2.0.x
 [submodule "contrib/googletest"]
 	path = contrib/googletest
-	url = https://github.com/google/googletest.git
+	url = https://github.com/google/googletest
 [submodule "contrib/capnproto"]
 	path = contrib/capnproto
-	url = https://github.com/capnproto/capnproto.git
+	url = https://github.com/ClickHouse/capnproto
 [submodule "contrib/double-conversion"]
 	path = contrib/double-conversion
-	url = https://github.com/google/double-conversion.git
+	url = https://github.com/google/double-conversion
 [submodule "contrib/re2"]
 	path = contrib/re2
-	url = https://github.com/google/re2.git
-[submodule "contrib/llvm"]
-	path = contrib/llvm
-	url = https://github.com/ClickHouse/llvm
+	url = https://github.com/google/re2
 [submodule "contrib/mariadb-connector-c"]
 	path = contrib/mariadb-connector-c
-	url = https://github.com/ClickHouse/mariadb-connector-c.git
+	url = https://github.com/ClickHouse/mariadb-connector-c
 [submodule "contrib/jemalloc"]
 	path = contrib/jemalloc
-	url = https://github.com/jemalloc/jemalloc.git
+	url = https://github.com/jemalloc/jemalloc
 [submodule "contrib/unixodbc"]
 	path = contrib/unixodbc
-	url = https://github.com/ClickHouse/UnixODBC.git
-[submodule "contrib/protobuf"]
-	path = contrib/protobuf
-	url = https://github.com/ClickHouse/protobuf.git
-	branch = v3.13.0.1
+	url = https://github.com/ClickHouse/UnixODBC
+[submodule "contrib/google-protobuf"]
+	path = contrib/google-protobuf
+	url = https://github.com/ClickHouse/google-protobuf.git
 [submodule "contrib/boost"]
 	path = contrib/boost
-	url = https://github.com/ClickHouse/boost.git
+	url = https://github.com/ClickHouse/boost
 [submodule "contrib/base64"]
 	path = contrib/base64
-	url = https://github.com/ClickHouse/Turbo-Base64.git
+	url = https://github.com/ClickHouse/Turbo-Base64
 [submodule "contrib/arrow"]
 	path = contrib/arrow
-	url = https://github.com/ClickHouse/arrow.git
+	url = https://github.com/ClickHouse/arrow
 	branch = blessed/release-6.0.1
 [submodule "contrib/thrift"]
 	path = contrib/thrift
-	url = https://github.com/apache/thrift.git
+	url = https://github.com/apache/thrift
 [submodule "contrib/libhdfs3"]
 	path = contrib/libhdfs3
-	url = https://github.com/ClickHouse/libhdfs3.git
+	url = https://github.com/ClickHouse/libhdfs3
 [submodule "contrib/libxml2"]
 	path = contrib/libxml2
-	url = https://github.com/GNOME/libxml2.git
+	url = https://github.com/GNOME/libxml2
 [submodule "contrib/libgsasl"]
 	path = contrib/libgsasl
-	url = https://github.com/ClickHouse/libgsasl.git
-[submodule "contrib/libcxx"]
-	path = contrib/libcxx
-	url = https://github.com/ClickHouse/libcxx.git
-[submodule "contrib/libcxxabi"]
-	path = contrib/libcxxabi
-	url = https://github.com/ClickHouse/libcxxabi.git
+	url = https://github.com/ClickHouse/libgsasl
 [submodule "contrib/snappy"]
 	path = contrib/snappy
-	url = https://github.com/ClickHouse/snappy.git
+	url = https://github.com/ClickHouse/snappy
 [submodule "contrib/cppkafka"]
 	path = contrib/cppkafka
-	url = https://github.com/mfontanini/cppkafka.git
+	url = https://github.com/mfontanini/cppkafka
 [submodule "contrib/brotli"]
 	path = contrib/brotli
-	url = https://github.com/google/brotli.git
+	url = https://github.com/google/brotli
 [submodule "contrib/h3"]
 	path = contrib/h3
 	url = https://github.com/ClickHouse/h3
 [submodule "contrib/libunwind"]
 	path = contrib/libunwind
-	url = https://github.com/ClickHouse/libunwind.git
+	url = https://github.com/ClickHouse/libunwind
 [submodule "contrib/simdjson"]
 	path = contrib/simdjson
-	url = https://github.com/simdjson/simdjson.git
+	url = https://github.com/simdjson/simdjson
 [submodule "contrib/rapidjson"]
 	path = contrib/rapidjson
 	url = https://github.com/ClickHouse/rapidjson
@@ -103,68 +89,68 @@
 	url = https://github.com/ClickHouse/orc
 [submodule "contrib/sparsehash-c11"]
 	path = contrib/sparsehash-c11
-	url = https://github.com/sparsehash/sparsehash-c11.git
+	url = https://github.com/sparsehash/sparsehash-c11
 [submodule "contrib/grpc"]
 	path = contrib/grpc
-	url = https://github.com/ClickHouse/grpc.git
+	url = https://github.com/ClickHouse/grpc
 	branch = v1.33.2
 [submodule "contrib/aws"]
 	path = contrib/aws
-	url = https://github.com/ClickHouse/aws-sdk-cpp.git
+	url = https://github.com/ClickHouse/aws-sdk-cpp
 [submodule "aws-c-event-stream"]
 	path = contrib/aws-c-event-stream
-	url = https://github.com/ClickHouse/aws-c-event-stream.git
+	url = https://github.com/awslabs/aws-c-event-stream
 [submodule "aws-c-common"]
 	path = contrib/aws-c-common
-	url = https://github.com/ClickHouse/aws-c-common.git
+	url = https://github.com/ClickHouse/aws-c-common
 [submodule "aws-checksums"]
 	path = contrib/aws-checksums
-	url = https://github.com/ClickHouse/aws-checksums.git
+	url = https://github.com/awslabs/aws-checksums
 [submodule "contrib/curl"]
 	path = contrib/curl
-	url = https://github.com/curl/curl.git
+	url = https://github.com/curl/curl
 [submodule "contrib/icudata"]
 	path = contrib/icudata
-	url = https://github.com/ClickHouse/icudata.git
+	url = https://github.com/ClickHouse/icudata
 [submodule "contrib/icu"]
 	path = contrib/icu
-	url = https://github.com/unicode-org/icu.git
+	url = https://github.com/unicode-org/icu
 [submodule "contrib/flatbuffers"]
 	path = contrib/flatbuffers
-	url = https://github.com/ClickHouse/flatbuffers.git
+	url = https://github.com/ClickHouse/flatbuffers
 [submodule "contrib/replxx"]
 	path = contrib/replxx
-	url = https://github.com/ClickHouse/replxx.git
+	url = https://github.com/ClickHouse/replxx
 [submodule "contrib/avro"]
 	path = contrib/avro
-	url = https://github.com/ClickHouse/avro.git
+	url = https://github.com/ClickHouse/avro
 	ignore = untracked
 [submodule "contrib/msgpack-c"]
 	path = contrib/msgpack-c
 	url = https://github.com/msgpack/msgpack-c
 [submodule "contrib/libcpuid"]
 	path = contrib/libcpuid
-	url = https://github.com/ClickHouse/libcpuid.git
+	url = https://github.com/ClickHouse/libcpuid
 [submodule "contrib/openldap"]
 	path = contrib/openldap
-	url = https://github.com/ClickHouse/openldap.git
+	url = https://github.com/ClickHouse/openldap
 [submodule "contrib/AMQP-CPP"]
 	path = contrib/AMQP-CPP
-	url = https://github.com/ClickHouse/AMQP-CPP.git
+	url = https://github.com/ClickHouse/AMQP-CPP
 [submodule "contrib/cassandra"]
 	path = contrib/cassandra
-	url = https://github.com/ClickHouse/cpp-driver.git
+	url = https://github.com/ClickHouse/cpp-driver
 	branch = clickhouse
 [submodule "contrib/libuv"]
 	path = contrib/libuv
-	url = https://github.com/ClickHouse/libuv.git
+	url = https://github.com/ClickHouse/libuv
 	branch = clickhouse
 [submodule "contrib/fmtlib"]
 	path = contrib/fmtlib
-	url = https://github.com/fmtlib/fmt.git
+	url = https://github.com/fmtlib/fmt
 [submodule "contrib/sentry-native"]
 	path = contrib/sentry-native
-	url = https://github.com/ClickHouse/sentry-native.git
+	url = https://github.com/ClickHouse/sentry-native
 [submodule "contrib/krb5"]
 	path = contrib/krb5
 	url = https://github.com/ClickHouse/krb5
@@ -181,17 +167,17 @@
 	url = https://github.com/danlark1/miniselect
 [submodule "contrib/rocksdb"]
 	path = contrib/rocksdb
-	url = https://github.com/ClickHouse/rocksdb.git
+	url = https://github.com/ClickHouse/rocksdb
 [submodule "contrib/xz"]
 	path = contrib/xz
 	url = https://github.com/xz-mirror/xz
 [submodule "contrib/abseil-cpp"]
 	path = contrib/abseil-cpp
-	url = https://github.com/abseil/abseil-cpp.git
+	url = https://github.com/abseil/abseil-cpp
 	branch = lts_2021_11_02
 [submodule "contrib/dragonbox"]
 	path = contrib/dragonbox
-	url = https://github.com/ClickHouse/dragonbox.git
+	url = https://github.com/ClickHouse/dragonbox
 [submodule "contrib/fast_float"]
 	path = contrib/fast_float
 	url = https://github.com/fastfloat/fast_float
@@ -200,44 +186,44 @@
 	url = https://github.com/ClickHouse/libpq
 [submodule "contrib/boringssl"]
 	path = contrib/boringssl
-	url = https://github.com/ClickHouse/boringssl.git
+	url = https://github.com/ClickHouse/boringssl
 	branch = unknown_branch_from_artur
 [submodule "contrib/NuRaft"]
 	path = contrib/NuRaft
-	url = https://github.com/ClickHouse/NuRaft.git
+	url = https://github.com/ClickHouse/NuRaft
 [submodule "contrib/nanodbc"]
 	path = contrib/nanodbc
-	url = https://github.com/ClickHouse/nanodbc.git
+	url = https://github.com/ClickHouse/nanodbc
 [submodule "contrib/datasketches-cpp"]
 	path = contrib/datasketches-cpp
-	url = https://github.com/ClickHouse/datasketches-cpp.git
+	url = https://github.com/ClickHouse/datasketches-cpp
 [submodule "contrib/yaml-cpp"]
 	path = contrib/yaml-cpp
-	url = https://github.com/ClickHouse/yaml-cpp.git
+	url = https://github.com/ClickHouse/yaml-cpp
 [submodule "contrib/cld2"]
 	path = contrib/cld2
-	url = https://github.com/ClickHouse/cld2.git
+	url = https://github.com/ClickHouse/cld2
 [submodule "contrib/libstemmer_c"]
 	path = contrib/libstemmer_c
-	url = https://github.com/ClickHouse/libstemmer_c.git
+	url = https://github.com/ClickHouse/libstemmer_c
 [submodule "contrib/wordnet-blast"]
 	path = contrib/wordnet-blast
-	url = https://github.com/ClickHouse/wordnet-blast.git
+	url = https://github.com/ClickHouse/wordnet-blast
 [submodule "contrib/lemmagen-c"]
 	path = contrib/lemmagen-c
-	url = https://github.com/ClickHouse/lemmagen-c.git
+	url = https://github.com/ClickHouse/lemmagen-c
 [submodule "contrib/libpqxx"]
 	path = contrib/libpqxx
-	url = https://github.com/ClickHouse/libpqxx.git
+	url = https://github.com/ClickHouse/libpqxx
 [submodule "contrib/sqlite-amalgamation"]
 	path = contrib/sqlite-amalgamation
-	url = https://github.com/azadkuh/sqlite-amalgamation
+	url = https://github.com/ClickHouse/sqlite-amalgamation
 [submodule "contrib/s2geometry"]
 	path = contrib/s2geometry
-	url = https://github.com/ClickHouse/s2geometry.git
+	url = https://github.com/ClickHouse/s2geometry
 [submodule "contrib/bzip2"]
 	path = contrib/bzip2
-	url = https://github.com/ClickHouse/bzip2.git
+	url = https://github.com/ClickHouse/bzip2
 [submodule "contrib/magic_enum"]
 	path = contrib/magic_enum
 	url = https://github.com/Neargye/magic_enum
@@ -246,37 +232,114 @@
 	url = https://github.com/google/libprotobuf-mutator
 [submodule "contrib/sysroot"]
 	path = contrib/sysroot
-	url = https://github.com/ClickHouse/sysroot.git
+	url = https://github.com/ClickHouse/sysroot
 [submodule "contrib/nlp-data"]
 	path = contrib/nlp-data
-	url = https://github.com/ClickHouse/nlp-data.git
+	url = https://github.com/ClickHouse/nlp-data
 [submodule "contrib/hive-metastore"]
 	path = contrib/hive-metastore
 	url = https://github.com/ClickHouse/hive-metastore
 [submodule "contrib/azure"]
 	path = contrib/azure
-	url = https://github.com/ClickHouse/azure-sdk-for-cpp.git
+	url = https://github.com/ClickHouse/azure-sdk-for-cpp
 [submodule "contrib/minizip-ng"]
 	path = contrib/minizip-ng
 	url = https://github.com/zlib-ng/minizip-ng
+[submodule "contrib/annoy"]
+	path = contrib/annoy
+	url = https://github.com/ClickHouse/annoy
+	branch = ClickHouse-master
 [submodule "contrib/qpl"]
 	path = contrib/qpl
-	url = https://github.com/intel/qpl.git
+	url = https://github.com/intel/qpl
+[submodule "contrib/idxd-config"]
+	path = contrib/idxd-config
+	url = https://github.com/intel/idxd-config
 [submodule "contrib/wyhash"]
 	path = contrib/wyhash
-	url = https://github.com/wangyi-fudan/wyhash.git
+	url = https://github.com/wangyi-fudan/wyhash
 [submodule "contrib/hashidsxx"]
 	path = contrib/hashidsxx
-	url = https://github.com/schoentoon/hashidsxx.git
+	url = https://github.com/schoentoon/hashidsxx
 [submodule "contrib/nats-io"]
 	path = contrib/nats-io
-	url = https://github.com/ClickHouse/nats.c.git
+	url = https://github.com/ClickHouse/nats.c
 [submodule "contrib/vectorscan"]
 	path = contrib/vectorscan
 	url = https://github.com/VectorCamp/vectorscan.git
+[submodule "contrib/llvm-project"]
+	path = contrib/llvm-project
+	url = https://github.com/ClickHouse/llvm-project
+[submodule "contrib/corrosion"]
+	path = contrib/corrosion
+	url = https://github.com/corrosion-rs/corrosion
+[submodule "contrib/morton-nd"]
+	path = contrib/morton-nd
+	url = https://github.com/morton-nd/morton-nd
+[submodule "contrib/xxHash"]
+	path = contrib/xxHash
+	url = https://github.com/Cyan4973/xxHash
+[submodule "contrib/crc32-s390x"]
+	path = contrib/crc32-s390x
+	url = https://github.com/linux-on-ibm-z/crc32-s390x
+[submodule "contrib/openssl"]
+	path = contrib/openssl
+	url = https://github.com/openssl/openssl
+	branch = openssl-3.0
+[submodule "contrib/google-benchmark"]
+	path = contrib/google-benchmark
+	url = https://github.com/google/benchmark
+[submodule "contrib/libdivide"]
+	path = contrib/libdivide
+	url = https://github.com/ridiculousfish/libdivide
+[submodule "contrib/libbcrypt"]
+	path = contrib/libbcrypt
+	url = https://github.com/rg3/libbcrypt.git
+[submodule "contrib/ulid-c"]
+	path = contrib/ulid-c
+	url = https://github.com/ClickHouse/ulid-c.git
+[submodule "contrib/aws-crt-cpp"]
+	path = contrib/aws-crt-cpp
+	url = https://github.com/ClickHouse/aws-crt-cpp
+[submodule "contrib/aws-c-io"]
+	path = contrib/aws-c-io
+	url = https://github.com/ClickHouse/aws-c-io
+[submodule "contrib/aws-c-mqtt"]
+	path = contrib/aws-c-mqtt
+	url = https://github.com/awslabs/aws-c-mqtt
+[submodule "contrib/aws-c-auth"]
+	path = contrib/aws-c-auth
+	url = https://github.com/awslabs/aws-c-auth
+[submodule "contrib/aws-c-cal"]
+	path = contrib/aws-c-cal
+	url = https://github.com/ClickHouse/aws-c-cal
+[submodule "contrib/aws-c-sdkutils"]
+	path = contrib/aws-c-sdkutils
+	url = https://github.com/awslabs/aws-c-sdkutils
+[submodule "contrib/aws-c-http"]
+	path = contrib/aws-c-http
+	url = https://github.com/awslabs/aws-c-http
+[submodule "contrib/aws-c-s3"]
+	path = contrib/aws-c-s3
+	url = https://github.com/awslabs/aws-c-s3
+[submodule "contrib/aws-c-compression"]
+	path = contrib/aws-c-compression
+	url = https://github.com/awslabs/aws-c-compression
+[submodule "contrib/aws-s2n-tls"]
+	path = contrib/aws-s2n-tls
+	url = https://github.com/ClickHouse/s2n-tls
+[submodule "contrib/crc32-vpmsum"]
+	path = contrib/crc32-vpmsum
+	url = https://github.com/antonblanchard/crc32-vpmsum.git
 [submodule "contrib/liburing"]
 	path = contrib/liburing
-	url = https://github.com/axboe/liburing.git
+	url = https://github.com/axboe/liburing
+[submodule "contrib/libfiu"]
+	path = contrib/libfiu
+	url = https://github.com/ClickHouse/libfiu.git
+[submodule "contrib/isa-l"]
+	path = contrib/isa-l
+	url = https://github.com/ClickHouse/isa-l.git
 [submodule "contrib/c-ares"]
 	path = contrib/c-ares
-	url = https://github.com/ClickHouse/c-ares
+	url = https://github.com/c-ares/c-ares.git

.snyk

@@ -0,0 +1,4 @@
+# Snyk (https://snyk.io) policy file
+exclude:
+  global:
+    - tests/**

.vimrc

@@ -1,2 +0,0 @@
-au BufRead,BufNewFile ./* set tabstop=4 softtabstop=0 expandtab shiftwidth=4 smarttab tags=tags,../tags
-

.yamllint

@@ -6,8 +6,10 @@ rules:
         level: warning
         indent-sequences: consistent
     line-length:
-        # there are some bash -c "", so this is OK
-        max: 300
+        # there are:
+        # - bash -c "", so this is OK
+        # - yaml in tests
+        max: 1000
         level: warning
     comments:
         min-spaces-from-content: 1

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.15)
+cmake_minimum_required(VERSION 3.20)
 
 project(ClickHouse LANGUAGES C CXX ASM)
 
@@ -18,7 +18,7 @@ include (cmake/target.cmake)
 include (cmake/tools.cmake)
 include (cmake/ccache.cmake)
 include (cmake/clang_tidy.cmake)
-include (cmake/git_status.cmake)
+include (cmake/git.cmake)
 
 # Ignore export() since we don't use it,
 # but it gets broken with a global targets via link_libraries()
@@ -57,8 +57,8 @@ if (ENABLE_CHECK_HEAVY_BUILDS)
     # set CPU time limit to 1000 seconds
     set (RLIMIT_CPU 1000)
 
-    # gcc10/gcc10/clang -fsanitize=memory is too heavy
-    if (SANITIZE STREQUAL "memory" OR COMPILER_GCC)
+    # -fsanitize=memory is too heavy
+    if (SANITIZE STREQUAL "memory")
        set (RLIMIT_DATA 10000000000) # 10G
     endif()
 
@@ -73,22 +73,7 @@ message (STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}")
 
 string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC)
 
-option(USE_STATIC_LIBRARIES "Disable to use shared libraries" ON)
-# DEVELOPER ONLY.
-# Faster linking if turned on.
-option(SPLIT_SHARED_LIBRARIES "Keep all internal libraries as separate .so files" OFF)
-
-if (USE_STATIC_LIBRARIES AND SPLIT_SHARED_LIBRARIES)
-    message(FATAL_ERROR "SPLIT_SHARED_LIBRARIES=1 must not be used together with USE_STATIC_LIBRARIES=1")
-endif()
-
-if (NOT USE_STATIC_LIBRARIES AND SPLIT_SHARED_LIBRARIES)
-    set(BUILD_SHARED_LIBS 1 CACHE INTERNAL "")
-endif ()
-
-if (USE_STATIC_LIBRARIES)
-    list(REVERSE CMAKE_FIND_LIBRARY_SUFFIXES)
-endif ()
+list(REVERSE CMAKE_FIND_LIBRARY_SUFFIXES)
 
 option (ENABLE_FUZZING "Fuzzy testing using libfuzzer" OFF)
 
@@ -111,11 +96,23 @@ if (ENABLE_FUZZING)
     set (ENABLE_JEMALLOC 0)
     set (ENABLE_CHECK_HEAVY_BUILDS 1)
     set (GLIBC_COMPATIBILITY OFF)
+    set (ENABLE_BENCHMARKS 0)
 
     # For codegen_select_fuzzer
     set (ENABLE_PROTOBUF 1)
 endif()
 
+option (ENABLE_WOBOQ_CODEBROWSER "Build for woboq codebrowser" OFF)
+
+if (ENABLE_WOBOQ_CODEBROWSER)
+    set (ENABLE_EMBEDDED_COMPILER 0)
+    set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-poison-system-directories")
+    # woboq codebrowser uses clang tooling, and they could add default system
+    # clang includes, and later clang will warn for those added by itself
+    # includes.
+    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-poison-system-directories")
+endif()
+
 # Global libraries
 # See:
 # - default_libs.cmake
@@ -135,6 +132,7 @@ if (ENABLE_COLORED_BUILD AND CMAKE_GENERATOR STREQUAL "Ninja")
     set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdiagnostics-color=always")
     # ... such manually setting of flags can be removed once CMake supports a variable to
     # activate colors in *all* build systems: https://gitlab.kitware.com/cmake/cmake/-/issues/15502
+    # --> available since CMake 3.24: https://stackoverflow.com/a/73349744
 endif ()
 
 include (cmake/check_flags.cmake)
@@ -143,32 +141,25 @@ include (cmake/add_warning.cmake)
 if (COMPILER_CLANG)
     # generate ranges for fast "addr2line" search
     if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE")
+        # NOTE: that clang has a bug because of it does not emit .debug_aranges
+        # with ThinLTO, so custom ld.lld wrapper is shipped in docker images.
         set(COMPILER_FLAGS "${COMPILER_FLAGS} -gdwarf-aranges")
     endif ()
 
-    if (HAS_USE_CTOR_HOMING)
-        # For more info see https://blog.llvm.org/posts/2021-04-05-constructor-homing-for-debug-info/
-        if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO")
-            set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xclang -fuse-ctor-homing")
-            set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Xclang -fuse-ctor-homing")
-        endif()
+    # See https://blog.llvm.org/posts/2021-04-05-constructor-homing-for-debug-info/
+    if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO")
+        set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xclang -fuse-ctor-homing")
+        set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Xclang -fuse-ctor-homing")
     endif()
 
     no_warning(enum-constexpr-conversion) # breaks Protobuf in clang-16
 endif ()
 
-# If compiler has support for -Wreserved-identifier. It is difficult to detect by clang version,
-# because there are two different branches of clang: clang and AppleClang.
-# (AppleClang is not supported by ClickHouse, but some developers have misfortune to use it).
-if (HAS_RESERVED_IDENTIFIER)
-    add_compile_definitions (HAS_RESERVED_IDENTIFIER)
-endif ()
-
-# If turned `ON`, assumes the user has either the system GTest library or the bundled one.
 option(ENABLE_TESTS "Provide unit_test_dbms target with Google.Test unit tests" ON)
 option(ENABLE_EXAMPLES "Build all example programs in 'examples' subdirectories" OFF)
+option(ENABLE_BENCHMARKS "Build all benchmark programs in 'benchmarks' subdirectories" OFF)
 
-if (OS_LINUX AND (ARCH_AMD64 OR ARCH_AARCH64) AND USE_STATIC_LIBRARIES AND NOT SPLIT_SHARED_LIBRARIES AND NOT USE_MUSL)
+if (OS_LINUX AND (ARCH_AMD64 OR ARCH_AARCH64) AND NOT USE_MUSL)
     # Only for Linux, x86_64 or aarch64.
     option(GLIBC_COMPATIBILITY "Enable compatibility with older glibc libraries." ON)
 elseif(GLIBC_COMPATIBILITY)
@@ -190,32 +181,11 @@ else ()
     set(NO_WHOLE_ARCHIVE --no-whole-archive)
 endif ()
 
-option(ENABLE_CURL_BUILD "Enable curl, azure, sentry build on by default except MacOS." ON)
-if (OS_DARWIN)
-    # Disable the curl, azure, senry build on MacOS
-    set (ENABLE_CURL_BUILD OFF)
-endif ()
-
-# Ignored if `lld` is used
-option(ADD_GDB_INDEX_FOR_GOLD "Add .gdb-index to resulting binaries for gold linker.")
-
 if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE")
-    # Can be lld or ld-lld.
-    if (LINKER_NAME MATCHES "lld$")
+    # Can be lld or ld-lld or lld-13 or /path/to/lld.
+    if (LINKER_NAME MATCHES "lld")
         set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gdb-index")
-        set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--gdb-index")
         message (STATUS "Adding .gdb-index via --gdb-index linker option.")
-    # we use another tool for gdb-index, because gold linker removes section .debug_aranges, which used inside clickhouse stacktraces
-    # http://sourceware-org.1504.n7.nabble.com/gold-No-debug-aranges-section-when-linking-with-gdb-index-td540965.html#a556932
-    elseif (LINKER_NAME MATCHES "gold$" AND ADD_GDB_INDEX_FOR_GOLD)
-        find_program (GDB_ADD_INDEX_EXE NAMES "gdb-add-index" DOC "Path to gdb-add-index executable")
-        if (NOT GDB_ADD_INDEX_EXE)
-            set (USE_GDB_ADD_INDEX 0)
-            message (WARNING "Cannot add gdb index to binaries, because gold linker is used, but gdb-add-index executable not found.")
-        else()
-            set (USE_GDB_ADD_INDEX 1)
-            message (STATUS "gdb-add-index found: ${GDB_ADD_INDEX_EXE}")
-        endif()
     endif ()
 endif()
 
@@ -246,7 +216,8 @@ else ()
 endif ()
 
 # Create BuildID when using lld. For other linkers it is created by default.
-if (LINKER_NAME MATCHES "lld$")
+# (NOTE: LINKER_NAME can be either path or name, and in different variants)
+if (LINKER_NAME MATCHES "lld")
     # SHA1 is not cryptographically secure but it is the best what lld is offering.
     set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--build-id=sha1")
 endif ()
@@ -299,8 +270,8 @@ endif ()
 option (ENABLE_BUILD_PATH_MAPPING "Enable remapping of file source paths in debug info, predefined preprocessor macros, and __builtin_FILE(). It's used to generate reproducible builds. See https://reproducible-builds.org/docs/build-path" ${ENABLE_BUILD_PATH_MAPPING_DEFAULT})
 
 if (ENABLE_BUILD_PATH_MAPPING)
-    set (COMPILER_FLAGS "${COMPILER_FLAGS} -ffile-prefix-map=${CMAKE_SOURCE_DIR}=.")
-    set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} -ffile-prefix-map=${CMAKE_SOURCE_DIR}=.")
+    set (COMPILER_FLAGS "${COMPILER_FLAGS} -ffile-prefix-map=${PROJECT_SOURCE_DIR}=.")
+    set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} -ffile-prefix-map=${PROJECT_SOURCE_DIR}=.")
 endif ()
 
 option (ENABLE_BUILD_PROFILING "Enable profiling of build time" OFF)
@@ -312,59 +283,39 @@ if (ENABLE_BUILD_PROFILING)
      endif ()
 endif ()
 
-set (CMAKE_CXX_STANDARD 20)
-set (CMAKE_CXX_EXTENSIONS ON) # Same as gnu++2a (ON) vs c++2a (OFF): https://cmake.org/cmake/help/latest/prop_tgt/CXX_EXTENSIONS.html
+set (CMAKE_CXX_STANDARD 23)
+set (CMAKE_CXX_EXTENSIONS OFF)
 set (CMAKE_CXX_STANDARD_REQUIRED ON)
 
 set (CMAKE_C_STANDARD 11)
-set (CMAKE_C_EXTENSIONS ON)
+set (CMAKE_C_EXTENSIONS ON) # required by most contribs written in C
 set (CMAKE_C_STANDARD_REQUIRED ON)
 
-if (COMPILER_GCC OR COMPILER_CLANG)
-    # Enable C++14 sized global deallocation functions. It should be enabled by setting -std=c++14 but I'm not sure.
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsized-deallocation")
-endif ()
-
-# falign-functions=32 prevents from random performance regressions with the code change. Thus, providing more stable
-# benchmarks.
-if (COMPILER_GCC OR COMPILER_CLANG)
-    set(COMPILER_FLAGS "${COMPILER_FLAGS} -falign-functions=32")
-endif ()
-
-if (ARCH_AMD64)
-    # align branches within a 32-Byte boundary to avoid the potential performance loss when code layout change,
-    # which makes benchmark results more stable.
-    set(BRANCHES_WITHIN_32B_BOUNDARIES "-mbranches-within-32B-boundaries")
-    if (COMPILER_GCC)
-        # gcc is in assembler, need to add "-Wa," prefix
-        set(BRANCHES_WITHIN_32B_BOUNDARIES "-Wa,${BRANCHES_WITHIN_32B_BOUNDARIES}")
-    endif()
-
-    include(CheckCXXCompilerFlag)
-    check_cxx_compiler_flag("${BRANCHES_WITHIN_32B_BOUNDARIES}" HAS_BRANCHES_WITHIN_32B_BOUNDARIES)
-    if (HAS_BRANCHES_WITHIN_32B_BOUNDARIES)
-        set(COMPILER_FLAGS "${COMPILER_FLAGS} ${BRANCHES_WITHIN_32B_BOUNDARIES}")
-    endif()
-endif()
-
-if (COMPILER_GCC)
-    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fcoroutines")
-endif ()
-
-# Compiler-specific coverage flags e.g. -fcoverage-mapping for gcc
+# Compiler-specific coverage flags e.g. -fcoverage-mapping
 option(WITH_COVERAGE "Profile the resulting binary/binaries" OFF)
 
-if (WITH_COVERAGE AND COMPILER_CLANG)
-    set(COMPILER_FLAGS "${COMPILER_FLAGS} -fprofile-instr-generate -fcoverage-mapping")
-    # If we want to disable coverage for specific translation units
-    set(WITHOUT_COVERAGE "-fno-profile-instr-generate -fno-coverage-mapping")
-endif()
+if (COMPILER_CLANG)
+    # Enable C++14 sized global deallocation functions. It should be enabled by setting -std=c++14 but I'm not sure.
+    # See https://reviews.llvm.org/D112921
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsized-deallocation")
 
-if (WITH_COVERAGE AND COMPILER_GCC)
-    set(COMPILER_FLAGS "${COMPILER_FLAGS} -fprofile-arcs -ftest-coverage")
-    set(COVERAGE_OPTION "-lgcov")
-    set(WITHOUT_COVERAGE "-fno-profile-arcs -fno-test-coverage")
-endif()
+    # falign-functions=32 prevents from random performance regressions with the code change. Thus, providing more stable
+    # benchmarks.
+    set(COMPILER_FLAGS "${COMPILER_FLAGS} -falign-functions=32")
+
+    if (ARCH_AMD64)
+        # align branches within a 32-Byte boundary to avoid the potential performance loss when code layout change,
+        # which makes benchmark results more stable.
+        set(BRANCHES_WITHIN_32B_BOUNDARIES "-mbranches-within-32B-boundaries")
+        set(COMPILER_FLAGS "${COMPILER_FLAGS} ${BRANCHES_WITHIN_32B_BOUNDARIES}")
+    endif()
+
+    if (WITH_COVERAGE)
+        set(COMPILER_FLAGS "${COMPILER_FLAGS} -fprofile-instr-generate -fcoverage-mapping")
+        # If we want to disable coverage for specific translation units
+        set(WITHOUT_COVERAGE "-fno-profile-instr-generate -fno-coverage-mapping")
+    endif()
+endif ()
 
 set (COMPILER_FLAGS "${COMPILER_FLAGS}")
 
@@ -373,15 +324,15 @@ set (DEBUG_INFO_FLAGS "-g -gdwarf-4")
 
 set (CMAKE_CXX_FLAGS                     "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS}")
 set (CMAKE_CXX_FLAGS_RELWITHDEBINFO      "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_CXX_FLAGS_ADD}")
-set (CMAKE_CXX_FLAGS_DEBUG               "${CMAKE_CXX_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} -fno-inline ${CMAKE_CXX_FLAGS_ADD}")
+set (CMAKE_CXX_FLAGS_DEBUG               "${CMAKE_CXX_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} ${CMAKE_CXX_FLAGS_ADD}")
 
 set (CMAKE_C_FLAGS                       "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} ${CMAKE_C_FLAGS_ADD}")
 set (CMAKE_C_FLAGS_RELWITHDEBINFO        "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_C_FLAGS_ADD}")
-set (CMAKE_C_FLAGS_DEBUG                 "${CMAKE_C_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} -fno-inline ${CMAKE_C_FLAGS_ADD}")
+set (CMAKE_C_FLAGS_DEBUG                 "${CMAKE_C_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} ${CMAKE_C_FLAGS_ADD}")
 
 set (CMAKE_ASM_FLAGS                     "${CMAKE_ASM_FLAGS} ${COMPILER_FLAGS} ${CMAKE_ASM_FLAGS_ADD}")
 set (CMAKE_ASM_FLAGS_RELWITHDEBINFO      "${CMAKE_ASM_FLAGS_RELWITHDEBINFO} -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_ASM_FLAGS_ADD}")
-set (CMAKE_ASM_FLAGS_DEBUG               "${CMAKE_ASM_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} -fno-inline ${CMAKE_ASM_FLAGS_ADD}")
+set (CMAKE_ASM_FLAGS_DEBUG               "${CMAKE_ASM_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} ${CMAKE_ASM_FLAGS_ADD}")
 
 if (COMPILER_CLANG)
     if (OS_DARWIN)
@@ -402,11 +353,6 @@ if (COMPILER_CLANG)
 
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fstrict-vtable-pointers")
 
-    # Set new experimental pass manager, it's a performance, build time and binary size win.
-    # Can be removed after https://reviews.llvm.org/D66490 merged and released to at least two versions of clang.
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fexperimental-new-pass-manager")
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fexperimental-new-pass-manager")
-
     # We cannot afford to use LTO when compiling unit tests, and it's not enough
     # to only supply -fno-lto at the final linking stage. So we disable it
     # completely.
@@ -440,19 +386,27 @@ elseif (OS_DARWIN)
     include(cmake/darwin/default_libs.cmake)
 elseif (OS_FREEBSD)
     include(cmake/freebsd/default_libs.cmake)
+else()
+    link_libraries(global-group)
 endif ()
-link_libraries(global-group)
 
-if (NOT (OS_LINUX OR OS_DARWIN))
-    # Using system libs can cause a lot of warnings in includes (on macro expansion).
-    option(WERROR "Enable -Werror compiler option" OFF)
-else ()
-    option(WERROR "Enable -Werror compiler option" ON)
+option (ENABLE_GWP_ASAN "Enable Gwp-Asan" ON)
+# We use mmap for allocations more heavily in debug builds,
+# but GWP-ASan also wants to use mmap frequently,
+# and due to a large number of memory mappings,
+# it does not work together well.
+if ((NOT OS_LINUX AND NOT OS_ANDROID) OR (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG"))
+    set(ENABLE_GWP_ASAN OFF)
 endif ()
 
+option (ENABLE_FIU "Enable Fiu" ON)
+
+option(WERROR "Enable -Werror compiler option" ON)
+
 if (WERROR)
     # Don't pollute CMAKE_CXX_FLAGS with -Werror as it will break some CMake checks.
     # Instead, adopt modern cmake usage requirement.
+    # TODO: Set CMAKE_COMPILE_WARNING_AS_ERROR (cmake 3.24)
     target_compile_options(global-group INTERFACE "-Werror")
 endif ()
 
@@ -467,22 +421,16 @@ endif ()
 
 set (CMAKE_POSTFIX_VARIABLE "CMAKE_${CMAKE_BUILD_TYPE_UC}_POSTFIX")
 
-if (USE_STATIC_LIBRARIES)
+if (NOT SANITIZE)
     set (CMAKE_POSITION_INDEPENDENT_CODE OFF)
-    if (OS_LINUX AND NOT ARCH_AARCH64)
-        # Slightly more efficient code can be generated
-        # It's disabled for ARM because otherwise ClickHouse cannot run on Android.
-        set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-pie")
-        set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -fno-pie")
-        set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -no-pie -Wl,-no-pie")
-    endif ()
-else ()
-    set (CMAKE_POSITION_INDEPENDENT_CODE ON)
-    # This is required for clang on Arch linux, that uses PIE by default.
-    # See enable-SSP-and-PIE-by-default.patch [1].
-    #
-    #   [1]: https://github.com/archlinux/svntogit-packages/blob/6e681aa860e65ad46a1387081482eb875c2200f2/trunk/enable-SSP-and-PIE-by-default.patch
-    set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -no-pie")
+endif()
+
+if (OS_LINUX AND NOT (ARCH_AARCH64 OR ARCH_S390X) AND NOT SANITIZE)
+    # Slightly more efficient code can be generated
+    # It's disabled for ARM because otherwise ClickHouse cannot run on Android.
+    set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-pie")
+    set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -fno-pie")
+    set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -no-pie -Wl,-no-pie")
 endif ()
 
 if (ENABLE_TESTS)
@@ -493,6 +441,15 @@ endif ()
 
 enable_testing() # Enable for tests without binary
 
+option(ENABLE_OPENSSL "This option performs a build with OpenSSL. NOTE! This option is insecure and should never be used. By default, ClickHouse uses and only supports BoringSSL" OFF)
+
+if (ARCH_S390X)
+    set(ENABLE_OPENSSL_DYNAMIC_DEFAULT ON)
+else ()
+    set(ENABLE_OPENSSL_DYNAMIC_DEFAULT OFF)
+endif ()
+option(ENABLE_OPENSSL_DYNAMIC "This option removes SSL from ClickHouse and will link to the OpenSSL version supplied by OS." ${ENABLE_OPENSSL_DYNAMIC_DEFAULT})
+
 # when installing to /usr - place configs to /etc but for /usr/local place to /usr/local/etc
 if (CMAKE_INSTALL_PREFIX STREQUAL "/usr")
     set (CLICKHOUSE_ETC_DIR "/etc")
@@ -500,10 +457,7 @@ else ()
     set (CLICKHOUSE_ETC_DIR "${CMAKE_INSTALL_PREFIX}/etc")
 endif ()
 
-message (STATUS
-    "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE} ;
-    USE_STATIC_LIBRARIES=${USE_STATIC_LIBRARIES}
-    SPLIT_SHARED_LIBRARIES=${SPLIT_SHARED_LIBRARIES}")
+message (STATUS "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE}")
 
 include (GNUInstallDirs)
 
@@ -549,15 +503,15 @@ macro (clickhouse_add_executable target)
         # - _je_zone_register due to JEMALLOC_PRIVATE_NAMESPACE=je_ under OS X.
         # - but jemalloc-cmake does not run private_namespace.sh
         #   so symbol name should be _zone_register
-        if (ENABLE_JEMALLOC AND USE_STATIC_LIBRARIES AND OS_DARWIN)
+        if (ENABLE_JEMALLOC AND OS_DARWIN)
             set_property(TARGET ${target} APPEND PROPERTY LINK_OPTIONS -u_zone_register)
         endif()
     endif()
 endmacro()
 
-# With cross-compiling, all targets are built for the target platform which usually different from the host 
-# platform. This is problematic if a build artifact X (e.g. a file or an executable) is generated by running 
-# another executable Y previously produced in the build. This is solved by compiling and running Y for/on 
+# With cross-compiling, all targets are built for the target platform which usually different from the host
+# platform. This is problematic if a build artifact X (e.g. a file or an executable) is generated by running
+# another executable Y previously produced in the build. This is solved by compiling and running Y for/on
 # the host platform. Add target to the list:
 #    add_native_target(<target> ...)
 set_property (GLOBAL PROPERTY NATIVE_BUILD_TARGETS)
@@ -565,24 +519,48 @@ function (add_native_target)
     set_property (GLOBAL APPEND PROPERTY NATIVE_BUILD_TARGETS ${ARGV})
 endfunction (add_native_target)
 
-set(ConfigIncludePath ${CMAKE_CURRENT_BINARY_DIR}/includes/configs CACHE INTERNAL "Path to generated configuration files.")
-include_directories(${ConfigIncludePath})
+set(CONFIG_INCLUDE_PATH ${CMAKE_CURRENT_BINARY_DIR}/includes/configs CACHE INTERNAL "Path to generated configuration files.")
+include_directories(${CONFIG_INCLUDE_PATH})
 
 # Add as many warnings as possible for our own code.
 include (cmake/warnings.cmake)
 include (cmake/print_flags.cmake)
 
+if (ENABLE_RUST)
+    add_subdirectory (rust)
+
+    # With LTO Rust adds few symbols with global visiblity, the most common is
+    # rust_eh_personality. And this leads to linking errors because multiple
+    # Rust libraries contains the same symbol.
+    #
+    # If it was shared library, that we could use version script for linker to
+    # hide this symbols, but libraries are static.
+    #
+    # we could in theory compile everything to one library but this will be a
+    # mess
+    #
+    # But this should be OK since CI has lots of other builds that are done
+    # without LTO and it will find multiple definitions if there will be any.
+    #
+    # More information about this behaviour in Rust can be found here
+    # - https://github.com/rust-lang/rust/issues/44322
+    # - https://alanwu.space/post/symbol-hygiene/
+    if (ENABLE_THINLTO)
+        set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--allow-multiple-definition")
+    endif()
+endif()
+
 add_subdirectory (base)
 add_subdirectory (src)
 add_subdirectory (programs)
 add_subdirectory (tests)
 add_subdirectory (utils)
 
-include (cmake/sanitize_target_link_libraries.cmake)
+include (cmake/sanitize_targets.cmake)
 
 # Build native targets if necessary
 get_property(NATIVE_BUILD_TARGETS GLOBAL PROPERTY NATIVE_BUILD_TARGETS)
-if (NATIVE_BUILD_TARGETS 
+if (NATIVE_BUILD_TARGETS
     AND NOT(
         CMAKE_HOST_SYSTEM_NAME STREQUAL CMAKE_SYSTEM_NAME
         AND CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL CMAKE_SYSTEM_PROCESSOR
@@ -590,7 +568,7 @@ if (NATIVE_BUILD_TARGETS
 )
     message (STATUS "Building native targets...")
 
-    set (NATIVE_BUILD_DIR "${CMAKE_BINARY_DIR}/native")
+    set (NATIVE_BUILD_DIR "${PROJECT_BINARY_DIR}/native")
 
     execute_process(
         COMMAND ${CMAKE_COMMAND} -E make_directory "${NATIVE_BUILD_DIR}"
@@ -600,7 +578,11 @@ if (NATIVE_BUILD_TARGETS
         COMMAND ${CMAKE_COMMAND}
             "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}"
             "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}"
-        ${CMAKE_SOURCE_DIR}
+            "-DCOMPILER_CACHE=${COMPILER_CACHE}"
+            # Avoid overriding .cargo/config.toml with native toolchain.
+            "-DENABLE_RUST=OFF"
+            "-DENABLE_CLICKHOUSE_SELF_EXTRACTING=${ENABLE_CLICKHOUSE_SELF_EXTRACTING}"
+        ${PROJECT_SOURCE_DIR}
         WORKING_DIRECTORY "${NATIVE_BUILD_DIR}"
         COMMAND_ECHO STDOUT)
 

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2016-2022 ClickHouse, Inc.
+Copyright 2016-2023 ClickHouse, Inc.
 
                                  Apache License
                            Version 2.0, January 2004
@@ -188,7 +188,7 @@ Copyright 2016-2022 ClickHouse, Inc.
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2016-2022 ClickHouse, Inc.
+   Copyright 2016-2023 ClickHouse, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

PreLoad.cmake

@@ -7,12 +7,8 @@
 # How to install Ninja on Ubuntu:
 #  sudo apt-get install ninja-build
 
-# CLion does not support Ninja
-# You can add your vote on CLion task tracker:
-# https://youtrack.jetbrains.com/issue/CPP-2659
-# https://youtrack.jetbrains.com/issue/CPP-870
 
-if (NOT DEFINED ENV{CLION_IDE} AND NOT DEFINED ENV{XCODE_IDE})
+if (NOT DEFINED ENV{XCODE_IDE})
     find_program(NINJA_PATH ninja)
     if (NINJA_PATH)
         set(CMAKE_GENERATOR "Ninja" CACHE INTERNAL "")
@@ -23,8 +19,8 @@ endif()
 if (NOT "$ENV{CFLAGS}" STREQUAL ""
     OR NOT "$ENV{CXXFLAGS}" STREQUAL ""
     OR NOT "$ENV{LDFLAGS}" STREQUAL ""
-    OR CMAKE_C_FLAGS OR CMAKE_CXX_FLAGS OR CMAKE_EXE_LINKER_FLAGS OR CMAKE_SHARED_LINKER_FLAGS OR CMAKE_MODULE_LINKER_FLAGS
-    OR CMAKE_C_FLAGS_INIT OR CMAKE_CXX_FLAGS_INIT OR CMAKE_EXE_LINKER_FLAGS_INIT OR CMAKE_SHARED_LINKER_FLAGS_INIT OR CMAKE_MODULE_LINKER_FLAGS_INIT)
+    OR CMAKE_C_FLAGS OR CMAKE_CXX_FLAGS OR CMAKE_EXE_LINKER_FLAGS OR CMAKE_MODULE_LINKER_FLAGS
+    OR CMAKE_C_FLAGS_INIT OR CMAKE_CXX_FLAGS_INIT OR CMAKE_EXE_LINKER_FLAGS_INIT OR CMAKE_MODULE_LINKER_FLAGS_INIT)
 
     # if $ENV
     message("CFLAGS: $ENV{CFLAGS}")
@@ -40,7 +36,6 @@ if (NOT "$ENV{CFLAGS}" STREQUAL ""
     message("CMAKE_C_FLAGS_INIT: ${CMAKE_C_FLAGS_INIT}")
     message("CMAKE_CXX_FLAGS_INIT: ${CMAKE_CXX_FLAGS_INIT}")
     message("CMAKE_EXE_LINKER_FLAGS_INIT: ${CMAKE_EXE_LINKER_FLAGS_INIT}")
-    message("CMAKE_SHARED_LINKER_FLAGS_INIT: ${CMAKE_SHARED_LINKER_FLAGS_INIT}")
     message("CMAKE_MODULE_LINKER_FLAGS_INIT: ${CMAKE_MODULE_LINKER_FLAGS_INIT}")
 
     message(FATAL_ERROR "
@@ -88,7 +83,10 @@ if (OS MATCHES "Linux"
         set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-aarch64.cmake" CACHE INTERNAL "")
     elseif (ARCH MATCHES "^(ppc64le.*|PPC64LE.*)")
         set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-ppc64le.cmake" CACHE INTERNAL "")
+    elseif (ARCH MATCHES "^(s390x.*|S390X.*)")
+        set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-s390x.cmake" CACHE INTERNAL "")
     else ()
         message (FATAL_ERROR "Unsupported architecture: ${ARCH}")
     endif ()
+
 endif()

README.md

@@ -1,18 +1,47 @@
-[![ClickHouse — open source distributed column-oriented DBMS](https://github.com/ClickHouse/clickhouse-presentations/raw/master/images/logo-400x240.png)](https://clickhouse.com)
+[<img alt="ClickHouse — open source distributed column-oriented DBMS" width="400px" src="https://clickhouse.com/images/ch_gh_logo_rounded.png" />](https://clickhouse.com?utm_source=github)
 
 ClickHouse® is an open-source column-oriented database management system that allows generating analytical data reports in real-time.
 
+## How To Install (Linux, macOS, FreeBSD)
+```
+curl https://clickhouse.com/ | sh
+```
+
 ## Useful Links
 
 * [Official website](https://clickhouse.com/) has a quick high-level overview of ClickHouse on the main page.
+* [ClickHouse Cloud](https://clickhouse.cloud) ClickHouse as a service, built by the creators and maintainers.
 * [Tutorial](https://clickhouse.com/docs/en/getting_started/tutorial/) shows how to set up and query a small ClickHouse cluster.
 * [Documentation](https://clickhouse.com/docs/en/) provides more in-depth information.
 * [YouTube channel](https://www.youtube.com/c/ClickHouseDB) has a lot of content about ClickHouse in video format.
-* [Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-rxm3rdrk-lIUmhLC3V8WTaL0TGxsOmg) and [Telegram](https://telegram.me/clickhouse_en) allow chatting with ClickHouse users in real-time.
-* [Blog](https://clickhouse.com/blog/en/) contains various ClickHouse-related articles, as well as announcements and reports about events.
-* [Code Browser (Woboq)](https://clickhouse.com/codebrowser/ClickHouse/index.html) with syntax highlight and navigation.
-* [Code Browser (github.dev)](https://github.dev/ClickHouse/ClickHouse) with syntax highlight, powered by github.dev.
+* [Slack](https://clickhouse.com/slack) and [Telegram](https://telegram.me/clickhouse_en) allow chatting with ClickHouse users in real-time.
+* [Blog](https://clickhouse.com/blog/) contains various ClickHouse-related articles, as well as announcements and reports about events.
+* [Code Browser (Woboq)](https://clickhouse.com/codebrowser/ClickHouse/index.html) with syntax highlighting and navigation.
+* [Code Browser (github.dev)](https://github.dev/ClickHouse/ClickHouse) with syntax highlighting, powered by github.dev.
+* [Static Analysis (SonarCloud)](https://sonarcloud.io/project/issues?resolved=false&id=ClickHouse_ClickHouse) proposes C++ quality improvements.
 * [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any.
 
-## Upcoming events
-* [**v22.8 Release Webinar**](https://clickhouse.com/company/events/v22-8-release-webinar) Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release, provide live demos, and share vision into what is coming in the roadmap.
+## Upcoming Events
+
+* [**v23.6 Release Webinar**](https://clickhouse.com/company/events/v23-6-release-call?utm_source=github&utm_medium=social&utm_campaign=release-webinar-2023-06) - Jun 29 - 23.6 is rapidly approaching. Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release.
+* [**ClickHouse Meetup in Boston**](https://www.meetup.com/clickhouse-boston-user-group/events/293913596) - Jul 18
+* [**ClickHouse Meetup in NYC**](https://www.meetup.com/clickhouse-new-york-user-group/events/293913441) - Jul 19
+* [**ClickHouse Meetup in Toronto**](https://www.meetup.com/clickhouse-toronto-user-group/events/294183127) - Jul 20
+* [**ClickHouse Meetup in Singapore**](https://www.meetup.com/clickhouse-singapore-meetup-group/events/294428050/) - Jul 27
+* [**ClickHouse Meetup in Paris**](https://www.meetup.com/clickhouse-france-user-group/events/294283460) - Sep 12
+
+Also, keep an eye out for upcoming meetups around the world. Somewhere else you want us to be? Please feel free to reach out to tyler <at> clickhouse <dot> com.
+
+## Recent Recordings
+* **Recent Meetup Videos**: [Meetup Playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3iNDUzpY1S3L_iV4nARda_U) Whenever possible recordings of the ClickHouse Community Meetups are edited and presented as individual talks. Current featuring "Modern SQL in 2023", "Fast, Concurrent, and Consistent Asynchronous INSERTS in ClickHouse", and "Full-Text Indices: Design and Experiments"
+* **Recording available**: [**v23.4 Release Webinar**](https://www.youtube.com/watch?v=4rrf6bk_mOg) Faster Parquet Reading, Asynchonous Connections to Reoplicas, Trailing Comma before FROM, extractKeyValuePairs, integrations updates, and so much more! Watch it now!
+* **All release webinar recordings**: [YouTube playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3jAlSy1JxyP8zluvXaN3nxU)
+
+  
+ ## Interested in joining ClickHouse and making it your full time job? 
+  
+We are a globally diverse and distributed team, united behind a common goal of creating industry-leading, real-time analytics. Here, you will have an opportunity to solve some of the most cutting edge technical challenges and have direct ownership of your work and vision. If you are a contributor by nature, a thinker as well as a doer - we’ll definitely click!
+  
+Check out our **current openings** here: https://clickhouse.com/company/careers 
+  
+Cant find what you are looking for, but want to let us know you are interested in joining ClickHouse? Email careers@clickhouse.com! 

SECURITY.md

@@ -1,3 +1,6 @@
+<!--
+the file is autogenerated by utils/security-generator/generate_security.py
+-->
 
 # Security Policy
 
@@ -10,26 +13,19 @@ The following versions of ClickHouse server are currently being supported with s
 
 | Version | Supported |
 |:-|:-|
+| 23.6 | ✔️ |
+| 23.5 | ✔️ |
+| 23.4 | ✔️ |
+| 23.3 | ✔️ |
+| 23.2 | ❌ |
+| 23.1 | ❌ |
+| 22.12 | ❌ |
+| 22.11 | ❌ |
+| 22.10 | ❌ |
+| 22.9 | ❌ |
 | 22.8 | ✔️ |
-| 22.7 | ✔️ |
-| 22.6 | ✔️ |
-| 22.5 | ❌ |
-| 22.4 | ❌ |
-| 22.3 | ✔️ |
-| 22.2 | ❌ |
-| 22.1 | ❌ |
-| 21.12 | ❌ |
-| 21.11 | ❌ |
-| 21.10 | ❌ |
-| 21.9 | ❌ |
-| 21.8 | ❌ |
-| 21.7 | ❌ |
-| 21.6 | ❌ |
-| 21.5 | ❌ |
-| 21.4 | ❌ |
-| 21.3 | ❌ |
-| 21.2 | ❌ |
-| 21.1 | ❌ |
+| 22.* | ❌ |
+| 21.* | ❌ |
 | 20.* | ❌ |
 | 19.* | ❌ |
 | 18.* | ❌ |
@@ -59,5 +55,5 @@ As the security issue moves from triage, to identified fix, to release planning
 
 ## Public Disclosure Timing
 
-A public disclosure date is negotiated by the ClickHouse maintainers and the bug submitter. We prefer to fully disclose the bug as soon as possible once a user mitigation is available. It is reasonable to delay disclosure when the bug or the fix is not yet fully understood, the solution is not well-tested, or for vendor coordination. The timeframe for disclosure is from immediate (especially if it's already publicly known) to 90 days. For a vulnerability with a straightforward mitigation, we expect the report date to disclosure date to be on the order of 7 days. 
+A public disclosure date is negotiated by the ClickHouse maintainers and the bug submitter. We prefer to fully disclose the bug as soon as possible once a user mitigation is available. It is reasonable to delay disclosure when the bug or the fix is not yet fully understood, the solution is not well-tested, or for vendor coordination. The timeframe for disclosure is from immediate (especially if it's already publicly known) to 90 days. For a vulnerability with a straightforward mitigation, we expect the report date to disclosure date to be on the order of 7 days.
 

base/CMakeLists.txt

@@ -1,8 +1,5 @@
-if (USE_CLANG_TIDY)
-    set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}")
-endif ()
-
 add_subdirectory (base)
 add_subdirectory (pcg-random)
+add_subdirectory (poco)
 add_subdirectory (widechar_width)
 add_subdirectory (readpassphrase)

base/base/BorrowedObjectPool.h

@@ -10,7 +10,7 @@
 #include <base/MoveOrCopyIfThrow.h>
 
 /** Pool for limited size objects that cannot be used from different threads simultaneously.
-  * The main use case is to have fixed size of objects that can be reused in difference threads during their lifetime
+  * The main use case is to have fixed size of objects that can be reused in different threads during their lifetime
   * and have to be initialized on demand.
   * Two main properties of pool are allocated objects size and borrowed objects size.
   * Allocated objects size is size of objects that are currently allocated by the pool.

base/base/CMakeLists.txt

@@ -1,3 +1,11 @@
+if (USE_CLANG_TIDY)
+    set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}")
+endif ()
+
+# TODO: Remove this. We like to compile with C++23 (set by top-level CMakeLists) but Clang crashes with our libcxx
+# when instantiated from JSON.cpp. Try again when libcxx(abi) and Clang are upgraded to 16.
+set (CMAKE_CXX_STANDARD 20)
+
 set (SRCS
     argsToConfig.cpp
     coverage.cpp
@@ -8,16 +16,13 @@ set (SRCS
     getPageSize.cpp
     getThreadId.cpp
     JSON.cpp
-    LineReader.cpp
     mremap.cpp
     phdr_cache.cpp
     preciseExp10.cpp
-    setTerminalEcho.cpp
     shift10.cpp
     sleep.cpp
     terminalColors.cpp
     errnoToString.cpp
-    ReplxxLineReader.cpp
     StringRef.cpp
     safeExit.cpp
     throwError.cpp
@@ -42,10 +47,6 @@ endif ()
 
 target_include_directories(common PUBLIC .. "${CMAKE_CURRENT_BINARY_DIR}/..")
 
-if (OS_DARWIN AND NOT USE_STATIC_LIBRARIES)
-    target_link_libraries(common PUBLIC -Wl,-U,_inside_main)
-endif()
-
 target_link_libraries (common
     PUBLIC
         ch_contrib::cityhash

base/base/CachedFn.h

@@ -1,75 +0,0 @@
-#pragma once
-
-#include <map>
-#include <tuple>
-#include <mutex>
-#include "FnTraits.h"
-
-/**
- * Caching proxy for a functor that decays to a pointer-to-function.
- * Saves pairs (func args, func result on args).
- * Cache size is unlimited. Cache items are evicted only on manual drop.
- * Invocation/update is O(log(saved cache values)).
- *
- * See Common/tests/cached_fn.cpp for examples.
-  */
-template <auto * Func>
-struct CachedFn
-{
-private:
-    using Traits = FnTraits<decltype(Func)>;
-    using DecayedArgs = TypeListMap<std::decay_t, typename Traits::Args>;
-    using Key = TypeListChangeRoot<std::tuple, DecayedArgs>;
-    using Result = typename Traits::Ret;
-
-    std::map<Key, Result> cache; // Can't use hashmap as tuples are unhashable by default
-    mutable std::mutex mutex;
-
-public:
-    template <class ...Args>
-    Result operator()(Args && ...args)
-    {
-        Key key{std::forward<Args>(args)...};
-
-        {
-            std::lock_guard lock(mutex);
-
-            if (auto it = cache.find(key); it != cache.end())
-                return it->second;
-        }
-
-        Result res = std::apply(Func, key);
-
-        {
-            std::lock_guard lock(mutex);
-            cache.emplace(std::move(key), res);
-        }
-
-        return res;
-    }
-
-    template <class ...Args>
-    void update(Args && ...args)
-    {
-        Key key{std::forward<Args>(args)...};
-        Result res = std::apply(Func, key);
-
-        {
-            std::lock_guard lock(mutex);
-            // TODO Can't use emplace(std::move(key), ..), causes test_host_ip_change errors.
-            cache[key] = std::move(res);
-        }
-    }
-
-    size_t size() const
-    {
-        std::lock_guard lock(mutex);
-        return cache.size();
-    }
-
-    void drop()
-    {
-        std::lock_guard lock(mutex);
-        cache.clear();
-    }
-};

base/base/Decimal.h

@@ -1,5 +1,6 @@
 #pragma once
 #include <base/extended_types.h>
+#include <base/Decimal_fwd.h>
 
 #if !defined(NO_SANITIZE_UNDEFINED)
 #if defined(__clang__)
@@ -19,23 +20,6 @@ using Decimal64 = Decimal<Int64>;
 using Decimal128 = Decimal<Int128>;
 using Decimal256 = Decimal<Int256>;
 
-template <class T>
-concept is_decimal =
-    std::is_same_v<T, Decimal32>
-    || std::is_same_v<T, Decimal64>
-    || std::is_same_v<T, Decimal128>
-    || std::is_same_v<T, Decimal256>
-    || std::is_same_v<T, DateTime64>;
-
-template <class T>
-concept is_over_big_int =
-    std::is_same_v<T, Int128>
-    || std::is_same_v<T, UInt128>
-    || std::is_same_v<T, Int256>
-    || std::is_same_v<T, UInt256>
-    || std::is_same_v<T, Decimal128>
-    || std::is_same_v<T, Decimal256>;
-
 template <class T> struct NativeTypeT { using Type = T; };
 template <is_decimal T> struct NativeTypeT<T> { using Type = typename T::NativeType; };
 template <class T> using NativeType = typename NativeTypeT<T>::Type;
@@ -52,15 +36,15 @@ struct Decimal
     constexpr Decimal(Decimal<T> &&) noexcept = default;
     constexpr Decimal(const Decimal<T> &) = default;
 
-    constexpr Decimal(const T & value_): value(value_) {}
+    constexpr Decimal(const T & value_): value(value_) {} // NOLINT(google-explicit-constructor)
 
     template <typename U>
-    constexpr Decimal(const Decimal<U> & x): value(x.value) {}
+    constexpr Decimal(const Decimal<U> & x): value(x.value) {} // NOLINT(google-explicit-constructor)
 
     constexpr Decimal<T> & operator=(Decimal<T> &&) noexcept = default;
     constexpr Decimal<T> & operator = (const Decimal<T> &) = default;
 
-    constexpr operator T () const { return value; }
+    constexpr operator T () const { return value; } // NOLINT(google-explicit-constructor)
 
     template <typename U>
     constexpr U convertTo() const
@@ -111,7 +95,7 @@ public:
     using Base::Base;
     using NativeType = Base::NativeType;
 
-    constexpr DateTime64(const Base & v): Base(v) {}
+    constexpr DateTime64(const Base & v): Base(v) {} // NOLINT(google-explicit-constructor)
 };
 }
 

base/base/Decimal_fwd.h

@@ -0,0 +1,46 @@
+#pragma once
+
+#include <base/types.h>
+
+namespace wide
+{
+
+template <size_t Bits, typename Signed>
+class integer;
+
+}
+
+using Int128 = wide::integer<128, signed>;
+using UInt128 = wide::integer<128, unsigned>;
+using Int256 = wide::integer<256, signed>;
+using UInt256 = wide::integer<256, unsigned>;
+
+namespace DB
+{
+
+template <class> struct Decimal;
+
+using Decimal32 = Decimal<Int32>;
+using Decimal64 = Decimal<Int64>;
+using Decimal128 = Decimal<Int128>;
+using Decimal256 = Decimal<Int256>;
+
+class DateTime64;
+
+template <class T>
+concept is_decimal =
+    std::is_same_v<T, Decimal32>
+    || std::is_same_v<T, Decimal64>
+    || std::is_same_v<T, Decimal128>
+    || std::is_same_v<T, Decimal256>
+    || std::is_same_v<T, DateTime64>;
+
+template <class T>
+concept is_over_big_int =
+    std::is_same_v<T, Int128>
+    || std::is_same_v<T, UInt128>
+    || std::is_same_v<T, Int256>
+    || std::is_same_v<T, UInt256>
+    || std::is_same_v<T, Decimal128>
+    || std::is_same_v<T, Decimal256>;
+}

base/base/DecomposedFloat.h

@@ -36,14 +36,14 @@ struct DecomposedFloat
 {
     using Traits = FloatTraits<T>;
 
-    DecomposedFloat(T x)
+    explicit DecomposedFloat(T x)
     {
         memcpy(&x_uint, &x, sizeof(x));
     }
 
     typename Traits::UInt x_uint;
 
-    bool is_negative() const
+    bool isNegative() const
     {
         return x_uint >> (Traits::bits - 1);
     }
@@ -53,7 +53,7 @@ struct DecomposedFloat
     {
         return (exponent() == 0 && mantissa() == 0)
             ? 0
-            : (is_negative()
+            : (isNegative()
                 ? -1
                 : 1);
     }
@@ -63,7 +63,7 @@ struct DecomposedFloat
         return (x_uint >> (Traits::mantissa_bits)) & (((1ull << (Traits::exponent_bits + 1)) - 1) >> 1);
     }
 
-    int16_t normalized_exponent() const
+    int16_t normalizedExponent() const
     {
         return int16_t(exponent()) - ((1ull << (Traits::exponent_bits - 1)) - 1);
     }
@@ -73,20 +73,20 @@ struct DecomposedFloat
         return x_uint & ((1ull << Traits::mantissa_bits) - 1);
     }
 
-    int64_t mantissa_with_sign() const
+    int64_t mantissaWithSign() const
     {
-        return is_negative() ? -mantissa() : mantissa();
+        return isNegative() ? -mantissa() : mantissa();
     }
 
     /// NOTE Probably floating point instructions can be better.
-    bool is_integer_in_representable_range() const
+    bool isIntegerInRepresentableRange() const
     {
         return x_uint == 0
-            || (normalized_exponent() >= 0  /// The number is not less than one
+            || (normalizedExponent() >= 0  /// The number is not less than one
                 /// The number is inside the range where every integer has exact representation in float
-                && normalized_exponent() <= static_cast<int16_t>(Traits::mantissa_bits)
+                && normalizedExponent() <= static_cast<int16_t>(Traits::mantissa_bits)
                 /// After multiplying by 2^exp, the fractional part becomes zero, means the number is integer
-                && ((mantissa() & ((1ULL << (Traits::mantissa_bits - normalized_exponent())) - 1)) == 0));
+                && ((mantissa() & ((1ULL << (Traits::mantissa_bits - normalizedExponent())) - 1)) == 0));
     }
 
 
@@ -102,15 +102,15 @@ struct DecomposedFloat
             return sign();
 
         /// Different signs
-        if (is_negative() && rhs > 0)
+        if (isNegative() && rhs > 0)
             return -1;
-        if (!is_negative() && rhs < 0)
+        if (!isNegative() && rhs < 0)
             return 1;
 
         /// Fractional number with magnitude less than one
-        if (normalized_exponent() < 0)
+        if (normalizedExponent() < 0)
         {
-            if (!is_negative())
+            if (!isNegative())
                 return rhs > 0 ? -1 : 1;
             else
                 return rhs >= 0 ? -1 : 1;
@@ -121,11 +121,11 @@ struct DecomposedFloat
         {
             if (rhs == std::numeric_limits<Int>::lowest())
             {
-                assert(is_negative());
+                assert(isNegative());
 
-                if (normalized_exponent() < static_cast<int16_t>(8 * sizeof(Int) - is_signed_v<Int>))
+                if (normalizedExponent() < static_cast<int16_t>(8 * sizeof(Int) - is_signed_v<Int>))
                     return 1;
-                if (normalized_exponent() > static_cast<int16_t>(8 * sizeof(Int) - is_signed_v<Int>))
+                if (normalizedExponent() > static_cast<int16_t>(8 * sizeof(Int) - is_signed_v<Int>))
                     return -1;
 
                 if (mantissa() == 0)
@@ -136,44 +136,44 @@ struct DecomposedFloat
         }
 
         /// Too large number: abs(float) > abs(rhs). Also the case with infinities and NaN.
-        if (normalized_exponent() >= static_cast<int16_t>(8 * sizeof(Int) - is_signed_v<Int>))
-            return is_negative() ? -1 : 1;
+        if (normalizedExponent() >= static_cast<int16_t>(8 * sizeof(Int) - is_signed_v<Int>))
+            return isNegative() ? -1 : 1;
 
         using UInt = std::conditional_t<(sizeof(Int) > sizeof(typename Traits::UInt)), make_unsigned_t<Int>, typename Traits::UInt>;
         UInt uint_rhs = rhs < 0 ? -rhs : rhs;
 
         /// Smaller octave: abs(rhs) < abs(float)
         /// FYI, TIL: octave is also called "binade", https://en.wikipedia.org/wiki/Binade
-        if (uint_rhs < (static_cast<UInt>(1) << normalized_exponent()))
-            return is_negative() ? -1 : 1;
+        if (uint_rhs < (static_cast<UInt>(1) << normalizedExponent()))
+            return isNegative() ? -1 : 1;
 
         /// Larger octave: abs(rhs) > abs(float)
-        if (normalized_exponent() + 1 < static_cast<int16_t>(8 * sizeof(Int) - is_signed_v<Int>)
-            && uint_rhs >= (static_cast<UInt>(1) << (normalized_exponent() + 1)))
-            return is_negative() ? 1 : -1;
+        if (normalizedExponent() + 1 < static_cast<int16_t>(8 * sizeof(Int) - is_signed_v<Int>)
+            && uint_rhs >= (static_cast<UInt>(1) << (normalizedExponent() + 1)))
+            return isNegative() ? 1 : -1;
 
         /// The same octave
-        /// uint_rhs == 2 ^ normalized_exponent + mantissa * 2 ^ (normalized_exponent - mantissa_bits)
+        /// uint_rhs == 2 ^ normalizedExponent + mantissa * 2 ^ (normalizedExponent - mantissa_bits)
 
-        bool large_and_always_integer = normalized_exponent() >= static_cast<int16_t>(Traits::mantissa_bits);
+        bool large_and_always_integer = normalizedExponent() >= static_cast<int16_t>(Traits::mantissa_bits);
 
         UInt a = large_and_always_integer
-            ? static_cast<UInt>(mantissa()) << (normalized_exponent() - Traits::mantissa_bits)
-            : static_cast<UInt>(mantissa()) >> (Traits::mantissa_bits - normalized_exponent());
+            ? static_cast<UInt>(mantissa()) << (normalizedExponent() - Traits::mantissa_bits)
+            : static_cast<UInt>(mantissa()) >> (Traits::mantissa_bits - normalizedExponent());
 
-        UInt b = uint_rhs - (static_cast<UInt>(1) << normalized_exponent());
+        UInt b = uint_rhs - (static_cast<UInt>(1) << normalizedExponent());
 
         if (a < b)
-            return is_negative() ? 1 : -1;
+            return isNegative() ? 1 : -1;
         if (a > b)
-            return is_negative() ? -1 : 1;
+            return isNegative() ? -1 : 1;
 
         /// Float has no fractional part means that the numbers are equal.
-        if (large_and_always_integer || (mantissa() & ((1ULL << (Traits::mantissa_bits - normalized_exponent())) - 1)) == 0)
+        if (large_and_always_integer || (mantissa() & ((1ULL << (Traits::mantissa_bits - normalizedExponent())) - 1)) == 0)
             return 0;
         else
             /// Float has fractional part means its abs value is larger.
-            return is_negative() ? -1 : 1;
+            return isNegative() ? -1 : 1;
     }
 
 

base/base/IPv4andIPv6.h

@@ -0,0 +1,77 @@
+#pragma once
+
+#include <base/strong_typedef.h>
+#include <base/extended_types.h>
+#include <Common/formatIPv6.h>
+#include <Common/memcmpSmall.h>
+
+namespace DB
+{
+
+    struct IPv4 : StrongTypedef<UInt32, struct IPv4Tag>
+    {
+        using StrongTypedef::StrongTypedef;
+        using StrongTypedef::operator=;
+        constexpr explicit IPv4(UInt64 value): StrongTypedef(static_cast<UnderlyingType>(value)) {}
+    };
+
+    struct IPv6 : StrongTypedef<UInt128, struct IPv6Tag>
+    {
+        using StrongTypedef::StrongTypedef;
+        using StrongTypedef::operator=;
+
+        bool operator<(const IPv6 & rhs) const
+        {
+            return
+                memcmp16(
+                    reinterpret_cast<const unsigned char *>(toUnderType().items),
+                    reinterpret_cast<const unsigned char *>(rhs.toUnderType().items)
+                ) < 0;
+        }
+
+        bool operator>(const IPv6 & rhs) const
+        {
+            return
+                memcmp16(
+                    reinterpret_cast<const unsigned char *>(toUnderType().items),
+                    reinterpret_cast<const unsigned char *>(rhs.toUnderType().items)
+                ) > 0;
+        }
+
+        bool operator==(const IPv6 & rhs) const
+        {
+            return
+                memcmp16(
+                    reinterpret_cast<const unsigned char *>(toUnderType().items),
+                    reinterpret_cast<const unsigned char *>(rhs.toUnderType().items)
+                ) == 0;
+        }
+
+        bool operator<=(const IPv6 & rhs) const { return !operator>(rhs); }
+        bool operator>=(const IPv6 & rhs) const { return !operator<(rhs); }
+        bool operator!=(const IPv6 & rhs) const { return !operator==(rhs); }
+    };
+
+}
+
+namespace std
+{
+    /// For historical reasons we hash IPv6 as a FixedString(16)
+    template <>
+    struct hash<DB::IPv6>
+    {
+        size_t operator()(const DB::IPv6 & x) const
+        {
+            return std::hash<std::string_view>{}(std::string_view(reinterpret_cast<const char*>(&x.toUnderType()), IPV6_BINARY_LENGTH));
+        }
+    };
+
+    template <>
+    struct hash<DB::IPv4>
+    {
+        size_t operator()(const DB::IPv4 & x) const
+        {
+            return std::hash<DB::IPv4::UnderlyingType>()(x.toUnderType());
+        }
+    };
+}

base/base/JSON.cpp

@@ -22,7 +22,7 @@ POCO_IMPLEMENT_EXCEPTION(JSONException, Poco::Exception, "JSONException") // NOL
 #endif
 
 
-/// Прочитать беззнаковое целое в простом формате из не-0-terminated строки.
+/// Read unsigned integer in a simple form from a non-0-terminated string.
 static UInt64 readUIntText(const char * buf, const char * end)
 {
     UInt64 x = 0;
@@ -59,7 +59,7 @@ static UInt64 readUIntText(const char * buf, const char * end)
 }
 
 
-/// Прочитать знаковое целое в простом формате из не-0-terminated строки.
+/// Read signed integer in a simple form from a non-0-terminated string.
 static Int64 readIntText(const char * buf, const char * end)
 {
     bool negative = false;
@@ -102,7 +102,7 @@ static Int64 readIntText(const char * buf, const char * end)
 }
 
 
-/// Прочитать число с плавающей запятой в простом формате, с грубым округлением, из не-0-terminated строки.
+/// Read floating point number in simple format, imprecisely, from a non-0-terminated string.
 static double readFloatText(const char * buf, const char * end)
 {
     bool negative = false;
@@ -151,8 +151,8 @@ static double readFloatText(const char * buf, const char * end)
             case 'E':
             {
                 ++buf;
-                Int32 exponent = readIntText(buf, end);
-                x *= preciseExp10(exponent);
+                auto exponent = readIntText(buf, end);
+                x *= preciseExp10(static_cast<double>(exponent));
 
                 run = false;
                 break;
@@ -207,7 +207,7 @@ JSON::ElementType JSON::getType() const
             return TYPE_NUMBER;
         case '"':
         {
-            /// Проверим - это просто строка или name-value pair
+            /// Is it a string or a name-value pair?
             Pos after_string = skipString();
             if (after_string < ptr_end && *after_string == ':')
                 return TYPE_NAME_VALUE_PAIR;
@@ -229,15 +229,13 @@ void JSON::checkPos(Pos pos) const
 
 JSON::Pos JSON::skipString() const
 {
-    //std::cerr << "skipString()\t" << data() << std::endl;
-
     Pos pos = ptr_begin;
     checkPos(pos);
     if (*pos != '"')
         throw JSONException(std::string("JSON: expected \", got ") + *pos);
     ++pos;
 
-    /// fast path: находим следующую двойную кавычку. Если перед ней нет бэкслеша - значит это конец строки (при допущении корректности JSON).
+    /// fast path: find next double quote. If it is not escaped by backslash - then it's an end of string (assuming JSON is valid).
     Pos closing_quote = reinterpret_cast<const char *>(memchr(reinterpret_cast<const void *>(pos), '\"', ptr_end - pos));
     if (nullptr != closing_quote && closing_quote[-1] != '\\')
         return closing_quote + 1;
@@ -269,8 +267,6 @@ JSON::Pos JSON::skipString() const
 
 JSON::Pos JSON::skipNumber() const
 {
-    //std::cerr << "skipNumber()\t" << data() << std::endl;
-
     Pos pos = ptr_begin;
 
     checkPos(pos);
@@ -296,8 +292,6 @@ JSON::Pos JSON::skipNumber() const
 
 JSON::Pos JSON::skipBool() const
 {
-    //std::cerr << "skipBool()\t" << data() << std::endl;
-
     Pos pos = ptr_begin;
     checkPos(pos);
 
@@ -314,16 +308,12 @@ JSON::Pos JSON::skipBool() const
 
 JSON::Pos JSON::skipNull() const
 {
-    //std::cerr << "skipNull()\t" << data() << std::endl;
-
     return ptr_begin + 4;
 }
 
 
 JSON::Pos JSON::skipNameValuePair() const
 {
-    //std::cerr << "skipNameValuePair()\t" << data() << std::endl;
-
     Pos pos = skipString();
     checkPos(pos);
 
@@ -338,8 +328,6 @@ JSON::Pos JSON::skipNameValuePair() const
 
 JSON::Pos JSON::skipArray() const
 {
-    //std::cerr << "skipArray()\t" << data() << std::endl;
-
     if (!isArray())
         throw JSONException("JSON: expected [");
     Pos pos = ptr_begin;
@@ -370,8 +358,6 @@ JSON::Pos JSON::skipArray() const
 
 JSON::Pos JSON::skipObject() const
 {
-    //std::cerr << "skipObject()\t" << data() << std::endl;
-
     if (!isObject())
         throw JSONException("JSON: expected {");
     Pos pos = ptr_begin;
@@ -402,8 +388,6 @@ JSON::Pos JSON::skipObject() const
 
 JSON::Pos JSON::skipElement() const
 {
-    //std::cerr << "skipElement()\t" << data() << std::endl;
-
     ElementType type = getType();
 
     switch (type)
@@ -482,9 +466,8 @@ JSON::Pos JSON::searchField(const char * data, size_t size) const
     {
         if (!it->hasEscapes())
         {
-            if (static_cast<int>(size) + 2 > it->dataEnd() - it->data())
-                continue;
-            if (!strncmp(data, it->data() + 1, size))
+            const auto current_name = it->getRawName();
+            if (current_name.size() == size && 0 == memcmp(current_name.data(), data, size))
                 break;
         }
         else
@@ -640,7 +623,7 @@ std::string JSON::getString() const
                         {
                             throw JSONException("JSON: incorrect syntax: incorrect HEX code.");
                         }
-                        buf.resize(buf.size() + 6);    /// максимальный размер UTF8 многобайтовой последовательности
+                        buf.resize(buf.size() + 6);    /// Max size of UTF-8 sequence, including pre-standard mapping of UCS-4 to UTF-8.
                         int res = utf8.convert(unicode,
                             reinterpret_cast<unsigned char *>(const_cast<char*>(buf.data())) + buf.size() - 6, 6);
                         if (!res)
@@ -754,8 +737,6 @@ JSON::iterator JSON::iterator::begin() const
     if (type != TYPE_ARRAY && type != TYPE_OBJECT)
         throw JSONException("JSON: not array or object when calling begin() method.");
 
-    //std::cerr << "begin()\t" << data() << std::endl;
-
     Pos pos = ptr_begin + 1;
     checkPos(pos);
     if (*pos == '}' || *pos == ']')
@@ -846,4 +827,3 @@ bool JSON::isType<bool>() const
 {
     return isBool();
 }
-

base/base/JSON.h

@@ -38,6 +38,7 @@
   */
 
 
+// NOLINTBEGIN(google-explicit-constructor)
 #ifdef __clang__
 #  pragma clang diagnostic push
 #  pragma clang diagnostic ignored "-Wdeprecated-dynamic-exception-spec"
@@ -46,6 +47,7 @@ POCO_DECLARE_EXCEPTION(Foundation_API, JSONException, Poco::Exception)
 #ifdef __clang__
 #  pragma clang diagnostic pop
 #endif
+// NOLINTEND(google-explicit-constructor)
 
 class JSON
 {
@@ -61,7 +63,7 @@ public:
         checkInit();
     }
 
-    JSON(const std::string & s) : ptr_begin(s.data()), ptr_end(s.data() + s.size()), level(0)
+    explicit JSON(std::string_view s) : ptr_begin(s.data()), ptr_end(s.data() + s.size()), level(0)
     {
         checkInit();
     }
@@ -71,13 +73,7 @@ public:
         *this = rhs;
     }
 
-    JSON & operator=(const JSON & rhs)
-    {
-        ptr_begin = rhs.ptr_begin;
-        ptr_end = rhs.ptr_end;
-        level = rhs.level;
-        return *this;
-    }
+    JSON & operator=(const JSON & rhs) = default;
 
     const char * data() const { return ptr_begin; }
     const char * dataEnd() const { return ptr_end; }
@@ -169,7 +165,7 @@ public:
 
     /// Перейти к следующему элементу массива или следующей name-value паре объекта.
     iterator & operator++();
-    iterator operator++(int);
+    iterator operator++(int); // NOLINT(cert-dcl21-cpp)
 
     /// Есть ли в строке escape-последовательности
     bool hasEscapes() const;

base/base/ReplxxLineReader.cpp

@@ -1,383 +0,0 @@
-#include <base/ReplxxLineReader.h>
-#include <base/errnoToString.h>
-
-#include <chrono>
-#include <cerrno>
-#include <cstring>
-#include <unistd.h>
-#include <functional>
-#include <sys/file.h>
-#include <sys/types.h>
-#include <sys/wait.h>
-#include <csignal>
-#include <dlfcn.h>
-#include <fcntl.h>
-#include <fstream>
-#include <fmt/format.h>
-
-
-namespace
-{
-
-/// Trim ending whitespace inplace
-void trim(String & s)
-{
-    s.erase(std::find_if(s.rbegin(), s.rend(), [](unsigned char ch) { return !std::isspace(ch); }).base(), s.end());
-}
-
-std::string getEditor()
-{
-    const char * editor = std::getenv("EDITOR");
-
-    if (!editor || !*editor)
-        editor = "vim";
-
-    return editor;
-}
-
-/// Copied from replxx::src/util.cxx::now_ms_str() under the terms of 3-clause BSD license of Replxx.
-/// Copyright (c) 2017-2018, Marcin Konarski (amok at codestation.org)
-/// Copyright (c) 2010, Salvatore Sanfilippo (antirez at gmail dot com)
-/// Copyright (c) 2010, Pieter Noordhuis (pcnoordhuis at gmail dot com)
-std::string replxx_now_ms_str()
-{
-    std::chrono::milliseconds ms(std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()));
-    time_t t = ms.count() / 1000;
-    tm broken;
-    if (!localtime_r(&t, &broken))
-        return {};
-
-    static int const BUFF_SIZE(32);
-    char str[BUFF_SIZE];
-    if (strftime(str, BUFF_SIZE, "%Y-%m-%d %H:%M:%S.", &broken) <= 0)
-        return {};
-
-    if (snprintf(str + sizeof("YYYY-mm-dd HH:MM:SS"), 5, "%03d", static_cast<int>(ms.count() % 1000)) <= 0)
-        return {};
-
-    return str;
-}
-
-/// Convert from readline to replxx format.
-///
-/// replxx requires each history line to prepended with time line:
-///
-///     ### YYYY-MM-DD HH:MM:SS.SSS
-///     select 1
-///
-/// And w/o those service lines it will load all lines from history file as
-/// one history line for suggestion. And if there are lots of lines in file it
-/// will take lots of time (getline() + tons of reallocations).
-///
-/// NOTE: this code uses std::ifstream/std::ofstream like original replxx code.
-void convertHistoryFile(const std::string & path, replxx::Replxx & rx)
-{
-    std::ifstream in(path);
-    if (!in)
-    {
-        rx.print("Cannot open %s reading (for conversion): %s\n",
-            path.c_str(), errnoToString(errno).c_str());
-        return;
-    }
-
-    std::string line;
-    if (getline(in, line).bad())
-    {
-        rx.print("Cannot read from %s (for conversion): %s\n",
-            path.c_str(), errnoToString(errno).c_str());
-        return;
-    }
-
-    /// This is the marker of the date, no need to convert.
-    static char const REPLXX_TIMESTAMP_PATTERN[] = "### dddd-dd-dd dd:dd:dd.ddd";
-    if (line.empty() || (line.starts_with("### ") && line.size() == strlen(REPLXX_TIMESTAMP_PATTERN)))
-    {
-        return;
-    }
-
-    std::vector<std::string> lines;
-    in.seekg(0);
-    while (getline(in, line).good())
-    {
-        lines.push_back(line);
-    }
-    in.close();
-
-    size_t lines_size = lines.size();
-    std::sort(lines.begin(), lines.end());
-    lines.erase(std::unique(lines.begin(), lines.end()), lines.end());
-    rx.print("The history file (%s) is in old format. %zu lines, %zu unique lines.\n",
-        path.c_str(), lines_size, lines.size());
-
-    std::ofstream out(path);
-    if (!out)
-    {
-        rx.print("Cannot open %s for writing (for conversion): %s\n",
-            path.c_str(), errnoToString(errno).c_str());
-        return;
-    }
-
-    const std::string & timestamp = replxx_now_ms_str();
-    for (const auto & out_line : lines)
-    {
-        out << "### " << timestamp << "\n" << out_line << std::endl;
-    }
-    out.close();
-}
-
-}
-
-static bool replxx_last_is_delimiter = false;
-void ReplxxLineReader::setLastIsDelimiter(bool flag)
-{
-    replxx_last_is_delimiter = flag;
-}
-
-ReplxxLineReader::ReplxxLineReader(
-    Suggest & suggest,
-    const String & history_file_path_,
-    bool multiline_,
-    Patterns extenders_,
-    Patterns delimiters_,
-    replxx::Replxx::highlighter_callback_t highlighter_)
-    : LineReader(history_file_path_, multiline_, std::move(extenders_), std::move(delimiters_)), highlighter(std::move(highlighter_))
-    , editor(getEditor())
-{
-    using namespace std::placeholders;
-    using Replxx = replxx::Replxx;
-
-    if (!history_file_path.empty())
-    {
-        history_file_fd = open(history_file_path.c_str(), O_RDWR);
-        if (history_file_fd < 0)
-        {
-            rx.print("Open of history file failed: %s\n", errnoToString(errno).c_str());
-        }
-        else
-        {
-            convertHistoryFile(history_file_path, rx);
-
-            if (flock(history_file_fd, LOCK_SH))
-            {
-                rx.print("Shared lock of history file failed: %s\n", errnoToString(errno).c_str());
-            }
-            else
-            {
-                if (!rx.history_load(history_file_path))
-                {
-                    rx.print("Loading history failed: %s\n", errnoToString(errno).c_str());
-                }
-
-                if (flock(history_file_fd, LOCK_UN))
-                {
-                    rx.print("Unlock of history file failed: %s\n", errnoToString(errno).c_str());
-                }
-            }
-        }
-    }
-
-    rx.install_window_change_handler();
-
-    auto callback = [&suggest] (const String & context, size_t context_size)
-    {
-        return suggest.getCompletions(context, context_size);
-    };
-
-    rx.set_completion_callback(callback);
-    rx.set_complete_on_empty(false);
-    rx.set_word_break_characters(word_break_characters);
-    rx.set_ignore_case(true);
-
-    if (highlighter)
-        rx.set_highlighter_callback(highlighter);
-
-    /// By default C-p/C-n binded to COMPLETE_NEXT/COMPLETE_PREV,
-    /// bind C-p/C-n to history-previous/history-next like readline.
-    rx.bind_key(Replxx::KEY::control('N'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::HISTORY_NEXT, code); });
-    rx.bind_key(Replxx::KEY::control('P'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::HISTORY_PREVIOUS, code); });
-
-    auto commit_action = [this](char32_t code)
-    {
-        /// If we allow multiline and there is already something in the input, start a newline.
-        /// NOTE: Lexer is only available if we use highlighter.
-        if (highlighter && multiline && !replxx_last_is_delimiter)
-            return rx.invoke(Replxx::ACTION::NEW_LINE, code);
-        replxx_last_is_delimiter = false;
-        return rx.invoke(Replxx::ACTION::COMMIT_LINE, code);
-    };
-    /// bind C-j to ENTER action.
-    rx.bind_key(Replxx::KEY::control('J'), commit_action);
-    rx.bind_key(Replxx::KEY::ENTER, commit_action);
-
-    /// By default COMPLETE_NEXT/COMPLETE_PREV was binded to C-p/C-n, re-bind
-    /// to M-P/M-N (that was used for HISTORY_COMMON_PREFIX_SEARCH before, but
-    /// it also binded to M-p/M-n).
-    rx.bind_key(Replxx::KEY::meta('N'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::COMPLETE_NEXT, code); });
-    rx.bind_key(Replxx::KEY::meta('P'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::COMPLETE_PREVIOUS, code); });
-    /// By default M-BACKSPACE is KILL_TO_WHITESPACE_ON_LEFT, while in readline it is backward-kill-word
-    rx.bind_key(Replxx::KEY::meta(Replxx::KEY::BACKSPACE), [this](char32_t code) { return rx.invoke(Replxx::ACTION::KILL_TO_BEGINING_OF_WORD, code); });
-    /// By default C-w is KILL_TO_BEGINING_OF_WORD, while in readline it is unix-word-rubout
-    rx.bind_key(Replxx::KEY::control('W'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::KILL_TO_WHITESPACE_ON_LEFT, code); });
-
-    rx.bind_key(Replxx::KEY::meta('E'), [this](char32_t) { openEditor(); return Replxx::ACTION_RESULT::CONTINUE; });
-}
-
-ReplxxLineReader::~ReplxxLineReader()
-{
-    if (close(history_file_fd))
-        rx.print("Close of history file failed: %s\n", errnoToString(errno).c_str());
-}
-
-LineReader::InputStatus ReplxxLineReader::readOneLine(const String & prompt)
-{
-    input.clear();
-
-    const char* cinput = rx.input(prompt);
-    if (cinput == nullptr)
-        return (errno != EAGAIN) ? ABORT : RESET_LINE;
-    input = cinput;
-
-    trim(input);
-    return INPUT_LINE;
-}
-
-void ReplxxLineReader::addToHistory(const String & line)
-{
-    // locking history file to prevent from inconsistent concurrent changes
-    //
-    // replxx::Replxx::history_save() already has lockf(),
-    // but replxx::Replxx::history_load() does not
-    // and that is why flock() is added here.
-    bool locked = false;
-    if (flock(history_file_fd, LOCK_EX))
-        rx.print("Lock of history file failed: %s\n", errnoToString(errno).c_str());
-    else
-        locked = true;
-
-    rx.history_add(line);
-
-    // flush changes to the disk
-    if (!rx.history_save(history_file_path))
-        rx.print("Saving history failed: %s\n", errnoToString(errno).c_str());
-
-    if (locked && 0 != flock(history_file_fd, LOCK_UN))
-        rx.print("Unlock of history file failed: %s\n", errnoToString(errno).c_str());
-}
-
-/// See comments in ShellCommand::executeImpl()
-/// (for the vfork via dlsym())
-int ReplxxLineReader::executeEditor(const std::string & path)
-{
-    std::vector<char> argv0(editor.data(), editor.data() + editor.size() + 1);
-    std::vector<char> argv1(path.data(), path.data() + path.size() + 1);
-    char * const argv[] = {argv0.data(), argv1.data(), nullptr};
-
-    static void * real_vfork = dlsym(RTLD_DEFAULT, "vfork");
-    if (!real_vfork)
-    {
-        rx.print("Cannot find symbol vfork in myself: %s\n", errnoToString(errno).c_str());
-        return -1;
-    }
-
-    pid_t pid = reinterpret_cast<pid_t (*)()>(real_vfork)();
-
-    if (-1 == pid)
-    {
-        rx.print("Cannot vfork: %s\n", errnoToString(errno).c_str());
-        return -1;
-    }
-
-    /// Child
-    if (0 == pid)
-    {
-        sigset_t mask;
-        sigemptyset(&mask);
-        sigprocmask(0, nullptr, &mask);
-        sigprocmask(SIG_UNBLOCK, &mask, nullptr);
-
-        execvp(editor.c_str(), argv);
-        rx.print("Cannot execute %s: %s\n", editor.c_str(), errnoToString(errno).c_str());
-        _exit(-1);
-    }
-
-    int status = 0;
-    do
-    {
-        int exited_pid = waitpid(pid, &status, 0);
-        if (exited_pid == -1)
-        {
-            if (errno == EINTR)
-                continue;
-
-            rx.print("Cannot waitpid: %s\n", errnoToString(errno).c_str());
-            return -1;
-        }
-        else
-            break;
-    } while (true);
-    return status;
-}
-
-void ReplxxLineReader::openEditor()
-{
-    char filename[] = "clickhouse_replxx_XXXXXX.sql";
-    int fd = ::mkstemps(filename, 4);
-    if (-1 == fd)
-    {
-        rx.print("Cannot create temporary file to edit query: %s\n", errnoToString(errno).c_str());
-        return;
-    }
-
-    replxx::Replxx::State state(rx.get_state());
-
-    size_t bytes_written = 0;
-    const char * begin = state.text();
-    size_t offset = strlen(state.text());
-    while (bytes_written != offset)
-    {
-        ssize_t res = ::write(fd, begin + bytes_written, offset - bytes_written);
-        if ((-1 == res || 0 == res) && errno != EINTR)
-        {
-            rx.print("Cannot write to temporary query file %s: %s\n", filename, errnoToString(errno).c_str());
-            break;
-        }
-        bytes_written += res;
-    }
-
-    if (0 != ::close(fd))
-    {
-        rx.print("Cannot close temporary query file %s: %s\n", filename, errnoToString(errno).c_str());
-        return;
-    }
-
-    if (0 == executeEditor(filename))
-    {
-        try
-        {
-            std::ifstream t(filename);
-            std::string str;
-            t.seekg(0, std::ios::end);
-            str.reserve(t.tellg());
-            t.seekg(0, std::ios::beg);
-            str.assign((std::istreambuf_iterator<char>(t)), std::istreambuf_iterator<char>());
-            rx.set_state(replxx::Replxx::State(str.c_str(), str.size()));
-        }
-        catch (...)
-        {
-            rx.print("Cannot read from temporary query file %s: %s\n", filename, errnoToString(errno).c_str());
-            return;
-        }
-    }
-
-    if (bracketed_paste_enabled)
-        enableBracketedPaste();
-
-    if (0 != ::unlink(filename))
-        rx.print("Cannot remove temporary query file %s: %s\n", filename, errnoToString(errno).c_str());
-}
-
-void ReplxxLineReader::enableBracketedPaste()
-{
-    bracketed_paste_enabled = true;
-    rx.enable_bracketed_paste();
-}

base/base/StringRef.h

@@ -3,6 +3,7 @@
 #include <cassert>
 #include <stdexcept> // for std::logic_error
 #include <string>
+#include <type_traits>
 #include <vector>
 #include <functional>
 #include <iosfwd>
@@ -61,11 +62,6 @@ struct StringRef
     constexpr explicit operator std::string_view() const { return std::string_view(data, size); }
 };
 
-/// Here constexpr doesn't implicate inline, see https://www.viva64.com/en/w/v1043/
-/// nullptr can't be used because the StringRef values are used in SipHash's pointer arithmetic
-/// and the UBSan thinks that something like nullptr + 8 is UB.
-constexpr const inline char empty_string_ref_addr{};
-constexpr const inline StringRef EMPTY_STRING_REF{&empty_string_ref_addr, 0};
 
 using StringRefs = std::vector<StringRef>;
 
@@ -227,7 +223,7 @@ inline UInt64 shiftMix(UInt64 val)
     return val ^ (val >> 47);
 }
 
-inline UInt64 rotateByAtLeast1(UInt64 val, int shift)
+inline UInt64 rotateByAtLeast1(UInt64 val, UInt8 shift)
 {
     return (val >> shift) | (val << (64 - shift));
 }
@@ -249,7 +245,7 @@ inline size_t hashLessThan8(const char * data, size_t size)
         uint8_t b = data[size >> 1];
         uint8_t c = data[size - 1];
         uint32_t y = static_cast<uint32_t>(a) + (static_cast<uint32_t>(b) << 8);
-        uint32_t z = size + (static_cast<uint32_t>(c) << 2);
+        uint32_t z = static_cast<uint32_t>(size) + (static_cast<uint32_t>(c) << 2);
         return shiftMix(y * k2 ^ z * k3) * k2;
     }
 
@@ -262,7 +258,7 @@ inline size_t hashLessThan16(const char * data, size_t size)
     {
         UInt64 a = unalignedLoad<UInt64>(data);
         UInt64 b = unalignedLoad<UInt64>(data + size - 8);
-        return hashLen16(a, rotateByAtLeast1(b + size, size)) ^ b;
+        return hashLen16(a, rotateByAtLeast1(b + size, static_cast<UInt8>(size))) ^ b;
     }
 
     return hashLessThan8(data, size);
@@ -270,7 +266,7 @@ inline size_t hashLessThan16(const char * data, size_t size)
 
 struct CRC32Hash
 {
-    size_t operator() (StringRef x) const
+    unsigned operator() (StringRef x) const
     {
         const char * pos = x.data;
         size_t size = x.size;
@@ -280,22 +276,22 @@ struct CRC32Hash
 
         if (size < 8)
         {
-            return hashLessThan8(x.data, x.size);
+            return static_cast<unsigned>(hashLessThan8(x.data, x.size));
         }
 
         const char * end = pos + size;
-        size_t res = -1ULL;
+        unsigned res = -1U;
 
         do
         {
             UInt64 word = unalignedLoad<UInt64>(pos);
-            res = CRC_INT(res, word);
+            res = static_cast<unsigned>(CRC_INT(res, word));
 
             pos += 8;
         } while (pos + 8 < end);
 
         UInt64 word = unalignedLoad<UInt64>(end - 8);    /// I'm not sure if this is normal.
-        res = CRC_INT(res, word);
+        res = static_cast<unsigned>(CRC_INT(res, word));
 
         return res;
     }
@@ -307,7 +303,7 @@ struct StringRefHash : CRC32Hash {};
 
 struct CRC32Hash
 {
-    size_t operator() (StringRef /* x */) const
+    unsigned operator() (StringRef /* x */) const
     {
        throw std::logic_error{"Not implemented CRC32Hash without SSE"};
     }
@@ -331,5 +327,16 @@ namespace ZeroTraits
     inline void set(StringRef & x) { x.size = 0; }
 }
 
+namespace PackedZeroTraits
+{
+    template <typename Second, template <typename, typename> class PackedPairNoInit>
+    inline bool check(const PackedPairNoInit<StringRef, Second> p)
+    { return 0 == p.key.size; }
+
+    template <typename Second, template <typename, typename> class PackedPairNoInit>
+    inline void set(PackedPairNoInit<StringRef, Second> & p)
+    { p.key.size = 0; }
+}
+
 
 std::ostream & operator<<(std::ostream & os, const StringRef & str);

base/base/TypeList.h

@@ -4,7 +4,6 @@
 #include <type_traits>
 #include <utility>
 #include "defines.h"
-#include "TypePair.h"
 
 /// General-purpose typelist. Easy on compilation times as it does not use recursion.
 template <typename ...Args>
@@ -28,7 +27,7 @@ namespace TypeListUtils /// In some contexts it's more handy to use functions in
     constexpr Root<Args...> changeRoot(TypeList<Args...>) { return {}; }
 
     template <typename F, typename ...Args>
-    constexpr void forEach(TypeList<Args...>, F && f) { (std::forward<F>(f)(Id<Args>{}), ...); }
+    constexpr void forEach(TypeList<Args...>, F && f) { (std::forward<F>(f)(TypeList<Args>{}), ...); }
 }
 
 template <typename TypeListLeft, typename TypeListRight>

base/base/TypeName.h

@@ -2,6 +2,7 @@
 
 #include "Decimal.h"
 #include "UUID.h"
+#include "IPv4andIPv6.h"
 
 namespace DB
 {
@@ -35,6 +36,8 @@ TN_MAP(Float32)
 TN_MAP(Float64)
 TN_MAP(String)
 TN_MAP(UUID)
+TN_MAP(IPv4)
+TN_MAP(IPv6)
 TN_MAP(Decimal32)
 TN_MAP(Decimal64)
 TN_MAP(Decimal128)

base/base/TypePair.h

@@ -1,4 +0,0 @@
-#pragma once
-
-template <typename T, typename V> struct TypePair {};
-template <typename T> struct Id {};

base/base/argsToConfig.cpp

@@ -3,13 +3,29 @@
 #include <Poco/Util/LayeredConfiguration.h>
 #include <Poco/Util/MapConfiguration.h>
 
-
-void argsToConfig(const Poco::Util::Application::ArgVec & argv, Poco::Util::LayeredConfiguration & config, int priority)
+void argsToConfig(const Poco::Util::Application::ArgVec & argv,
+                  Poco::Util::LayeredConfiguration & config,
+                  int priority,
+                  const std::unordered_set<std::string>* alias_names)
 {
     /// Parsing all args and converting to config layer
     /// Test: -- --1=1 --1=2 --3 5 7 8 -9 10 -11=12 14= 15== --16==17 --=18 --19= --20 21 22 --23 --24 25 --26 -27 28 ---29=30 -- ----31 32 --33 3-4
     Poco::AutoPtr<Poco::Util::MapConfiguration> map_config = new Poco::Util::MapConfiguration;
     std::string key;
+
+    auto add_arg = [&map_config, &alias_names](const std::string & k, const std::string & v)
+    {
+        map_config->setString(k, v);
+
+        if (alias_names && !alias_names->contains(k))
+        {
+            std::string alias_key = k;
+            std::replace(alias_key.begin(), alias_key.end(), '-', '_');
+            if (alias_names->contains(alias_key))
+                map_config->setString(alias_key, v);
+        }
+    };
+
     for (const auto & arg : argv)
     {
         auto key_start = arg.find_first_not_of('-');
@@ -19,7 +35,7 @@ void argsToConfig(const Poco::Util::Application::ArgVec & argv, Poco::Util::Laye
         // old saved '--key', will set to some true value "1"
         if (!key.empty() && pos_minus != std::string::npos && pos_minus < key_start)
         {
-            map_config->setString(key, "1");
+            add_arg(key, "1");
             key = "";
         }
 
@@ -29,7 +45,7 @@ void argsToConfig(const Poco::Util::Application::ArgVec & argv, Poco::Util::Laye
             {
                 if (pos_minus == std::string::npos || pos_minus > key_start)
                 {
-                    map_config->setString(key, arg);
+                    add_arg(key, arg);
                 }
                 key = "";
             }
@@ -55,7 +71,7 @@ void argsToConfig(const Poco::Util::Application::ArgVec & argv, Poco::Util::Laye
         if (arg.size() > pos_eq)
             value = arg.substr(pos_eq + 1);
 
-        map_config->setString(key, value);
+        add_arg(key, value);
         key = "";
     }
 

base/base/argsToConfig.h

@@ -1,6 +1,8 @@
 #pragma once
 
 #include <Poco/Util/Application.h>
+#include <string>
+#include <unordered_set>
 
 namespace Poco::Util
 {
@@ -8,4 +10,7 @@ class LayeredConfiguration; // NOLINT(cppcoreguidelines-virtual-class-destructor
 }
 
 /// Import extra command line arguments to configuration. These are command line arguments after --.
-void argsToConfig(const Poco::Util::Application::ArgVec & argv, Poco::Util::LayeredConfiguration & config, int priority);
+void argsToConfig(const Poco::Util::Application::ArgVec & argv,
+                  Poco::Util::LayeredConfiguration & config,
+                  int priority,
+                  const std::unordered_set<std::string>* registered_alias_names = nullptr);

base/base/arithmeticOverflow.h

@@ -3,6 +3,7 @@
 #include <base/extended_types.h>
 #include <base/defines.h>
 
+// NOLINTBEGIN(google-runtime-int)
 
 namespace common
 {
@@ -206,3 +207,5 @@ namespace common
         return false;
     }
 }
+
+// NOLINTEND(google-runtime-int)

base/base/bit_cast.h

@@ -1,27 +1,39 @@
 #pragma once
 
-#include <string.h>
+#include <cstring>
 #include <algorithm>
 #include <type_traits>
 
 
-/** \brief Returns value `from` converted to type `To` while retaining bit representation.
-  *    `To` and `From` must satisfy `CopyConstructible`.
+/** Returns value `from` converted to type `To` while retaining bit representation.
+  * `To` and `From` must satisfy `CopyConstructible`.
+  *
+  * In contrast to std::bit_cast can cast types of different width.
+  *
+  * Note: for signed types of narrower size, the casted result is zero-extended
+  * instead of sign-extended as with regular static_cast.
+  * For example, -1 Int8 (represented as 0xFF) bit_casted to UInt64
+  * gives 255 (represented as 0x00000000000000FF) instead of 0xFFFFFFFFFFFFFFFF
   */
 template <typename To, typename From>
 std::decay_t<To> bit_cast(const From & from)
 {
+    /** Assume the source value is 0xAABBCCDD (i.e. sizeof(from) == 4).
+      * Its BE representation is 0xAABBCCDD, the LE representation is 0xDDCCBBAA.
+      * Further assume, sizeof(res) == 8 and that res is initially zeroed out.
+      * With LE, the result after bit_cast will be 0xDDCCBBAA00000000 --> input value == output value.
+      * With BE, the result after bit_cast will be 0x00000000AABBCCDD --> input value == output value.
+      */
     To res {};
-    memcpy(static_cast<void*>(&res), &from, std::min(sizeof(res), sizeof(from)));
+    if constexpr (std::endian::native == std::endian::little)
+    {
+        memcpy(static_cast<void*>(&res), &from, std::min(sizeof(res), sizeof(from)));
+    }
+    else
+    {
+        uint32_t offset_to = (sizeof(res) > sizeof(from)) ? (sizeof(res) - sizeof(from)) : 0;
+        uint32_t offset_from = (sizeof(from) > sizeof(res)) ? (sizeof(from) - sizeof(res)) : 0;
+        memcpy(reinterpret_cast<char *>(&res) + offset_to, reinterpret_cast<const char *>(&from) + offset_from, std::min(sizeof(res), sizeof(from)));
+    }
     return res;
 }
-
-/** \brief Returns value `from` converted to type `To` while retaining bit representation.
-  *    `To` and `From` must satisfy `CopyConstructible`.
-  */
-template <typename To, typename From>
-std::decay_t<To> safe_bit_cast(const From & from)
-{
-    static_assert(sizeof(To) == sizeof(From), "bit cast on types of different width");
-    return bit_cast<To, From>(from);
-}

base/base/coverage.cpp

@@ -2,6 +2,8 @@
 
 #if WITH_COVERAGE
 
+#pragma GCC diagnostic ignored "-Wreserved-identifier"
+
 #    include <mutex>
 #    include <unistd.h>
 

base/base/defines.h

@@ -28,8 +28,8 @@
 #define NO_INLINE __attribute__((__noinline__))
 #define MAY_ALIAS __attribute__((__may_alias__))
 
-#if !defined(__x86_64__) && !defined(__aarch64__) && !defined(__PPC__) && !(defined(__riscv) && (__riscv_xlen == 64))
-#    error "The only supported platforms are x86_64 and AArch64, PowerPC (work in progress) and RISC-V 64 (experimental)"
+#if !defined(__x86_64__) && !defined(__aarch64__) && !defined(__PPC__) && !defined(__s390x__) && !(defined(__riscv) && (__riscv_xlen == 64))
+#    error "The only supported platforms are x86_64 and AArch64, PowerPC (work in progress), s390x (work in progress) and RISC-V 64 (experimental)"
 #endif
 
 /// Check for presence of address sanitizer
@@ -73,18 +73,6 @@
 #    endif
 #endif
 
-#if defined(ADDRESS_SANITIZER)
-#    define BOOST_USE_ASAN 1
-#    define BOOST_USE_UCONTEXT 1
-#endif
-
-#if defined(THREAD_SANITIZER)
-#    define BOOST_USE_TSAN 1
-#    define BOOST_USE_UCONTEXT 1
-#endif
-
-/// TODO: Strange enough, there is no way to detect UB sanitizer.
-
 /// Explicitly allow undefined behaviour for certain functions. Use it as a function attribute.
 /// It is useful in case when compiler cannot see (and exploit) it, but UBSan can.
 /// Example: multiplication of signed integers with possibility of overflow when both sides are from user input.
@@ -123,11 +111,19 @@
 ///     - tries to print failed assertion into server log
 /// It can be used for all assertions except heavy ones.
 /// Heavy assertions (that run loops or call complex functions) are allowed in debug builds only.
+/// Also it makes sense to call abort() instead of __builtin_unreachable() in debug builds,
+/// because SIGABRT is easier to debug than SIGTRAP (the second one makes gdb crazy)
 #if !defined(chassert)
     #if defined(ABORT_ON_LOGICAL_ERROR)
-        #define chassert(x) static_cast<bool>(x) ? void(0) : abortOnFailedAssertion(#x)
+        #define chassert(x) static_cast<bool>(x) ? void(0) : ::DB::abortOnFailedAssertion(#x)
+        #define UNREACHABLE() abort()
     #else
-        #define chassert(x) ((void)0)
+        /// Here sizeof() trick is used to suppress unused warning for result,
+        /// since simple "(void)x" will evaluate the expression, while
+        /// "sizeof(!(x))" will not.
+        #define NIL_EXPRESSION(x) (void)sizeof(!(x))
+        #define chassert(x) NIL_EXPRESSION(x)
+        #define UNREACHABLE() __builtin_unreachable()
     #endif
 #endif
 
@@ -140,11 +136,20 @@
 #    define TSA_REQUIRES_SHARED(...) __attribute__((requires_shared_capability(__VA_ARGS__)))  /// thread needs shared possession of given capability
 #    define TSA_ACQUIRED_AFTER(...) __attribute__((acquired_after(__VA_ARGS__)))               /// annotated lock must be locked after given lock
 #    define TSA_NO_THREAD_SAFETY_ANALYSIS __attribute__((no_thread_safety_analysis))           /// disable TSA for a function
+#    define TSA_CAPABILITY(...) __attribute__((capability(__VA_ARGS__)))                       /// object of a class can be used as capability
+#    define TSA_ACQUIRE(...) __attribute__((acquire_capability(__VA_ARGS__)))                        /// function acquires a capability, but does not release it
+#    define TSA_TRY_ACQUIRE(...) __attribute__((try_acquire_capability(__VA_ARGS__)))                /// function tries to acquire a capability and returns a boolean value indicating success or failure
+#    define TSA_RELEASE(...) __attribute__((release_capability(__VA_ARGS__)))                        /// function releases the given capability
+#    define TSA_ACQUIRE_SHARED(...) __attribute__((acquire_shared_capability(__VA_ARGS__)))          /// function acquires a shared capability, but does not release it
+#    define TSA_TRY_ACQUIRE_SHARED(...) __attribute__((try_acquire_shared_capability(__VA_ARGS__)))  /// function tries to acquire a shared capability and returns a boolean value indicating success or failure
+#    define TSA_RELEASE_SHARED(...) __attribute__((release_shared_capability(__VA_ARGS__)))          /// function releases the given shared capability
 
 /// Macros for suppressing TSA warnings for specific reads/writes (instead of suppressing it for the whole function)
-/// Consider adding a comment before using these macros.
-#   define TSA_SUPPRESS_WARNING_FOR_READ(x) [&]() TSA_NO_THREAD_SAFETY_ANALYSIS -> const auto & { return (x); }()
-#   define TSA_SUPPRESS_WARNING_FOR_WRITE(x) [&]() TSA_NO_THREAD_SAFETY_ANALYSIS -> auto & { return (x); }()
+/// They use a lambda function to apply function attribute to a single statement. This enable us to suppress warnings locally instead of
+/// suppressing them in the whole function
+/// Consider adding a comment when using these macros.
+#   define TSA_SUPPRESS_WARNING_FOR_READ(x) ([&]() TSA_NO_THREAD_SAFETY_ANALYSIS -> const auto & { return (x); }())
+#   define TSA_SUPPRESS_WARNING_FOR_WRITE(x) ([&]() TSA_NO_THREAD_SAFETY_ANALYSIS -> auto & { return (x); }())
 
 /// This macro is useful when only one thread writes to a member
 /// and you want to read this member from the same thread without locking a mutex.
@@ -158,10 +163,17 @@
 #    define TSA_REQUIRES(...)
 #    define TSA_REQUIRES_SHARED(...)
 #    define TSA_NO_THREAD_SAFETY_ANALYSIS
+#    define TSA_CAPABILITY(...)
+#    define TSA_ACQUIRE(...)
+#    define TSA_TRY_ACQUIRE(...)
+#    define TSA_RELEASE(...)
+#    define TSA_ACQUIRE_SHARED(...)
+#    define TSA_TRY_ACQUIRE_SHARED(...)
+#    define TSA_RELEASE_SHARED(...)
 
-#    define TSA_SUPPRESS_WARNING_FOR_READ(x)
-#    define TSA_SUPPRESS_WARNING_FOR_WRITE(x)
-#    define TSA_READ_ONE_THREAD(x)
+#    define TSA_SUPPRESS_WARNING_FOR_READ(x) (x)
+#    define TSA_SUPPRESS_WARNING_FOR_WRITE(x) (x)
+#    define TSA_READ_ONE_THREAD(x) TSA_SUPPRESS_WARNING_FOR_READ(x)
 #endif
 
 /// A template function for suppressing warnings about unused variables or function results.

base/base/errnoToString.cpp

@@ -3,10 +3,11 @@
 #include <fmt/format.h>
 
 
-std::string errnoToString(int code, int the_errno)
+std::string errnoToString(int the_errno)
 {
     const size_t buf_size = 128;
     char buf[buf_size];
+
 #ifndef _GNU_SOURCE
     int rc = strerror_r(the_errno, buf, buf_size);
 #ifdef OS_DARWIN
@@ -15,7 +16,7 @@ std::string errnoToString(int code, int the_errno)
     if (rc != 0)
 #endif
     {
-        std::string tmp = std::to_string(code);
+        std::string tmp = std::to_string(the_errno);
         const char * code_str = tmp.c_str();
         const char * unknown_message = "Unknown error ";
         strcpy(buf, unknown_message);
@@ -23,7 +24,6 @@ std::string errnoToString(int code, int the_errno)
     }
     return fmt::format("errno: {}, strerror: {}", the_errno, buf);
 #else
-    (void)code;
     return fmt::format("errno: {}, strerror: {}", the_errno, strerror_r(the_errno, buf, sizeof(buf)));
 #endif
 }

base/base/errnoToString.h

@@ -3,4 +3,4 @@
 #include <cerrno>
 #include <string>
 
-std::string errnoToString(int code, int the_errno = errno);
+std::string errnoToString(int the_errno = errno);

base/base/extended_types.h

@@ -5,7 +5,6 @@
 #include <base/types.h>
 #include <base/wide_integer.h>
 
-
 using Int128 = wide::integer<128, signed>;
 using UInt128 = wide::integer<128, unsigned>;
 using Int256 = wide::integer<256, signed>;
@@ -18,7 +17,7 @@ static_assert(sizeof(UInt256) == 32);
 /// (std::common_type), are "set in stone". Attempting to specialize them causes undefined behavior.
 /// So instead of using the std type_traits, we use our own version which allows extension.
 template <typename T>
-struct is_signed
+struct is_signed // NOLINT(readability-identifier-naming)
 {
     static constexpr bool value = std::is_signed_v<T>;
 };
@@ -30,7 +29,7 @@ template <typename T>
 inline constexpr bool is_signed_v = is_signed<T>::value;
 
 template <typename T>
-struct is_unsigned
+struct is_unsigned // NOLINT(readability-identifier-naming)
 {
     static constexpr bool value = std::is_unsigned_v<T>;
 };
@@ -51,7 +50,7 @@ template <class T> concept is_integer =
 template <class T> concept is_floating_point = std::is_floating_point_v<T>;
 
 template <typename T>
-struct is_arithmetic
+struct is_arithmetic // NOLINT(readability-identifier-naming)
 {
     static constexpr bool value = std::is_arithmetic_v<T>;
 };
@@ -66,9 +65,9 @@ template <typename T>
 inline constexpr bool is_arithmetic_v = is_arithmetic<T>::value;
 
 template <typename T>
-struct make_unsigned
+struct make_unsigned // NOLINT(readability-identifier-naming)
 {
-    typedef std::make_unsigned_t<T> type;
+    using type = std::make_unsigned_t<T>;
 };
 
 template <> struct make_unsigned<Int128> { using type = UInt128; };
@@ -79,9 +78,9 @@ template <> struct make_unsigned<UInt256> { using type = UInt256; };
 template <typename T> using make_unsigned_t = typename make_unsigned<T>::type;
 
 template <typename T>
-struct make_signed
+struct make_signed // NOLINT(readability-identifier-naming)
 {
-    typedef std::make_signed_t<T> type;
+    using type = std::make_signed_t<T>;
 };
 
 template <> struct make_signed<Int128>  { using type = Int128; };
@@ -92,7 +91,7 @@ template <> struct make_signed<UInt256> { using type = Int256; };
 template <typename T> using make_signed_t = typename make_signed<T>::type;
 
 template <typename T>
-struct is_big_int
+struct is_big_int // NOLINT(readability-identifier-naming)
 {
     static constexpr bool value = false;
 };
@@ -104,4 +103,3 @@ template <> struct is_big_int<UInt256> { static constexpr bool value = true; };
 
 template <typename T>
 inline constexpr bool is_big_int_v = is_big_int<T>::value;
-

base/base/find_symbols.h

@@ -2,6 +2,7 @@
 
 #include <cstdint>
 #include <string>
+#include <array>
 
 #if defined(__SSE2__)
     #include <emmintrin.h>
@@ -15,7 +16,7 @@
   *
   * Allow to search for next character from the set of 'symbols...' in a string.
   * It is similar to 'strpbrk', 'strcspn' (and 'strchr', 'memchr' in the case of one symbol and '\0'),
-  * but with the following differencies:
+  * but with the following differences:
   * - works with any memory ranges, including containing zero bytes;
   * - doesn't require terminating zero byte: end of memory range is passed explicitly;
   * - if not found, returns pointer to end instead of nullptr;
@@ -34,9 +35,51 @@
   * If no such characters, returns nullptr.
   */
 
+struct SearchSymbols
+{
+    static constexpr auto BUFFER_SIZE = 16;
+
+    SearchSymbols() = default;
+
+    explicit SearchSymbols(std::string in)
+        : str(std::move(in))
+    {
+#if defined(__SSE4_2__)
+        if (str.size() > BUFFER_SIZE)
+        {
+            throw std::runtime_error("SearchSymbols can contain at most " + std::to_string(BUFFER_SIZE) + " symbols and " + std::to_string(str.size()) + " was provided\n");
+        }
+
+        char tmp_safety_buffer[BUFFER_SIZE] = {0};
+
+        memcpy(tmp_safety_buffer, str.data(), str.size());
+
+        simd_vector = _mm_loadu_si128(reinterpret_cast<const __m128i *>(tmp_safety_buffer));
+#endif
+    }
+
+#if defined(__SSE4_2__)
+    __m128i simd_vector;
+#endif
+    std::string str;
+};
+
 namespace detail
 {
-template <char ...chars> constexpr bool is_in(char x) { return ((x == chars) || ...); }
+template <char ...chars> constexpr bool is_in(char x) { return ((x == chars) || ...); } // NOLINT(misc-redundant-expression)
+
+static bool is_in(char c, const char * symbols, size_t num_chars)
+{
+    for (size_t i = 0u; i < num_chars; ++i)
+    {
+        if (c == symbols[i])
+        {
+            return true;
+        }
+    }
+
+    return false;
+}
 
 #if defined(__SSE2__)
 template <char s0>
@@ -53,6 +96,43 @@ inline __m128i mm_is_in(__m128i bytes)
     __m128i eq = mm_is_in<s1, tail...>(bytes);
     return _mm_or_si128(eq0, eq);
 }
+
+inline __m128i mm_is_in(__m128i bytes, const char * symbols, size_t num_chars)
+{
+    __m128i accumulator = _mm_setzero_si128();
+    for (size_t i = 0; i < num_chars; ++i)
+    {
+        __m128i eq = _mm_cmpeq_epi8(bytes, _mm_set1_epi8(symbols[i]));
+        accumulator = _mm_or_si128(accumulator, eq);
+    }
+
+    return accumulator;
+}
+
+inline std::array<__m128i, 16u> mm_is_in_prepare(const char * symbols, size_t num_chars)
+{
+    std::array<__m128i, 16u> result {};
+
+    for (size_t i = 0; i < num_chars; ++i)
+    {
+        result[i] = _mm_set1_epi8(symbols[i]);
+    }
+
+    return result;
+}
+
+inline __m128i mm_is_in_execute(__m128i bytes, const std::array<__m128i, 16u> & needles)
+{
+    __m128i accumulator = _mm_setzero_si128();
+
+    for (const auto & needle : needles)
+    {
+        __m128i eq = _mm_cmpeq_epi8(bytes, needle);
+        accumulator = _mm_or_si128(accumulator, eq);
+    }
+
+    return accumulator;
+}
 #endif
 
 template <bool positive>
@@ -99,6 +179,32 @@ inline const char * find_first_symbols_sse2(const char * const begin, const char
     return return_mode == ReturnMode::End ? end : nullptr;
 }
 
+template <bool positive, ReturnMode return_mode>
+inline const char * find_first_symbols_sse2(const char * const begin, const char * const end, const char * symbols, size_t num_chars)
+{
+    const char * pos = begin;
+
+#if defined(__SSE2__)
+    const auto needles = mm_is_in_prepare(symbols, num_chars);
+    for (; pos + 15 < end; pos += 16)
+    {
+        __m128i bytes = _mm_loadu_si128(reinterpret_cast<const __m128i *>(pos));
+
+        __m128i eq = mm_is_in_execute(bytes, needles);
+
+        uint16_t bit_mask = maybe_negate<positive>(uint16_t(_mm_movemask_epi8(eq)));
+        if (bit_mask)
+            return pos + __builtin_ctz(bit_mask);
+    }
+#endif
+
+    for (; pos < end; ++pos)
+        if (maybe_negate<positive>(is_in(*pos, symbols, num_chars)))
+            return pos;
+
+    return return_mode == ReturnMode::End ? end : nullptr;
+}
+
 
 template <bool positive, ReturnMode return_mode, char... symbols>
 inline const char * find_last_symbols_sse2(const char * const begin, const char * const end)
@@ -159,26 +265,61 @@ inline const char * find_first_symbols_sse42(const char * const begin, const cha
 #endif
 
     for (; pos < end; ++pos)
-        if (   (num_chars >= 1 && maybe_negate<positive>(*pos == c01))
-            || (num_chars >= 2 && maybe_negate<positive>(*pos == c02))
-            || (num_chars >= 3 && maybe_negate<positive>(*pos == c03))
-            || (num_chars >= 4 && maybe_negate<positive>(*pos == c04))
-            || (num_chars >= 5 && maybe_negate<positive>(*pos == c05))
-            || (num_chars >= 6 && maybe_negate<positive>(*pos == c06))
-            || (num_chars >= 7 && maybe_negate<positive>(*pos == c07))
-            || (num_chars >= 8 && maybe_negate<positive>(*pos == c08))
-            || (num_chars >= 9 && maybe_negate<positive>(*pos == c09))
-            || (num_chars >= 10 && maybe_negate<positive>(*pos == c10))
-            || (num_chars >= 11 && maybe_negate<positive>(*pos == c11))
-            || (num_chars >= 12 && maybe_negate<positive>(*pos == c12))
-            || (num_chars >= 13 && maybe_negate<positive>(*pos == c13))
-            || (num_chars >= 14 && maybe_negate<positive>(*pos == c14))
-            || (num_chars >= 15 && maybe_negate<positive>(*pos == c15))
-            || (num_chars >= 16 && maybe_negate<positive>(*pos == c16)))
+        if (   (num_chars == 1 && maybe_negate<positive>(is_in<c01>(*pos)))
+            || (num_chars == 2 && maybe_negate<positive>(is_in<c01, c02>(*pos)))
+            || (num_chars == 3 && maybe_negate<positive>(is_in<c01, c02, c03>(*pos)))
+            || (num_chars == 4 && maybe_negate<positive>(is_in<c01, c02, c03, c04>(*pos)))
+            || (num_chars == 5 && maybe_negate<positive>(is_in<c01, c02, c03, c04, c05>(*pos)))
+            || (num_chars == 6 && maybe_negate<positive>(is_in<c01, c02, c03, c04, c05, c06>(*pos)))
+            || (num_chars == 7 && maybe_negate<positive>(is_in<c01, c02, c03, c04, c05, c06, c07>(*pos)))
+            || (num_chars == 8 && maybe_negate<positive>(is_in<c01, c02, c03, c04, c05, c06, c07, c08>(*pos)))
+            || (num_chars == 9 && maybe_negate<positive>(is_in<c01, c02, c03, c04, c05, c06, c07, c08, c09>(*pos)))
+            || (num_chars == 10 && maybe_negate<positive>(is_in<c01, c02, c03, c04, c05, c06, c07, c08, c09, c10>(*pos)))
+            || (num_chars == 11 && maybe_negate<positive>(is_in<c01, c02, c03, c04, c05, c06, c07, c08, c09, c10, c11>(*pos)))
+            || (num_chars == 12 && maybe_negate<positive>(is_in<c01, c02, c03, c04, c05, c06, c07, c08, c09, c10, c11, c12>(*pos)))
+            || (num_chars == 13 && maybe_negate<positive>(is_in<c01, c02, c03, c04, c05, c06, c07, c08, c09, c10, c11, c12, c13>(*pos)))
+            || (num_chars == 14 && maybe_negate<positive>(is_in<c01, c02, c03, c04, c05, c06, c07, c08, c09, c10, c11, c12, c13, c14>(*pos)))
+            || (num_chars == 15 && maybe_negate<positive>(is_in<c01, c02, c03, c04, c05, c06, c07, c08, c09, c10, c11, c12, c13, c14, c15>(*pos)))
+            || (num_chars == 16 && maybe_negate<positive>(is_in<c01, c02, c03, c04, c05, c06, c07, c08, c09, c10, c11, c12, c13, c14, c15, c16>(*pos))))
             return pos;
     return return_mode == ReturnMode::End ? end : nullptr;
 }
 
+template <bool positive, ReturnMode return_mode>
+inline const char * find_first_symbols_sse42(const char * const begin, const char * const end, const SearchSymbols & symbols)
+{
+    const char * pos = begin;
+
+    const auto num_chars = symbols.str.size();
+
+#if defined(__SSE4_2__)
+    constexpr int mode = _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT;
+
+    const __m128i set = symbols.simd_vector;
+
+    for (; pos + 15 < end; pos += 16)
+    {
+        __m128i bytes = _mm_loadu_si128(reinterpret_cast<const __m128i *>(pos));
+
+        if constexpr (positive)
+        {
+            if (_mm_cmpestrc(set, num_chars, bytes, 16, mode))
+                return pos + _mm_cmpestri(set, num_chars, bytes, 16, mode);
+        }
+        else
+        {
+            if (_mm_cmpestrc(set, num_chars, bytes, 16, mode | _SIDD_NEGATIVE_POLARITY))
+                return pos + _mm_cmpestri(set, num_chars, bytes, 16, mode | _SIDD_NEGATIVE_POLARITY);
+        }
+    }
+#endif
+
+    for (; pos < end; ++pos)
+        if (maybe_negate<positive>(is_in(*pos, symbols.str.data(), num_chars)))
+            return pos;
+
+    return return_mode == ReturnMode::End ? end : nullptr;
+}
 
 /// NOTE No SSE 4.2 implementation for find_last_symbols_or_null. Not worth to do.
 
@@ -194,6 +335,17 @@ inline const char * find_first_symbols_dispatch(const char * begin, const char *
         return find_first_symbols_sse2<positive, return_mode, symbols...>(begin, end);
 }
 
+template <bool positive, ReturnMode return_mode>
+inline const char * find_first_symbols_dispatch(const std::string_view haystack, const SearchSymbols & symbols)
+{
+#if defined(__SSE4_2__)
+    if (symbols.str.size() >= 5)
+        return find_first_symbols_sse42<positive, return_mode>(haystack.begin(), haystack.end(), symbols);
+    else
+#endif
+        return find_first_symbols_sse2<positive, return_mode>(haystack.begin(), haystack.end(), symbols.str.data(), symbols.str.size());
+}
+
 }
 
 
@@ -211,6 +363,11 @@ inline char * find_first_symbols(char * begin, char * end)
     return const_cast<char *>(detail::find_first_symbols_dispatch<true, detail::ReturnMode::End, symbols...>(begin, end));
 }
 
+inline const char * find_first_symbols(std::string_view haystack, const SearchSymbols & symbols)
+{
+    return detail::find_first_symbols_dispatch<true, detail::ReturnMode::End>(haystack, symbols);
+}
+
 template <char... symbols>
 inline const char * find_first_not_symbols(const char * begin, const char * end)
 {
@@ -223,6 +380,11 @@ inline char * find_first_not_symbols(char * begin, char * end)
     return const_cast<char *>(detail::find_first_symbols_dispatch<false, detail::ReturnMode::End, symbols...>(begin, end));
 }
 
+inline const char * find_first_not_symbols(std::string_view haystack, const SearchSymbols & symbols)
+{
+    return detail::find_first_symbols_dispatch<false, detail::ReturnMode::End>(haystack, symbols);
+}
+
 template <char... symbols>
 inline const char * find_first_symbols_or_null(const char * begin, const char * end)
 {
@@ -235,6 +397,11 @@ inline char * find_first_symbols_or_null(char * begin, char * end)
     return const_cast<char *>(detail::find_first_symbols_dispatch<true, detail::ReturnMode::Nullptr, symbols...>(begin, end));
 }
 
+inline const char * find_first_symbols_or_null(std::string_view haystack, const SearchSymbols & symbols)
+{
+    return detail::find_first_symbols_dispatch<true, detail::ReturnMode::Nullptr>(haystack, symbols);
+}
+
 template <char... symbols>
 inline const char * find_first_not_symbols_or_null(const char * begin, const char * end)
 {
@@ -247,6 +414,10 @@ inline char * find_first_not_symbols_or_null(char * begin, char * end)
     return const_cast<char *>(detail::find_first_symbols_dispatch<false, detail::ReturnMode::Nullptr, symbols...>(begin, end));
 }
 
+inline const char * find_first_not_symbols_or_null(std::string_view haystack, const SearchSymbols & symbols)
+{
+    return detail::find_first_symbols_dispatch<false, detail::ReturnMode::Nullptr>(haystack, symbols);
+}
 
 template <char... symbols>
 inline const char * find_last_symbols_or_null(const char * begin, const char * end)

base/base/getMemoryAmount.cpp

@@ -28,14 +28,28 @@ uint64_t getMemoryAmountOrZero()
 
 #if defined(OS_LINUX)
     // Try to lookup at the Cgroup limit
-    std::ifstream cgroup_limit("/sys/fs/cgroup/memory/memory.limit_in_bytes");
-    if (cgroup_limit.is_open())
+
+    // CGroups v2
+    std::ifstream cgroupv2_limit("/sys/fs/cgroup/memory.max");
+    if (cgroupv2_limit.is_open())
     {
-        uint64_t memory_limit = 0; // in case of read error
-        cgroup_limit >> memory_limit;
+        uint64_t memory_limit = 0;
+        cgroupv2_limit >> memory_limit;
         if (memory_limit > 0 && memory_limit < memory_amount)
             memory_amount = memory_limit;
     }
+    else
+    {
+        // CGroups v1
+        std::ifstream cgroup_limit("/sys/fs/cgroup/memory/memory.limit_in_bytes");
+        if (cgroup_limit.is_open())
+        {
+            uint64_t memory_limit = 0; // in case of read error
+            cgroup_limit >> memory_limit;
+            if (memory_limit > 0 && memory_limit < memory_amount)
+                memory_amount = memory_limit;
+        }
+    }
 #endif
 
     return memory_amount;

base/base/getThreadId.cpp

@@ -22,7 +22,7 @@ uint64_t getThreadId()
 #if defined(OS_ANDROID)
         current_tid = gettid();
 #elif defined(OS_LINUX)
-        current_tid = syscall(SYS_gettid); /// This call is always successful. - man gettid
+        current_tid = static_cast<uint64_t>(syscall(SYS_gettid)); /// This call is always successful. - man gettid
 #elif defined(OS_FREEBSD)
         current_tid = pthread_getthreadid_np();
 #elif defined(OS_SUNOS)

base/base/hex.h

@@ -0,0 +1,291 @@
+#pragma once
+
+#include <bit>
+#include <cstring>
+#include "types.h"
+
+namespace CityHash_v1_0_2 { struct uint128; }
+
+namespace wide
+{
+    template <size_t Bits, typename Signed>
+    class integer;
+}
+
+namespace impl
+{
+    /// Maps 0..15 to 0..9A..F or 0..9a..f correspondingly.
+    constexpr inline std::string_view hex_digit_to_char_uppercase_table = "0123456789ABCDEF";
+    constexpr inline std::string_view hex_digit_to_char_lowercase_table = "0123456789abcdef";
+
+    /// Maps 0..255 to 00..FF or 00..ff correspondingly.
+    constexpr inline std::string_view hex_byte_to_char_uppercase_table = //
+        "000102030405060708090A0B0C0D0E0F"
+        "101112131415161718191A1B1C1D1E1F"
+        "202122232425262728292A2B2C2D2E2F"
+        "303132333435363738393A3B3C3D3E3F"
+        "404142434445464748494A4B4C4D4E4F"
+        "505152535455565758595A5B5C5D5E5F"
+        "606162636465666768696A6B6C6D6E6F"
+        "707172737475767778797A7B7C7D7E7F"
+        "808182838485868788898A8B8C8D8E8F"
+        "909192939495969798999A9B9C9D9E9F"
+        "A0A1A2A3A4A5A6A7A8A9AAABACADAEAF"
+        "B0B1B2B3B4B5B6B7B8B9BABBBCBDBEBF"
+        "C0C1C2C3C4C5C6C7C8C9CACBCCCDCECF"
+        "D0D1D2D3D4D5D6D7D8D9DADBDCDDDEDF"
+        "E0E1E2E3E4E5E6E7E8E9EAEBECEDEEEF"
+        "F0F1F2F3F4F5F6F7F8F9FAFBFCFDFEFF";
+
+    constexpr inline std::string_view hex_byte_to_char_lowercase_table = //
+        "000102030405060708090a0b0c0d0e0f"
+        "101112131415161718191a1b1c1d1e1f"
+        "202122232425262728292a2b2c2d2e2f"
+        "303132333435363738393a3b3c3d3e3f"
+        "404142434445464748494a4b4c4d4e4f"
+        "505152535455565758595a5b5c5d5e5f"
+        "606162636465666768696a6b6c6d6e6f"
+        "707172737475767778797a7b7c7d7e7f"
+        "808182838485868788898a8b8c8d8e8f"
+        "909192939495969798999a9b9c9d9e9f"
+        "a0a1a2a3a4a5a6a7a8a9aaabacadaeaf"
+        "b0b1b2b3b4b5b6b7b8b9babbbcbdbebf"
+        "c0c1c2c3c4c5c6c7c8c9cacbcccdcecf"
+        "d0d1d2d3d4d5d6d7d8d9dadbdcdddedf"
+        "e0e1e2e3e4e5e6e7e8e9eaebecedeeef"
+        "f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff";
+
+    /// Maps 0..255 to 00000000..11111111 correspondingly.
+    constexpr inline std::string_view bin_byte_to_char_table = //
+        "0000000000000001000000100000001100000100000001010000011000000111"
+        "0000100000001001000010100000101100001100000011010000111000001111"
+        "0001000000010001000100100001001100010100000101010001011000010111"
+        "0001100000011001000110100001101100011100000111010001111000011111"
+        "0010000000100001001000100010001100100100001001010010011000100111"
+        "0010100000101001001010100010101100101100001011010010111000101111"
+        "0011000000110001001100100011001100110100001101010011011000110111"
+        "0011100000111001001110100011101100111100001111010011111000111111"
+        "0100000001000001010000100100001101000100010001010100011001000111"
+        "0100100001001001010010100100101101001100010011010100111001001111"
+        "0101000001010001010100100101001101010100010101010101011001010111"
+        "0101100001011001010110100101101101011100010111010101111001011111"
+        "0110000001100001011000100110001101100100011001010110011001100111"
+        "0110100001101001011010100110101101101100011011010110111001101111"
+        "0111000001110001011100100111001101110100011101010111011001110111"
+        "0111100001111001011110100111101101111100011111010111111001111111"
+        "1000000010000001100000101000001110000100100001011000011010000111"
+        "1000100010001001100010101000101110001100100011011000111010001111"
+        "1001000010010001100100101001001110010100100101011001011010010111"
+        "1001100010011001100110101001101110011100100111011001111010011111"
+        "1010000010100001101000101010001110100100101001011010011010100111"
+        "1010100010101001101010101010101110101100101011011010111010101111"
+        "1011000010110001101100101011001110110100101101011011011010110111"
+        "1011100010111001101110101011101110111100101111011011111010111111"
+        "1100000011000001110000101100001111000100110001011100011011000111"
+        "1100100011001001110010101100101111001100110011011100111011001111"
+        "1101000011010001110100101101001111010100110101011101011011010111"
+        "1101100011011001110110101101101111011100110111011101111011011111"
+        "1110000011100001111000101110001111100100111001011110011011100111"
+        "1110100011101001111010101110101111101100111011011110111011101111"
+        "1111000011110001111100101111001111110100111101011111011011110111"
+        "1111100011111001111110101111101111111100111111011111111011111111";
+
+    /// Maps 0..9, A..F, a..f to 0..15. Other chars are mapped to implementation specific value.
+    constexpr inline std::string_view hex_char_to_digit_table
+        = {"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\xff\xff\xff\xff\xff\xff" //0-9
+        "\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" //A-Z
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" //a-z
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff",
+        256};
+
+    /// Converts a hex digit '0'..'f' or '0'..'F' to its value 0..15.
+    constexpr UInt8 unhexDigit(char c)
+    {
+        return hex_char_to_digit_table[static_cast<UInt8>(c)];
+    }
+
+    /// Converts an unsigned integer in the native endian to hexadecimal representation and back. Used as a base class for HexConversion<T>.
+    template <typename TUInt, typename = void>
+    struct HexConversionUInt
+    {
+        static const constexpr size_t num_hex_digits = sizeof(TUInt) * 2;
+
+        static void hex(TUInt uint_, char * out, std::string_view table)
+        {
+            union
+            {
+                TUInt value;
+                UInt8 uint8[sizeof(TUInt)];
+            };
+
+            value = uint_;
+
+            for (size_t i = 0; i < sizeof(TUInt); ++i)
+            {
+                if constexpr (std::endian::native == std::endian::little)
+                    memcpy(out + i * 2, &table[static_cast<size_t>(uint8[sizeof(TUInt) - 1 - i]) * 2], 2);
+                else
+                    memcpy(out + i * 2, &table[static_cast<size_t>(uint8[i]) * 2], 2);
+            }
+        }
+
+        static TUInt unhex(const char * data)
+        {
+            TUInt res;
+            if constexpr (sizeof(TUInt) == 1)
+            {
+                res = static_cast<UInt8>(unhexDigit(data[0])) * 0x10 + static_cast<UInt8>(unhexDigit(data[1]));
+            }
+            else if constexpr (sizeof(TUInt) == 2)
+            {
+                res = static_cast<UInt16>(unhexDigit(data[0])) * 0x1000 + static_cast<UInt16>(unhexDigit(data[1])) * 0x100
+                    + static_cast<UInt16>(unhexDigit(data[2])) * 0x10 + static_cast<UInt16>(unhexDigit(data[3]));
+            }
+            else if constexpr ((sizeof(TUInt) <= 8) || ((sizeof(TUInt) % 8) != 0))
+            {
+                res = 0;
+                for (size_t i = 0; i < sizeof(TUInt) * 2; ++i, ++data)
+                {
+                    res <<= 4;
+                    res += unhexDigit(*data);
+                }
+            }
+            else
+            {
+                res = 0;
+                for (size_t i = 0; i < sizeof(TUInt) / 8; ++i, data += 16)
+                {
+                    res <<= 64;
+                    res += HexConversionUInt<UInt64>::unhex(data);
+                }
+            }
+            return res;
+        }
+    };
+
+    /// Helper template class to convert a value of any supported type to hexadecimal representation and back.
+    template <typename T, typename SFINAE = void>
+    struct HexConversion;
+
+    template <typename TUInt>
+    struct HexConversion<TUInt, std::enable_if_t<std::is_integral_v<TUInt>>> : public HexConversionUInt<TUInt> {};
+
+    template <size_t Bits, typename Signed>
+    struct HexConversion<wide::integer<Bits, Signed>> : public HexConversionUInt<wide::integer<Bits, Signed>> {};
+
+    template <typename CityHashUInt128> /// Partial specialization here allows not to include <city.h> in this header.
+    struct HexConversion<CityHashUInt128, std::enable_if_t<std::is_same_v<CityHashUInt128, typename CityHash_v1_0_2::uint128>>>
+    {
+        static const constexpr size_t num_hex_digits = 32;
+
+        static void hex(const CityHashUInt128 & uint_, char * out, std::string_view table)
+        {
+            HexConversion<UInt64>::hex(uint_.high64, out, table);
+            HexConversion<UInt64>::hex(uint_.low64, out + 16, table);
+        }
+
+        static CityHashUInt128 unhex(const char * data)
+        {
+            CityHashUInt128 res;
+            res.high64 = HexConversion<UInt64>::unhex(data);
+            res.low64 = HexConversion<UInt64>::unhex(data + 16);
+            return res;
+        }
+    };
+}
+
+/// Produces a hexadecimal representation of an integer value with leading zeros (for checksums).
+/// The function supports native integer types, wide::integer, CityHash_v1_0_2::uint128.
+/// It can be used with signed types as well, however they are written as corresponding unsigned numbers
+/// using two's complement (i.e. for example "-1" is written as "0xFF", not as "-0x01").
+template <typename T>
+void writeHexUIntUppercase(const T & value, char * out)
+{
+    impl::HexConversion<T>::hex(value, out, impl::hex_byte_to_char_uppercase_table);
+}
+
+template <typename T>
+void writeHexUIntLowercase(const T & value, char * out)
+{
+    impl::HexConversion<T>::hex(value, out, impl::hex_byte_to_char_lowercase_table);
+}
+
+template <typename T>
+std::string getHexUIntUppercase(const T & value)
+{
+    std::string res(impl::HexConversion<T>::num_hex_digits, '\0');
+    writeHexUIntUppercase(value, res.data());
+    return res;
+}
+
+template <typename T>
+std::string getHexUIntLowercase(const T & value)
+{
+    std::string res(impl::HexConversion<T>::num_hex_digits, '\0');
+    writeHexUIntLowercase(value, res.data());
+    return res;
+}
+
+constexpr char hexDigitUppercase(unsigned char c)
+{
+    return impl::hex_digit_to_char_uppercase_table[c];
+}
+
+constexpr char hexDigitLowercase(unsigned char c)
+{
+    return impl::hex_digit_to_char_lowercase_table[c];
+}
+
+inline void writeHexByteUppercase(UInt8 byte, void * out)
+{
+    memcpy(out, &impl::hex_byte_to_char_uppercase_table[static_cast<size_t>(byte) * 2], 2);
+}
+
+inline void writeHexByteLowercase(UInt8 byte, void * out)
+{
+    memcpy(out, &impl::hex_byte_to_char_lowercase_table[static_cast<size_t>(byte) * 2], 2);
+}
+
+/// Converts a hex representation with leading zeros back to an integer value.
+/// The function supports native integer types, wide::integer, CityHash_v1_0_2::uint128.
+template <typename T>
+constexpr T unhexUInt(const char * data)
+{
+    return impl::HexConversion<T>::unhex(data);
+}
+
+/// Converts a hexadecimal digit '0'..'f' or '0'..'F' to UInt8.
+constexpr UInt8 unhex(char c)
+{
+    return impl::unhexDigit(c);
+}
+
+/// Converts two hexadecimal digits to UInt8.
+constexpr UInt8 unhex2(const char * data)
+{
+    return unhexUInt<UInt8>(data);
+}
+
+/// Converts four hexadecimal digits to UInt16.
+constexpr UInt16 unhex4(const char * data)
+{
+    return unhexUInt<UInt16>(data);
+}
+
+/// Produces a binary representation of a single byte.
+inline void writeBinByte(UInt8 byte, void * out)
+{
+    memcpy(out, &impl::bin_byte_to_char_table[static_cast<size_t>(byte) * 8], 8);
+}

base/base/interpolate.h

@@ -0,0 +1,18 @@
+#pragma once
+#include <cassert>
+#include <cmath>
+
+/** Linear interpolation in logarithmic coordinates.
+  * Exponential interpolation is related to linear interpolation
+  * exactly in same way as geometric mean is related to arithmetic mean.
+  */
+constexpr double interpolateExponential(double min, double max, double ratio)
+{
+    assert(min > 0 && ratio >= 0 && ratio <= 1);
+    return min * std::pow(max / min, ratio);
+}
+
+constexpr double interpolateLinear(double min, double max, double ratio)
+{
+    return std::lerp(min, max, ratio);
+}

base/base/iostream_debug_helpers.h

@@ -120,6 +120,7 @@ Out & dumpDispatchPriorities(Out & out, T && x, std::decay_t<decltype(dumpImpl<p
     return dumpImpl<priority>(out, x);
 }
 
+// NOLINTNEXTLINE(google-explicit-constructor)
 struct LowPriority { LowPriority(void *) {} };
 
 template <int priority, typename Out, typename T>

base/base/itoa.h

@@ -33,6 +33,41 @@
 #include <base/extended_types.h>
 
 
+template <typename T>
+inline int digits10(T x)
+{
+    if (x < 10ULL)
+        return 1;
+    if (x < 100ULL)
+        return 2;
+    if (x < 1000ULL)
+        return 3;
+
+    if (x < 1000000000000ULL)
+    {
+        if (x < 100000000ULL)
+        {
+            if (x < 1000000ULL)
+            {
+                if (x < 10000ULL)
+                    return 4;
+                else
+                    return 5 + (x >= 100000ULL);
+            }
+
+            return 7 + (x >= 10000000ULL);
+        }
+
+        if (x < 10000000000ULL)
+            return 9 + (x >= 1000000000ULL);
+
+        return 11 + (x >= 100000000000ULL);
+    }
+
+    return 12 + digits10(x / 1000000000000ULL);
+}
+
+
 namespace impl
 {
 
@@ -91,10 +126,10 @@ template <size_t N>
 using DivisionBy10PowN = typename SelectType
 <
     N,
-    Division<uint8_t, 0, 205U, 11>,                           /// divide by 10
-    Division<uint16_t, 1, 41943U, 22>,                        /// divide by 100
-    Division<uint32_t, 0, 3518437209U, 45>,                   /// divide by 10000
-    Division<uint64_t, 0, 12379400392853802749ULL, 90>        /// divide by 100000000
+    Division<uint8_t, false, 205U, 11>,                           /// divide by 10
+    Division<uint16_t, true, 41943U, 22>,                         /// divide by 100
+    Division<uint32_t, false, 3518437209U, 45>,                   /// divide by 10000
+    Division<uint64_t, false, 12379400392853802749ULL, 90>        /// divide by 100000000
 >::Result;
 
 template <size_t N>
@@ -122,7 +157,7 @@ QuotientAndRemainder<N> static inline split(UnsignedOfSize<N> value)
     constexpr DivisionBy10PowN<N> division;
 
     UnsignedOfSize<N> quotient = (division.multiplier * (UnsignedOfSize<2 * N>(value) + division.add)) >> division.shift;
-    UnsignedOfSize<N / 2> remainder = value - quotient * pow10<UnsignedOfSize<N / 2>>(N);
+    UnsignedOfSize<N / 2> remainder = static_cast<UnsignedOfSize<N / 2>>(value - quotient * pow10<UnsignedOfSize<N / 2>>(N));
 
     return {quotient, remainder};
 }
@@ -312,39 +347,6 @@ namespace convert
     }
 }
 
-template <typename T>
-static inline int digits10(T x)
-{
-    if (x < 10ULL)
-        return 1;
-    if (x < 100ULL)
-        return 2;
-    if (x < 1000ULL)
-        return 3;
-
-    if (x < 1000000000000ULL)
-    {
-        if (x < 100000000ULL)
-        {
-            if (x < 1000000ULL)
-            {
-                if (x < 10000ULL)
-                    return 4;
-                else
-                    return 5 + (x >= 100000ULL);
-            }
-
-            return 7 + (x >= 10000000ULL);
-        }
-
-        if (x < 10000000000ULL)
-            return 9 + (x >= 1000000000ULL);
-
-        return 11 + (x >= 100000000000ULL);
-    }
-
-    return 12 + digits10(x / 1000000000000ULL);
-}
 
 template <typename T>
 static inline char * writeUIntText(T x, char * p)
@@ -352,7 +354,7 @@ static inline char * writeUIntText(T x, char * p)
     static_assert(is_unsigned_v<T>);
 
     int len = digits10(x);
-    auto pp = p + len;
+    auto * pp = p + len;
     while (x >= 100)
     {
         const auto i = x % 100;

base/base/phdr_cache.cpp

@@ -1,6 +1,4 @@
-#ifdef HAS_RESERVED_IDENTIFIER
 #pragma clang diagnostic ignored "-Wreserved-identifier"
-#endif
 
 /// This code was based on the code by Fedor Korotkiy https://www.linkedin.com/in/fedor-korotkiy-659a1838/
 

base/base/phdr_cache.h

@@ -8,6 +8,7 @@
   * As a drawback, this only works if no dynamic object unloading happens after this point.
   * This function is thread-safe. You should call it to update cache after loading new shared libraries.
   * Otherwise exception handling from dlopened libraries won't work (will call std::terminate immediately).
+  * NOTE: dlopen is forbidden in our code.
   *
   * NOTE: It is disabled with Thread Sanitizer because TSan can only use original "dl_iterate_phdr" function.
   */

base/base/preciseExp10.cpp

@@ -1,8 +1,8 @@
+/** Origin: musl-libc /src/math/exp10.c
+  * https://www.musl-libc.org/
+  * http://git.musl-libc.org/cgit/musl/tree/src/math/exp10.c
+  */
 /*
-
-https://www.musl-libc.org/
-http://git.musl-libc.org/cgit/musl/tree/src/math/exp10.c
-
 musl as a whole is licensed under the following standard MIT license:
 
 ----------------------------------------------------------------------
@@ -27,146 +27,6 @@ CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 ----------------------------------------------------------------------
-
-Authors/contributors include:
-
-Alex Dowad
-Alexander Monakov
-Anthony G. Basile
-Arvid Picciani
-Bobby Bingham
-Boris Brezillon
-Brent Cook
-Chris Spiegel
-Clément Vasseur
-Daniel Micay
-Denys Vlasenko
-Emil Renner Berthing
-Felix Fietkau
-Felix Janda
-Gianluca Anzolin
-Hauke Mehrtens
-Hiltjo Posthuma
-Isaac Dunham
-Jaydeep Patil
-Jens Gustedt
-Jeremy Huntwork
-Jo-Philipp Wich
-Joakim Sindholt
-John Spencer
-Josiah Worcester
-Justin Cormack
-Khem Raj
-Kylie McClain
-Luca Barbato
-Luka Perkov
-M Farkas-Dyck (Strake)
-Mahesh Bodapati
-Michael Forney
-Natanael Copa
-Nicholas J. Kain
-orc
-Pascal Cuoq
-Petr Hosek
-Pierre Carrier
-Rich Felker
-Richard Pennington
-Shiz
-sin
-Solar Designer
-Stefan Kristiansson
-Szabolcs Nagy
-Timo Teräs
-Trutz Behn
-Valentin Ochs
-William Haddon
-
-Portions of this software are derived from third-party works licensed
-under terms compatible with the above MIT license:
-
-The TRE regular expression implementation (src/regex/reg* and
-src/regex/tre*) is Copyright © 2001-2008 Ville Laurikari and licensed
-under a 2-clause BSD license (license text in the source files). The
-included version has been heavily modified by Rich Felker in 2012, in
-the interests of size, simplicity, and namespace cleanliness.
-
-Much of the math library code (src/math/ * and src/complex/ *) is
-Copyright © 1993,2004 Sun Microsystems or
-Copyright © 2003-2011 David Schultz or
-Copyright © 2003-2009 Steven G. Kargl or
-Copyright © 2003-2009 Bruce D. Evans or
-Copyright © 2008 Stephen L. Moshier
-and labelled as such in comments in the individual source files. All
-have been licensed under extremely permissive terms.
-
-The ARM memcpy code (src/string/arm/memcpy_el.S) is Copyright © 2008
-The Android Open Source Project and is licensed under a two-clause BSD
-license. It was taken from Bionic libc, used on Android.
-
-The implementation of DES for crypt (src/crypt/crypt_des.c) is
-Copyright © 1994 David Burren. It is licensed under a BSD license.
-
-The implementation of blowfish crypt (src/crypt/crypt_blowfish.c) was
-originally written by Solar Designer and placed into the public
-domain. The code also comes with a fallback permissive license for use
-in jurisdictions that may not recognize the public domain.
-
-The smoothsort implementation (src/stdlib/qsort.c) is Copyright © 2011
-Valentin Ochs and is licensed under an MIT-style license.
-
-The BSD PRNG implementation (src/prng/random.c) and XSI search API
-(src/search/ *.c) functions are Copyright © 2011 Szabolcs Nagy and
-licensed under following terms: "Permission to use, copy, modify,
-and/or distribute this code for any purpose with or without fee is
-hereby granted. There is no warranty."
-
-The x86_64 port was written by Nicholas J. Kain and is licensed under
-the standard MIT terms.
-
-The mips and microblaze ports were originally written by Richard
-Pennington for use in the ellcc project. The original code was adapted
-by Rich Felker for build system and code conventions during upstream
-integration. It is licensed under the standard MIT terms.
-
-The mips64 port was contributed by Imagination Technologies and is
-licensed under the standard MIT terms.
-
-The powerpc port was also originally written by Richard Pennington,
-and later supplemented and integrated by John Spencer. It is licensed
-under the standard MIT terms.
-
-All other files which have no copyright comments are original works
-produced specifically for use as part of this library, written either
-by Rich Felker, the main author of the library, or by one or more
-contibutors listed above. Details on authorship of individual files
-can be found in the git version control history of the project. The
-omission of copyright and license comments in each file is in the
-interest of source tree size.
-
-In addition, permission is hereby granted for all public header files
-(include/ * and arch/ * /bits/ *) and crt files intended to be linked into
-applications (crt/ *, ldso/dlstart.c, and arch/ * /crt_arch.h) to omit
-the copyright notice and permission notice otherwise required by the
-license, and to use these files without any requirement of
-attribution. These files include substantial contributions from:
-
-Bobby Bingham
-John Spencer
-Nicholas J. Kain
-Rich Felker
-Richard Pennington
-Stefan Kristiansson
-Szabolcs Nagy
-
-all of whom have explicitly granted such permission.
-
-This file previously contained text expressing a belief that most of
-the files covered by the above exception were sufficiently trivial not
-to be subject to copyright, resulting in confusion over whether it
-negated the permissions granted in the license. In the spirit of
-permissive licensing, and of not having licensing issues being an
-obstacle to adoption, that text has been removed.
-
 */
 
 #include <cmath>

base/base/preciseExp10.h

@@ -3,9 +3,9 @@
 /** exp10 from GNU libm fails to give precise result for integer arguments.
   * For example, exp10(3) gives 1000.0000000000001
   *  despite the fact that 1000 is exactly representable in double and float.
-  * Better to always use implementation from MUSL.
+  * Better to always use our own implementation based on a MUSL's one.
   *
-  * Note: the function names are different to avoid confusion with symbols from the system libm.
+  * Note: the function name is different to avoid confusion with symbols from the system libm.
   */
 
 double preciseExp10(double x);

base/base/safeExit.cpp

@@ -1,9 +1,10 @@
 #if defined(OS_LINUX)
 #    include <sys/syscall.h>
 #endif
+#include <cstdlib>
 #include <unistd.h>
 #include <base/safeExit.h>
-#include <base/defines.h>
+#include <base/defines.h> /// for THREAD_SANITIZER
 
 [[noreturn]] void safeExit(int code)
 {
@@ -11,7 +12,7 @@
     /// Thread sanitizer tries to do something on exit that we don't need if we want to exit immediately,
     /// while connection handling threads are still run.
     (void)syscall(SYS_exit_group, code);
-    __builtin_unreachable();
+    UNREACHABLE();
 #else
     _exit(code);
 #endif

base/base/scope_guard.h

@@ -5,13 +5,13 @@
 #include <utility>
 
 template <class F>
-class [[nodiscard]] basic_scope_guard
+class [[nodiscard]] BasicScopeGuard
 {
 public:
-    constexpr basic_scope_guard() = default;
-    constexpr basic_scope_guard(basic_scope_guard && src) : function{src.release()} {}
+    constexpr BasicScopeGuard() = default;
+    constexpr BasicScopeGuard(BasicScopeGuard && src) : function{src.release()} {} // NOLINT(hicpp-noexcept-move, performance-noexcept-move-constructor)
 
-    constexpr basic_scope_guard & operator=(basic_scope_guard && src)
+    constexpr BasicScopeGuard & operator=(BasicScopeGuard && src) // NOLINT(hicpp-noexcept-move, performance-noexcept-move-constructor)
     {
         if (this != &src)
         {
@@ -23,11 +23,11 @@ public:
 
     template <typename G>
     requires std::is_convertible_v<G, F>
-    constexpr basic_scope_guard(basic_scope_guard<G> && src) : function{src.release()} {}
+    constexpr BasicScopeGuard(BasicScopeGuard<G> && src) : function{src.release()} {} // NOLINT(google-explicit-constructor)
 
     template <typename G>
     requires std::is_convertible_v<G, F>
-    constexpr basic_scope_guard & operator=(basic_scope_guard<G> && src)
+    constexpr BasicScopeGuard & operator=(BasicScopeGuard<G> && src)
     {
         if (this != &src)
         {
@@ -39,13 +39,13 @@ public:
 
     template <typename G>
     requires std::is_convertible_v<G, F>
-    constexpr basic_scope_guard(const G & function_) : function{function_} {}
+    constexpr BasicScopeGuard(const G & function_) : function{function_} {} // NOLINT(google-explicit-constructor)
 
     template <typename G>
     requires std::is_convertible_v<G, F>
-    constexpr basic_scope_guard(G && function_) : function{std::move(function_)} {}
+    constexpr BasicScopeGuard(G && function_) : function{std::move(function_)} {} // NOLINT(google-explicit-constructor, bugprone-forwarding-reference-overload, bugprone-move-forwarding-reference)
 
-    ~basic_scope_guard() { invoke(); }
+    ~BasicScopeGuard() { invoke(); }
 
     static constexpr bool is_nullable = std::is_constructible_v<bool, F>;
 
@@ -70,7 +70,7 @@ public:
 
     template <typename G>
     requires std::is_convertible_v<G, F>
-    basic_scope_guard<F> & join(basic_scope_guard<G> && other)
+    BasicScopeGuard<F> & join(BasicScopeGuard<G> && other)
     {
         if (other.function)
         {
@@ -102,14 +102,13 @@ private:
     F function = F{};
 };
 
-using scope_guard = basic_scope_guard<std::function<void(void)>>;
+using scope_guard = BasicScopeGuard<std::function<void(void)>>;
 
 
 template <class F>
-inline basic_scope_guard<F> make_scope_guard(F && function_) { return std::forward<F>(function_); }
+inline BasicScopeGuard<F> make_scope_guard(F && function_) { return std::forward<F>(function_); }
 
 #define SCOPE_EXIT_CONCAT(n, ...) \
 const auto scope_exit##n = make_scope_guard([&] { __VA_ARGS__; })
 #define SCOPE_EXIT_FWD(n, ...) SCOPE_EXIT_CONCAT(n, __VA_ARGS__)
 #define SCOPE_EXIT(...) SCOPE_EXIT_FWD(__LINE__, __VA_ARGS__)
-

base/base/setTerminalEcho.cpp

@@ -1,28 +0,0 @@
-#include <base/setTerminalEcho.h>
-#include <base/errnoToString.h>
-#include <stdexcept>
-#include <cstring>
-#include <string>
-#include <termios.h>
-#include <unistd.h>
-
-
-void setTerminalEcho(bool enable)
-{
-    /// Obtain terminal attributes,
-    /// toggle the ECHO flag
-    /// and set them back.
-
-    struct termios tty{};
-
-    if (0 != tcgetattr(STDIN_FILENO, &tty))
-        throw std::runtime_error(std::string("setTerminalEcho failed get: ") + errnoToString(errno));
-
-    if (enable)
-        tty.c_lflag |= ECHO;
-    else
-        tty.c_lflag &= ~ECHO;
-
-    if (0 != tcsetattr(STDIN_FILENO, TCSANOW, &tty))
-        throw std::runtime_error(std::string("setTerminalEcho failed set: ") + errnoToString(errno));
-}

base/base/setTerminalEcho.h

@@ -1,4 +0,0 @@
-#pragma once
-
-/// Enable or disable echoing of typed characters. Throws std::runtime_error on error.
-void setTerminalEcho(bool enable);

base/base/shift10.cpp

@@ -14,37 +14,37 @@ static T shift10Impl(T x, int exponent)
     static const long double powers10[] =
     {
         1e-323L, 1e-322L, 1e-321L, 1e-320L, 1e-319L, 1e-318L, 1e-317L, 1e-316L, 1e-315L, 1e-314L, 1e-313L, 1e-312L, 1e-311L,
-        1e-310L,1e-309L,1e-308L,1e-307L,1e-306L,1e-305L,1e-304L,1e-303L,1e-302L,1e-301L,1e-300L,1e-299L,1e-298L,1e-297L,1e-296L,1e-295L,1e-294L,1e-293L,1e-292L,1e-291L,
-        1e-290L,1e-289L,1e-288L,1e-287L,1e-286L,1e-285L,1e-284L,1e-283L,1e-282L,1e-281L,1e-280L,1e-279L,1e-278L,1e-277L,1e-276L,1e-275L,1e-274L,1e-273L,1e-272L,1e-271L,
-        1e-270L,1e-269L,1e-268L,1e-267L,1e-266L,1e-265L,1e-264L,1e-263L,1e-262L,1e-261L,1e-260L,1e-259L,1e-258L,1e-257L,1e-256L,1e-255L,1e-254L,1e-253L,1e-252L,1e-251L,
-        1e-250L,1e-249L,1e-248L,1e-247L,1e-246L,1e-245L,1e-244L,1e-243L,1e-242L,1e-241L,1e-240L,1e-239L,1e-238L,1e-237L,1e-236L,1e-235L,1e-234L,1e-233L,1e-232L,1e-231L,
-        1e-230L,1e-229L,1e-228L,1e-227L,1e-226L,1e-225L,1e-224L,1e-223L,1e-222L,1e-221L,1e-220L,1e-219L,1e-218L,1e-217L,1e-216L,1e-215L,1e-214L,1e-213L,1e-212L,1e-211L,
-        1e-210L,1e-209L,1e-208L,1e-207L,1e-206L,1e-205L,1e-204L,1e-203L,1e-202L,1e-201L,1e-200L,1e-199L,1e-198L,1e-197L,1e-196L,1e-195L,1e-194L,1e-193L,1e-192L,1e-191L,
-        1e-190L,1e-189L,1e-188L,1e-187L,1e-186L,1e-185L,1e-184L,1e-183L,1e-182L,1e-181L,1e-180L,1e-179L,1e-178L,1e-177L,1e-176L,1e-175L,1e-174L,1e-173L,1e-172L,1e-171L,
-        1e-170L,1e-169L,1e-168L,1e-167L,1e-166L,1e-165L,1e-164L,1e-163L,1e-162L,1e-161L,1e-160L,1e-159L,1e-158L,1e-157L,1e-156L,1e-155L,1e-154L,1e-153L,1e-152L,1e-151L,
-        1e-150L,1e-149L,1e-148L,1e-147L,1e-146L,1e-145L,1e-144L,1e-143L,1e-142L,1e-141L,1e-140L,1e-139L,1e-138L,1e-137L,1e-136L,1e-135L,1e-134L,1e-133L,1e-132L,1e-131L,
-        1e-130L,1e-129L,1e-128L,1e-127L,1e-126L,1e-125L,1e-124L,1e-123L,1e-122L,1e-121L,1e-120L,1e-119L,1e-118L,1e-117L,1e-116L,1e-115L,1e-114L,1e-113L,1e-112L,1e-111L,
-        1e-110L,1e-109L,1e-108L,1e-107L,1e-106L,1e-105L,1e-104L,1e-103L,1e-102L,1e-101L,1e-100L,1e-99L,1e-98L,1e-97L,1e-96L,1e-95L,1e-94L,1e-93L,1e-92L,1e-91L,1e-90L,
-        1e-89L,1e-88L,1e-87L,1e-86L,1e-85L,1e-84L,1e-83L,1e-82L,1e-81L,1e-80L,1e-79L,1e-78L,1e-77L,1e-76L,1e-75L,1e-74L,1e-73L,1e-72L,1e-71L,1e-70,
-        1e-69L,1e-68L,1e-67L,1e-66L,1e-65L,1e-64L,1e-63L,1e-62L,1e-61L,1e-60L,1e-59L,1e-58L,1e-57L,1e-56L,1e-55L,1e-54L,1e-53L,1e-52L,1e-51L,1e-50,
-        1e-49L,1e-48L,1e-47L,1e-46L,1e-45L,1e-44L,1e-43L,1e-42L,1e-41L,1e-40L,1e-39L,1e-38L,1e-37L,1e-36L,1e-35L,1e-34L,1e-33L,1e-32L,1e-31L,1e-30,
-        1e-29L,1e-28L,1e-27L,1e-26L,1e-25L,1e-24L,1e-23L,1e-22L,1e-21L,1e-20L,1e-19L,1e-18L,1e-17L,1e-16L,1e-15L,1e-14L,1e-13L,1e-12L,1e-11L,1e-10,
-        1e-9L,1e-8L,1e-7L,1e-6L,1e-5L,1e-4L,1e-3L,1e-2L,1e-1L,1e0L,1e1L,1e2L,1e3L,1e4L,1e5L,1e6L,1e7L,1e8L,1e9L,1e10,
-        1e11L,1e12L,1e13L,1e14L,1e15L,1e16L,1e17L,1e18L,1e19L,1e20L,1e21L,1e22L,1e23L,1e24L,1e25L,1e26L,1e27L,1e28L,1e29L,1e30,
-        1e31L,1e32L,1e33L,1e34L,1e35L,1e36L,1e37L,1e38L,1e39L,1e40L,1e41L,1e42L,1e43L,1e44L,1e45L,1e46L,1e47L,1e48L,1e49L,1e50,
-        1e51L,1e52L,1e53L,1e54L,1e55L,1e56L,1e57L,1e58L,1e59L,1e60L,1e61L,1e62L,1e63L,1e64L,1e65L,1e66L,1e67L,1e68L,1e69L,1e70,
-        1e71L,1e72L,1e73L,1e74L,1e75L,1e76L,1e77L,1e78L,1e79L,1e80L,1e81L,1e82L,1e83L,1e84L,1e85L,1e86L,1e87L,1e88L,1e89L,1e90,
-        1e91L,1e92L,1e93L,1e94L,1e95L,1e96L,1e97L,1e98L,1e99L,1e100L,1e101L,1e102L,1e103L,1e104L,1e105L,1e106L,1e107L,1e108L,1e109L,1e110,
-        1e111L,1e112L,1e113L,1e114L,1e115L,1e116L,1e117L,1e118L,1e119L,1e120L,1e121L,1e122L,1e123L,1e124L,1e125L,1e126L,1e127L,1e128L,1e129L,1e130,
-        1e131L,1e132L,1e133L,1e134L,1e135L,1e136L,1e137L,1e138L,1e139L,1e140L,1e141L,1e142L,1e143L,1e144L,1e145L,1e146L,1e147L,1e148L,1e149L,1e150,
-        1e151L,1e152L,1e153L,1e154L,1e155L,1e156L,1e157L,1e158L,1e159L,1e160L,1e161L,1e162L,1e163L,1e164L,1e165L,1e166L,1e167L,1e168L,1e169L,1e170,
-        1e171L,1e172L,1e173L,1e174L,1e175L,1e176L,1e177L,1e178L,1e179L,1e180L,1e181L,1e182L,1e183L,1e184L,1e185L,1e186L,1e187L,1e188L,1e189L,1e190,
-        1e191L,1e192L,1e193L,1e194L,1e195L,1e196L,1e197L,1e198L,1e199L,1e200L,1e201L,1e202L,1e203L,1e204L,1e205L,1e206L,1e207L,1e208L,1e209L,1e210,
-        1e211L,1e212L,1e213L,1e214L,1e215L,1e216L,1e217L,1e218L,1e219L,1e220L,1e221L,1e222L,1e223L,1e224L,1e225L,1e226L,1e227L,1e228L,1e229L,1e230,
-        1e231L,1e232L,1e233L,1e234L,1e235L,1e236L,1e237L,1e238L,1e239L,1e240L,1e241L,1e242L,1e243L,1e244L,1e245L,1e246L,1e247L,1e248L,1e249L,1e250,
-        1e251L,1e252L,1e253L,1e254L,1e255L,1e256L,1e257L,1e258L,1e259L,1e260L,1e261L,1e262L,1e263L,1e264L,1e265L,1e266L,1e267L,1e268L,1e269L,1e270,
-        1e271L,1e272L,1e273L,1e274L,1e275L,1e276L,1e277L,1e278L,1e279L,1e280L,1e281L,1e282L,1e283L,1e284L,1e285L,1e286L,1e287L,1e288L,1e289L,1e290,
-        1e291L,1e292L,1e293L,1e294L,1e295L,1e296L,1e297L,1e298L,1e299L,1e300L,1e301L,1e302L,1e303L,1e304L,1e305L,1e306L,1e307L,1e308L
+        1e-310L, 1e-309L, 1e-308L, 1e-307L, 1e-306L, 1e-305L, 1e-304L, 1e-303L, 1e-302L, 1e-301L, 1e-300L, 1e-299L, 1e-298L, 1e-297L, 1e-296L, 1e-295L, 1e-294L, 1e-293L, 1e-292L, 1e-291L,
+        1e-290L, 1e-289L, 1e-288L, 1e-287L, 1e-286L, 1e-285L, 1e-284L, 1e-283L, 1e-282L, 1e-281L, 1e-280L, 1e-279L, 1e-278L, 1e-277L, 1e-276L, 1e-275L, 1e-274L, 1e-273L, 1e-272L, 1e-271L,
+        1e-270L, 1e-269L, 1e-268L, 1e-267L, 1e-266L, 1e-265L, 1e-264L, 1e-263L, 1e-262L, 1e-261L, 1e-260L, 1e-259L, 1e-258L, 1e-257L, 1e-256L, 1e-255L, 1e-254L, 1e-253L, 1e-252L, 1e-251L,
+        1e-250L, 1e-249L, 1e-248L, 1e-247L, 1e-246L, 1e-245L, 1e-244L, 1e-243L, 1e-242L, 1e-241L, 1e-240L, 1e-239L, 1e-238L, 1e-237L, 1e-236L, 1e-235L, 1e-234L, 1e-233L, 1e-232L, 1e-231L,
+        1e-230L, 1e-229L, 1e-228L, 1e-227L, 1e-226L, 1e-225L, 1e-224L, 1e-223L, 1e-222L, 1e-221L, 1e-220L, 1e-219L, 1e-218L, 1e-217L, 1e-216L, 1e-215L, 1e-214L, 1e-213L, 1e-212L, 1e-211L,
+        1e-210L, 1e-209L, 1e-208L, 1e-207L, 1e-206L, 1e-205L, 1e-204L, 1e-203L, 1e-202L, 1e-201L, 1e-200L, 1e-199L, 1e-198L, 1e-197L, 1e-196L, 1e-195L, 1e-194L, 1e-193L, 1e-192L, 1e-191L,
+        1e-190L, 1e-189L, 1e-188L, 1e-187L, 1e-186L, 1e-185L, 1e-184L, 1e-183L, 1e-182L, 1e-181L, 1e-180L, 1e-179L, 1e-178L, 1e-177L, 1e-176L, 1e-175L, 1e-174L, 1e-173L, 1e-172L, 1e-171L,
+        1e-170L, 1e-169L, 1e-168L, 1e-167L, 1e-166L, 1e-165L, 1e-164L, 1e-163L, 1e-162L, 1e-161L, 1e-160L, 1e-159L, 1e-158L, 1e-157L, 1e-156L, 1e-155L, 1e-154L, 1e-153L, 1e-152L, 1e-151L,
+        1e-150L, 1e-149L, 1e-148L, 1e-147L, 1e-146L, 1e-145L, 1e-144L, 1e-143L, 1e-142L, 1e-141L, 1e-140L, 1e-139L, 1e-138L, 1e-137L, 1e-136L, 1e-135L, 1e-134L, 1e-133L, 1e-132L, 1e-131L,
+        1e-130L, 1e-129L, 1e-128L, 1e-127L, 1e-126L, 1e-125L, 1e-124L, 1e-123L, 1e-122L, 1e-121L, 1e-120L, 1e-119L, 1e-118L, 1e-117L, 1e-116L, 1e-115L, 1e-114L, 1e-113L, 1e-112L, 1e-111L,
+        1e-110L, 1e-109L, 1e-108L, 1e-107L, 1e-106L, 1e-105L, 1e-104L, 1e-103L, 1e-102L, 1e-101L, 1e-100L, 1e-99L, 1e-98L, 1e-97L, 1e-96L, 1e-95L, 1e-94L, 1e-93L, 1e-92L, 1e-91L, 1e-90L,
+        1e-89L, 1e-88L, 1e-87L, 1e-86L, 1e-85L, 1e-84L, 1e-83L, 1e-82L, 1e-81L, 1e-80L, 1e-79L, 1e-78L, 1e-77L, 1e-76L, 1e-75L, 1e-74L, 1e-73L, 1e-72L, 1e-71L, 1e-70,
+        1e-69L, 1e-68L, 1e-67L, 1e-66L, 1e-65L, 1e-64L, 1e-63L, 1e-62L, 1e-61L, 1e-60L, 1e-59L, 1e-58L, 1e-57L, 1e-56L, 1e-55L, 1e-54L, 1e-53L, 1e-52L, 1e-51L, 1e-50,
+        1e-49L, 1e-48L, 1e-47L, 1e-46L, 1e-45L, 1e-44L, 1e-43L, 1e-42L, 1e-41L, 1e-40L, 1e-39L, 1e-38L, 1e-37L, 1e-36L, 1e-35L, 1e-34L, 1e-33L, 1e-32L, 1e-31L, 1e-30,
+        1e-29L, 1e-28L, 1e-27L, 1e-26L, 1e-25L, 1e-24L, 1e-23L, 1e-22L, 1e-21L, 1e-20L, 1e-19L, 1e-18L, 1e-17L, 1e-16L, 1e-15L, 1e-14L, 1e-13L, 1e-12L, 1e-11L, 1e-10,
+        1e-9L, 1e-8L, 1e-7L, 1e-6L, 1e-5L, 1e-4L, 1e-3L, 1e-2L, 1e-1L, 1e0L, 1e1L, 1e2L, 1e3L, 1e4L, 1e5L, 1e6L, 1e7L, 1e8L, 1e9L, 1e10,
+        1e11L, 1e12L, 1e13L, 1e14L, 1e15L, 1e16L, 1e17L, 1e18L, 1e19L, 1e20L, 1e21L, 1e22L, 1e23L, 1e24L, 1e25L, 1e26L, 1e27L, 1e28L, 1e29L, 1e30,
+        1e31L, 1e32L, 1e33L, 1e34L, 1e35L, 1e36L, 1e37L, 1e38L, 1e39L, 1e40L, 1e41L, 1e42L, 1e43L, 1e44L, 1e45L, 1e46L, 1e47L, 1e48L, 1e49L, 1e50,
+        1e51L, 1e52L, 1e53L, 1e54L, 1e55L, 1e56L, 1e57L, 1e58L, 1e59L, 1e60L, 1e61L, 1e62L, 1e63L, 1e64L, 1e65L, 1e66L, 1e67L, 1e68L, 1e69L, 1e70,
+        1e71L, 1e72L, 1e73L, 1e74L, 1e75L, 1e76L, 1e77L, 1e78L, 1e79L, 1e80L, 1e81L, 1e82L, 1e83L, 1e84L, 1e85L, 1e86L, 1e87L, 1e88L, 1e89L, 1e90,
+        1e91L, 1e92L, 1e93L, 1e94L, 1e95L, 1e96L, 1e97L, 1e98L, 1e99L, 1e100L, 1e101L, 1e102L, 1e103L, 1e104L, 1e105L, 1e106L, 1e107L, 1e108L, 1e109L, 1e110,
+        1e111L, 1e112L, 1e113L, 1e114L, 1e115L, 1e116L, 1e117L, 1e118L, 1e119L, 1e120L, 1e121L, 1e122L, 1e123L, 1e124L, 1e125L, 1e126L, 1e127L, 1e128L, 1e129L, 1e130,
+        1e131L, 1e132L, 1e133L, 1e134L, 1e135L, 1e136L, 1e137L, 1e138L, 1e139L, 1e140L, 1e141L, 1e142L, 1e143L, 1e144L, 1e145L, 1e146L, 1e147L, 1e148L, 1e149L, 1e150,
+        1e151L, 1e152L, 1e153L, 1e154L, 1e155L, 1e156L, 1e157L, 1e158L, 1e159L, 1e160L, 1e161L, 1e162L, 1e163L, 1e164L, 1e165L, 1e166L, 1e167L, 1e168L, 1e169L, 1e170,
+        1e171L, 1e172L, 1e173L, 1e174L, 1e175L, 1e176L, 1e177L, 1e178L, 1e179L, 1e180L, 1e181L, 1e182L, 1e183L, 1e184L, 1e185L, 1e186L, 1e187L, 1e188L, 1e189L, 1e190,
+        1e191L, 1e192L, 1e193L, 1e194L, 1e195L, 1e196L, 1e197L, 1e198L, 1e199L, 1e200L, 1e201L, 1e202L, 1e203L, 1e204L, 1e205L, 1e206L, 1e207L, 1e208L, 1e209L, 1e210,
+        1e211L, 1e212L, 1e213L, 1e214L, 1e215L, 1e216L, 1e217L, 1e218L, 1e219L, 1e220L, 1e221L, 1e222L, 1e223L, 1e224L, 1e225L, 1e226L, 1e227L, 1e228L, 1e229L, 1e230,
+        1e231L, 1e232L, 1e233L, 1e234L, 1e235L, 1e236L, 1e237L, 1e238L, 1e239L, 1e240L, 1e241L, 1e242L, 1e243L, 1e244L, 1e245L, 1e246L, 1e247L, 1e248L, 1e249L, 1e250,
+        1e251L, 1e252L, 1e253L, 1e254L, 1e255L, 1e256L, 1e257L, 1e258L, 1e259L, 1e260L, 1e261L, 1e262L, 1e263L, 1e264L, 1e265L, 1e266L, 1e267L, 1e268L, 1e269L, 1e270,
+        1e271L, 1e272L, 1e273L, 1e274L, 1e275L, 1e276L, 1e277L, 1e278L, 1e279L, 1e280L, 1e281L, 1e282L, 1e283L, 1e284L, 1e285L, 1e286L, 1e287L, 1e288L, 1e289L, 1e290,
+        1e291L, 1e292L, 1e293L, 1e294L, 1e295L, 1e296L, 1e297L, 1e298L, 1e299L, 1e300L, 1e301L, 1e302L, 1e303L, 1e304L, 1e305L, 1e306L, 1e307L, 1e308L
     };
 
     if (unlikely(exponent < min_exponent))  /// Note: there are some values below MIN_EXPONENT that is greater than zero.
@@ -52,7 +52,7 @@ static T shift10Impl(T x, int exponent)
     else if (unlikely(exponent > max_exponent))
         x *= std::numeric_limits<T>::infinity();  /// Multiplying to keep the sign of infinity.
     else
-        x *= powers10[exponent - min_exponent];
+        x *= static_cast<T>(powers10[exponent - min_exponent]);
 
     return x;
 }
@@ -68,12 +68,12 @@ float shift10(float x, int exponent)
     return shift10Impl(x, exponent);
 }
 
-double shift10(UInt64 x, int exponent)
+long double shift10(UInt64 x, int exponent)
 {
     return shift10Impl(static_cast<long double>(x), exponent);
 }
 
-double shift10(Int64 x, int exponent)
+long double shift10(Int64 x, int exponent)
 {
     return shift10Impl(static_cast<long double>(x), exponent);
 }

base/base/shift10.h

@@ -12,5 +12,5 @@
 double shift10(double x, int exponent);
 float shift10(float x, int exponent);
 
-double shift10(UInt64 x, int exponent);
-double shift10(Int64 x, int exponent);
+long double shift10(UInt64 x, int exponent);
+long double shift10(Int64 x, int exponent);

base/base/sort.h

@@ -14,7 +14,7 @@ template <typename Comparator>
 class DebugLessComparator
 {
 public:
-    constexpr DebugLessComparator(Comparator & cmp_)
+    constexpr DebugLessComparator(Comparator & cmp_) // NOLINT(google-explicit-constructor)
         : cmp(cmp_)
     {}
 

base/base/strong_typedef.h

@@ -34,8 +34,10 @@ public:
     template <class Enable = typename std::is_move_assignable<T>::type>
     Self & operator=(T && rhs) { t = std::move(rhs); return *this;}
 
-    operator const T & () const { return t; }
+    // NOLINTBEGIN(google-explicit-constructor)
+    constexpr operator const T & () const { return t; }
     operator T & () { return t; }
+    // NOLINTEND(google-explicit-constructor)
 
     bool operator==(const Self & rhs) const { return t == rhs.t; }
     bool operator<(const Self & rhs) const { return t < rhs.t; }
@@ -58,7 +60,10 @@ namespace std
     };
 }
 
+// NOLINTBEGIN(bugprone-macro-parentheses)
+
 #define STRONG_TYPEDEF(T, D) \
     struct D ## Tag {}; \
     using D = StrongTypedef<T, D ## Tag>; \
 
+// NOLINTEND(bugprone-macro-parentheses)

base/base/terminalColors.cpp

@@ -11,12 +11,12 @@ std::string setColor(UInt64 hash)
     /// It still looks awesome.
     UInt8 y = 128;
 
-    UInt8 cb = hash % 256;
-    UInt8 cr = hash / 256 % 256;
+    UInt8 cb = static_cast<UInt8>(hash % 256);
+    UInt8 cr = static_cast<UInt8>(hash / 256 % 256);
 
-    UInt8 r = std::max(0.0, std::min(255.0, y + 1.402 * (cr - 128)));
-    UInt8 g = std::max(0.0, std::min(255.0, y - 0.344136 * (cb - 128) - 0.714136 * (cr - 128)));
-    UInt8 b = std::max(0.0, std::min(255.0, y + 1.772 * (cb - 128)));
+    UInt8 r = static_cast<UInt8>(std::max(0.0, std::min(255.0, y + 1.402 * (cr - 128))));
+    UInt8 g = static_cast<UInt8>(std::max(0.0, std::min(255.0, y - 0.344136 * (cb - 128) - 0.714136 * (cr - 128))));
+    UInt8 b = static_cast<UInt8>(std::max(0.0, std::min(255.0, y + 1.772 * (cb - 128))));
 
     /// ANSI escape sequence to set 24-bit foreground font color in terminal.
     return "\033[38;2;" + std::to_string(r) + ";" + std::to_string(g) + ";" + std::to_string(b) + "m";

base/base/types.h

@@ -13,11 +13,7 @@ using char8_t = unsigned char;
 #endif
 
 /// This is needed for more strict aliasing. https://godbolt.org/z/xpJBSb https://stackoverflow.com/a/57453713
-#if !defined(PVS_STUDIO) /// But PVS-Studio does not treat it correctly.
 using UInt8 = char8_t;
-#else
-using UInt8 = uint8_t;
-#endif
 
 using UInt16 = uint16_t;
 using UInt32 = uint32_t;

base/base/unaligned.h

@@ -5,44 +5,6 @@
 #include <bit>
 
 
-inline void reverseMemcpy(void * dst, const void * src, size_t size)
-{
-    uint8_t * uint_dst = reinterpret_cast<uint8_t *>(dst);
-    const uint8_t * uint_src = reinterpret_cast<const uint8_t *>(src);
-
-    uint_dst += size;
-    while (size)
-    {
-        --uint_dst;
-        *uint_dst = *uint_src;
-        ++uint_src;
-        --size;
-    }
-}
-
-template <typename T>
-inline T unalignedLoadLE(const void * address)
-{
-    T res {};
-    if constexpr (std::endian::native == std::endian::little)
-        memcpy(&res, address, sizeof(res));
-    else
-        reverseMemcpy(&res, address, sizeof(res));
-    return res;
-}
-
-
-template <typename T>
-inline void unalignedStoreLE(void * address,
-                           const typename std::enable_if<true, T>::type & src)
-{
-    static_assert(std::is_trivially_copyable_v<T>);
-    if constexpr (std::endian::native == std::endian::little)
-        memcpy(address, &src, sizeof(src));
-    else
-        reverseMemcpy(address, &src, sizeof(src));
-}
-
 template <typename T>
 inline T unalignedLoad(const void * address)
 {
@@ -62,3 +24,70 @@ inline void unalignedStore(void * address,
     static_assert(std::is_trivially_copyable_v<T>);
     memcpy(address, &src, sizeof(src));
 }
+
+
+inline void reverseMemcpy(void * dst, const void * src, size_t size)
+{
+    uint8_t * uint_dst = reinterpret_cast<uint8_t *>(dst);
+    const uint8_t * uint_src = reinterpret_cast<const uint8_t *>(src);
+
+    uint_dst += size;
+    while (size)
+    {
+        --uint_dst;
+        *uint_dst = *uint_src;
+        ++uint_src;
+        --size;
+    }
+}
+
+template <std::endian endian, typename T>
+inline T unalignedLoadEndian(const void * address)
+{
+    T res {};
+    if constexpr (std::endian::native == endian)
+        memcpy(&res, address, sizeof(res));
+    else
+        reverseMemcpy(&res, address, sizeof(res));
+    return res;
+}
+
+
+template <std::endian endian, typename T>
+inline void unalignedStoreEndian(void * address, T & src)
+{
+    static_assert(std::is_trivially_copyable_v<T>);
+    if constexpr (std::endian::native == endian)
+        memcpy(address, &src, sizeof(src));
+    else
+        reverseMemcpy(address, &src, sizeof(src));
+}
+
+
+template <typename T>
+inline T unalignedLoadLittleEndian(const void * address)
+{
+    return unalignedLoadEndian<std::endian::little, T>(address);
+}
+
+
+template <typename T>
+inline void unalignedStoreLittleEndian(void * address,
+    const typename std::enable_if<true, T>::type & src)
+{
+    unalignedStoreEndian<std::endian::little>(address, src);
+}
+
+template <typename T>
+inline T unalignedLoadBigEndian(const void * address)
+{
+    return unalignedLoadEndian<std::endian::big, T>(address);
+}
+
+
+template <typename T>
+inline void unalignedStoreBigEndian(void * address,
+    const typename std::enable_if<true, T>::type & src)
+{
+    unalignedStoreEndian<std::endian::big>(address, src);
+}

base/base/unit.h

@@ -5,15 +5,13 @@ constexpr size_t KiB = 1024;
 constexpr size_t MiB = 1024 * KiB;
 constexpr size_t GiB = 1024 * MiB;
 
-#ifdef HAS_RESERVED_IDENTIFIER
-#  pragma clang diagnostic push
-#  pragma clang diagnostic ignored "-Wreserved-identifier"
-#endif
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wreserved-identifier"
 
+// NOLINTBEGIN(google-runtime-int)
 constexpr size_t operator"" _KiB(unsigned long long val) { return val * KiB; }
 constexpr size_t operator"" _MiB(unsigned long long val) { return val * MiB; }
 constexpr size_t operator"" _GiB(unsigned long long val) { return val * GiB; }
+// NOLINTEND(google-runtime-int)
 
-#ifdef HAS_RESERVED_IDENTIFIER
-#  pragma clang diagnostic pop
-#endif
+#pragma clang diagnostic pop

base/base/wide_integer_impl.h

@@ -27,6 +27,8 @@ using FromDoubleIntermediateType = long double;
 using FromDoubleIntermediateType = boost::multiprecision::cpp_bin_float_double_extended;
 #endif
 
+namespace CityHash_v1_0_2 { struct uint128; }
+
 namespace wide
 {
 
@@ -155,13 +157,13 @@ struct common_type<wide::integer<Bits, Signed>, Arithmetic>
         std::is_floating_point_v<Arithmetic>,
         Arithmetic,
         std::conditional_t<
-            sizeof(Arithmetic) < Bits * sizeof(long),
+            sizeof(Arithmetic) * 8 < Bits,
             wide::integer<Bits, Signed>,
             std::conditional_t<
-                Bits * sizeof(long) < sizeof(Arithmetic),
+                Bits < sizeof(Arithmetic) * 8,
                 Arithmetic,
                 std::conditional_t<
-                    Bits * sizeof(long) == sizeof(Arithmetic) && (std::is_same_v<Signed, signed> || std::is_signed_v<Arithmetic>),
+                    Bits == sizeof(Arithmetic) * 8 && (std::is_same_v<Signed, signed> || std::is_signed_v<Arithmetic>),
                     Arithmetic,
                     wide::integer<Bits, Signed>>>>>;
 };
@@ -187,8 +189,20 @@ struct integer<Bits, Signed>::_impl
     static_assert(Bits % base_bits == 0);
 
     /// Simple iteration in both directions
-    static constexpr unsigned little(unsigned idx) { return idx; }
-    static constexpr unsigned big(unsigned idx) { return item_count - 1 - idx; }
+    static constexpr unsigned little(unsigned idx)
+    {
+        if constexpr (std::endian::native == std::endian::little)
+            return idx;
+        else
+            return item_count - 1 - idx;
+    }
+    static constexpr unsigned big(unsigned idx)
+    {
+        if constexpr (std::endian::native == std::endian::little)
+            return item_count - 1 - idx;
+        else
+            return idx;
+    }
     static constexpr unsigned any(unsigned idx) { return idx; }
 
     template <class T>
@@ -227,6 +241,8 @@ struct integer<Bits, Signed>::_impl
     template <typename T>
     __attribute__((no_sanitize("undefined"))) constexpr static auto to_Integral(T f) noexcept
     {
+        /// NOTE: this can be called with DB::Decimal, and in this case, result
+        /// will be wrong
         if constexpr (std::is_signed_v<T>)
             return static_cast<int64_t>(f);
         else
@@ -238,20 +254,20 @@ struct integer<Bits, Signed>::_impl
     {
         static_assert(sizeof(Integral) <= sizeof(base_type));
 
-        self.items[0] = _impl::to_Integral(rhs);
+        self.items[little(0)] = _impl::to_Integral(rhs);
 
         if constexpr (std::is_signed_v<Integral>)
         {
             if (rhs < 0)
             {
-                for (size_t i = 1; i < item_count; ++i)
-                    self.items[i] = -1;
+                for (unsigned i = 1; i < item_count; ++i)
+                    self.items[little(i)] = -1;
                 return;
             }
         }
 
-        for (size_t i = 1; i < item_count; ++i)
-            self.items[i] = 0;
+        for (unsigned i = 1; i < item_count; ++i)
+            self.items[little(i)] = 0;
     }
 
     template <typename TupleLike, size_t i = 0>
@@ -267,6 +283,17 @@ struct integer<Bits, Signed>::_impl
         }
     }
 
+    template <typename CityHashUInt128 = CityHash_v1_0_2::uint128>
+    constexpr static void wide_integer_from_cityhash_uint128(integer<Bits, Signed> & self, const CityHashUInt128 & value) noexcept
+    {
+        static_assert(sizeof(item_count) >= 2);
+
+        if constexpr (std::endian::native == std::endian::little)
+            wide_integer_from_tuple_like(self, std::make_pair(value.low64, value.high64));
+        else
+            wide_integer_from_tuple_like(self, std::make_pair(value.high64, value.low64));
+    }
+
     /**
      * N.B. t is constructed from double, so max(t) = max(double) ~ 2^310
      * the recursive call happens when t / 2^64 > 2^64, so there won't be more than 5 of them.
@@ -300,7 +327,14 @@ struct integer<Bits, Signed>::_impl
 
         const T alpha = t / static_cast<T>(max_int);
 
-        if (alpha <= static_cast<T>(max_int))
+        /** Here we have to use strict comparison.
+          * The max_int is 2^64 - 1.
+          * When casted to floating point type, it will be rounded to the closest representable number,
+          * which is 2^64.
+          * But 2^64 is not representable in uint64_t,
+          * so the maximum representable number will be strictly less.
+          */
+        if (alpha < static_cast<T>(max_int))
             self = static_cast<uint64_t>(alpha);
         else // max(double) / 2^64 will surely contain less than 52 precision bits, so speed up computations.
             set_multiplier<double>(self, static_cast<double>(alpha));
@@ -346,7 +380,7 @@ struct integer<Bits, Signed>::_impl
         constexpr const unsigned to_copy = min_bits / base_bits;
 
         for (unsigned i = 0; i < to_copy; ++i)
-            self.items[i] = rhs.items[i];
+            self.items[little(i)] = rhs.items[integer<Bits2, Signed2>::_impl::little(i)];
 
         if constexpr (Bits > Bits2)
         {
@@ -355,13 +389,13 @@ struct integer<Bits, Signed>::_impl
                 if (rhs < 0)
                 {
                     for (unsigned i = to_copy; i < item_count; ++i)
-                        self.items[i] = -1;
+                        self.items[little(i)] = -1;
                     return;
                 }
             }
 
             for (unsigned i = to_copy; i < item_count; ++i)
-                self.items[i] = 0;
+                self.items[little(i)] = 0;
         }
     }
 
@@ -452,8 +486,8 @@ private:
         {
             if constexpr (sizeof(T) <= sizeof(base_type))
             {
-                if (0 == idx)
-                    return x;
+                if (little(0) == idx)
+                    return static_cast<base_type>(x);
             }
             else if (idx * sizeof(base_type) < sizeof(T))
                 return x >> (idx * base_bits); // & std::numeric_limits<base_type>::max()
@@ -473,7 +507,7 @@ private:
 
         for (unsigned i = 0; i < op_items; ++i)
         {
-            base_type rhs_item = get_item(rhs, i);
+            base_type rhs_item = get_item(rhs, little(i));
             base_type & res_item = res.items[little(i)];
 
             underflows[i] = res_item < rhs_item;
@@ -506,7 +540,7 @@ private:
 
         for (unsigned i = 0; i < op_items; ++i)
         {
-            base_type rhs_item = get_item(rhs, i);
+            base_type rhs_item = get_item(rhs, little(i));
             base_type & res_item = res.items[little(i)];
 
             res_item += rhs_item;
@@ -578,12 +612,12 @@ private:
         else if constexpr (Bits == 128 && sizeof(base_type) == 8)
         {
             using CompilerUInt128 = unsigned __int128;
-            CompilerUInt128 a = (CompilerUInt128(lhs.items[1]) << 64) + lhs.items[0]; // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult)
-            CompilerUInt128 b = (CompilerUInt128(rhs.items[1]) << 64) + rhs.items[0]; // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult)
+            CompilerUInt128 a = (CompilerUInt128(lhs.items[little(1)]) << 64) + lhs.items[little(0)]; // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult)
+            CompilerUInt128 b = (CompilerUInt128(rhs.items[little(1)]) << 64) + rhs.items[little(0)]; // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult)
             CompilerUInt128 c = a * b;
             integer<Bits, Signed> res;
-            res.items[0] = c;
-            res.items[1] = c >> 64;
+            res.items[little(0)] = c;
+            res.items[little(1)] = c >> 64;
             return res;
         }
         else
@@ -595,7 +629,7 @@ private:
 #endif
             for (unsigned i = 0; i < item_count; ++i)
             {
-                base_type rhs_item = get_item(rhs, i);
+                base_type rhs_item = get_item(rhs, little(i));
                 unsigned pos = i * base_bits;
 
                 while (rhs_item)
@@ -718,9 +752,10 @@ public:
             if (std::numeric_limits<T>::is_signed && (is_negative(lhs) != is_negative(rhs)))
                 return is_negative(rhs);
 
+            integer<Bits, Signed> t = rhs;
             for (unsigned i = 0; i < item_count; ++i)
             {
-                base_type rhs_item = get_item(rhs, big(i));
+                base_type rhs_item = get_item(t, big(i));
 
                 if (lhs.items[big(i)] != rhs_item)
                     return lhs.items[big(i)] > rhs_item;
@@ -743,9 +778,10 @@ public:
             if (std::numeric_limits<T>::is_signed && (is_negative(lhs) != is_negative(rhs)))
                 return is_negative(lhs);
 
+            integer<Bits, Signed> t = rhs;
             for (unsigned i = 0; i < item_count; ++i)
             {
-                base_type rhs_item = get_item(rhs, big(i));
+                base_type rhs_item = get_item(t, big(i));
 
                 if (lhs.items[big(i)] != rhs_item)
                     return lhs.items[big(i)] < rhs_item;
@@ -765,9 +801,10 @@ public:
     {
         if constexpr (should_keep_size<T>())
         {
+            integer<Bits, Signed> t = rhs;
             for (unsigned i = 0; i < item_count; ++i)
             {
-                base_type rhs_item = get_item(rhs, any(i));
+                base_type rhs_item = get_item(t, any(i));
 
                 if (lhs.items[any(i)] != rhs_item)
                     return false;
@@ -790,7 +827,7 @@ public:
             integer<Bits, Signed> res;
 
             for (unsigned i = 0; i < item_count; ++i)
-                res.items[little(i)] = lhs.items[little(i)] | get_item(rhs, i);
+                res.items[little(i)] = lhs.items[little(i)] | get_item(rhs, little(i));
             return res;
         }
         else
@@ -808,7 +845,7 @@ public:
             integer<Bits, Signed> res;
 
             for (unsigned i = 0; i < item_count; ++i)
-                res.items[little(i)] = lhs.items[little(i)] & get_item(rhs, i);
+                res.items[little(i)] = lhs.items[little(i)] & get_item(rhs, little(i));
             return res;
         }
         else
@@ -843,17 +880,17 @@ public:
         {
             using CompilerUInt128 = unsigned __int128;
 
-            CompilerUInt128 a = (CompilerUInt128(numerator.items[1]) << 64) + numerator.items[0]; // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult)
-            CompilerUInt128 b = (CompilerUInt128(denominator.items[1]) << 64) + denominator.items[0]; // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult)
+            CompilerUInt128 a = (CompilerUInt128(numerator.items[little(1)]) << 64) + numerator.items[little(0)]; // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult)
+            CompilerUInt128 b = (CompilerUInt128(denominator.items[little(1)]) << 64) + denominator.items[little(0)]; // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult)
             CompilerUInt128 c = a / b; // NOLINT
 
             integer<Bits, Signed> res;
-            res.items[0] = c;
-            res.items[1] = c >> 64;
+            res.items[little(0)] = c;
+            res.items[little(1)] = c >> 64;
 
             CompilerUInt128 remainder = a - b * c;
-            numerator.items[0] = remainder;
-            numerator.items[1] = remainder >> 64;
+            numerator.items[little(0)] = remainder;
+            numerator.items[little(1)] = remainder >> 64;
 
             return res;
         }
@@ -1012,6 +1049,8 @@ constexpr integer<Bits, Signed>::integer(T rhs) noexcept
         _impl::wide_integer_from_wide_integer(*this, rhs);
     else if  constexpr (IsTupleLike<T>::value)
         _impl::wide_integer_from_tuple_like(*this, rhs);
+    else if constexpr (std::is_same_v<std::remove_cvref_t<T>, CityHash_v1_0_2::uint128>)
+        _impl::wide_integer_from_cityhash_uint128(*this, rhs);
     else
         _impl::wide_integer_from_builtin(*this, rhs);
 }
@@ -1027,6 +1066,8 @@ constexpr integer<Bits, Signed>::integer(std::initializer_list<T> il) noexcept
             _impl::wide_integer_from_wide_integer(*this, *il.begin());
         else if  constexpr (IsTupleLike<T>::value)
             _impl::wide_integer_from_tuple_like(*this, *il.begin());
+        else if constexpr (std::is_same_v<std::remove_cvref_t<T>, CityHash_v1_0_2::uint128>)
+            _impl::wide_integer_from_cityhash_uint128(*this, *il.begin());
         else
             _impl::wide_integer_from_builtin(*this, *il.begin());
     }
@@ -1037,15 +1078,15 @@ constexpr integer<Bits, Signed>::integer(std::initializer_list<T> il) noexcept
     else
     {
         auto it = il.begin();
-        for (size_t i = 0; i < _impl::item_count; ++i)
+        for (unsigned i = 0; i < _impl::item_count; ++i)
         {
             if (it < il.end())
             {
-                items[i] = *it;
+                items[_impl::little(i)] = *it;
                 ++it;
             }
             else
-                items[i] = 0;
+                items[_impl::little(i)] = 0;
         }
     }
 }
@@ -1064,6 +1105,8 @@ constexpr integer<Bits, Signed> & integer<Bits, Signed>::operator=(T rhs) noexce
 {
     if  constexpr (IsTupleLike<T>::value)
         _impl::wide_integer_from_tuple_like(*this, rhs);
+    else if constexpr (std::is_same_v<std::remove_cvref_t<T>, CityHash_v1_0_2::uint128>)
+        _impl::wide_integer_from_cityhash_uint128(*this, rhs);
     else
         _impl::wide_integer_from_builtin(*this, rhs);
     return *this;
@@ -1206,7 +1249,7 @@ constexpr integer<Bits, Signed>::operator T() const noexcept
 
     UnsignedT res{};
     for (unsigned i = 0; i < _impl::item_count && i < (sizeof(T) + sizeof(base_type) - 1) / sizeof(base_type); ++i)
-        res += UnsignedT(items[i]) << (sizeof(base_type) * 8 * i); // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult)
+        res += UnsignedT(items[_impl::little(i)]) << (sizeof(base_type) * 8 * i); // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult)
 
     return res;
 }
@@ -1225,7 +1268,7 @@ constexpr integer<Bits, Signed>::operator long double() const noexcept
     for (unsigned i = 0; i < _impl::item_count; ++i)
     {
         long double t = res;
-        res *= std::numeric_limits<base_type>::max();
+        res *= static_cast<long double>(std::numeric_limits<base_type>::max());
         res += t;
         res += tmp.items[_impl::big(i)];
     }
@@ -1239,13 +1282,13 @@ constexpr integer<Bits, Signed>::operator long double() const noexcept
 template <size_t Bits, typename Signed>
 constexpr integer<Bits, Signed>::operator double() const noexcept
 {
-    return static_cast<long double>(*this);
+    return static_cast<double>(static_cast<long double>(*this));
 }
 
 template <size_t Bits, typename Signed>
 constexpr integer<Bits, Signed>::operator float() const noexcept
 {
-    return static_cast<long double>(*this);
+    return static_cast<float>(static_cast<long double>(*this));
 }
 
 // Unary operators

base/base/wide_integer_to_string.h

@@ -51,8 +51,8 @@ struct fmt::formatter<wide::integer<Bits, Signed>>
 {
     constexpr auto parse(format_parse_context & ctx)
     {
-        auto it = ctx.begin();
-        auto end = ctx.end();
+        const auto * it = ctx.begin();
+        const auto * end = ctx.end();
 
         /// Only support {}.
         if (it != end && *it != '}')
@@ -64,6 +64,6 @@ struct fmt::formatter<wide::integer<Bits, Signed>>
     template <typename FormatContext>
     auto format(const wide::integer<Bits, Signed> & value, FormatContext & ctx)
     {
-        return format_to(ctx.out(), "{}", to_string(value));
+        return fmt::format_to(ctx.out(), "{}", to_string(value));
     }
 };

base/glibc-compatibility/CMakeLists.txt

@@ -37,7 +37,7 @@ if (GLIBC_COMPATIBILITY)
 
     target_include_directories(glibc-compatibility PRIVATE libcxxabi ${musl_arch_include_dir})
 
-    if (NOT USE_STATIC_LIBRARIES AND NOT USE_STATIC_LIBRARIES)
+    if (ENABLE_OPENSSL_DYNAMIC)
         target_compile_options(glibc-compatibility PRIVATE -fPIC)
     endif ()
 

base/glibc-compatibility/glibc-compatibility.c

@@ -176,6 +176,258 @@ void __explicit_bzero_chk(void * buf, size_t len, size_t unused)
 }
 
 
+#include <unistd.h>
+#include "syscall.h"
+
+ssize_t copy_file_range(int fd_in, off_t *off_in, int fd_out, off_t *off_out, size_t len, unsigned flags)
+{
+	return syscall(SYS_copy_file_range, fd_in, off_in, fd_out, off_out, len, flags);
+}
+
+
+long splice(int fd_in, off_t *off_in, int fd_out, off_t *off_out, size_t len, unsigned flags)
+{
+	return syscall(SYS_splice, fd_in, off_in, fd_out, off_out, len, flags);
+}
+
+
+#define _BSD_SOURCE
+#include <sys/stat.h>
+#include <stdint.h>
+
+struct statx {
+	uint32_t stx_mask;
+	uint32_t stx_blksize;
+	uint64_t stx_attributes;
+	uint32_t stx_nlink;
+	uint32_t stx_uid;
+	uint32_t stx_gid;
+	uint16_t stx_mode;
+	uint16_t pad1;
+	uint64_t stx_ino;
+	uint64_t stx_size;
+	uint64_t stx_blocks;
+	uint64_t stx_attributes_mask;
+	struct {
+		int64_t tv_sec;
+		uint32_t tv_nsec;
+		int32_t pad;
+	} stx_atime, stx_btime, stx_ctime, stx_mtime;
+	uint32_t stx_rdev_major;
+	uint32_t stx_rdev_minor;
+	uint32_t stx_dev_major;
+	uint32_t stx_dev_minor;
+	uint64_t spare[14];
+};
+
+int statx(int fd, const char *restrict path, int flag,
+                 unsigned int mask, struct statx *restrict statxbuf)
+{
+	return syscall(SYS_statx, fd, path, flag, mask, statxbuf);
+}
+
+
+#include <syscall.h>
+
+ssize_t getrandom(void *buf, size_t buflen, unsigned flags)
+{
+    /// There was cancellable syscall (syscall_cp), but I don't care too.
+    return syscall(SYS_getrandom, buf, buflen, flags);
+}
+
+/* Structure for scatter/gather I/O.  */
+struct iovec
+{
+    void *iov_base;    /* Pointer to data.  */
+    size_t iov_len;    /* Length of data.  */
+};
+
+ssize_t preadv(int __fd, const struct iovec *__iovec, int __count, __off_t __offset)
+{
+    return syscall(SYS_preadv, __fd, __iovec, __count, (long)(__offset), (long)(__offset>>32));
+}
+
+#include <errno.h>
+#include <limits.h>
+
+#define ALIGN (sizeof(size_t))
+#define ONES ((size_t)-1/UCHAR_MAX)
+#define HIGHS (ONES * (UCHAR_MAX/2+1))
+#define HASZERO(x) ((x)-ONES & ~(x) & HIGHS)
+
+char *__strchrnul(const char *s, int c)
+{
+	c = (unsigned char)c;
+	if (!c) return (char *)s + strlen(s);
+
+#ifdef __GNUC__
+	typedef size_t __attribute__((__may_alias__)) word;
+	const word *w;
+	for (; (uintptr_t)s % ALIGN; s++)
+		if (!*s || *(unsigned char *)s == c) return (char *)s;
+	size_t k = ONES * c;
+	for (w = (void *)s; !HASZERO(*w) && !HASZERO(*w^k); w++);
+	s = (void *)w;
+#endif
+	for (; *s && *(unsigned char *)s != c; s++);
+	return (char *)s;
+}
+
+int __execvpe(const char *file, char *const argv[], char *const envp[])
+{
+	const char *p, *z, *path = getenv("PATH");
+	size_t l, k;
+	int seen_eacces = 0;
+
+	errno = ENOENT;
+	if (!*file) return -1;
+
+	if (strchr(file, '/'))
+		return execve(file, argv, envp);
+
+	if (!path) path = "/usr/local/bin:/bin:/usr/bin";
+	k = strnlen(file, NAME_MAX+1);
+	if (k > NAME_MAX) {
+		errno = ENAMETOOLONG;
+		return -1;
+	}
+	l = strnlen(path, PATH_MAX-1)+1;
+
+	for(p=path; ; p=z) {
+		char b[l+k+1];
+		z = __strchrnul(p, ':');
+		if (z-p >= l) {
+			if (!*z++) break;
+			continue;
+		}
+		memcpy(b, p, z-p);
+		b[z-p] = '/';
+		memcpy(b+(z-p)+(z>p), file, k+1);
+		execve(b, argv, envp);
+		switch (errno) {
+		case EACCES:
+			seen_eacces = 1;
+		case ENOENT:
+		case ENOTDIR:
+			break;
+		default:
+			return -1;
+		}
+		if (!*z++) break;
+	}
+	if (seen_eacces) errno = EACCES;
+	return -1;
+}
+
+
+#include "spawn.h"
+
+int posix_spawnp(pid_t *restrict res, const char *restrict file,
+	const posix_spawn_file_actions_t *fa,
+	const posix_spawnattr_t *restrict attr,
+	char *const argv[restrict], char *const envp[restrict])
+{
+	posix_spawnattr_t spawnp_attr = { 0 };
+	if (attr) spawnp_attr = *attr;
+	spawnp_attr.__fn = (void *)__execvpe;
+	return posix_spawn(res, file, fa, &spawnp_attr, argv, envp);
+}
+
+#define FDOP_CLOSE 1
+#define FDOP_DUP2 2
+#define FDOP_OPEN 3
+#define FDOP_CHDIR 4
+#define FDOP_FCHDIR 5
+
+#define ENOMEM 12
+#define EBADF 9
+
+struct fdop {
+	struct fdop *next, *prev;
+	int cmd, fd, srcfd, oflag;
+	mode_t mode;
+	char path[];
+};
+
+int posix_spawn_file_actions_init(posix_spawn_file_actions_t *fa) {
+	fa->__actions = 0;
+	return 0;
+}
+
+int posix_spawn_file_actions_addchdir_np(posix_spawn_file_actions_t *restrict fa, const char *restrict path) {
+	struct fdop *op = malloc(sizeof *op + strlen(path) + 1);
+	if (!op) return ENOMEM;
+	op->cmd = FDOP_CHDIR;
+	op->fd = -1;
+	strcpy(op->path, path);
+	if ((op->next = fa->__actions)) op->next->prev = op;
+	op->prev = 0;
+	fa->__actions = op;
+	return 0;
+}
+
+int posix_spawn_file_actions_addclose(posix_spawn_file_actions_t *fa, int fd) {
+	if (fd < 0) return EBADF;
+	struct fdop *op = malloc(sizeof *op);
+	if (!op) return ENOMEM;
+	op->cmd = FDOP_CLOSE;
+	op->fd = fd;
+	if ((op->next = fa->__actions)) op->next->prev = op;
+	op->prev = 0;
+	fa->__actions = op;
+	return 0;
+}
+
+int posix_spawn_file_actions_adddup2(posix_spawn_file_actions_t *fa, int srcfd, int fd) {
+	if (srcfd < 0 || fd < 0) return EBADF;
+	struct fdop *op = malloc(sizeof *op);
+	if (!op) return ENOMEM;
+	op->cmd = FDOP_DUP2;
+	op->srcfd = srcfd;
+	op->fd = fd;
+	if ((op->next = fa->__actions)) op->next->prev = op;
+	op->prev = 0;
+	fa->__actions = op;
+	return 0;
+}
+
+int posix_spawn_file_actions_addfchdir_np(posix_spawn_file_actions_t *fa, int fd) {
+	if (fd < 0) return EBADF;
+	struct fdop *op = malloc(sizeof *op);
+	if (!op) return ENOMEM;
+	op->cmd = FDOP_FCHDIR;
+	op->fd = fd;
+	if ((op->next = fa->__actions)) op->next->prev = op;
+	op->prev = 0;
+	fa->__actions = op;
+	return 0;
+}
+
+int posix_spawn_file_actions_addopen(posix_spawn_file_actions_t *restrict fa, int fd, const char *restrict path, int flags, mode_t mode) {
+	if (fd < 0) return EBADF;
+	struct fdop *op = malloc(sizeof *op + strlen(path) + 1);
+	if (!op) return ENOMEM;
+	op->cmd = FDOP_OPEN;
+	op->fd = fd;
+	op->oflag = flags;
+	op->mode = mode;
+	strcpy(op->path, path);
+	if ((op->next = fa->__actions)) op->next->prev = op;
+	op->prev = 0;
+	fa->__actions = op;
+	return 0;
+}
+
+int posix_spawn_file_actions_destroy(posix_spawn_file_actions_t *fa) {
+	struct fdop *op = fa->__actions, *next;
+	while (op) {
+		next = op->next;
+		free(op);
+		op = next;
+	}
+	return 0;
+}
+
 #if defined (__cplusplus)
 }
 #endif

base/glibc-compatibility/memcpy/memcpy.h

@@ -63,7 +63,7 @@
   * Very large size of memcpy typically indicates suboptimal (not cache friendly) algorithms in code or unrealistic scenarios,
   * so we don't pay attention to using non-temporary stores.
   *
-  * On recent Intel CPUs, the presence of "erms" makes "rep movsb" the most benefitial,
+  * On recent Intel CPUs, the presence of "erms" makes "rep movsb" the most beneficial,
   * even comparing to non-temporary aligned unrolled stores even with the most wide registers.
   *
   * memcpy can be written in asm, C or C++. The latter can also use inline asm.
@@ -214,4 +214,3 @@ tail:
 
     return ret;
 }
-

base/glibc-compatibility/musl/dup3.c

@@ -0,0 +1,22 @@
+#define _GNU_SOURCE
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include "syscall.h"
+
+int dup3(int old, int new, int flags)
+{
+	int r;
+#ifdef SYS_dup2
+	if (old==new) return __syscall_ret(-EINVAL);
+	if (flags & O_CLOEXEC) {
+		while ((r=__syscall(SYS_dup3, old, new, flags))==-EBUSY);
+		if (r!=-ENOSYS) return __syscall_ret(r);
+	}
+	while ((r=__syscall(SYS_dup2, old, new))==-EBUSY);
+	if (flags & O_CLOEXEC) __syscall(SYS_fcntl, new, F_SETFD, FD_CLOEXEC);
+#else
+	while ((r=__syscall(SYS_dup3, old, new, flags))==-EBUSY);
+#endif
+	return __syscall_ret(r);
+}

base/glibc-compatibility/musl/expf.c

@@ -0,0 +1,81 @@
+/* origin: FreeBSD /usr/src/lib/msun/src/e_expf.c */
+/*
+ * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com.
+ */
+/*
+ * ====================================================
+ * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved.
+ *
+ * Developed at SunPro, a Sun Microsystems, Inc. business.
+ * Permission to use, copy, modify, and distribute this
+ * software is freely granted, provided that this notice
+ * is preserved.
+ * ====================================================
+ */
+
+#include "libm.h"
+
+static const float
+    half[2] = {0.5,-0.5},
+    ln2hi   = 6.9314575195e-1f,  /* 0x3f317200 */
+    ln2lo   = 1.4286067653e-6f,  /* 0x35bfbe8e */
+    invln2  = 1.4426950216e+0f,  /* 0x3fb8aa3b */
+    /*
+ * Domain [-0.34568, 0.34568], range ~[-4.278e-9, 4.447e-9]:
+ * |x*(exp(x)+1)/(exp(x)-1) - p(x)| < 2**-27.74
+ */
+    P1 =  1.6666625440e-1f, /*  0xaaaa8f.0p-26 */
+    P2 = -2.7667332906e-3f; /* -0xb55215.0p-32 */
+
+float expf(float x)
+{
+    float_t hi, lo, c, xx, y;
+    int k, sign;
+    uint32_t hx;
+
+    GET_FLOAT_WORD(hx, x);
+    sign = hx >> 31;   /* sign bit of x */
+    hx &= 0x7fffffff;  /* high word of |x| */
+
+    /* special cases */
+    if (hx >= 0x42aeac50) {  /* if |x| >= -87.33655f or NaN */
+        if (hx >= 0x42b17218 && !sign) {  /* x >= 88.722839f */
+            /* overflow */
+            x *= 0x1p127f;
+            return x;
+        }
+        if (sign) {
+            /* underflow */
+            FORCE_EVAL(-0x1p-149f/x);
+            if (hx >= 0x42cff1b5)  /* x <= -103.972084f */
+                return 0;
+        }
+    }
+
+    /* argument reduction */
+    if (hx > 0x3eb17218) {  /* if |x| > 0.5 ln2 */
+        if (hx > 0x3f851592)  /* if |x| > 1.5 ln2 */
+            k = invln2*x + half[sign];
+        else
+            k = 1 - sign - sign;
+        hi = x - k*ln2hi;  /* k*ln2hi is exact here */
+        lo = k*ln2lo;
+        x = hi - lo;
+    } else if (hx > 0x39000000) {  /* |x| > 2**-14 */
+        k = 0;
+        hi = x;
+        lo = 0;
+    } else {
+        /* raise inexact */
+        FORCE_EVAL(0x1p127f + x);
+        return 1 + x;
+    }
+
+    /* x is now in primary range */
+    xx = x*x;
+    c = x - xx*(P1+xx*P2);
+    y = 1 + (x*c/(2-c) - lo + hi);
+    if (k == 0)
+        return y;
+    return scalbnf(y, k);
+}

base/glibc-compatibility/musl/fallocate.c

@@ -8,3 +8,8 @@ int fallocate(int fd, int mode, off_t base, off_t len)
 {
 	return syscall(SYS_fallocate, fd, mode, base, len);
 }
+
+int fallocate64(int fd, int mode, off_t base, off_t len)
+{
+	return fallocate(fd, mode, base, len);
+}

base/glibc-compatibility/musl/getauxval.c

@@ -8,6 +8,14 @@
 #include <link.h> // ElfW
 #include <errno.h>
 
+#include "syscall.h"
+
+#if defined(__has_feature)
+#if __has_feature(memory_sanitizer)
+#include <sanitizer/msan_interface.h>
+#endif
+#endif
+
 #define ARRAY_SIZE(a) sizeof((a))/sizeof((a[0]))
 
 /// Suppress TSan since it is possible for this code to be called from multiple threads,
@@ -39,7 +47,9 @@ ssize_t __retry_read(int fd, void * buf, size_t count)
 {
     for (;;)
     {
-        ssize_t ret = read(fd, buf, count);
+        // We cannot use the read syscall as it will be intercept by sanitizers, which aren't
+        // initialized yet. Emit syscall directly.
+        ssize_t ret = __syscall_ret(__syscall(SYS_read, fd, buf, count));
         if (ret == -1)
         {
             if (errno == EINTR)
@@ -90,6 +100,11 @@ static unsigned long NO_SANITIZE_THREAD __auxv_init_procfs(unsigned long type)
     _Static_assert(sizeof(aux) < 4096, "Unexpected sizeof(aux)");
     while (__retry_read(fd, &aux, sizeof(aux)) == sizeof(aux))
     {
+#if defined(__has_feature)
+#if __has_feature(memory_sanitizer)
+        __msan_unpoison(&aux, sizeof(aux));
+#endif
+#endif
         if (aux.a_type == AT_NULL)
         {
             break;

base/glibc-compatibility/musl/inotify.c

@@ -0,0 +1,26 @@
+#include <sys/inotify.h>
+#include <errno.h>
+#include "syscall.h"
+
+int inotify_init()
+{
+	return inotify_init1(0);
+}
+int inotify_init1(int flags)
+{
+	int r = __syscall(SYS_inotify_init1, flags);
+#ifdef SYS_inotify_init
+	if (r==-ENOSYS && !flags) r = __syscall(SYS_inotify_init);
+#endif
+	return __syscall_ret(r);
+}
+
+int inotify_add_watch(int fd, const char *pathname, uint32_t mask)
+{
+	return syscall(SYS_inotify_add_watch, fd, pathname, mask);
+}
+
+int inotify_rm_watch(int fd, int wd)
+{
+	return syscall(SYS_inotify_rm_watch, fd, wd);
+}

base/glibc-compatibility/musl/lgamma.c

@@ -78,9 +78,6 @@
  *
  */
 
-// Disable warnings by PVS-Studio
-//-V::GA
-
 static const double
 pi  =  3.14159265358979311600e+00, /* 0x400921FB, 0x54442D18 */
 a0  =  7.72156649015328655494e-02, /* 0x3FB3C467, 0xE37DB0C8 */

base/glibc-compatibility/musl/lgammal.c

@@ -85,9 +85,6 @@
  *
  */
 
-// Disable warnings by PVS-Studio
-//-V::GA
-
 #include <stdint.h>
 #include <math.h>
 #include "libm.h"

base/glibc-compatibility/musl/libm.h

@@ -155,7 +155,7 @@ static inline long double fp_barrierl(long double x)
 static inline void fp_force_evalf(float x)
 {
 	volatile float y;
-	y = x; //-V1001
+	y = x;
 }
 #endif
 
@@ -164,7 +164,7 @@ static inline void fp_force_evalf(float x)
 static inline void fp_force_eval(double x)
 {
 	volatile double y;
-	y = x; //-V1001
+	y = x;
 }
 #endif
 
@@ -173,7 +173,7 @@ static inline void fp_force_eval(double x)
 static inline void fp_force_evall(long double x)
 {
 	volatile long double y;
-	y = x; //-V1001
+	y = x;
 }
 #endif
 

base/glibc-compatibility/musl/logf.c

@@ -53,7 +53,7 @@ float logf(float x)
 	tmp = ix - OFF;
 	i = (tmp >> (23 - LOGF_TABLE_BITS)) % N;
 	k = (int32_t)tmp >> 23; /* arithmetic shift */
-	iz = ix - (tmp & 0x1ff << 23);
+	iz = ix - (tmp & 0xff800000);
 	invc = T[i].invc;
 	logc = T[i].logc;
 	z = (double_t)asfloat(iz);

base/glibc-compatibility/musl/powf.c

@@ -3,9 +3,6 @@
  * SPDX-License-Identifier: MIT
  */
 
-// Disable warnings by PVS-Studio
-//-V::GA
-
 #include <math.h>
 #include <stdint.h>
 #include "libm.h"

base/glibc-compatibility/musl/pwritev.c

@@ -9,3 +9,8 @@ ssize_t pwritev(int fd, const struct iovec *iov, int count, off_t ofs)
     /// There was cancellable syscall (syscall_cp), but I don't care.
 	return syscall(SYS_pwritev, fd, iov, count, (long)(ofs), (long)(ofs>>32));
 }
+
+ssize_t pwritev64(int fd, const struct iovec *iov, int count, off_t ofs)
+{
+	return pwritev(fd, iov, count, ofs);
+}

base/glibc-compatibility/musl/scalbnf.c

@@ -0,0 +1,31 @@
+#include <math.h>
+#include <stdint.h>
+
+float scalbnf(float x, int n)
+{
+    union {float f; uint32_t i;} u;
+    float_t y = x;
+
+    if (n > 127) {
+        y *= 0x1p127f;
+        n -= 127;
+        if (n > 127) {
+            y *= 0x1p127f;
+            n -= 127;
+            if (n > 127)
+                n = 127;
+        }
+    } else if (n < -126) {
+        y *= 0x1p-126f;
+        n += 126;
+        if (n < -126) {
+            y *= 0x1p-126f;
+            n += 126;
+            if (n < -126)
+                n = -126;
+        }
+    }
+    u.i = (uint32_t)(0x7f+n)<<23;
+    x = y * u.f;
+    return x;
+}