Merge branch 'executable-table-function' into add-executable-table-function

.gitattributes

@@ -1,2 +1,4 @@
 contrib/* linguist-vendored
 *.h linguist-language=C++
+# to avoid frequent conflicts
+tests/queries/0_stateless/arcadia_skip_list.txt text merge=union

.github/ISSUE_TEMPLATE/10_question.md

@@ -0,0 +1,12 @@
+---
+name: Question
+about: Ask a question about ClickHouse
+title: ''
+labels: question
+assignees: ''
+
+---
+
+> Make sure to check documentation https://clickhouse.yandex/docs/en/ first. If the question is concise and probably has a short answer, asking it in Telegram chat https://telegram.me/clickhouse_en is probably the fastest way to find the answer. For more complicated questions, consider asking them on StackOverflow with "clickhouse" tag https://stackoverflow.com/questions/tagged/clickhouse 
+
+> If you still prefer GitHub issues, remove all this text and ask your question here.

.github/ISSUE_TEMPLATE/20_feature-request.md

@@ -0,0 +1,26 @@
+---
+name: Feature request
+about: Suggest an idea for ClickHouse
+title: ''
+labels: feature
+assignees: ''
+
+---
+
+> (you don't have to strictly follow this form)
+
+**Use case**
+
+> A clear and concise description of what is the intended usage scenario is.
+
+**Describe the solution you'd like**
+
+> A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+
+> A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+
+> Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/30_unexpected-behaviour.md

@@ -0,0 +1,30 @@
+---
+name: Unexpected behaviour
+about: Some feature is working in non-obvious way
+title: ''
+labels: unexpected behaviour
+assignees: ''
+
+---
+
+(you don't have to strictly follow this form)
+
+**Describe the unexpected behaviour**
+A clear and concise description of what works not as it is supposed to.
+
+**How to reproduce**
+* Which ClickHouse server version to use
+* Which interface to use, if matters
+* Non-default settings, if any
+* `CREATE TABLE` statements for all tables involved
+* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
+* Queries to run that lead to unexpected result
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Error message and/or stacktrace**
+If applicable, add screenshots to help explain your problem.
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/35_incomplete_implementation.md

@@ -0,0 +1,30 @@
+---
+name: Incomplete implementation
+about: Implementation of existing feature is not finished
+title: ''
+labels: unfinished code
+assignees: ''
+
+---
+
+(you don't have to strictly follow this form)
+
+**Describe the unexpected behaviour**
+A clear and concise description of what works not as it is supposed to.
+
+**How to reproduce**
+* Which ClickHouse server version to use
+* Which interface to use, if matters
+* Non-default settings, if any
+* `CREATE TABLE` statements for all tables involved
+* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
+* Queries to run that lead to unexpected result
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Error message and/or stacktrace**
+If applicable, add screenshots to help explain your problem.
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/45_usability-issue.md

@@ -0,0 +1,30 @@
+---
+name: Usability issue
+about: Report something can be made more convenient to use
+title: ''
+labels: usability
+assignees: ''
+
+---
+
+(you don't have to strictly follow this form)
+
+**Describe the issue**
+A clear and concise description of what works not as it is supposed to.
+
+**How to reproduce**
+* Which ClickHouse server version to use
+* Which interface to use, if matters
+* Non-default settings, if any
+* `CREATE TABLE` statements for all tables involved
+* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
+* Queries to run that lead to unexpected result
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Error message and/or stacktrace**
+If applicable, add screenshots to help explain your problem.
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/50_build-issue.md

@@ -0,0 +1,22 @@
+---
+name: Build issue
+about: Report failed ClickHouse build from master
+title: ''
+labels: build
+assignees: ''
+
+---
+
+> Make sure that `git diff` result is empty and you've just pulled fresh master. Try cleaning up cmake cache. Just in case, official build instructions are published here: https://clickhouse.yandex/docs/en/development/build/
+
+**Operating system**
+
+> OS kind or distribution, specific version/release, non-standard kernel if any. If you are trying to build inside virtual machine, please mention it too.
+
+**Cmake version**
+
+**Ninja version**
+
+**Compiler name and version**
+
+**Full cmake and/or ninja output**

.github/ISSUE_TEMPLATE/60_documentation-issue.md

@@ -0,0 +1,15 @@
+---
+name: Documentation issue
+about: Report something incorrect or missing in documentation
+title: ''
+labels: comp-documentation
+
+---
+
+(you don't have to strictly follow this form)
+
+**Describe the issue**
+A clear and concise description of what's wrong in documentation.
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/80_backward-compatibility.md

@@ -0,0 +1,27 @@
+---
+name: Backward compatibility issue
+about: Report the case when the behaviour of a new version can break existing use cases
+title: ''
+labels: backward compatibility
+assignees: ''
+
+---
+
+(you don't have to strictly follow this form)
+
+**Describe the issue**
+A clear and concise description of what works not as it is supposed to.
+
+**How to reproduce**
+* Which ClickHouse server versions are incompatible
+* Which interface to use, if matters
+* Non-default settings, if any
+* `CREATE TABLE` statements for all tables involved
+* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
+* Queries to run that lead to unexpected result
+
+**Error message and/or stacktrace**
+If applicable, add screenshots to help explain your problem.
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/85_bug-report.md

@@ -0,0 +1,50 @@
+---
+name: Bug report
+about: Wrong behaviour (visible to users) in official ClickHouse release.
+title: ''
+labels: 'potential bug'
+assignees: ''
+
+---
+
+> You have to provide the following information whenever possible.
+
+**Describe the bug**
+
+> A clear and concise description of what works not as it is supposed to.
+
+**Does it reproduce on recent release?**
+
+[The list of releases](https://github.com/ClickHouse/ClickHouse/blob/master/utils/list-versions/version_date.tsv)
+
+**Enable crash reporting**
+
+> If possible, change "enabled" to true in "send_crash_reports" section in `config.xml`:
+
+```
+<send_crash_reports>
+        <!-- Changing <enabled> to true allows sending crash reports to -->
+        <!-- the ClickHouse core developers team via Sentry https://sentry.io -->
+        <enabled>false</enabled>
+```
+
+**How to reproduce**
+
+* Which ClickHouse server version to use
+* Which interface to use, if matters
+* Non-default settings, if any
+* `CREATE TABLE` statements for all tables involved
+* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
+* Queries to run that lead to unexpected result
+
+**Expected behavior**
+
+> A clear and concise description of what you expected to happen.
+
+**Error message and/or stacktrace**
+
+> If applicable, add screenshots to help explain your problem.
+
+**Additional context**
+
+> Add any other context about the problem here.

.github/ISSUE_TEMPLATE/90_fuzzing-report.md

@@ -0,0 +1,16 @@
+---
+name: Assertion found via fuzzing
+about: Potential issue has been found via Fuzzer or Stress tests
+title: ''
+labels: fuzz
+assignees: ''
+
+---
+
+(you don't have to strictly follow this form)
+
+**Describe the bug**
+A link to the report
+
+**How to reproduce**
+Try to reproduce the report and copy the tables and queries involved.

.github/ISSUE_TEMPLATE/95_sanitizer-report.md

@@ -0,0 +1,19 @@
+---
+name: Sanitizer alert
+about: Potential issue has been found by special code instrumentation
+title: ''
+labels: testing
+assignees: ''
+
+---
+
+(you don't have to strictly follow this form)
+
+**Describe the bug**
+A link to the report
+
+**How to reproduce**
+Try to reproduce the report and copy the tables and queries involved.
+
+**Error message and/or stacktrace**
+You can find additional information in server logs.

.github/ISSUE_TEMPLATE/backward-compatibility.md

@@ -1,27 +0,0 @@
----
-name: Backward compatibility issue
-about: Create a report to help us improve ClickHouse
-title: ''
-labels: backward compatibility
-assignees: ''
-
----
-
-(you don't have to strictly follow this form)
-
-**Describe the issue**
-A clear and concise description of what works not as it is supposed to.
-
-**How to reproduce**
-* Which ClickHouse server versions are incompatible
-* Which interface to use, if matters
-* Non-default settings, if any
-* `CREATE TABLE` statements for all tables involved
-* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
-* Queries to run that lead to unexpected result
-
-**Error message and/or stacktrace**
-If applicable, add screenshots to help explain your problem.
-
-**Additional context**
-Add any other context about the problem here.

.github/ISSUE_TEMPLATE/bug-report.md

@@ -1,30 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve ClickHouse
-title: ''
-labels: bug
-assignees: ''
-
----
-
-(you don't have to strictly follow this form)
-
-**Describe the bug**
-A clear and concise description of what works not as it is supposed to.
-
-**How to reproduce**
-* Which ClickHouse server version to use
-* Which interface to use, if matters
-* Non-default settings, if any
-* `CREATE TABLE` statements for all tables involved
-* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
-* Queries to run that lead to unexpected result
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Error message and/or stacktrace**
-If applicable, add screenshots to help explain your problem.
-
-**Additional context**
-Add any other context about the problem here.

.github/ISSUE_TEMPLATE/build-issue.md

@@ -1,21 +0,0 @@
----
-name: Build issue
-about: Report failed ClickHouse build from master
-title: ''
-labels: build
-assignees: ''
-
----
-
-Make sure that `git diff` result is empty and you've just pulled fresh master. Try cleaning up cmake cache. Just in case, official build instructions are published here: https://clickhouse.yandex/docs/en/development/build/
-
-**Operating system**
-OS kind or distribution, specific version/release, non-standard kernel if any. If you are trying to build inside virtual machine, please mention it too.
-
-**Cmake version**
-
-**Ninja version**
-
-**Compiler name and version**
-
-**Full cmake and/or ninja output**

.github/ISSUE_TEMPLATE/documentation-issue.md

@@ -1,16 +0,0 @@
----
-name: Documentation issue
-about: Report something incorrect or missing in documentation
-title: ''
-labels: documentation
-assignees: BayoNet
-
----
-
-(you don't have to strictly follow this form)
-
-**Describe the issue**
-A clear and concise description of what's wrong in documentation.
-
-**Additional context**
-Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature-request.md

@@ -1,22 +0,0 @@
----
-name: Feature request
-about: Suggest an idea for ClickHouse
-title: ''
-labels: feature
-assignees: ''
-
----
-
-(you don't have to strictly follow this form)
-
-**Use case**
-A clear and concise description of what is the intended usage scenario is.
-
-**Describe the solution you'd like**
-A clear and concise description of what you want to happen.
-
-**Describe alternatives you've considered**
-A clear and concise description of any alternative solutions or features you've considered.
-
-**Additional context**
-Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/question.md

@@ -1,12 +0,0 @@
----
-name: Question
-about: Ask question about ClickHouse
-title: ''
-labels: question
-assignees: ''
-
----
-
-Make sure to check documentation https://clickhouse.yandex/docs/en/ first. If the question is concise and probably has a short answer, asking it in Telegram chat https://telegram.me/clickhouse_en is probably the fastest way to find the answer. For more complicated questions, consider asking them on StackOverflow with "clickhouse" tag https://stackoverflow.com/questions/tagged/clickhouse 
-
-If you still prefer GitHub issues, remove all this text and ask your question here.

.github/ISSUE_TEMPLATE/unexpected-behaviour.md

@@ -1,30 +0,0 @@
----
-name: Unexpected behaviour
-about: Create a report to help us improve ClickHouse
-title: ''
-labels: unexpected behaviour
-assignees: ''
-
----
-
-(you don't have to strictly follow this form)
-
-**Describe the unexpected behaviour**
-A clear and concise description of what works not as it is supposed to.
-
-**How to reproduce**
-* Which ClickHouse server version to use
-* Which interface to use, if matters
-* Non-default settings, if any
-* `CREATE TABLE` statements for all tables involved
-* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
-* Queries to run that lead to unexpected result
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Error message and/or stacktrace**
-If applicable, add screenshots to help explain your problem.
-
-**Additional context**
-Add any other context about the problem here.

.github/ISSUE_TEMPLATE/usability-issue.md

@@ -1,30 +0,0 @@
----
-name: Usability issue
-about: Create a report to help us improve ClickHouse
-title: ''
-labels: usability
-assignees: ''
-
----
-
-(you don't have to strictly follow this form)
-
-**Describe the issue**
-A clear and concise description of what works not as it is supposed to.
-
-**How to reproduce**
-* Which ClickHouse server version to use
-* Which interface to use, if matters
-* Non-default settings, if any
-* `CREATE TABLE` statements for all tables involved
-* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
-* Queries to run that lead to unexpected result
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Error message and/or stacktrace**
-If applicable, add screenshots to help explain your problem.
-
-**Additional context**
-Add any other context about the problem here.

.github/PULL_REQUEST_TEMPLATE.md

@@ -2,28 +2,26 @@ I hereby agree to the terms of the CLA available at: https://yandex.ru/legal/cla
 
 Changelog category (leave one):
 - New Feature
-- Bug Fix
 - Improvement
+- Bug Fix (user-visible misbehaviour in official stable or prestable release)
 - Performance Improvement
 - Backward Incompatible Change
 - Build/Testing/Packaging Improvement
 - Documentation (changelog entry is not required)
-- Other
 - Not for changelog (changelog entry is not required)
 
 
 Changelog entry (a user-readable short description of the changes that goes to CHANGELOG.md):
-
 ...
 
 
 Detailed description / Documentation draft:
-
 ...
 
-By adding documentation, you'll allow users to try your new feature immediately, not when someone else will have time to document it later. Documentation is necessary for all features that affect user experience in any way. You can add brief documentation draft above, or add documentation right into your patch as Markdown files in [docs](https://github.com/ClickHouse/ClickHouse/tree/master/docs) folder.
 
-If you are doing this for the first time, it's recommended to read the lightweight [Contributing to ClickHouse Documentation](https://github.com/ClickHouse/ClickHouse/tree/master/docs/README.md) guide first.
+> By adding documentation, you'll allow users to try your new feature immediately, not when someone else will have time to document it later. Documentation is necessary for all features that affect user experience in any way. You can add brief documentation draft above, or add documentation right into your patch as Markdown files in [docs](https://github.com/ClickHouse/ClickHouse/tree/master/docs) folder.
+
+> If you are doing this for the first time, it's recommended to read the lightweight [Contributing to ClickHouse Documentation](https://github.com/ClickHouse/ClickHouse/tree/master/docs/README.md) guide first.
 
 
-Information about CI checks: https://clickhouse.tech/docs/en/development/continuous-integration/
+> Information about CI checks: https://clickhouse.tech/docs/en/development/continuous-integration/

.github/codecov.yml

@@ -1,5 +1,5 @@
 codecov:
-  max_report_age: off
+  max_report_age: "off"
   strict_yaml_branch: "master"
 
 ignore:

.github/workflows/anchore-analysis.yml

@@ -1,14 +1,14 @@
 # This workflow checks out code, performs an Anchore container image
 # vulnerability and compliance scan, and integrates the results with
-# GitHub Advanced Security code scanning feature.  For more information on
+# GitHub Advanced Security code scanning feature. For more information on
 # the Anchore scan action usage and parameters, see
-# https://github.com/anchore/scan-action.  For more information on
+# https://github.com/anchore/scan-action. For more information on
 # Anchore container image scanning in general, see
 # https://docs.anchore.com.
 
 name: Docker Container Scan (clickhouse-server)
 
-on:
+"on":
   pull_request:
     paths:
       - docker/server/Dockerfile
@@ -20,26 +20,20 @@ jobs:
   Anchore-Build-Scan:
     runs-on: ubuntu-latest
     steps:
-    - name: Checkout the code
+      - name: Checkout the code
-      uses: actions/checkout@v2
+        uses: actions/checkout@v2
-    - name: Build the Docker image
+      - name: Build the Docker image
-      run: |
+        run: |
-        cd docker/server
+          cd docker/server
-        perl -pi -e 's|=\$version||g' Dockerfile
+          perl -pi -e 's|=\$version||g' Dockerfile
-        docker build . --file Dockerfile --tag localbuild/testimage:latest      
+          docker build . --file Dockerfile --tag localbuild/testimage:latest
-    - name: Run the local Anchore scan action itself with GitHub Advanced Security code scanning integration enabled
+      - name: Run the local Anchore scan action itself with GitHub Advanced Security code scanning integration enabled
-      uses: anchore/scan-action@master
+        uses: anchore/scan-action@v2
-      with:
+        id: scan
-        image-reference: "localbuild/testimage:latest"
+        with:
-        dockerfile-path: "docker/server/Dockerfile"
+          image: "localbuild/testimage:latest"
-        acs-report-enable: true
+          acs-report-enable: true
-        fail-build: true
+      - name: Upload Anchore Scan Report
-    - name: Upload artifact
+        uses: github/codeql-action/upload-sarif@v1
-      uses: actions/upload-artifact@v1.0.0
+        with:
-      with:
+          sarif_file: ${{ steps.scan.outputs.sarif }}
-        name: AnchoreReports
-        path: ./anchore-reports/
-    - name: Upload Anchore Scan Report
-      uses: github/codeql-action/upload-sarif@v1
-      with:
-        sarif_file: results.sarif

.github/workflows/codeql-analysis.yml

@@ -1,33 +0,0 @@
-name: "CodeQL Scanning"
-
-on:
-  schedule:
-    - cron: '0 19 * * *'
-jobs:
-  CodeQL-Build:
-
-    runs-on: self-hosted
-    timeout-minutes: 1440
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
-      with:
-        fetch-depth: 2
-        submodules: 'recursive'
-
-    - run: git checkout HEAD^2
-      if: ${{ github.event_name == 'pull_request' }}
-      
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
-
-      with:
-        languages: cpp
-
-    - run: sudo apt-get update && sudo apt-get install -y git cmake python ninja-build gcc-9 g++-9 && mkdir build
-    - run: cd build && CC=gcc-9 CXX=g++-9 cmake ..
-    - run: cd build && ninja
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1

.gitignore

@@ -14,6 +14,11 @@
 /build-*
 /tests/venv
 
+# logs
+*.log
+*.stderr
+*.stdout
+
 /docs/build
 /docs/publish
 /docs/edit
@@ -27,6 +32,7 @@
 /docs/zh/single.md
 /docs/ja/single.md
 /docs/fa/single.md
+/docs/en/development/cmake-in-clickhouse.md
 
 # callgrind files
 callgrind.out.*
@@ -124,3 +130,22 @@ website/package-lock.json
 
 # Toolchains
 /cmake/toolchain/*
+
+# ANTLR extension cache
+.antlr
+
+# ANTLR generated files
+/src/Parsers/New/*.interp
+/src/Parsers/New/*.tokens
+/src/Parsers/New/ClickHouseParserBaseVisitor.*
+
+# pytest-profiling
+/prof
+
+*.iml
+
+# data store
+/programs/server/data
+/programs/server/metadata
+/programs/server/store
+

.gitmodules

@@ -17,6 +17,7 @@
 [submodule "contrib/zlib-ng"]
 	path = contrib/zlib-ng
 	url = https://github.com/ClickHouse-Extras/zlib-ng.git
+	branch = clickhouse-new
 [submodule "contrib/googletest"]
 	path = contrib/googletest
 	url = https://github.com/google/googletest.git
@@ -44,6 +45,7 @@
 [submodule "contrib/protobuf"]
 	path = contrib/protobuf
 	url = https://github.com/ClickHouse-Extras/protobuf.git
+	branch = v3.13.0.1
 [submodule "contrib/boost"]
 	path = contrib/boost
 	url = https://github.com/ClickHouse-Extras/boost.git
@@ -52,7 +54,8 @@
 	url = https://github.com/ClickHouse-Extras/Turbo-Base64.git
 [submodule "contrib/arrow"]
 	path = contrib/arrow
-	url = https://github.com/apache/arrow
+	url = https://github.com/ClickHouse-Extras/arrow
+	branch = clickhouse-arrow-2.0.0
 [submodule "contrib/thrift"]
 	path = contrib/thrift
 	url = https://github.com/apache/thrift.git
@@ -82,7 +85,7 @@
 	url = https://github.com/google/brotli.git
 [submodule "contrib/h3"]
 	path = contrib/h3
-	url = https://github.com/uber/h3
+	url = https://github.com/ClickHouse-Extras/h3
 [submodule "contrib/hyperscan"]
 	path = contrib/hyperscan
 	url = https://github.com/ClickHouse-Extras/hyperscan.git
@@ -91,7 +94,7 @@
 	url = https://github.com/ClickHouse-Extras/libunwind.git
 [submodule "contrib/simdjson"]
 	path = contrib/simdjson
-	url = https://github.com/ClickHouse-Extras/simdjson.git
+	url = https://github.com/simdjson/simdjson.git
 [submodule "contrib/rapidjson"]
 	path = contrib/rapidjson
 	url = https://github.com/ClickHouse-Extras/rapidjson
@@ -100,13 +103,14 @@
 	url = https://github.com/ClickHouse-Extras/fastops
 [submodule "contrib/orc"]
 	path = contrib/orc
-	url = https://github.com/apache/orc
+	url = https://github.com/ClickHouse-Extras/orc
 [submodule "contrib/sparsehash-c11"]
 	path = contrib/sparsehash-c11
 	url = https://github.com/sparsehash/sparsehash-c11.git
 [submodule "contrib/grpc"]
 	path = contrib/grpc
 	url = https://github.com/ClickHouse-Extras/grpc.git
+	branch = v1.33.2
 [submodule "contrib/aws"]
 	path = contrib/aws
 	url = https://github.com/ClickHouse-Extras/aws-sdk-cpp.git
@@ -122,9 +126,6 @@
 [submodule "contrib/curl"]
 	path = contrib/curl
 	url = https://github.com/curl/curl.git
-[submodule "contrib/openssl"]
-	path = contrib/openssl
-	url = https://github.com/ClickHouse-Extras/openssl.git
 [submodule "contrib/icudata"]
 	path = contrib/icudata
 	url = https://github.com/ClickHouse-Extras/icudata.git
@@ -133,16 +134,13 @@
 	url = https://github.com/unicode-org/icu.git
 [submodule "contrib/flatbuffers"]
 	path = contrib/flatbuffers
-	url = https://github.com/google/flatbuffers.git
+	url = https://github.com/ClickHouse-Extras/flatbuffers.git
 [submodule "contrib/libc-headers"]
 	path = contrib/libc-headers
 	url = https://github.com/ClickHouse-Extras/libc-headers.git
 [submodule "contrib/replxx"]
 	path = contrib/replxx
 	url = https://github.com/ClickHouse-Extras/replxx.git
-[submodule "contrib/ryu"]
-	path = contrib/ryu
-	url = https://github.com/ClickHouse-Extras/ryu.git
 [submodule "contrib/avro"]
 	path = contrib/avro
 	url = https://github.com/ClickHouse-Extras/avro.git
@@ -155,7 +153,7 @@
 	url = https://github.com/ClickHouse-Extras/libcpuid.git
 [submodule "contrib/openldap"]
 	path = contrib/openldap
-	url = https://github.com/openldap/openldap.git
+	url = https://github.com/ClickHouse-Extras/openldap.git
 [submodule "contrib/AMQP-CPP"]
 	path = contrib/AMQP-CPP
 	url = https://github.com/ClickHouse-Extras/AMQP-CPP.git
@@ -181,10 +179,10 @@
 	url = https://github.com/kthohr/stats.git
 [submodule "contrib/krb5"]
 	path = contrib/krb5
-	url = https://github.com/krb5/krb5
+	url = https://github.com/ClickHouse-Extras/krb5
 [submodule "contrib/cyrus-sasl"]
 	path = contrib/cyrus-sasl
-	url = https://github.com/cyrusimap/cyrus-sasl
+	url = https://github.com/ClickHouse-Extras/cyrus-sasl
 	branch = cyrus-sasl-2.1
 [submodule "contrib/croaring"]
 	path = contrib/croaring
@@ -193,3 +191,58 @@
 [submodule "contrib/miniselect"]
 	path = contrib/miniselect
 	url = https://github.com/danlark1/miniselect
+[submodule "contrib/rocksdb"]
+	path = contrib/rocksdb
+	url = https://github.com/ClickHouse-Extras/rocksdb.git
+[submodule "contrib/xz"]
+	path = contrib/xz
+	url = https://github.com/xz-mirror/xz
+[submodule "contrib/abseil-cpp"]
+	path = contrib/abseil-cpp
+	url = https://github.com/ClickHouse-Extras/abseil-cpp.git
+	branch = lts_2020_02_25
+[submodule "contrib/dragonbox"]
+	path = contrib/dragonbox
+	url = https://github.com/ClickHouse-Extras/dragonbox.git
+[submodule "contrib/fast_float"]
+	path = contrib/fast_float
+	url = https://github.com/fastfloat/fast_float
+[submodule "contrib/libpq"]
+	path = contrib/libpq
+	url = https://github.com/ClickHouse-Extras/libpq
+[submodule "contrib/boringssl"]
+	path = contrib/boringssl
+	url = https://github.com/ClickHouse-Extras/boringssl.git
+[submodule "contrib/NuRaft"]
+	path = contrib/NuRaft
+	url = https://github.com/ClickHouse-Extras/NuRaft.git
+[submodule "contrib/nanodbc"]
+	path = contrib/nanodbc
+	url = https://github.com/ClickHouse-Extras/nanodbc.git
+[submodule "contrib/datasketches-cpp"]
+	path = contrib/datasketches-cpp
+	url = https://github.com/ClickHouse-Extras/datasketches-cpp.git
+[submodule "contrib/yaml-cpp"]
+	path = contrib/yaml-cpp
+	url = https://github.com/ClickHouse-Extras/yaml-cpp.git
+[submodule "contrib/libstemmer_c"]
+	path = contrib/libstemmer_c
+	url = https://github.com/ClickHouse-Extras/libstemmer_c.git
+[submodule "contrib/wordnet-blast"]
+	path = contrib/wordnet-blast
+	url = https://github.com/ClickHouse-Extras/wordnet-blast.git
+[submodule "contrib/lemmagen-c"]
+	path = contrib/lemmagen-c
+	url = https://github.com/ClickHouse-Extras/lemmagen-c.git
+[submodule "contrib/libpqxx"]
+	path = contrib/libpqxx
+	url = https://github.com/ClickHouse-Extras/libpqxx.git
+[submodule "contrib/sqlite-amalgamation"]
+	path = contrib/sqlite-amalgamation
+	url = https://github.com/azadkuh/sqlite-amalgamation
+[submodule "contrib/s2geometry"]
+	path = contrib/s2geometry
+	url = https://github.com/ClickHouse-Extras/s2geometry.git
+[submodule "contrib/bzip2"]
+	path = contrib/bzip2
+	url = https://github.com/ClickHouse-Extras/bzip2.git

.potato.yml

@@ -14,14 +14,14 @@ handlers:
         # The trigger for creating the Yandex.Tracker issue. When the specified event occurs, it transfers PR data to Yandex.Tracker.
         github:pullRequest:labeled:
           data:
-              # The Yandex.Tracker queue to create the issue in. Each issue in Tracker belongs to one of the project queues.
+          # The Yandex.Tracker queue to create the issue in. Each issue in Tracker belongs to one of the project queues.
-          	queue: CLICKHOUSEDOCS
+          queue: CLICKHOUSEDOCS
-              # The issue title.
+          # The issue title.
-          	summary: '[Potato] Pull Request #{{pullRequest.number}}'
+          summary: '[Potato] Pull Request #{{pullRequest.number}}'
-              # The issue description.
+          # The issue description.
-          	description: >
+          description: >
                 {{pullRequest.description}}
 
                 Ссылка на Pull Request: {{pullRequest.webUrl}}
-            # The condition for creating the Yandex.Tracker issue.
+          # The condition for creating the Yandex.Tracker issue.
-        	condition: eventPayload.labels.filter(label => ['pr-feature'].includes(label.name)).length
+          condition: eventPayload.labels.filter(label => ['pr-feature'].includes(label.name)).length

.pylintrc

@@ -0,0 +1,45 @@
+# vim: ft=config
+
+[BASIC]
+max-module-lines=2000
+# due to SQL
+max-line-length=200
+# Drop/decrease them one day:
+max-branches=50
+max-nested-blocks=10
+max-statements=200
+
+[FORMAT]
+ignore-long-lines = (# )?<?https?://\S+>?$
+
+[MESSAGES CONTROL]
+disable = bad-continuation,
+          missing-docstring,
+          bad-whitespace,
+          too-few-public-methods,
+          invalid-name,
+          too-many-arguments,
+          keyword-arg-before-vararg,
+          too-many-locals,
+          too-many-instance-attributes,
+          cell-var-from-loop,
+          fixme,
+          too-many-public-methods,
+          wildcard-import,
+          unused-wildcard-import,
+          singleton-comparison,
+          # pytest.mark.parametrize is not callable (not-callable)
+          not-callable,
+          # https://github.com/PyCQA/pylint/issues/3882
+          # [Python 3.9] Value 'Optional' is unsubscriptable (unsubscriptable-object) (also Union)
+          unsubscriptable-object,
+          # Drop them one day:
+          redefined-outer-name,
+          broad-except,
+          bare-except,
+          no-else-return,
+          global-statement
+
+[SIMILARITIES]
+# due to SQL
+min-similarity-lines=1000

.yamllint

@@ -0,0 +1,15 @@
+# vi: ft=yaml
+extends: default
+
+rules:
+    indentation:
+        level: warning
+        indent-sequences: consistent
+    line-length:
+        # there are some bash -c "", so this is OK
+        max: 300
+        level: warning
+    comments:
+        min-spaces-from-content: 1
+    document-start:
+        present: false

CMakeLists.txt

@@ -36,13 +36,16 @@ option(FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION
 if(FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION)
     set(RECONFIGURE_MESSAGE_LEVEL FATAL_ERROR)
 else()
-    set(RECONFIGURE_MESSAGE_LEVEL STATUS)
+    set(RECONFIGURE_MESSAGE_LEVEL WARNING)
 endif()
 
+enable_language(C CXX ASM)
+
 include (cmake/arch.cmake)
 include (cmake/target.cmake)
 include (cmake/tools.cmake)
 include (cmake/analysis.cmake)
+include (cmake/git_status.cmake)
 
 # Ignore export() since we don't use it,
 # but it gets broken with a global targets via link_libraries()
@@ -66,9 +69,30 @@ endif ()
 
 include (cmake/find/ccache.cmake)
 
-option(ENABLE_CHECK_HEAVY_BUILDS "Don't allow C++ translation units to compile too long or to take too much memory while compiling" OFF)
+# Take care to add prlimit in command line before ccache, or else ccache thinks that
+# prlimit is compiler, and clang++ is its input file, and refuses to work  with
+# multiple inputs, e.g in ccache log:
+# [2021-03-31T18:06:32.655327 36900] Command line: /usr/bin/ccache prlimit --as=10000000000 --data=5000000000 --cpu=600 /usr/bin/clang++-11 - ...... std=gnu++2a -MD -MT src/CMakeFiles/dbms.dir/Storages/MergeTree/IMergeTreeDataPart.cpp.o -MF src/CMakeFiles/dbms.dir/Storages/MergeTree/IMergeTreeDataPart.cpp.o.d -o src/CMakeFiles/dbms.dir/Storages/MergeTree/IMergeTreeDataPart.cpp.o -c ../src/Storages/MergeTree/IMergeTreeDataPart.cpp
+#
+# [2021-03-31T18:06:32.656704 36900] Multiple input files: /usr/bin/clang++-11 and ../src/Storages/MergeTree/IMergeTreeDataPart.cpp
+#
+# Another way would be to use --ccache-skip option before clang++-11 to make
+# ccache ignore it.
+option(ENABLE_CHECK_HEAVY_BUILDS "Don't allow C++ translation units to compile too long or to take too much memory while compiling." OFF)
 if (ENABLE_CHECK_HEAVY_BUILDS)
-    set (CMAKE_CXX_COMPILER_LAUNCHER prlimit --rss=10000000 --cpu=600)
+    # set DATA (since RSS does not work since 2.6.x+) to 2G
+    set (RLIMIT_DATA 5000000000)
+    # set VIRT (RLIMIT_AS) to 10G (DATA*10)
+    set (RLIMIT_AS 10000000000)
+    # set CPU time limit to 600 seconds
+    set (RLIMIT_CPU 600)
+
+    # gcc10/gcc10/clang -fsanitize=memory is too heavy
+    if (SANITIZE STREQUAL "memory" OR COMPILER_GCC)
+       set (RLIMIT_DATA 10000000000)
+    endif()
+
+    set (CMAKE_CXX_COMPILER_LAUNCHER prlimit --as=${RLIMIT_AS} --data=${RLIMIT_DATA} --cpu=${RLIMIT_CPU} ${CMAKE_CXX_COMPILER_LAUNCHER})
 endif ()
 
 if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "None")
@@ -112,6 +136,12 @@ if (ENABLE_FUZZING)
     set (FUZZER "libfuzzer")
 endif()
 
+# Global libraries
+# See:
+# - default_libs.cmake
+# - sanitize.cmake
+add_library(global-libs INTERFACE)
+
 include (cmake/fuzzer.cmake)
 include (cmake/sanitize.cmake)
 
@@ -138,10 +168,10 @@ endif ()
 
 # If turned `ON`, assumes the user has either the system GTest library or the bundled one.
 option(ENABLE_TESTS "Provide unit_test_dbms target with Google.Test unit tests" ON)
+option(ENABLE_EXAMPLES "Build all example programs in 'examples' subdirectories" OFF)
 
-if (OS_LINUX AND NOT UNBUNDLED AND MAKE_STATIC_LIBRARIES AND NOT SPLIT_SHARED_LIBRARIES AND CMAKE_VERSION VERSION_GREATER "3.9.0")
+if (OS_LINUX AND (ARCH_AMD64 OR ARCH_AARCH64) AND NOT UNBUNDLED AND MAKE_STATIC_LIBRARIES AND NOT SPLIT_SHARED_LIBRARIES AND CMAKE_VERSION VERSION_GREATER "3.9.0")
-    # Only for Linux, x86_64.
+    # Only for Linux, x86_64 or aarch64.
-    # Implies ${ENABLE_FASTMEMCPY}
     option(GLIBC_COMPATIBILITY "Enable compatibility with older glibc libraries." ON)
 elseif(GLIBC_COMPATIBILITY)
     message (${RECONFIGURE_MESSAGE_LEVEL} "Glibc compatibility cannot be enabled in current configuration")
@@ -154,22 +184,37 @@ endif ()
 # Make sure the final executable has symbols exported
 set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -rdynamic")
 
-find_program (OBJCOPY_PATH NAMES "llvm-objcopy" "llvm-objcopy-11" "llvm-objcopy-10" "llvm-objcopy-9" "llvm-objcopy-8" "objcopy")
+find_program (OBJCOPY_PATH NAMES "llvm-objcopy" "llvm-objcopy-12" "llvm-objcopy-11" "llvm-objcopy-10" "llvm-objcopy-9" "llvm-objcopy-8" "objcopy")
-if (OBJCOPY_PATH)
-    message(STATUS "Using objcopy: ${OBJCOPY_PATH}.")
 
-    if (ARCH_AMD64)
+if (NOT OBJCOPY_PATH AND OS_DARWIN)
-        set(OBJCOPY_ARCH_OPTIONS -O elf64-x86-64 -B i386)
+    find_program (BREW_PATH NAMES "brew")
-    elseif (ARCH_AARCH64)
+    if (BREW_PATH)
-        set(OBJCOPY_ARCH_OPTIONS -O elf64-aarch64 -B aarch64)
+        execute_process (COMMAND ${BREW_PATH} --prefix llvm ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE LLVM_PREFIX)
+        if (LLVM_PREFIX)
+            find_program (OBJCOPY_PATH NAMES "llvm-objcopy" PATHS "${LLVM_PREFIX}/bin" NO_DEFAULT_PATH)
+        endif ()
+        if (NOT OBJCOPY_PATH)
+            execute_process (COMMAND ${BREW_PATH} --prefix binutils ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE BINUTILS_PREFIX)
+            if (BINUTILS_PREFIX)
+                find_program (OBJCOPY_PATH NAMES "objcopy" PATHS "${BINUTILS_PREFIX}/bin" NO_DEFAULT_PATH)
+            endif ()
+        endif ()
     endif ()
+endif ()
+
+if (OBJCOPY_PATH)
+    message (STATUS "Using objcopy: ${OBJCOPY_PATH}")
 else ()
-    message(FATAL_ERROR "Cannot find objcopy.")
+    message (FATAL_ERROR "Cannot find objcopy.")
 endif ()
 
 if (OS_DARWIN)
-    set(WHOLE_ARCHIVE -all_load)
+    # The `-all_load` flag forces loading of all symbols from all libraries,
-    set(NO_WHOLE_ARCHIVE -noall_load)
+    # and leads to multiply-defined symbols. This flag allows force loading
+    # from a _specific_ library, which is what we need.
+    set(WHOLE_ARCHIVE -force_load)
+    # The `-noall_load` flag is the default and now obsolete.
+    set(NO_WHOLE_ARCHIVE "")
 else ()
     set(WHOLE_ARCHIVE --whole-archive)
     set(NO_WHOLE_ARCHIVE --no-whole-archive)
@@ -179,13 +224,14 @@ endif ()
 option(ADD_GDB_INDEX_FOR_GOLD "Add .gdb-index to resulting binaries for gold linker.")
 
 if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE")
-    if (LINKER_NAME STREQUAL "lld")
+    # Can be lld or ld-lld.
+    if (LINKER_NAME MATCHES "lld$")
         set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gdb-index")
         set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--gdb-index")
         message (STATUS "Adding .gdb-index via --gdb-index linker option.")
     # we use another tool for gdb-index, because gold linker removes section .debug_aranges, which used inside clickhouse stacktraces
     # http://sourceware-org.1504.n7.nabble.com/gold-No-debug-aranges-section-when-linking-with-gdb-index-td540965.html#a556932
-    elseif (LINKER_NAME STREQUAL "gold" AND ADD_GDB_INDEX_FOR_GOLD)
+    elseif (LINKER_NAME MATCHES "gold$" AND ADD_GDB_INDEX_FOR_GOLD)
         find_program (GDB_ADD_INDEX_EXE NAMES "gdb-add-index" DOC "Path to gdb-add-index executable")
         if (NOT GDB_ADD_INDEX_EXE)
             set (USE_GDB_ADD_INDEX 0)
@@ -197,6 +243,19 @@ if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE")
     endif ()
 endif()
 
+# Create BuildID when using lld. For other linkers it is created by default.
+if (LINKER_NAME MATCHES "lld$")
+    # SHA1 is not cryptographically secure but it is the best what lld is offering.
+    set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--build-id=sha1")
+endif ()
+
+# Add a section with the hash of the compiled machine code for integrity checks.
+# Only for official builds, because adding a section can be time consuming (rewrite of several GB).
+# And cross compiled binaries are not supported (since you cannot execute clickhouse hash-binary)
+if (OBJCOPY_PATH AND YANDEX_OFFICIAL_BUILD AND (NOT CMAKE_TOOLCHAIN_FILE))
+    set (USE_BINARY_HASH 1)
+endif ()
+
 cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_PHYSICAL_MEMORY) # Not available under freebsd
 
 
@@ -211,35 +270,46 @@ else()
     message(STATUS "Disabling compiler -pipe option (have only ${AVAILABLE_PHYSICAL_MEMORY} mb of memory)")
 endif()
 
-if(NOT DISABLE_CPU_OPTIMIZE)
+include(cmake/cpu_features.cmake)
-    include(cmake/cpu_features.cmake)
-endif()
 
-option(ARCH_NATIVE "Add -march=native compiler flag")
+# Asynchronous unwind tables are needed for Query Profiler.
+# They are already by default on some platforms but possibly not on all platforms.
+# Enable it explicitly.
+set (COMPILER_FLAGS "${COMPILER_FLAGS} -fasynchronous-unwind-tables")
 
-if (ARCH_NATIVE)
+if (${CMAKE_VERSION} VERSION_LESS "3.12.4")
-    set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=native")
+    # CMake < 3.12 doesn't support setting 20 as a C++ standard version.
+    # We will add C++ standard controlling flag in CMAKE_CXX_FLAGS manually for now.
+
+    if (COMPILER_GCC OR COMPILER_CLANG)
+        # to make numeric_limits<__int128> works with GCC
+        set (_CXX_STANDARD "gnu++2a")
+    else ()
+        set (_CXX_STANDARD "c++2a")
+    endif ()
+
+    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=${_CXX_STANDARD}")
+else ()
+    set (CMAKE_CXX_STANDARD 20)
+    set (CMAKE_CXX_EXTENSIONS ON) # Same as gnu++2a (ON) vs c++2a (OFF): https://cmake.org/cmake/help/latest/prop_tgt/CXX_EXTENSIONS.html
+    set (CMAKE_CXX_STANDARD_REQUIRED ON)
 endif ()
 
-if (UNBUNDLED AND (COMPILER_GCC OR COMPILER_CLANG))
+set (CMAKE_C_STANDARD 11)
-    # to make numeric_limits<__int128> works for unbundled build
+set (CMAKE_C_EXTENSIONS ON)
-    set (_CXX_STANDARD "-std=gnu++2a")
+set (CMAKE_C_STANDARD_REQUIRED ON)
-else()
-    set (_CXX_STANDARD "-std=c++2a")
-endif()
-
-# cmake < 3.12 doesn't support 20. We'll set CMAKE_CXX_FLAGS for now
-# set (CMAKE_CXX_STANDARD 20)
-set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${_CXX_STANDARD}")
-
-set (CMAKE_CXX_EXTENSIONS 0) # https://cmake.org/cmake/help/latest/prop_tgt/CXX_EXTENSIONS.html#prop_tgt:CXX_EXTENSIONS
-set (CMAKE_CXX_STANDARD_REQUIRED ON)
 
 if (COMPILER_GCC OR COMPILER_CLANG)
     # Enable C++14 sized global deallocation functions. It should be enabled by setting -std=c++14 but I'm not sure.
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsized-deallocation")
 endif ()
 
+# falign-functions=32 prevents from random performance regressions with the code change. Thus, providing more stable
+# benchmarks.
+if (COMPILER_GCC OR COMPILER_CLANG)
+    set(COMPILER_FLAGS "${COMPILER_FLAGS} -falign-functions=32")
+endif ()
+
 # Compiler-specific coverage flags e.g. -fcoverage-mapping for gcc
 option(WITH_COVERAGE "Profile the resulting binary/binaries" OFF)
 
@@ -255,6 +325,8 @@ if (WITH_COVERAGE AND COMPILER_GCC)
     set(WITHOUT_COVERAGE "-fno-profile-arcs -fno-test-coverage")
 endif()
 
+set(COMPILER_FLAGS "${COMPILER_FLAGS}")
+
 set (CMAKE_BUILD_COLOR_MAKEFILE          ON)
 set (CMAKE_CXX_FLAGS                     "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS} ${PLATFORM_EXTRA_CXX_FLAG} ${COMMON_WARNING_FLAGS} ${CXX_WARNING_FLAGS}")
 set (CMAKE_CXX_FLAGS_RELWITHDEBINFO      "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_CXX_FLAGS_ADD}")
@@ -299,7 +371,7 @@ if (COMPILER_CLANG)
     endif ()
 
     # Always prefer llvm tools when using clang. For instance, we cannot use GNU ar when llvm LTO is enabled
-    find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-11" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8")
+    find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-12" "llvm-ar-11" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8")
 
     if (LLVM_AR_PATH)
         message(STATUS "Using llvm-ar: ${LLVM_AR_PATH}.")
@@ -308,7 +380,7 @@ if (COMPILER_CLANG)
         message(WARNING "Cannot find llvm-ar. System ar will be used instead. It does not work with ThinLTO.")
     endif ()
 
-    find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-11" "llvm-ranlib-10" "llvm-ranlib-9" "llvm-ranlib-8")
+    find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-12" "llvm-ranlib-11" "llvm-ranlib-10" "llvm-ranlib-9" "llvm-ranlib-8")
 
     if (LLVM_RANLIB_PATH)
         message(STATUS "Using llvm-ranlib: ${LLVM_RANLIB_PATH}.")
@@ -324,9 +396,10 @@ endif ()
 # Turns on all external libs like s3, kafka, ODBC, ...
 option(ENABLE_LIBRARIES "Enable all external libraries by default" ON)
 
-# We recommend avoiding this mode for production builds because we can't guarantee all needed libraries exist in your
+# We recommend avoiding this mode for production builds because we can't guarantee
-# system.
+# all needed libraries exist in your system.
 # This mode exists for enthusiastic developers who are searching for trouble.
+# The whole idea of using unknown version of libraries from the OS distribution is deeply flawed.
 # Useful for maintainers of OS packages.
 option (UNBUNDLED "Use system libraries instead of ones in contrib/" OFF)
 
@@ -425,6 +498,7 @@ find_contrib_lib(double-conversion) # Must be before parquet
 include (cmake/find/ssl.cmake)
 include (cmake/find/ldap.cmake) # after ssl
 include (cmake/find/icu.cmake)
+include (cmake/find/xz.cmake)
 include (cmake/find/zlib.cmake)
 include (cmake/find/zstd.cmake)
 include (cmake/find/ltdl.cmake) # for odbc
@@ -435,10 +509,10 @@ include (cmake/find/krb5.cmake)
 include (cmake/find/libgsasl.cmake)
 include (cmake/find/cyrus-sasl.cmake)
 include (cmake/find/rdkafka.cmake)
+include (cmake/find/libuv.cmake) # for amqpcpp and cassandra
 include (cmake/find/amqpcpp.cmake)
 include (cmake/find/capnp.cmake)
 include (cmake/find/llvm.cmake)
-include (cmake/find/termcap.cmake) # for external static llvm
 include (cmake/find/h3.cmake)
 include (cmake/find/libxml2.cmake)
 include (cmake/find/brotli.cmake)
@@ -453,9 +527,19 @@ include (cmake/find/s3.cmake)
 include (cmake/find/base64.cmake)
 include (cmake/find/parquet.cmake)
 include (cmake/find/simdjson.cmake)
+include (cmake/find/fast_float.cmake)
 include (cmake/find/rapidjson.cmake)
 include (cmake/find/fastops.cmake)
 include (cmake/find/odbc.cmake)
+include (cmake/find/nanodbc.cmake)
+include (cmake/find/sqlite.cmake)
+include (cmake/find/rocksdb.cmake)
+include (cmake/find/libpqxx.cmake)
+include (cmake/find/nuraft.cmake)
+include (cmake/find/yaml-cpp.cmake)
+include (cmake/find/s2geometry.cmake)
+include (cmake/find/nlp.cmake)
+include (cmake/find/bzip2.cmake)
 
 if(NOT USE_INTERNAL_PARQUET_LIBRARY)
     set (ENABLE_ORC OFF CACHE INTERNAL "")
@@ -467,15 +551,13 @@ include (cmake/find/msgpack.cmake)
 include (cmake/find/cassandra.cmake)
 include (cmake/find/sentry.cmake)
 include (cmake/find/stats.cmake)
+include (cmake/find/datasketches.cmake)
 
 set (USE_INTERNAL_CITYHASH_LIBRARY ON CACHE INTERNAL "")
 find_contrib_lib(cityhash)
 
 find_contrib_lib(farmhash)
 
-set (USE_INTERNAL_BTRIE_LIBRARY ON CACHE INTERNAL "")
-find_contrib_lib(btrie)
-
 if (ENABLE_TESTS)
     include (cmake/find/gtest.cmake)
 endif ()
@@ -502,15 +584,34 @@ macro (add_executable target)
     # explicitly acquire and interpose malloc symbols by clickhouse_malloc
     # if GLIBC_COMPATIBILITY is ON and ENABLE_THINLTO is on than provide memcpy symbol explicitly to neutrialize thinlto's libcall generation.
     if (GLIBC_COMPATIBILITY AND ENABLE_THINLTO)
-        _add_executable (${ARGV} $<TARGET_OBJECTS:clickhouse_malloc> $<TARGET_OBJECTS:clickhouse_memcpy>)
+        _add_executable (${ARGV} $<TARGET_OBJECTS:clickhouse_malloc> $<TARGET_OBJECTS:memcpy>)
     else ()
         _add_executable (${ARGV} $<TARGET_OBJECTS:clickhouse_malloc>)
     endif ()
 
     get_target_property (type ${target} TYPE)
     if (${type} STREQUAL EXECUTABLE)
-        # operator::new/delete for executables (MemoryTracker stuff)
+        # disabled for TSAN and gcc since libtsan.a provides overrides too
-        target_link_libraries (${target} PRIVATE clickhouse_new_delete ${MALLOC_LIBRARIES})
+        if (TARGET clickhouse_new_delete)
+            # operator::new/delete for executables (MemoryTracker stuff)
+            target_link_libraries (${target} PRIVATE clickhouse_new_delete)
+        endif()
+
+        # In case of static jemalloc, because zone_register() is located in zone.c and
+        # is never used outside (it is declared as constructor) it is omitted
+        # by the linker, and so jemalloc will not be registered as system
+        # allocator under osx [1], and clickhouse will SIGSEGV.
+        #
+        #   [1]: https://github.com/jemalloc/jemalloc/issues/708
+        #
+        # About symbol name:
+        # - _zone_register not zone_register due to Mach-O binary format,
+        # - _je_zone_register due to JEMALLOC_PRIVATE_NAMESPACE=je_ under OS X.
+        # - but jemalloc-cmake does not run private_namespace.sh
+        #   so symbol name should be _zone_register
+        if (ENABLE_JEMALLOC AND MAKE_STATIC_LIBRARIES AND OS_DARWIN)
+            set_property(TARGET ${target} APPEND PROPERTY LINK_OPTIONS -u_zone_register)
+        endif()
     endif()
 endmacro()
 
@@ -520,9 +621,12 @@ include_directories(${ConfigIncludePath})
 # Add as many warnings as possible for our own code.
 include (cmake/warnings.cmake)
 
+# Check if needed compiler flags are supported
+include (cmake/check_flags.cmake)
+
 add_subdirectory (base)
-add_subdirectory (programs)
 add_subdirectory (src)
+add_subdirectory (programs)
 add_subdirectory (tests)
 add_subdirectory (utils)
 

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2016-2020 Yandex LLC
+Copyright 2016-2021 Yandex LLC
 
                                  Apache License
                            Version 2.0, January 2004
@@ -188,7 +188,7 @@ Copyright 2016-2020 Yandex LLC
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2016-2020 Yandex LLC
+   Copyright 2016-2021 Yandex LLC
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

PreLoad.cmake

@@ -12,7 +12,7 @@
 # https://youtrack.jetbrains.com/issue/CPP-2659
 # https://youtrack.jetbrains.com/issue/CPP-870
 
-if (NOT DEFINED ENV{CLION_IDE})
+if (NOT DEFINED ENV{CLION_IDE} AND NOT DEFINED ENV{XCODE_IDE})
     find_program(NINJA_PATH ninja)
     if (NINJA_PATH)
         set(CMAKE_GENERATOR "Ninja" CACHE INTERNAL "" FORCE)

README.md

@@ -1,6 +1,6 @@
 [![ClickHouse — open source distributed column-oriented DBMS](https://github.com/ClickHouse/ClickHouse/raw/master/website/images/logo-400x240.png)](https://clickhouse.tech)
 
-ClickHouse is an open-source column-oriented database management system that allows generating analytical data reports in real time.
+ClickHouse® is an open-source column-oriented database management system that allows generating analytical data reports in real time.
 
 ## Useful Links
 
@@ -8,15 +8,11 @@ ClickHouse is an open-source column-oriented database management system that all
 * [Tutorial](https://clickhouse.tech/docs/en/getting_started/tutorial/) shows how to set up and query small ClickHouse cluster.
 * [Documentation](https://clickhouse.tech/docs/en/) provides more in-depth information.
 * [YouTube channel](https://www.youtube.com/c/ClickHouseDB) has a lot of content about ClickHouse in video format.
-* [Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-d2zxkf9e-XyxDa_ucfPxzuH4SJIm~Ng) and [Telegram](https://telegram.me/clickhouse_en) allow to chat with ClickHouse users in real-time.
+* [Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-rxm3rdrk-lIUmhLC3V8WTaL0TGxsOmg) and [Telegram](https://telegram.me/clickhouse_en) allow to chat with ClickHouse users in real-time.
 * [Blog](https://clickhouse.yandex/blog/en/) contains various ClickHouse-related articles, as well as announcements and reports about events.
 * [Code Browser](https://clickhouse.tech/codebrowser/html_report/ClickHouse/index.html) with syntax highlight and navigation.
-* [Yandex.Messenger channel](https://yandex.ru/chat/#/join/20e380d9-c7be-4123-ab06-e95fb946975e) shares announcements and useful links in Russian.
 * [Contacts](https://clickhouse.tech/#contacts) can help to get your questions answered if there are any.
 * You can also [fill this form](https://clickhouse.tech/#meet) to meet Yandex ClickHouse team in person.
 
 ## Upcoming Events
-
+* [SF Bay Area ClickHouse August Community Meetup (online)](https://www.meetup.com/San-Francisco-Bay-Area-ClickHouse-Meetup/events/279109379/) on 25 August 2021.
-* [The Second ClickHouse Meetup East (online)](https://www.eventbrite.com/e/the-second-clickhouse-meetup-east-tickets-126787955187) on October 31, 2020.
-* [ClickHouse for Enterprise Meetup (online in Russian)](https://arenadata-events.timepad.ru/event/1465249/) on November 10, 2020.
-

SECURITY.md

@@ -1,9 +1,11 @@
 # Security Policy
 
-## Supported Versions
+## Security Announcements
+Security fixes will be announced by posting them in the [security changelog](https://clickhouse.tech/docs/en/whats-new/security-changelog/)
 
-The following versions of ClickHouse server are
+## Scope and Supported Versions
-currently being supported with security updates:
+
+The following versions of ClickHouse server are currently being supported with security updates:
 
 | Version | Supported          |
 | ------- | ------------------ |
@@ -11,14 +13,49 @@ currently being supported with security updates:
 | 18.x   | :x:                |
 | 19.x   | :x:                |
 | 20.1   | :x: |
-| 20.3   | :white_check_mark: |
+| 20.3   | :x: |
 | 20.4   | :x: |
 | 20.5   | :x: |
 | 20.6   | :x: |
-| 20.7   | :white_check_mark: |
+| 20.7   | :x: |
-| 20.8   | :white_check_mark: |
+| 20.8   | :x: |
-| 20.9   | :white_check_mark: |
+| 20.9   | :x: |
+| 20.10  | :x: |
+| 20.11  | :x: |
+| 20.12  | :x: |
+| 21.1   | :x: |
+| 21.2   | :x: |
+| 21.3   | ✅ |
+| 21.4   | :x: |
+| 21.5   | :x: |
+| 21.6   | ✅ |
+| 21.7   | ✅ |
+| 21.8   | ✅ |
 
 ## Reporting a Vulnerability
 
+We're extremely grateful for security researchers and users that report vulnerabilities to the ClickHouse Open Source Community. All reports are thoroughly investigated by developers.
+
 To report a potential vulnerability in ClickHouse please send the details about it to [clickhouse-feedback@yandex-team.com](mailto:clickhouse-feedback@yandex-team.com).
+
+### When Should I Report a Vulnerability?
+
+- You think you discovered a potential security vulnerability in ClickHouse
+- You are unsure how a vulnerability affects ClickHouse
+
+### When Should I NOT Report a Vulnerability?
+
+- You need help tuning ClickHouse components for security
+- You need help applying security related updates
+- Your issue is not security related
+
+## Security Vulnerability Response
+
+Each report is acknowledged and analyzed by ClickHouse maintainers within 5 working days.
+As the security issue moves from triage, to identified fix, to release planning we will keep the reporter updated.
+
+## Public Disclosure Timing
+
+A public disclosure date is negotiated by the ClickHouse maintainers and the bug submitter. We prefer to fully disclose the bug as soon as possible once a user mitigation is available. It is reasonable to delay disclosure when the bug or the fix is not yet fully understood, the solution is not well-tested, or for vendor coordination. The timeframe for disclosure is from immediate (especially if it's already publicly known) to 90 days. For a vulnerability with a straightforward mitigation, we expect report date to disclosure date to be on the order of 7 days. 
+
+

base/CMakeLists.txt

@@ -8,6 +8,7 @@ add_subdirectory (loggers)
 add_subdirectory (pcg-random)
 add_subdirectory (widechar_width)
 add_subdirectory (readpassphrase)
+add_subdirectory (bridge)
 
 if (USE_MYSQL)
     add_subdirectory (mysqlxx)

base/bridge/CMakeLists.txt

@@ -0,0 +1,13 @@
+add_library (bridge
+    IBridge.cpp
+)
+
+target_include_directories (daemon PUBLIC ..)
+target_link_libraries (bridge 
+    PRIVATE 
+	daemon
+	dbms
+	Poco::Data
+	Poco::Data::ODBC
+)
+

base/bridge/IBridge.cpp

@@ -0,0 +1,266 @@
+#include "IBridge.h"
+
+#include <boost/program_options.hpp>
+#include <Poco/Net/NetException.h>
+#include <Poco/Util/HelpFormatter.h>
+
+#include <common/logger_useful.h>
+#include <common/range.h>
+
+#include <Common/StringUtils/StringUtils.h>
+#include <Common/SensitiveDataMasker.h>
+#include <common/errnoToString.h>
+#include <IO/ReadHelpers.h>
+#include <Formats/registerFormats.h>
+#include <Server/HTTP/HTTPServer.h>
+#include <IO/WriteBufferFromFile.h>
+#include <IO/WriteHelpers.h>
+#include <sys/time.h>
+#include <sys/resource.h>
+
+#if USE_ODBC
+#    include <Poco/Data/ODBC/Connector.h>
+#endif
+
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int ARGUMENT_OUT_OF_BOUND;
+}
+
+namespace
+{
+    Poco::Net::SocketAddress makeSocketAddress(const std::string & host, UInt16 port, Poco::Logger * log)
+    {
+        Poco::Net::SocketAddress socket_address;
+        try
+        {
+            socket_address = Poco::Net::SocketAddress(host, port);
+        }
+        catch (const Poco::Net::DNSException & e)
+        {
+            const auto code = e.code();
+            if (code == EAI_FAMILY
+#if defined(EAI_ADDRFAMILY)
+                || code == EAI_ADDRFAMILY
+#endif
+            )
+            {
+                LOG_ERROR(log, "Cannot resolve listen_host ({}), error {}: {}. If it is an IPv6 address and your host has disabled IPv6, then consider to specify IPv4 address to listen in <listen_host> element of configuration file. Example: <listen_host>0.0.0.0</listen_host>", host, e.code(), e.message());
+            }
+
+            throw;
+        }
+        return socket_address;
+    }
+
+    Poco::Net::SocketAddress socketBindListen(Poco::Net::ServerSocket & socket, const std::string & host, UInt16 port, Poco::Logger * log)
+    {
+        auto address = makeSocketAddress(host, port, log);
+#if POCO_VERSION < 0x01080000
+        socket.bind(address, /* reuseAddress = */ true);
+#else
+        socket.bind(address, /* reuseAddress = */ true, /* reusePort = */ false);
+#endif
+
+        socket.listen(/* backlog = */ 64);
+
+        return address;
+    }
+}
+
+
+void IBridge::handleHelp(const std::string &, const std::string &)
+{
+    Poco::Util::HelpFormatter help_formatter(options());
+    help_formatter.setCommand(commandName());
+    help_formatter.setHeader("HTTP-proxy for odbc requests");
+    help_formatter.setUsage("--http-port <port>");
+    help_formatter.format(std::cerr);
+
+    stopOptionsProcessing();
+}
+
+
+void IBridge::defineOptions(Poco::Util::OptionSet & options)
+{
+    options.addOption(
+        Poco::Util::Option("http-port", "", "port to listen").argument("http-port", true) .binding("http-port"));
+
+    options.addOption(
+        Poco::Util::Option("listen-host", "", "hostname or address to listen, default 127.0.0.1").argument("listen-host").binding("listen-host"));
+
+    options.addOption(
+        Poco::Util::Option("http-timeout", "", "http timeout for socket, default 1800").argument("http-timeout").binding("http-timeout"));
+
+    options.addOption(
+        Poco::Util::Option("max-server-connections", "", "max connections to server, default 1024").argument("max-server-connections").binding("max-server-connections"));
+
+    options.addOption(
+        Poco::Util::Option("keep-alive-timeout", "", "keepalive timeout, default 10").argument("keep-alive-timeout").binding("keep-alive-timeout"));
+
+    options.addOption(
+        Poco::Util::Option("log-level", "", "sets log level, default info") .argument("log-level").binding("logger.level"));
+
+    options.addOption(
+        Poco::Util::Option("log-path", "", "log path for all logs, default console").argument("log-path").binding("logger.log"));
+
+    options.addOption(
+        Poco::Util::Option("err-log-path", "", "err log path for all logs, default no").argument("err-log-path").binding("logger.errorlog"));
+
+    options.addOption(
+        Poco::Util::Option("stdout-path", "", "stdout log path, default console").argument("stdout-path").binding("logger.stdout"));
+
+    options.addOption(
+        Poco::Util::Option("stderr-path", "", "stderr log path, default console").argument("stderr-path").binding("logger.stderr"));
+
+    using Me = std::decay_t<decltype(*this)>;
+
+    options.addOption(
+        Poco::Util::Option("help", "", "produce this help message").binding("help").callback(Poco::Util::OptionCallback<Me>(this, &Me::handleHelp)));
+
+    ServerApplication::defineOptions(options); // NOLINT Don't need complex BaseDaemon's .xml config
+}
+
+
+void IBridge::initialize(Application & self)
+{
+    BaseDaemon::closeFDs();
+    is_help = config().has("help");
+
+    if (is_help)
+        return;
+
+    config().setString("logger", bridgeName());
+
+    /// Redirect stdout, stderr to specified files.
+    /// Some libraries and sanitizers write to stderr in case of errors.
+    const auto stdout_path = config().getString("logger.stdout", "");
+    if (!stdout_path.empty())
+    {
+        if (!freopen(stdout_path.c_str(), "a+", stdout))
+            throw Poco::OpenFileException("Cannot attach stdout to " + stdout_path);
+
+        /// Disable buffering for stdout.
+        setbuf(stdout, nullptr);
+    }
+    const auto stderr_path = config().getString("logger.stderr", "");
+    if (!stderr_path.empty())
+    {
+        if (!freopen(stderr_path.c_str(), "a+", stderr))
+            throw Poco::OpenFileException("Cannot attach stderr to " + stderr_path);
+
+        /// Disable buffering for stderr.
+        setbuf(stderr, nullptr);
+    }
+
+    buildLoggers(config(), logger(), self.commandName());
+
+    BaseDaemon::logRevision();
+
+    log = &logger();
+    hostname = config().getString("listen-host", "127.0.0.1");
+    port = config().getUInt("http-port");
+    if (port > 0xFFFF)
+        throw Exception("Out of range 'http-port': " + std::to_string(port), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
+
+    http_timeout = config().getUInt64("http-timeout", DEFAULT_HTTP_READ_BUFFER_TIMEOUT);
+    max_server_connections = config().getUInt("max-server-connections", 1024);
+    keep_alive_timeout = config().getUInt64("keep-alive-timeout", 10);
+
+    struct rlimit limit;
+    const UInt64 gb = 1024 * 1024 * 1024;
+
+    /// Set maximum RSS to 1 GiB.
+    limit.rlim_max = limit.rlim_cur = gb;
+    if (setrlimit(RLIMIT_RSS, &limit))
+        LOG_WARNING(log, "Unable to set maximum RSS to 1GB: {} (current rlim_cur={}, rlim_max={})",
+                    errnoToString(errno), limit.rlim_cur, limit.rlim_max);
+
+    if (!getrlimit(RLIMIT_RSS, &limit))
+        LOG_INFO(log, "RSS limit: cur={}, max={}", limit.rlim_cur, limit.rlim_max);
+
+    try
+    {
+        const auto oom_score = toString(config().getUInt64("bridge_oom_score", 500));
+        WriteBufferFromFile buf("/proc/self/oom_score_adj");
+        buf.write(oom_score.data(), oom_score.size());
+        buf.close();
+        LOG_INFO(log, "OOM score is set to {}", oom_score);
+    }
+    catch (const Exception & e)
+    {
+        LOG_WARNING(log, "Failed to set OOM score, error: {}", e.what());
+    }
+
+    initializeTerminationAndSignalProcessing();
+
+    ServerApplication::initialize(self); // NOLINT
+}
+
+
+void IBridge::uninitialize()
+{
+    BaseDaemon::uninitialize();
+}
+
+
+int IBridge::main(const std::vector<std::string> & /*args*/)
+{
+    if (is_help)
+        return Application::EXIT_OK;
+
+    registerFormats();
+    LOG_INFO(log, "Starting up {} on host: {}, port: {}", bridgeName(), hostname, port);
+
+    Poco::Net::ServerSocket socket;
+    auto address = socketBindListen(socket, hostname, port, log);
+    socket.setReceiveTimeout(http_timeout);
+    socket.setSendTimeout(http_timeout);
+
+    Poco::ThreadPool server_pool(3, max_server_connections);
+
+    Poco::Net::HTTPServerParams::Ptr http_params = new Poco::Net::HTTPServerParams;
+    http_params->setTimeout(http_timeout);
+    http_params->setKeepAliveTimeout(keep_alive_timeout);
+
+    auto shared_context = Context::createShared();
+    auto context = Context::createGlobal(shared_context.get());
+    context->makeGlobalContext();
+
+    if (config().has("query_masking_rules"))
+        SensitiveDataMasker::setInstance(std::make_unique<SensitiveDataMasker>(config(), "query_masking_rules"));
+
+    auto server = HTTPServer(
+        context,
+        getHandlerFactoryPtr(context),
+        server_pool,
+        socket,
+        http_params);
+
+    SCOPE_EXIT({
+        LOG_DEBUG(log, "Received termination signal.");
+        LOG_DEBUG(log, "Waiting for current connections to close.");
+
+        server.stop();
+
+        for (size_t count : collections::range(1, 6))
+        {
+            if (server.currentConnections() == 0)
+                break;
+            LOG_DEBUG(log, "Waiting for {} connections, try {}", server.currentConnections(), count);
+            std::this_thread::sleep_for(std::chrono::milliseconds(1000));
+        }
+    });
+
+    server.start();
+    LOG_INFO(log, "Listening http://{}", address.toString());
+
+    waitForTerminationRequest();
+    return Application::EXIT_OK;
+}
+
+}

base/bridge/IBridge.h

@@ -0,0 +1,51 @@
+#pragma once
+
+#include <Interpreters/Context.h>
+#include <Server/HTTP/HTTPRequestHandlerFactory.h>
+#include <daemon/BaseDaemon.h>
+
+#include <Poco/Logger.h>
+#include <Poco/Util/ServerApplication.h>
+
+
+namespace DB
+{
+
+/// Class represents base for clickhouse-odbc-bridge and clickhouse-library-bridge servers.
+/// Listens to incoming HTTP POST and GET requests on specified port and host.
+/// Has two handlers '/' for all incoming POST requests and /ping for GET request about service status.
+class IBridge : public BaseDaemon
+{
+
+public:
+    /// Define command line arguments
+    void defineOptions(Poco::Util::OptionSet & options) override;
+
+protected:
+    using HandlerFactoryPtr = std::shared_ptr<HTTPRequestHandlerFactory>;
+
+    void initialize(Application & self) override;
+
+    void uninitialize() override;
+
+    int main(const std::vector<std::string> & args) override;
+
+    virtual std::string bridgeName() const = 0;
+
+    virtual HandlerFactoryPtr getHandlerFactoryPtr(ContextPtr context) const = 0;
+
+    size_t keep_alive_timeout;
+
+private:
+    void handleHelp(const std::string &, const std::string &);
+
+    bool is_help;
+    std::string hostname;
+    size_t port;
+    std::string log_level;
+    size_t max_server_connections;
+    size_t http_timeout;
+
+    Poco::Logger * log;
+};
+}

base/common/BorrowedObjectPool.h

@@ -0,0 +1,156 @@
+#pragma once
+
+#include <cstdint>
+#include <vector>
+#include <chrono>
+#include <mutex>
+#include <condition_variable>
+
+#include <common/defines.h>
+#include <common/MoveOrCopyIfThrow.h>
+
+/** Pool for limited size objects that cannot be used from different threads simultaneously.
+  * The main use case is to have fixed size of objects that can be reused in difference threads during their lifetime
+  * and have to be initialized on demand.
+  * Two main properties of pool are allocated objects size and borrowed objects size.
+  * Allocated objects size is size of objects that are currently allocated by the pool.
+  * Borrowed objects size is size of objects that are borrowed by clients.
+  * If max_size == 0 then pool has unlimited size and objects will be allocated without limit.
+  *
+  * Pool provides following strategy for borrowing object:
+  * If max_size == 0 then pool has unlimited size and objects will be allocated without limit.
+  * 1. If pool has objects that can be borrowed increase borrowed objects size and return it.
+  * 2. If pool allocatedObjectsSize is lower than max objects size or pool has unlimited size
+  * allocate new object, increase borrowed objects size and return it.
+  * 3. If pool is full wait on condition variable with or without timeout until some object
+  * will be returned to the pool.
+  */
+template <typename T>
+class BorrowedObjectPool final
+{
+public:
+    explicit BorrowedObjectPool(size_t max_size_) : max_size(max_size_) {}
+
+    /// Borrow object from pool. If pull is full and all objects were borrowed
+    /// then calling thread will wait until some object will be returned into pool.
+    template <typename FactoryFunc>
+    void borrowObject(T & dest, FactoryFunc && func)
+    {
+        std::unique_lock<std::mutex> lock(objects_mutex);
+
+        if (!objects.empty())
+        {
+            dest = borrowFromObjects(lock);
+            return;
+        }
+
+        bool has_unlimited_size = (max_size == 0);
+
+        if (unlikely(has_unlimited_size) || allocated_objects_size < max_size)
+        {
+            dest = allocateObjectForBorrowing(lock, std::forward<FactoryFunc>(func));
+            return;
+        }
+
+        condition_variable.wait(lock, [this] { return !objects.empty(); });
+        dest = borrowFromObjects(lock);
+    }
+
+    /// Same as borrowObject function, but wait with timeout.
+    /// Returns true if object was borrowed during timeout.
+    template <typename FactoryFunc>
+    bool tryBorrowObject(T & dest, FactoryFunc && func, size_t timeout_in_milliseconds = 0)
+    {
+        std::unique_lock<std::mutex> lock(objects_mutex);
+
+        if (!objects.empty())
+        {
+            dest = borrowFromObjects(lock);
+            return true;
+        }
+
+        bool has_unlimited_size = (max_size == 0);
+
+        if (unlikely(has_unlimited_size) || allocated_objects_size < max_size)
+        {
+            dest = allocateObjectForBorrowing(lock, std::forward<FactoryFunc>(func));
+            return true;
+        }
+
+        bool wait_result = condition_variable.wait_for(lock, std::chrono::milliseconds(timeout_in_milliseconds), [this] { return !objects.empty(); });
+
+        if (wait_result)
+            dest = borrowFromObjects(lock);
+
+        return wait_result;
+    }
+
+    /// Return object into pool. Client must return same object that was borrowed.
+    inline void returnObject(T && object_to_return)
+    {
+        std::unique_lock<std::mutex> lck(objects_mutex);
+
+        objects.emplace_back(std::move(object_to_return));
+        --borrowed_objects_size;
+
+        condition_variable.notify_one();
+    }
+
+    /// Max pool size
+    inline size_t maxSize() const
+    {
+        return max_size;
+    }
+
+    /// Allocated objects size by the pool. If allocatedObjectsSize == maxSize then pool is full.
+    inline size_t allocatedObjectsSize() const
+    {
+        std::unique_lock<std::mutex> lock(objects_mutex);
+        return allocated_objects_size;
+    }
+
+    /// Returns allocatedObjectsSize == maxSize
+    inline bool isFull() const
+    {
+        std::unique_lock<std::mutex> lock(objects_mutex);
+        return allocated_objects_size == max_size;
+    }
+
+    /// Borrowed objects size. If borrowedObjectsSize == allocatedObjectsSize and pool is full.
+    /// Then client will wait during borrowObject function call.
+    inline size_t borrowedObjectsSize() const
+    {
+        std::unique_lock<std::mutex> lock(objects_mutex);
+        return borrowed_objects_size;
+    }
+
+private:
+
+    template <typename FactoryFunc>
+    inline T allocateObjectForBorrowing(const std::unique_lock<std::mutex> &, FactoryFunc && func)
+    {
+        ++allocated_objects_size;
+        ++borrowed_objects_size;
+
+        return std::forward<FactoryFunc>(func)();
+    }
+
+    inline T borrowFromObjects(const std::unique_lock<std::mutex> &)
+    {
+        T dst;
+        detail::moveOrCopyIfThrow(std::move(objects.back()), dst);
+        objects.pop_back();
+
+        ++borrowed_objects_size;
+
+        return dst;
+    }
+
+    size_t max_size;
+
+    mutable std::mutex objects_mutex;
+    std::condition_variable condition_variable;
+    size_t allocated_objects_size = 0;
+    size_t borrowed_objects_size = 0;
+    std::vector<T> objects;
+};

base/common/CMakeLists.txt

@@ -6,6 +6,7 @@ set (SRCS
     demangle.cpp
     getFQDNOrHostName.cpp
     getMemoryAmount.cpp
+    getPageSize.cpp
     getThreadId.cpp
     JSON.cpp
     LineReader.cpp
@@ -28,7 +29,7 @@ elseif (ENABLE_READLINE)
 endif ()
 
 if (USE_DEBUG_HELPERS)
-    set (INCLUDE_DEBUG_HELPERS "-include ${ClickHouse_SOURCE_DIR}/base/common/iostream_debug_helpers.h")
+    set (INCLUDE_DEBUG_HELPERS "-include \"${ClickHouse_SOURCE_DIR}/base/common/iostream_debug_helpers.h\"")
     set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${INCLUDE_DEBUG_HELPERS}")
 endif ()
 
@@ -44,7 +45,11 @@ if (USE_INTERNAL_CCTZ)
     set_source_files_properties(DateLUTImpl.cpp PROPERTIES COMPILE_DEFINITIONS USE_INTERNAL_CCTZ)
 endif()
 
-target_include_directories(common PUBLIC .. ${CMAKE_CURRENT_BINARY_DIR}/..)
+target_include_directories(common PUBLIC .. "${CMAKE_CURRENT_BINARY_DIR}/..")
+
+if (OS_DARWIN AND NOT MAKE_STATIC_LIBRARIES)
+    target_link_libraries(common PUBLIC -Wl,-U,_inside_main)
+endif()
 
 # Allow explicit fallback to readline
 if (NOT ENABLE_REPLXX AND ENABLE_READLINE)
@@ -73,7 +78,6 @@ target_link_libraries (common
         ${CITYHASH_LIBRARIES}
         boost::headers_only
         boost::system
-        FastMemcpy
         Poco::Net
         Poco::Net::SSL
         Poco::Util

base/common/DateLUT.cpp

@@ -152,7 +152,7 @@ const DateLUTImpl & DateLUT::getImplementation(const std::string & time_zone) co
 
     auto it = impls.emplace(time_zone, nullptr).first;
     if (!it->second)
-        it->second = std::make_unique<DateLUTImpl>(time_zone);
+        it->second = std::unique_ptr<DateLUTImpl>(new DateLUTImpl(time_zone));
 
     return *it->second;
 }

base/common/DateLUT.h

@@ -17,7 +17,7 @@ class DateLUT : private boost::noncopyable
 {
 public:
     /// Return singleton DateLUTImpl instance for the default time zone.
-    static ALWAYS_INLINE const DateLUTImpl & instance()
+    static ALWAYS_INLINE const DateLUTImpl & instance()  // -V1071
     {
         const auto & date_lut = getInstance();
         return *date_lut.default_impl.load(std::memory_order_acquire);
@@ -32,7 +32,6 @@ public:
 
         return date_lut.getImplementation(time_zone);
     }
-
     static void setDefaultTimezone(const std::string & time_zone)
     {
         auto & date_lut = getInstance();

base/common/DateLUTImpl.cpp

@@ -46,24 +46,42 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_)
     if (&inside_main)
         assert(inside_main);
 
-    size_t i = 0;
-    time_t start_of_day = 0;
-
     cctz::time_zone cctz_time_zone;
     if (!cctz::load_time_zone(time_zone, &cctz_time_zone))
         throw Poco::Exception("Cannot load time zone " + time_zone_);
 
-    cctz::time_zone::absolute_lookup start_of_epoch_lookup = cctz_time_zone.lookup(std::chrono::system_clock::from_time_t(start_of_day));
+    constexpr cctz::civil_day epoch{1970, 1, 1};
-    offset_at_start_of_epoch = start_of_epoch_lookup.offset;
+    constexpr cctz::civil_day lut_start{DATE_LUT_MIN_YEAR, 1, 1};
-    offset_is_whole_number_of_hours_everytime = true;
+    time_t start_of_day;
 
-    cctz::civil_day date{1970, 1, 1};
+    /// Note: it's validated against all timezones in the system.
+    static_assert((epoch - lut_start) == daynum_offset_epoch);
 
+    offset_at_start_of_epoch = cctz_time_zone.lookup(cctz_time_zone.lookup(epoch).pre).offset;
+    offset_at_start_of_lut = cctz_time_zone.lookup(cctz_time_zone.lookup(lut_start).pre).offset;
+    offset_is_whole_number_of_hours_during_epoch = true;
+    offset_is_whole_number_of_minutes_during_epoch = true;
+
+    cctz::civil_day date = lut_start;
+
+    UInt32 i = 0;
     do
     {
         cctz::time_zone::civil_lookup lookup = cctz_time_zone.lookup(date);
 
-        start_of_day = std::chrono::system_clock::to_time_t(lookup.pre);    /// Ambiguity is possible.
+        /// Ambiguity is possible if time was changed backwards at the midnight
+        /// or after midnight time has been changed back to midnight, for example one hour backwards at 01:00
+        /// or after midnight time has been changed to the previous day, for example two hours backwards at 01:00
+        /// Then midnight appears twice. Usually time change happens exactly at 00:00 or 01:00.
+
+        /// If transition did not involve previous day, we should use the first midnight as the start of the day,
+        /// otherwise it's better to use the second midnight.
+
+        std::chrono::time_point start_of_day_time_point = lookup.trans < lookup.post
+            ? lookup.post /* Second midnight appears after transition, so there was a piece of previous day after transition */
+            : lookup.pre;
+
+        start_of_day = std::chrono::system_clock::to_time_t(start_of_day_time_point);
 
         Values & values = lut[i];
         values.year = date.year();
@@ -72,7 +90,7 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_)
         values.day_of_week = getDayOfWeek(date);
         values.date = start_of_day;
 
-        assert(values.year >= DATE_LUT_MIN_YEAR && values.year <= DATE_LUT_MAX_YEAR);
+        assert(values.year >= DATE_LUT_MIN_YEAR && values.year <= DATE_LUT_MAX_YEAR + 1);
         assert(values.month >= 1 && values.month <= 12);
         assert(values.day_of_month >= 1 && values.day_of_month <= 31);
         assert(values.day_of_week >= 1 && values.day_of_week <= 7);
@@ -85,50 +103,45 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_)
         else
             values.days_in_month = i != 0 ? lut[i - 1].days_in_month : 31;
 
-        values.time_at_offset_change = 0;
+        values.time_at_offset_change_value = 0;
-        values.amount_of_offset_change = 0;
+        values.amount_of_offset_change_value = 0;
 
-        if (start_of_day % 3600)
+        if (offset_is_whole_number_of_hours_during_epoch && start_of_day > 0 && start_of_day % 3600)
-            offset_is_whole_number_of_hours_everytime = false;
+            offset_is_whole_number_of_hours_during_epoch = false;
 
-        /// If UTC offset was changed in previous day.
+        if (offset_is_whole_number_of_minutes_during_epoch && start_of_day > 0 && start_of_day % 60)
-        if (i != 0)
+            offset_is_whole_number_of_minutes_during_epoch = false;
+
+        /// If UTC offset was changed this day.
+        /// Change in time zone without transition is possible, e.g. Moscow 1991 Sun, 31 Mar, 02:00 MSK to EEST
+        cctz::time_zone::civil_transition transition{};
+        if (cctz_time_zone.next_transition(start_of_day_time_point - std::chrono::seconds(1), &transition)
+            && (cctz::civil_day(transition.from) == date || cctz::civil_day(transition.to) == date)
+            && transition.from != transition.to)
         {
-            auto amount_of_offset_change_at_prev_day = 86400 - (lut[i].date - lut[i - 1].date);
+            values.time_at_offset_change_value = (transition.from - cctz::civil_second(date)) / Values::OffsetChangeFactor;
-            if (amount_of_offset_change_at_prev_day)
+            values.amount_of_offset_change_value = (transition.to - transition.from) / Values::OffsetChangeFactor;
-            {
-                lut[i - 1].amount_of_offset_change = amount_of_offset_change_at_prev_day;
 
-                const auto utc_offset_at_beginning_of_day = cctz_time_zone.lookup(std::chrono::system_clock::from_time_t(lut[i - 1].date)).offset;
+//            std::cerr << time_zone << ", " << date << ": change from " << transition.from << " to " << transition.to << "\n";
+//            std::cerr << time_zone << ", " << date << ": change at " << values.time_at_offset_change() << " with " << values.amount_of_offset_change() << "\n";
 
-                /// Find a time (timestamp offset from beginning of day),
+            /// We don't support too large changes.
-                ///  when UTC offset was changed. Search is performed with 15-minute granularity, assuming it is enough.
+            if (values.amount_of_offset_change_value > 24 * 4)
+                values.amount_of_offset_change_value = 24 * 4;
+            else if (values.amount_of_offset_change_value < -24 * 4)
+                values.amount_of_offset_change_value = -24 * 4;
 
-                time_t time_at_offset_change = 900;
+            /// We don't support cases when time change results in switching to previous day.
-                while (time_at_offset_change < 86400)
+            /// Shift the point of time change later.
-                {
+            if (values.time_at_offset_change_value + values.amount_of_offset_change_value < 0)
-                    auto utc_offset_at_current_time = cctz_time_zone.lookup(std::chrono::system_clock::from_time_t(
+                values.time_at_offset_change_value = -values.amount_of_offset_change_value;
-                        lut[i - 1].date + time_at_offset_change)).offset;
-
-                    if (utc_offset_at_current_time != utc_offset_at_beginning_of_day)
-                        break;
-
-                    time_at_offset_change += 900;
-                }
-
-                lut[i - 1].time_at_offset_change = time_at_offset_change;
-
-                /// We doesn't support cases when time change results in switching to previous day.
-                if (static_cast<int>(lut[i - 1].time_at_offset_change) + static_cast<int>(lut[i - 1].amount_of_offset_change) < 0)
-                    lut[i - 1].time_at_offset_change = -lut[i - 1].amount_of_offset_change;
-            }
         }
 
         /// Going to next day.
         ++date;
         ++i;
     }
-    while (start_of_day <= DATE_LUT_MAX && i <= DATE_LUT_MAX_DAY_NUM);
+    while (i < DATE_LUT_SIZE && lut[i - 1].year <= DATE_LUT_MAX_YEAR);
 
     /// Fill excessive part of lookup table. This is needed only to simplify handling of overflow cases.
     while (i < DATE_LUT_SIZE)

base/common/DayNum.h

@@ -7,3 +7,8 @@
   * See DateLUTImpl for usage examples.
   */
 STRONG_TYPEDEF(UInt16, DayNum)
+
+/** Represent number of days since 1970-01-01 but in extended range,
+ * for dates before 1970-01-01 and after 2105
+ */
+STRONG_TYPEDEF(Int32, ExtendedDayNum)

base/common/DecomposedFloat.h

@@ -0,0 +1,219 @@
+#pragma once
+
+#include <cstdint>
+#include <cstddef>
+#include <cstring>
+#include <common/extended_types.h>
+
+
+/// Allows to check the internals of IEEE-754 floating point number.
+
+template <typename T> struct FloatTraits;
+
+template <>
+struct FloatTraits<float>
+{
+    using UInt = uint32_t;
+    static constexpr size_t bits = 32;
+    static constexpr size_t exponent_bits = 8;
+    static constexpr size_t mantissa_bits = bits - exponent_bits - 1;
+};
+
+template <>
+struct FloatTraits<double>
+{
+    using UInt = uint64_t;
+    static constexpr size_t bits = 64;
+    static constexpr size_t exponent_bits = 11;
+    static constexpr size_t mantissa_bits = bits - exponent_bits - 1;
+};
+
+
+/// x = sign * (2 ^ normalized_exponent) * (1 + mantissa * 2 ^ -mantissa_bits)
+/// x = sign * (2 ^ normalized_exponent + mantissa * 2 ^ (normalized_exponent - mantissa_bits))
+template <typename T>
+struct DecomposedFloat
+{
+    using Traits = FloatTraits<T>;
+
+    DecomposedFloat(T x)
+    {
+        memcpy(&x_uint, &x, sizeof(x));
+    }
+
+    typename Traits::UInt x_uint;
+
+    bool is_negative() const
+    {
+        return x_uint >> (Traits::bits - 1);
+    }
+
+    /// Returns 0 for both +0. and -0.
+    int sign() const
+    {
+        return (exponent() == 0 && mantissa() == 0)
+            ? 0
+            : (is_negative()
+                ? -1
+                : 1);
+    }
+
+    uint16_t exponent() const
+    {
+        return (x_uint >> (Traits::mantissa_bits)) & (((1ull << (Traits::exponent_bits + 1)) - 1) >> 1);
+    }
+
+    int16_t normalized_exponent() const
+    {
+        return int16_t(exponent()) - ((1ull << (Traits::exponent_bits - 1)) - 1);
+    }
+
+    uint64_t mantissa() const
+    {
+        return x_uint & ((1ull << Traits::mantissa_bits) - 1);
+    }
+
+    int64_t mantissa_with_sign() const
+    {
+        return is_negative() ? -mantissa() : mantissa();
+    }
+
+    /// NOTE Probably floating point instructions can be better.
+    bool is_integer_in_representable_range() const
+    {
+        return x_uint == 0
+            || (normalized_exponent() >= 0  /// The number is not less than one
+                /// The number is inside the range where every integer has exact representation in float
+                && normalized_exponent() <= static_cast<int16_t>(Traits::mantissa_bits)
+                /// After multiplying by 2^exp, the fractional part becomes zero, means the number is integer
+                && ((mantissa() & ((1ULL << (Traits::mantissa_bits - normalized_exponent())) - 1)) == 0));
+    }
+
+
+    /// Compare float with integer of arbitrary width (both signed and unsigned are supported). Assuming two's complement arithmetic.
+    /// This function is generic, big integers (128, 256 bit) are supported as well.
+    /// Infinities are compared correctly. NaNs are treat similarly to infinities, so they can be less than all numbers.
+    /// (note that we need total order)
+    /// Returns -1, 0 or 1.
+    template <typename Int>
+    int compare(Int rhs) const
+    {
+        if (rhs == 0)
+            return sign();
+
+        /// Different signs
+        if (is_negative() && rhs > 0)
+            return -1;
+        if (!is_negative() && rhs < 0)
+            return 1;
+
+        /// Fractional number with magnitude less than one
+        if (normalized_exponent() < 0)
+        {
+            if (!is_negative())
+                return rhs > 0 ? -1 : 1;
+            else
+                return rhs >= 0 ? -1 : 1;
+        }
+
+        /// The case of the most negative integer
+        if constexpr (is_signed_v<Int>)
+        {
+            if (rhs == std::numeric_limits<Int>::lowest())
+            {
+                assert(is_negative());
+
+                if (normalized_exponent() < static_cast<int16_t>(8 * sizeof(Int) - is_signed_v<Int>))
+                    return 1;
+                if (normalized_exponent() > static_cast<int16_t>(8 * sizeof(Int) - is_signed_v<Int>))
+                    return -1;
+
+                if (mantissa() == 0)
+                    return 0;
+                else
+                    return -1;
+            }
+        }
+
+        /// Too large number: abs(float) > abs(rhs). Also the case with infinities and NaN.
+        if (normalized_exponent() >= static_cast<int16_t>(8 * sizeof(Int) - is_signed_v<Int>))
+            return is_negative() ? -1 : 1;
+
+        using UInt = std::conditional_t<(sizeof(Int) > sizeof(typename Traits::UInt)), make_unsigned_t<Int>, typename Traits::UInt>;
+        UInt uint_rhs = rhs < 0 ? -rhs : rhs;
+
+        /// Smaller octave: abs(rhs) < abs(float)
+        /// FYI, TIL: octave is also called "binade", https://en.wikipedia.org/wiki/Binade
+        if (uint_rhs < (static_cast<UInt>(1) << normalized_exponent()))
+            return is_negative() ? -1 : 1;
+
+        /// Larger octave: abs(rhs) > abs(float)
+        if (normalized_exponent() + 1 < static_cast<int16_t>(8 * sizeof(Int) - is_signed_v<Int>)
+            && uint_rhs >= (static_cast<UInt>(1) << (normalized_exponent() + 1)))
+            return is_negative() ? 1 : -1;
+
+        /// The same octave
+        /// uint_rhs == 2 ^ normalized_exponent + mantissa * 2 ^ (normalized_exponent - mantissa_bits)
+
+        bool large_and_always_integer = normalized_exponent() >= static_cast<int16_t>(Traits::mantissa_bits);
+
+        UInt a = large_and_always_integer
+            ? static_cast<UInt>(mantissa()) << (normalized_exponent() - Traits::mantissa_bits)
+            : static_cast<UInt>(mantissa()) >> (Traits::mantissa_bits - normalized_exponent());
+
+        UInt b = uint_rhs - (static_cast<UInt>(1) << normalized_exponent());
+
+        if (a < b)
+            return is_negative() ? 1 : -1;
+        if (a > b)
+            return is_negative() ? -1 : 1;
+
+        /// Float has no fractional part means that the numbers are equal.
+        if (large_and_always_integer || (mantissa() & ((1ULL << (Traits::mantissa_bits - normalized_exponent())) - 1)) == 0)
+            return 0;
+        else
+            /// Float has fractional part means its abs value is larger.
+            return is_negative() ? -1 : 1;
+    }
+
+
+    template <typename Int>
+    bool equals(Int rhs) const
+    {
+        return compare(rhs) == 0;
+    }
+
+    template <typename Int>
+    bool notEquals(Int rhs) const
+    {
+        return compare(rhs) != 0;
+    }
+
+    template <typename Int>
+    bool less(Int rhs) const
+    {
+        return compare(rhs) < 0;
+    }
+
+    template <typename Int>
+    bool greater(Int rhs) const
+    {
+        return compare(rhs) > 0;
+    }
+
+    template <typename Int>
+    bool lessOrEquals(Int rhs) const
+    {
+        return compare(rhs) <= 0;
+    }
+
+    template <typename Int>
+    bool greaterOrEquals(Int rhs) const
+    {
+        return compare(rhs) >= 0;
+    }
+};
+
+
+using DecomposedFloat64 = DecomposedFloat<double>;
+using DecomposedFloat32 = DecomposedFloat<float>;

base/common/FunctorToStaticMethodAdaptor.h

@@ -0,0 +1,41 @@
+#include <functional>
+
+/** Adapt functor to static method where functor passed as context.
+  * Main use case to convert lambda into function that can be passed into JIT code.
+  */
+template <typename Functor>
+class FunctorToStaticMethodAdaptor : public FunctorToStaticMethodAdaptor<decltype(&Functor::operator())>
+{
+};
+
+template <typename R, typename C, typename ...Args>
+class FunctorToStaticMethodAdaptor<R (C::*)(Args...) const>
+{
+public:
+    static R call(C * ptr, Args &&... arguments)
+    {
+        return std::invoke(&C::operator(), ptr, std::forward<Args>(arguments)...);
+    }
+
+    static R unsafeCall(char * ptr, Args &&... arguments)
+    {
+        C * ptr_typed = reinterpret_cast<C*>(ptr);
+        return std::invoke(&C::operator(), ptr_typed, std::forward<Args>(arguments)...);
+    }
+};
+
+template <typename R, typename C, typename ...Args>
+class FunctorToStaticMethodAdaptor<R (C::*)(Args...)>
+{
+public:
+    static R call(C * ptr, Args &&... arguments)
+    {
+        return std::invoke(&C::operator(), ptr, std::forward<Args>(arguments)...);
+    }
+
+    static R unsafeCall(char * ptr, Args &&... arguments)
+    {
+        C * ptr_typed = static_cast<C*>(ptr);
+        return std::invoke(&C::operator(), ptr_typed, std::forward<Args>(arguments)...);
+    }
+};

base/common/LineReader.cpp

@@ -127,7 +127,7 @@ String LineReader::readLine(const String & first_prompt, const String & second_p
         }
 #endif
 
-        line += (line.empty() ? "" : " ") + input;
+        line += (line.empty() ? "" : "\n") + input;
 
         if (!need_next_line)
             break;

base/common/LocalDate.h

@@ -70,6 +70,14 @@ public:
         m_day   = values.day_of_month;
     }
 
+    explicit LocalDate(ExtendedDayNum day_num)
+    {
+        const auto & values = DateLUT::instance().getValues(day_num);
+        m_year  = values.year;
+        m_month = values.month;
+        m_day   = values.day_of_month;
+    }
+
     LocalDate(unsigned short year_, unsigned char month_, unsigned char day_)
         : m_year(year_), m_month(month_), m_day(day_)
     {
@@ -92,20 +100,16 @@ public:
     LocalDate(const LocalDate &) noexcept = default;
     LocalDate & operator= (const LocalDate &) noexcept = default;
 
-    LocalDate & operator= (time_t time)
-    {
-        init(time);
-        return *this;
-    }
-
-    operator time_t() const
-    {
-        return DateLUT::instance().makeDate(m_year, m_month, m_day);
-    }
-
     DayNum getDayNum() const
     {
-        return DateLUT::instance().makeDayNum(m_year, m_month, m_day);
+        const auto & lut = DateLUT::instance();
+        return DayNum(lut.makeDayNum(m_year, m_month, m_day).toUnderType());
+    }
+
+    ExtendedDayNum  getExtenedDayNum() const
+    {
+        const auto & lut = DateLUT::instance();
+        return ExtendedDayNum (lut.makeDayNum(m_year, m_month, m_day).toUnderType());
     }
 
     operator DayNum() const
@@ -166,20 +170,3 @@ public:
 };
 
 static_assert(sizeof(LocalDate) == 4);
-
-
-inline std::ostream & operator<< (std::ostream & ostr, const LocalDate & date)
-{
-    return ostr << date.year()
-        << '-' << (date.month() / 10) << (date.month() % 10)
-        << '-' << (date.day() / 10) << (date.day() % 10);
-}
-
-
-namespace std
-{
-inline string to_string(const LocalDate & date)
-{
-    return date.toString();
-}
-}

base/common/LocalDateTime.h

@@ -29,29 +29,16 @@ private:
     /// NOTE We may use attribute packed instead, but it is less portable.
     unsigned char pad = 0;
 
-    void init(time_t time)
+    void init(time_t time, const DateLUTImpl & time_zone)
     {
-        if (unlikely(time > DATE_LUT_MAX || time == 0))
+        DateLUTImpl::DateTimeComponents components = time_zone.toDateTimeComponents(time);
-        {
-            m_year = 0;
-            m_month = 0;
-            m_day = 0;
-            m_hour = 0;
-            m_minute = 0;
-            m_second = 0;
 
-            return;
+        m_year = components.date.year;
-        }
+        m_month = components.date.month;
-
+        m_day = components.date.day;
-        const auto & date_lut = DateLUT::instance();
+        m_hour = components.time.hour;
-        const auto & values = date_lut.getValues(time);
+        m_minute = components.time.minute;
-
+        m_second = components.time.second;
-        m_year = values.year;
-        m_month = values.month;
-        m_day = values.day_of_month;
-        m_hour = date_lut.toHour(time);
-        m_minute = date_lut.toMinute(time);
-        m_second = date_lut.toSecond(time);
 
         (void)pad;  /// Suppress unused private field warning.
     }
@@ -73,9 +60,9 @@ private:
     }
 
 public:
-    explicit LocalDateTime(time_t time)
+    explicit LocalDateTime(time_t time, const DateLUTImpl & time_zone = DateLUT::instance())
     {
-        init(time);
+        init(time, time_zone);
     }
 
     LocalDateTime(unsigned short year_, unsigned char month_, unsigned char day_,
@@ -104,19 +91,6 @@ public:
     LocalDateTime(const LocalDateTime &) noexcept = default;
     LocalDateTime & operator= (const LocalDateTime &) noexcept = default;
 
-    LocalDateTime & operator= (time_t time)
-    {
-        init(time);
-        return *this;
-    }
-
-    operator time_t() const
-    {
-        return m_year == 0
-            ? 0
-            : DateLUT::instance().makeDateTime(m_year, m_month, m_day, m_hour, m_minute, m_second);
-    }
-
     unsigned short year() const { return m_year; }
     unsigned char month() const { return m_month; }
     unsigned char day() const { return m_day; }
@@ -132,8 +106,30 @@ public:
     void second(unsigned char x) { m_second = x; }
 
     LocalDate toDate() const { return LocalDate(m_year, m_month, m_day); }
+    LocalDateTime toStartOfDate() const { return LocalDateTime(m_year, m_month, m_day, 0, 0, 0); }
 
-    LocalDateTime toStartOfDate() { return LocalDateTime(m_year, m_month, m_day, 0, 0, 0); }
+    std::string toString() const
+    {
+        std::string s{"0000-00-00 00:00:00"};
+
+        s[0] += m_year / 1000;
+        s[1] += (m_year / 100) % 10;
+        s[2] += (m_year / 10) % 10;
+        s[3] += m_year % 10;
+        s[5] += m_month / 10;
+        s[6] += m_month % 10;
+        s[8] += m_day / 10;
+        s[9] += m_day % 10;
+
+        s[11] += m_hour / 10;
+        s[12] += m_hour % 10;
+        s[14] += m_minute / 10;
+        s[15] += m_minute % 10;
+        s[17] += m_second / 10;
+        s[18] += m_second % 10;
+
+        return s;
+    }
 
     bool operator< (const LocalDateTime & other) const
     {
@@ -167,28 +163,3 @@ public:
 };
 
 static_assert(sizeof(LocalDateTime) == 8);
-
-
-inline std::ostream & operator<< (std::ostream & ostr, const LocalDateTime & datetime)
-{
-    ostr << std::setfill('0') << std::setw(4) << datetime.year();
-
-    ostr << '-' << (datetime.month() / 10) << (datetime.month() % 10)
-        << '-' << (datetime.day() / 10) << (datetime.day() % 10)
-        << ' ' << (datetime.hour() / 10) << (datetime.hour() % 10)
-        << ':' << (datetime.minute() / 10) << (datetime.minute() % 10)
-        << ':' << (datetime.second() / 10) << (datetime.second() % 10);
-
-    return ostr;
-}
-
-
-namespace std
-{
-inline string to_string(const LocalDateTime & datetime)
-{
-    stringstream str;
-    str << datetime;
-    return str.str();
-}
-}

base/common/MoveOrCopyIfThrow.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#include <common/types.h>
+
+namespace detail
+{
+    template <typename T, bool is_nothrow_move_assignable = std::is_nothrow_move_assignable_v<T>>
+    struct MoveOrCopyIfThrow;
+
+    template <typename T>
+    struct MoveOrCopyIfThrow<T, true>
+    {
+        void operator()(T && src, T & dst) const
+        {
+            dst = std::forward<T>(src);
+        }
+    };
+
+    template <typename T>
+    struct MoveOrCopyIfThrow<T, false>
+    {
+        void operator()(T && src, T & dst) const
+        {
+            dst = src;
+        }
+    };
+
+    template <typename T>
+    void moveOrCopyIfThrow(T && src, T & dst)
+    {
+        MoveOrCopyIfThrow<T>()(std::forward<T>(src), dst);
+    }
+}

base/common/ReadlineLineReader.cpp

@@ -1,5 +1,6 @@
 #include <common/ReadlineLineReader.h>
-#include <ext/scope_guard.h>
+#include <common/errnoToString.h>
+#include <common/scope_guard.h>
 
 #include <errno.h>
 #include <signal.h>
@@ -69,7 +70,7 @@ ReadlineLineReader::ReadlineLineReader(
     {
         int res = read_history(history_file_path.c_str());
         if (res)
-            std::cerr << "Cannot read history from file " + history_file_path + ": "+ strerror(errno) << std::endl;
+            std::cerr << "Cannot read history from file " + history_file_path + ": "+ errnoToString(errno) << std::endl;
     }
 
     /// Added '.' to the default list. Because it is used to separate database and table.
@@ -107,7 +108,7 @@ ReadlineLineReader::ReadlineLineReader(
     };
 
     if (signal(SIGINT, clear_prompt_or_exit) == SIG_ERR)
-        throw std::runtime_error(std::string("Cannot set signal handler for readline: ") + strerror(errno));
+        throw std::runtime_error(std::string("Cannot set signal handler for readline: ") + errnoToString(errno));
 
     rl_variable_bind("completion-ignore-case", "on");
     // TODO: it doesn't work

base/common/ReplxxLineReader.cpp

@@ -1,11 +1,20 @@
 #include <common/ReplxxLineReader.h>
 #include <common/errnoToString.h>
 
-#include <errno.h>
+#include <chrono>
-#include <string.h>
+#include <cerrno>
+#include <cstring>
 #include <unistd.h>
 #include <functional>
 #include <sys/file.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <csignal>
+#include <dlfcn.h>
+#include <fcntl.h>
+#include <fstream>
+#include <fmt/format.h>
+
 
 namespace
 {
@@ -16,6 +25,94 @@ void trim(String & s)
     s.erase(std::find_if(s.rbegin(), s.rend(), [](int ch) { return !std::isspace(ch); }).base(), s.end());
 }
 
+/// Copied from replxx::src/util.cxx::now_ms_str() under the terms of 3-clause BSD license of Replxx.
+/// Copyright (c) 2017-2018, Marcin Konarski (amok at codestation.org)
+/// Copyright (c) 2010, Salvatore Sanfilippo (antirez at gmail dot com)
+/// Copyright (c) 2010, Pieter Noordhuis (pcnoordhuis at gmail dot com)
+std::string replxx_now_ms_str()
+{
+    std::chrono::milliseconds ms(std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()));
+    time_t t = ms.count() / 1000;
+    tm broken;
+    if (!localtime_r(&t, &broken))
+    {
+        return std::string();
+    }
+
+    static int const BUFF_SIZE(32);
+    char str[BUFF_SIZE];
+    strftime(str, BUFF_SIZE, "%Y-%m-%d %H:%M:%S.", &broken);
+    snprintf(str + sizeof("YYYY-mm-dd HH:MM:SS"), 5, "%03d", static_cast<int>(ms.count() % 1000));
+    return str;
+}
+
+/// Convert from readline to replxx format.
+///
+/// replxx requires each history line to prepended with time line:
+///
+///     ### YYYY-MM-DD HH:MM:SS.SSS
+///     select 1
+///
+/// And w/o those service lines it will load all lines from history file as
+/// one history line for suggestion. And if there are lots of lines in file it
+/// will take lots of time (getline() + tons of reallocations).
+///
+/// NOTE: this code uses std::ifstream/std::ofstream like original replxx code.
+void convertHistoryFile(const std::string & path, replxx::Replxx & rx)
+{
+    std::ifstream in(path);
+    if (!in)
+    {
+        rx.print("Cannot open %s reading (for conversion): %s\n",
+            path.c_str(), errnoToString(errno).c_str());
+        return;
+    }
+
+    std::string line;
+    if (getline(in, line).bad())
+    {
+        rx.print("Cannot read from %s (for conversion): %s\n",
+            path.c_str(), errnoToString(errno).c_str());
+        return;
+    }
+
+    /// This is the marker of the date, no need to convert.
+    static char const REPLXX_TIMESTAMP_PATTERN[] = "### dddd-dd-dd dd:dd:dd.ddd";
+    if (line.empty() || (line.starts_with("### ") && line.size() == strlen(REPLXX_TIMESTAMP_PATTERN)))
+    {
+        return;
+    }
+
+    std::vector<std::string> lines;
+    in.seekg(0);
+    while (getline(in, line).good())
+    {
+        lines.push_back(line);
+    }
+    in.close();
+
+    size_t lines_size = lines.size();
+    std::sort(lines.begin(), lines.end());
+    lines.erase(std::unique(lines.begin(), lines.end()), lines.end());
+    rx.print("The history file (%s) is in old format. %zu lines, %zu unique lines.\n",
+        path.c_str(), lines_size, lines.size());
+
+    std::ofstream out(path);
+    if (!out)
+    {
+        rx.print("Cannot open %s for writing (for conversion): %s\n",
+            path.c_str(), errnoToString(errno).c_str());
+        return;
+    }
+
+    const std::string & timestamp = replxx_now_ms_str();
+    for (const auto & out_line : lines)
+    {
+        out << "### " << timestamp << "\n" << out_line << std::endl;
+    }
+    out.close();
+}
+
 }
 
 ReplxxLineReader::ReplxxLineReader(
@@ -39,6 +136,8 @@ ReplxxLineReader::ReplxxLineReader(
         }
         else
         {
+            convertHistoryFile(history_file_path, rx);
+
             if (flock(history_file_fd, LOCK_SH))
             {
                 rx.print("Shared lock of history file failed: %s\n", errnoToString(errno).c_str());
@@ -47,7 +146,7 @@ ReplxxLineReader::ReplxxLineReader(
             {
                 if (!rx.history_load(history_file_path))
                 {
-                    rx.print("Loading history failed: %s\n", strerror(errno));
+                    rx.print("Loading history failed: %s\n", errnoToString(errno).c_str());
                 }
 
                 if (flock(history_file_fd, LOCK_UN))
@@ -58,6 +157,8 @@ ReplxxLineReader::ReplxxLineReader(
         }
     }
 
+    rx.install_window_change_handler();
+
     auto callback = [&suggest] (const String & context, size_t context_size)
     {
         if (auto range = suggest.getCompletions(context, context_size))
@@ -81,12 +182,18 @@ ReplxxLineReader::ReplxxLineReader(
     /// it also binded to M-p/M-n).
     rx.bind_key(Replxx::KEY::meta('N'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::COMPLETE_NEXT, code); });
     rx.bind_key(Replxx::KEY::meta('P'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::COMPLETE_PREVIOUS, code); });
+    /// By default M-BACKSPACE is KILL_TO_WHITESPACE_ON_LEFT, while in readline it is backward-kill-word
+    rx.bind_key(Replxx::KEY::meta(Replxx::KEY::BACKSPACE), [this](char32_t code) { return rx.invoke(Replxx::ACTION::KILL_TO_BEGINING_OF_WORD, code); });
+    /// By default C-w is KILL_TO_BEGINING_OF_WORD, while in readline it is unix-word-rubout
+    rx.bind_key(Replxx::KEY::control('W'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::KILL_TO_WHITESPACE_ON_LEFT, code); });
+
+    rx.bind_key(Replxx::KEY::meta('E'), [this](char32_t) { openEditor(); return Replxx::ACTION_RESULT::CONTINUE; });
 }
 
 ReplxxLineReader::~ReplxxLineReader()
 {
     if (close(history_file_fd))
-        rx.print("Close of history file failed: %s\n", strerror(errno));
+        rx.print("Close of history file failed: %s\n", errnoToString(errno).c_str());
 }
 
 LineReader::InputStatus ReplxxLineReader::readOneLine(const String & prompt)
@@ -111,7 +218,7 @@ void ReplxxLineReader::addToHistory(const String & line)
     // and that is why flock() is added here.
     bool locked = false;
     if (flock(history_file_fd, LOCK_EX))
-        rx.print("Lock of history file failed: %s\n", strerror(errno));
+        rx.print("Lock of history file failed: %s\n", errnoToString(errno).c_str());
     else
         locked = true;
 
@@ -119,13 +226,120 @@ void ReplxxLineReader::addToHistory(const String & line)
 
     // flush changes to the disk
     if (!rx.history_save(history_file_path))
-        rx.print("Saving history failed: %s\n", strerror(errno));
+        rx.print("Saving history failed: %s\n", errnoToString(errno).c_str());
 
     if (locked && 0 != flock(history_file_fd, LOCK_UN))
-        rx.print("Unlock of history file failed: %s\n", strerror(errno));
+        rx.print("Unlock of history file failed: %s\n", errnoToString(errno).c_str());
+}
+
+int ReplxxLineReader::execute(const std::string & command)
+{
+    std::vector<char> argv0("sh", &("sh"[3]));
+    std::vector<char> argv1("-c", &("-c"[3]));
+    std::vector<char> argv2(command.data(), command.data() + command.size() + 1);
+
+    const char * filename = "/bin/sh";
+    char * const argv[] = {argv0.data(), argv1.data(), argv2.data(), nullptr};
+
+    static void * real_vfork = dlsym(RTLD_DEFAULT, "vfork");
+    if (!real_vfork)
+    {
+        rx.print("Cannot find symbol vfork in myself: %s\n", errnoToString(errno).c_str());
+        return -1;
+    }
+
+    pid_t pid = reinterpret_cast<pid_t (*)()>(real_vfork)();
+
+    if (-1 == pid)
+    {
+        rx.print("Cannot vfork: %s\n", errnoToString(errno).c_str());
+        return -1;
+    }
+
+    if (0 == pid)
+    {
+        sigset_t mask;
+        sigemptyset(&mask);
+        sigprocmask(0, nullptr, &mask);
+        sigprocmask(SIG_UNBLOCK, &mask, nullptr);
+
+        execv(filename, argv);
+        _exit(-1);
+    }
+
+    int status = 0;
+    if (-1 == waitpid(pid, &status, 0))
+    {
+        rx.print("Cannot waitpid: %s\n", errnoToString(errno).c_str());
+        return -1;
+    }
+    return status;
+}
+
+void ReplxxLineReader::openEditor()
+{
+    char filename[] = "clickhouse_replxx_XXXXXX.sql";
+    int fd = ::mkstemps(filename, 4);
+    if (-1 == fd)
+    {
+        rx.print("Cannot create temporary file to edit query: %s\n", errnoToString(errno).c_str());
+        return;
+    }
+
+    const char * editor = std::getenv("EDITOR");
+    if (!editor || !*editor)
+        editor = "vim";
+
+    replxx::Replxx::State state(rx.get_state());
+
+    size_t bytes_written = 0;
+    const char * begin = state.text();
+    size_t offset = strlen(state.text());
+    while (bytes_written != offset)
+    {
+        ssize_t res = ::write(fd, begin + bytes_written, offset - bytes_written);
+        if ((-1 == res || 0 == res) && errno != EINTR)
+        {
+            rx.print("Cannot write to temporary query file %s: %s\n", filename, errnoToString(errno).c_str());
+            break;
+        }
+        bytes_written += res;
+    }
+
+    if (0 != ::close(fd))
+    {
+        rx.print("Cannot close temporary query file %s: %s\n", filename, errnoToString(errno).c_str());
+        return;
+    }
+
+    if (0 == execute(fmt::format("{} {}", editor, filename)))
+    {
+        try
+        {
+            std::ifstream t(filename);
+            std::string str;
+            t.seekg(0, std::ios::end);
+            str.reserve(t.tellg());
+            t.seekg(0, std::ios::beg);
+            str.assign((std::istreambuf_iterator<char>(t)), std::istreambuf_iterator<char>());
+            rx.set_state(replxx::Replxx::State(str.c_str(), str.size()));
+        }
+        catch (...)
+        {
+            rx.print("Cannot read from temporary query file %s: %s\n", filename, errnoToString(errno).c_str());
+            return;
+        }
+    }
+
+    if (bracketed_paste_enabled)
+        enableBracketedPaste();
+
+    if (0 != ::unlink(filename))
+        rx.print("Cannot remove temporary query file %s: %s\n", filename, errnoToString(errno).c_str());
 }
 
 void ReplxxLineReader::enableBracketedPaste()
 {
+    bracketed_paste_enabled = true;
     rx.enable_bracketed_paste();
 };

base/common/ReplxxLineReader.h

@@ -22,10 +22,13 @@ public:
 private:
     InputStatus readOneLine(const String & prompt) override;
     void addToHistory(const String & line) override;
+    int execute(const std::string & command);
+    void openEditor();
 
     replxx::Replxx rx;
     replxx::Replxx::highlighter_callback_t highlighter;
 
     // used to call flock() to synchronize multiple clients using same history file
     int history_file_fd = -1;
+    bool bracketed_paste_enabled = false;
 };

base/common/SimpleCache.h

@@ -3,7 +3,7 @@
 #include <map>
 #include <tuple>
 #include <mutex>
-#include <ext/function_traits.h>
+#include <common/function_traits.h>
 
 
 /** The simplest cache for a free function.
@@ -32,10 +32,11 @@ public:
     template <typename... Args>
     Result operator() (Args &&... args)
     {
+        Key key{std::forward<Args>(args)...};
+
         {
             std::lock_guard lock(mutex);
 
-            Key key{std::forward<Args>(args)...};
             auto it = cache.find(key);
 
             if (cache.end() != it)
@@ -43,7 +44,7 @@ public:
         }
 
         /// The calculations themselves are not done under mutex.
-        Result res = f(std::forward<Args>(args)...);
+        Result res = std::apply(f, key);
 
         {
             std::lock_guard lock(mutex);
@@ -57,11 +58,12 @@ public:
     template <typename... Args>
     void update(Args &&... args)
     {
-        Result res = f(std::forward<Args>(args)...);
+        Key key{std::forward<Args>(args)...};
+
+        Result res = std::apply(f, key);
+
         {
             std::lock_guard lock(mutex);
-
-            Key key{std::forward<Args>(args)...};
             cache[key] = std::move(res);
         }
     }

base/common/StringRef.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <cassert>
+#include <stdexcept> // for std::logic_error
 #include <string>
 #include <vector>
 #include <functional>

base/common/arithmeticOverflow.h

@@ -1,9 +1,36 @@
 #pragma once
 
 #include <common/extended_types.h>
+#include <common/defines.h>
+
 
 namespace common
 {
+    /// Multiply and ignore overflow.
+    template <typename T1, typename T2>
+    inline auto NO_SANITIZE_UNDEFINED mulIgnoreOverflow(T1 x, T2 y)
+    {
+        return x * y;
+    }
+
+    template <typename T1, typename T2>
+    inline auto NO_SANITIZE_UNDEFINED addIgnoreOverflow(T1 x, T2 y)
+    {
+        return x + y;
+    }
+
+    template <typename T1, typename T2>
+    inline auto NO_SANITIZE_UNDEFINED subIgnoreOverflow(T1 x, T2 y)
+    {
+        return x - y;
+    }
+
+    template <typename T>
+    inline auto NO_SANITIZE_UNDEFINED negateIgnoreOverflow(T x)
+    {
+        return -x;
+    }
+
     template <typename T>
     inline bool addOverflow(T x, T y, T & res)
     {
@@ -29,27 +56,33 @@ namespace common
     }
 
     template <>
-    inline bool addOverflow(__int128 x, __int128 y, __int128 & res)
+    inline bool addOverflow(Int128 x, Int128 y, Int128 & res)
     {
-        static constexpr __int128 min_int128 = minInt128();
+        res = addIgnoreOverflow(x, y);
-        static constexpr __int128 max_int128 = maxInt128();
+        return (y > 0 && x > std::numeric_limits<Int128>::max() - y) ||
-        res = x + y;
+            (y < 0 && x < std::numeric_limits<Int128>::min() - y);
-        return (y > 0 && x > max_int128 - y) || (y < 0 && x < min_int128 - y);
     }
 
     template <>
-    inline bool addOverflow(wInt256 x, wInt256 y, wInt256 & res)
+    inline bool addOverflow(UInt128 x, UInt128 y, UInt128 & res)
     {
-        res = x + y;
+        res = addIgnoreOverflow(x, y);
-        return (y > 0 && x > std::numeric_limits<wInt256>::max() - y) ||
+        return x > std::numeric_limits<UInt128>::max() - y;
-            (y < 0 && x < std::numeric_limits<wInt256>::min() - y);
     }
 
     template <>
-    inline bool addOverflow(wUInt256 x, wUInt256 y, wUInt256 & res)
+    inline bool addOverflow(Int256 x, Int256 y, Int256 & res)
     {
-        res = x + y;
+        res = addIgnoreOverflow(x, y);
-        return x > std::numeric_limits<wUInt256>::max() - y;
+        return (y > 0 && x > std::numeric_limits<Int256>::max() - y) ||
+            (y < 0 && x < std::numeric_limits<Int256>::min() - y);
+    }
+
+    template <>
+    inline bool addOverflow(UInt256 x, UInt256 y, UInt256 & res)
+    {
+        res = addIgnoreOverflow(x, y);
+        return x > std::numeric_limits<UInt256>::max() - y;
     }
 
     template <typename T>
@@ -77,26 +110,32 @@ namespace common
     }
 
     template <>
-    inline bool subOverflow(__int128 x, __int128 y, __int128 & res)
+    inline bool subOverflow(Int128 x, Int128 y, Int128 & res)
     {
-        static constexpr __int128 min_int128 = minInt128();
+        res = subIgnoreOverflow(x, y);
-        static constexpr __int128 max_int128 = maxInt128();
+        return (y < 0 && x > std::numeric_limits<Int128>::max() + y) ||
-        res = x - y;
+            (y > 0 && x < std::numeric_limits<Int128>::min() + y);
-        return (y < 0 && x > max_int128 + y) || (y > 0 && x < min_int128 + y);
     }
 
     template <>
-    inline bool subOverflow(wInt256 x, wInt256 y, wInt256 & res)
+    inline bool subOverflow(UInt128 x, UInt128 y, UInt128 & res)
     {
-        res = x - y;
+        res = subIgnoreOverflow(x, y);
-        return (y < 0 && x > std::numeric_limits<wInt256>::max() + y) ||
+        return x < y;
-            (y > 0 && x < std::numeric_limits<wInt256>::min() + y);
     }
 
     template <>
-    inline bool subOverflow(wUInt256 x, wUInt256 y, wUInt256 & res)
+    inline bool subOverflow(Int256 x, Int256 y, Int256 & res)
     {
-        res = x - y;
+        res = subIgnoreOverflow(x, y);
+        return (y < 0 && x > std::numeric_limits<Int256>::max() + y) ||
+            (y > 0 && x < std::numeric_limits<Int256>::min() + y);
+    }
+
+    template <>
+    inline bool subOverflow(UInt256 x, UInt256 y, UInt256 & res)
+    {
+        res = subIgnoreOverflow(x, y);
         return x < y;
     }
 
@@ -124,36 +163,33 @@ namespace common
         return __builtin_smulll_overflow(x, y, &res);
     }
 
-    template <>
+    /// Overflow check is not implemented for big integers.
-    inline bool mulOverflow(__int128 x, __int128 y, __int128 & res)
-    {
-        res = static_cast<unsigned __int128>(x) * static_cast<unsigned __int128>(y);    /// Avoid signed integer overflow.
-        if (!x || !y)
-            return false;
 
-        unsigned __int128 a = (x > 0) ? x : -x;
+    template <>
-        unsigned __int128 b = (y > 0) ? y : -y;
+    inline bool mulOverflow(Int128 x, Int128 y, Int128 & res)
-        return (a * b) / b != a;
+    {
+        res = mulIgnoreOverflow(x, y);
+        return false;
     }
 
     template <>
-    inline bool mulOverflow(wInt256 x, wInt256 y, wInt256 & res)
+    inline bool mulOverflow(Int256 x, Int256 y, Int256 & res)
     {
-        res = x * y;
+        res = mulIgnoreOverflow(x, y);
-        if (!x || !y)
+        return false;
-            return false;
-
-        wInt256 a = (x > 0) ? x : -x;
-        wInt256 b = (y > 0) ? y : -y;
-        return (a * b) / b != a;
     }
 
     template <>
-    inline bool mulOverflow(wUInt256 x, wUInt256 y, wUInt256 & res)
+    inline bool mulOverflow(UInt128 x, UInt128 y, UInt128 & res)
     {
-        res = x * y;
+        res = mulIgnoreOverflow(x, y);
-        if (!x || !y)
+        return false;
-            return false;
+    }
-        return (x * y) / y != x;
+
+    template <>
+    inline bool mulOverflow(UInt256 x, UInt256 y, UInt256 & res)
+    {
+        res = mulIgnoreOverflow(x, y);
+        return false;
     }
 }

base/common/arraySize.h

@@ -0,0 +1,7 @@
+#pragma once
+
+#include <cstdlib>
+
+/** \brief Returns number of elements in an automatic array. */
+template <typename T, std::size_t N>
+constexpr size_t arraySize(const T (&)[N]) noexcept { return N; }

base/common/bit_cast.h

@@ -0,0 +1,27 @@
+#pragma once
+
+#include <string.h>
+#include <algorithm>
+#include <type_traits>
+
+
+/** \brief Returns value `from` converted to type `To` while retaining bit representation.
+  *    `To` and `From` must satisfy `CopyConstructible`.
+  */
+template <typename To, typename From>
+std::decay_t<To> bit_cast(const From & from)
+{
+    To res {};
+    memcpy(static_cast<void*>(&res), &from, std::min(sizeof(res), sizeof(from)));
+    return res;
+}
+
+/** \brief Returns value `from` converted to type `To` while retaining bit representation.
+  *    `To` and `From` must satisfy `CopyConstructible`.
+  */
+template <typename To, typename From>
+std::decay_t<To> safe_bit_cast(const From & from)
+{
+    static_assert(sizeof(To) == sizeof(From), "bit cast on types of different width");
+    return bit_cast<To, From>(from);
+}

base/common/chrono_io.h

@@ -0,0 +1,46 @@
+#pragma once
+
+#include <chrono>
+#include <string>
+#include <sstream>
+#include <cctz/time_zone.h>
+
+
+inline std::string to_string(const std::time_t & time)
+{
+    return cctz::format("%Y-%m-%d %H:%M:%S", std::chrono::system_clock::from_time_t(time), cctz::local_time_zone());
+}
+
+template <typename Clock, typename Duration = typename Clock::duration>
+std::string to_string(const std::chrono::time_point<Clock, Duration> & tp)
+{
+    // Don't use DateLUT because it shows weird characters for
+    // TimePoint::max(). I wish we could use C++20 format, but it's not
+    // there yet.
+    // return DateLUT::instance().timeToString(std::chrono::system_clock::to_time_t(tp));
+
+    auto in_time_t = std::chrono::system_clock::to_time_t(tp);
+    return to_string(in_time_t);
+}
+
+template <typename Rep, typename Period = std::ratio<1>>
+std::string to_string(const std::chrono::duration<Rep, Period> & duration)
+{
+    auto seconds_as_int = std::chrono::duration_cast<std::chrono::seconds>(duration);
+    if (seconds_as_int == duration)
+        return std::to_string(seconds_as_int.count()) + "s";
+    auto seconds_as_double = std::chrono::duration_cast<std::chrono::duration<double>>(duration);
+    return std::to_string(seconds_as_double.count()) + "s";
+}
+
+template <typename Clock, typename Duration = typename Clock::duration>
+std::ostream & operator<<(std::ostream & o, const std::chrono::time_point<Clock, Duration> & tp)
+{
+    return o << to_string(tp);
+}
+
+template <typename Rep, typename Period = std::ratio<1>>
+std::ostream & operator<<(std::ostream & o, const std::chrono::duration<Rep, Period> & duration)
+{
+    return o << to_string(duration);
+}

base/common/defines.h

@@ -1,5 +1,20 @@
 #pragma once
 
+/// __has_feature supported only by clang.
+///
+/// But libcxx/libcxxabi overrides it to 0,
+/// thus the checks for __has_feature will be wrong.
+///
+/// NOTE:
+/// - __has_feature cannot be simply undefined,
+///   since this will be broken if some C++ header will be included after
+///   including <common/defines.h>
+/// - it should not have fallback to 0,
+///   since this may create false-positive detection (common problem)
+#if defined(__clang__) && defined(__has_feature)
+#    define ch_has_feature __has_feature
+#endif
+
 #if defined(_MSC_VER)
 #   if !defined(likely)
 #      define likely(x)   (x)
@@ -32,8 +47,8 @@
 
 /// Check for presence of address sanitizer
 #if !defined(ADDRESS_SANITIZER)
-#    if defined(__has_feature)
+#    if defined(ch_has_feature)
-#        if __has_feature(address_sanitizer)
+#        if ch_has_feature(address_sanitizer)
 #            define ADDRESS_SANITIZER 1
 #        endif
 #    elif defined(__SANITIZE_ADDRESS__)
@@ -42,8 +57,8 @@
 #endif
 
 #if !defined(THREAD_SANITIZER)
-#    if defined(__has_feature)
+#    if defined(ch_has_feature)
-#        if __has_feature(thread_sanitizer)
+#        if ch_has_feature(thread_sanitizer)
 #            define THREAD_SANITIZER 1
 #        endif
 #    elif defined(__SANITIZE_THREAD__)
@@ -52,8 +67,8 @@
 #endif
 
 #if !defined(MEMORY_SANITIZER)
-#    if defined(__has_feature)
+#    if defined(ch_has_feature)
-#        if __has_feature(memory_sanitizer)
+#        if ch_has_feature(memory_sanitizer)
 #            define MEMORY_SANITIZER 1
 #        endif
 #    elif defined(__MEMORY_SANITIZER__)
@@ -61,6 +76,30 @@
 #    endif
 #endif
 
+#if !defined(UNDEFINED_BEHAVIOR_SANITIZER)
+#    if defined(__has_feature)
+#        if __has_feature(undefined_behavior_sanitizer)
+#            define UNDEFINED_BEHAVIOR_SANITIZER 1
+#        endif
+#    elif defined(__UNDEFINED_BEHAVIOR_SANITIZER__)
+#        define UNDEFINED_BEHAVIOR_SANITIZER 1
+#    endif
+#endif
+
+#if defined(ADDRESS_SANITIZER)
+#    define BOOST_USE_ASAN 1
+#    define BOOST_USE_UCONTEXT 1
+#endif
+
+#if defined(THREAD_SANITIZER)
+#    define BOOST_USE_TSAN 1
+#    define BOOST_USE_UCONTEXT 1
+#endif
+
+#if defined(ARCADIA_BUILD) && defined(BOOST_USE_UCONTEXT)
+#    undef BOOST_USE_UCONTEXT
+#endif
+
 /// TODO: Strange enough, there is no way to detect UB sanitizer.
 
 /// Explicitly allow undefined behaviour for certain functions. Use it as a function attribute.
@@ -70,18 +109,16 @@
 #    define NO_SANITIZE_UNDEFINED __attribute__((__no_sanitize__("undefined")))
 #    define NO_SANITIZE_ADDRESS __attribute__((__no_sanitize__("address")))
 #    define NO_SANITIZE_THREAD __attribute__((__no_sanitize__("thread")))
+#    define ALWAYS_INLINE_NO_SANITIZE_UNDEFINED __attribute__((__always_inline__, __no_sanitize__("undefined")))
 #else  /// It does not work in GCC. GCC 7 cannot recognize this attribute and GCC 8 simply ignores it.
 #    define NO_SANITIZE_UNDEFINED
 #    define NO_SANITIZE_ADDRESS
 #    define NO_SANITIZE_THREAD
+#    define ALWAYS_INLINE_NO_SANITIZE_UNDEFINED ALWAYS_INLINE
 #endif
 
-#if defined __GNUC__ && !defined __clang__
+/// A template function for suppressing warnings about unused variables or function results.
-#    define OPTIMIZE(x) __attribute__((__optimize__(x)))
+template <typename... Args>
-#else
+constexpr void UNUSED(Args &&... args [[maybe_unused]])
-#    define OPTIMIZE(x)
+{
-#endif
+}
-
-/// A macro for suppressing warnings about unused variables or function results.
-/// Useful for structured bindings which have no standard way to declare this.
-#define UNUSED(...) (void)(__VA_ARGS__)

base/common/extended_types.h

@@ -5,16 +5,14 @@
 #include <common/types.h>
 #include <common/wide_integer.h>
 
-using Int128 = __int128;
 
-using wInt256 = wide::integer<256, signed>;
+using Int128 = wide::integer<128, signed>;
-using wUInt256 = wide::integer<256, unsigned>;
+using UInt128 = wide::integer<128, unsigned>;
+using Int256 = wide::integer<256, signed>;
+using UInt256 = wide::integer<256, unsigned>;
 
-static_assert(sizeof(wInt256) == 32);
+static_assert(sizeof(Int256) == 32);
-static_assert(sizeof(wUInt256) == 32);
+static_assert(sizeof(UInt256) == 32);
-
-static constexpr __int128 minInt128() { return static_cast<unsigned __int128>(1) << 127; }
-static constexpr __int128 maxInt128() { return (static_cast<unsigned __int128>(1) << 127) - 1; }
 
 /// The standard library type traits, such as std::is_arithmetic, with one exception
 /// (std::common_type), are "set in stone". Attempting to specialize them causes undefined behavior.
@@ -26,7 +24,7 @@ struct is_signed
 };
 
 template <> struct is_signed<Int128> { static constexpr bool value = true; };
-template <> struct is_signed<wInt256> { static constexpr bool value = true; };
+template <> struct is_signed<Int256> { static constexpr bool value = true; };
 
 template <typename T>
 inline constexpr bool is_signed_v = is_signed<T>::value;
@@ -37,7 +35,8 @@ struct is_unsigned
     static constexpr bool value = std::is_unsigned_v<T>;
 };
 
-template <> struct is_unsigned<wUInt256> { static constexpr bool value = true; };
+template <> struct is_unsigned<UInt128> { static constexpr bool value = true; };
+template <> struct is_unsigned<UInt256> { static constexpr bool value = true; };
 
 template <typename T>
 inline constexpr bool is_unsigned_v = is_unsigned<T>::value;
@@ -51,8 +50,9 @@ struct is_integer
 };
 
 template <> struct is_integer<Int128> { static constexpr bool value = true; };
-template <> struct is_integer<wInt256> { static constexpr bool value = true; };
+template <> struct is_integer<UInt128> { static constexpr bool value = true; };
-template <> struct is_integer<wUInt256> { static constexpr bool value = true; };
+template <> struct is_integer<Int256> { static constexpr bool value = true; };
+template <> struct is_integer<UInt256> { static constexpr bool value = true; };
 
 template <typename T>
 inline constexpr bool is_integer_v = is_integer<T>::value;
@@ -64,7 +64,11 @@ struct is_arithmetic
     static constexpr bool value = std::is_arithmetic_v<T>;
 };
 
-template <> struct is_arithmetic<__int128> { static constexpr bool value = true; };
+template <> struct is_arithmetic<Int128> { static constexpr bool value = true; };
+template <> struct is_arithmetic<UInt128> { static constexpr bool value = true; };
+template <> struct is_arithmetic<Int256> { static constexpr bool value = true; };
+template <> struct is_arithmetic<UInt256> { static constexpr bool value = true; };
+
 
 template <typename T>
 inline constexpr bool is_arithmetic_v = is_arithmetic<T>::value;
@@ -75,9 +79,10 @@ struct make_unsigned
     typedef std::make_unsigned_t<T> type;
 };
 
-template <> struct make_unsigned<Int128> { using type = unsigned __int128; };
+template <> struct make_unsigned<Int128> { using type = UInt128; };
-template <> struct make_unsigned<wInt256>  { using type = wUInt256; };
+template <> struct make_unsigned<UInt128> { using type = UInt128; };
-template <> struct make_unsigned<wUInt256> { using type = wUInt256; };
+template <> struct make_unsigned<Int256>  { using type = UInt256; };
+template <> struct make_unsigned<UInt256> { using type = UInt256; };
 
 template <typename T> using make_unsigned_t = typename make_unsigned<T>::type;
 
@@ -87,8 +92,10 @@ struct make_signed
     typedef std::make_signed_t<T> type;
 };
 
-template <> struct make_signed<wInt256>  { using type = wInt256; };
+template <> struct make_signed<Int128>  { using type = Int128; };
-template <> struct make_signed<wUInt256> { using type = wInt256; };
+template <> struct make_signed<UInt128> { using type = Int128; };
+template <> struct make_signed<Int256>  { using type = Int256; };
+template <> struct make_signed<UInt256> { using type = Int256; };
 
 template <typename T> using make_signed_t = typename make_signed<T>::type;
 
@@ -98,14 +105,11 @@ struct is_big_int
     static constexpr bool value = false;
 };
 
-template <> struct is_big_int<wInt256> { static constexpr bool value = true; };
+template <> struct is_big_int<Int128> { static constexpr bool value = true; };
-template <> struct is_big_int<wUInt256> { static constexpr bool value = true; };
+template <> struct is_big_int<UInt128> { static constexpr bool value = true; };
+template <> struct is_big_int<Int256> { static constexpr bool value = true; };
+template <> struct is_big_int<UInt256> { static constexpr bool value = true; };
 
 template <typename T>
 inline constexpr bool is_big_int_v = is_big_int<T>::value;
 
-template <typename To, typename From>
-inline To bigint_cast(const From & x [[maybe_unused]])
-{
-    return static_cast<To>(x);
-}

base/common/getMemoryAmount.cpp

@@ -1,100 +1,29 @@
 #include <stdexcept>
 #include "common/getMemoryAmount.h"
+#include "common/getPageSize.h"
 
-// http://nadeausoftware.com/articles/2012/09/c_c_tip_how_get_physical_memory_size_system
-
-/*
- * Author:  David Robert Nadeau
- * Site:    http://NadeauSoftware.com/
- * License: Creative Commons Attribution 3.0 Unported License
- *          http://creativecommons.org/licenses/by/3.0/deed.en_US
- */
-
-#if defined(WIN32) || defined(_WIN32)
-#include <Windows.h>
-#else
 #include <unistd.h>
 #include <sys/types.h>
 #include <sys/param.h>
 #if defined(BSD)
 #include <sys/sysctl.h>
 #endif
-#endif
 
 
-/**
+/** Returns the size of physical memory (RAM) in bytes.
- * Returns the size of physical memory (RAM) in bytes.
+  * Returns 0 on unsupported platform
- * Returns 0 on unsupported platform
+  */
- */
 uint64_t getMemoryAmountOrZero()
 {
-#if defined(_WIN32) && (defined(__CYGWIN__) || defined(__CYGWIN32__))
+    int64_t num_pages = sysconf(_SC_PHYS_PAGES);
-    /* Cygwin under Windows. ------------------------------------ */
+    if (num_pages <= 0)
-    /* New 64-bit MEMORYSTATUSEX isn't available.  Use old 32.bit */
+        return 0;
-    MEMORYSTATUS status;
-    status.dwLength = sizeof(status);
-    GlobalMemoryStatus(&status);
-    return status.dwTotalPhys;
 
-#elif defined(WIN32) || defined(_WIN32)
+    int64_t page_size = getPageSize();
-    /* Windows. ------------------------------------------------- */
+    if (page_size <= 0)
-    /* Use new 64-bit MEMORYSTATUSEX, not old 32-bit MEMORYSTATUS */
+        return 0;
-    MEMORYSTATUSEX status;
-    status.dwLength = sizeof(status);
-    GlobalMemoryStatusEx(&status);
-    return status.ullTotalPhys;
 
-#else
+    return num_pages * page_size;
-    /* UNIX variants. ------------------------------------------- */
-    /* Prefer sysctl() over sysconf() except sysctl() HW_REALMEM and HW_PHYSMEM */
-
-#if defined(CTL_HW) && (defined(HW_MEMSIZE) || defined(HW_PHYSMEM64))
-    int mib[2];
-    mib[0] = CTL_HW;
-#if defined(HW_MEMSIZE)
-    mib[1] = HW_MEMSIZE; /* OSX. --------------------- */
-#elif defined(HW_PHYSMEM64)
-    mib[1] = HW_PHYSMEM64; /* NetBSD, OpenBSD. --------- */
-#endif
-    uint64_t size = 0; /* 64-bit */
-    size_t len = sizeof(size);
-    if (sysctl(mib, 2, &size, &len, nullptr, 0) == 0)
-        return size;
-
-    return 0; /* Failed? */
-
-#elif defined(_SC_AIX_REALMEM)
-    /* AIX. ----------------------------------------------------- */
-    return sysconf(_SC_AIX_REALMEM) * 1024;
-
-#elif defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE)
-    /* FreeBSD, Linux, OpenBSD, and Solaris. -------------------- */
-    return uint64_t(sysconf(_SC_PHYS_PAGES))
-           *uint64_t(sysconf(_SC_PAGESIZE));
-
-#elif defined(_SC_PHYS_PAGES) && defined(_SC_PAGE_SIZE)
-    /* Legacy. -------------------------------------------------- */
-    return uint64_t(sysconf(_SC_PHYS_PAGES))
-           * uint64_t(sysconf(_SC_PAGE_SIZE));
-
-#elif defined(CTL_HW) && (defined(HW_PHYSMEM) || defined(HW_REALMEM))
-    /* DragonFly BSD, FreeBSD, NetBSD, OpenBSD, and OSX. -------- */
-    int mib[2];
-    mib[0] = CTL_HW;
-#if defined(HW_REALMEM)
-    mib[1] = HW_REALMEM; /* FreeBSD. ----------------- */
-#elif defined(HW_PYSMEM)
-    mib[1] = HW_PHYSMEM; /* Others. ------------------ */
-#endif
-    unsigned int size = 0; /* 32-bit */
-    size_t len = sizeof(size);
-    if (sysctl(mib, 2, &size, &len, nullptr, 0) == 0)
-        return size;
-
-    return 0; /* Failed? */
-#endif /* sysctl and sysconf variants */
-
-#endif
 }
 
 

base/common/getPageSize.cpp

@@ -0,0 +1,8 @@
+#include "common/getPageSize.h"
+
+#include <unistd.h>
+
+Int64 getPageSize()
+{
+    return sysconf(_SC_PAGESIZE);
+}

base/common/getPageSize.h

@@ -0,0 +1,6 @@
+#pragma once
+
+#include "common/types.h"
+
+/// Get memory page size
+Int64 getPageSize();

base/common/getResource.cpp

@@ -4,23 +4,42 @@
 #include <string>
 #include <boost/algorithm/string/replace.hpp>
 
-
 std::string_view getResource(std::string_view name)
 {
+    // Convert the resource file name into the form generated by `ld -r -b binary`.
     std::string name_replaced(name);
     std::replace(name_replaced.begin(), name_replaced.end(), '/', '_');
     std::replace(name_replaced.begin(), name_replaced.end(), '-', '_');
     std::replace(name_replaced.begin(), name_replaced.end(), '.', '_');
     boost::replace_all(name_replaced, "+", "_PLUS_");
 
-    /// These are the names that are generated by "ld -r -b binary"
+    // In most `dlsym(3)` APIs, one passes the symbol name as it appears via
-    std::string symbol_name_data = "_binary_" + name_replaced + "_start";
+    // something like `nm` or `objdump -t`. For example, a symbol `_foo` would be
-    std::string symbol_name_size = "_binary_" + name_replaced + "_size";
+    // looked up with the string `"_foo"`.
+    //
+    // Apple's linker is confusingly different. The NOTES on the man page for
+    // `dlsym(3)` claim that one looks up the symbol with "the name used in C
+    // source code". In this example, that would mean using the string `"foo"`.
+    // This apparently applies even in the case where the symbol did not originate
+    // from C source, such as the embedded binary resource files used here. So
+    // the symbol name must not have a leading `_` on Apple platforms. It's not
+    // clear how this applies to other symbols, such as those which _have_ a leading
+    // underscore in them by design, many leading underscores, etc.
+#if defined OS_DARWIN
+    std::string prefix = "binary_";
+#else
+    std::string prefix = "_binary_";
+#endif
+    std::string symbol_name_start = prefix + name_replaced + "_start";
+    std::string symbol_name_end = prefix + name_replaced + "_end";
 
-    const void * sym_data = dlsym(RTLD_DEFAULT, symbol_name_data.c_str());
+    const char* sym_start = reinterpret_cast<const char*>(dlsym(RTLD_DEFAULT, symbol_name_start.c_str()));
-    const void * sym_size = dlsym(RTLD_DEFAULT, symbol_name_size.c_str());
+    const char* sym_end = reinterpret_cast<const char*>(dlsym(RTLD_DEFAULT, symbol_name_end.c_str()));
 
-    if (sym_data && sym_size)
+    if (sym_start && sym_end)
-        return { static_cast<const char *>(sym_data), unalignedLoad<size_t>(&sym_size) };
+    {
+        auto resource_size = static_cast<size_t>(std::distance(sym_start, sym_end));
+        return { sym_start, resource_size };
+    }
     return {};
 }

base/common/getThreadId.cpp

@@ -25,6 +25,10 @@ uint64_t getThreadId()
         current_tid = syscall(SYS_gettid); /// This call is always successful. - man gettid
 #elif defined(OS_FREEBSD)
         current_tid = pthread_getthreadid_np();
+#elif defined(OS_SUNOS)
+        // On Solaris-derived systems, this returns the ID of the LWP, analogous
+        // to a thread.
+        current_tid = static_cast<uint64_t>(pthread_self());
 #else
         if (0 != pthread_threadid_np(nullptr, &current_tid))
             throw std::logic_error("pthread_threadid_np returned error");

base/common/insertAtEnd.h

@@ -0,0 +1,28 @@
+#pragma once
+
+#include <vector>
+
+/// Appends a specified vector with elements of another vector.
+template <typename T>
+void insertAtEnd(std::vector<T> & dest, const std::vector<T> & src)
+{
+    if (src.empty())
+        return;
+    dest.reserve(dest.size() + src.size());
+    dest.insert(dest.end(), src.begin(), src.end());
+}
+
+template <typename T>
+void insertAtEnd(std::vector<T> & dest, std::vector<T> && src)
+{
+    if (src.empty())
+        return;
+    if (dest.empty())
+    {
+        dest.swap(src);
+        return;
+    }
+    dest.reserve(dest.size() + src.size());
+    dest.insert(dest.end(), std::make_move_iterator(src.begin()), std::make_move_iterator(src.end()));
+    src.clear();
+}

base/common/itoa.h

@@ -30,9 +30,8 @@
 #include <cstddef>
 #include <cstring>
 #include <type_traits>
+#include <common/extended_types.h>
 
-using int128_t = __int128;
-using uint128_t = unsigned __int128;
 
 namespace impl
 {
@@ -106,7 +105,7 @@ using UnsignedOfSize = typename SelectType
     uint16_t,
     uint32_t,
     uint64_t,
-    uint128_t
+    __uint128_t
 >::Result;
 
 /// Holds the result of dividing an unsigned N-byte variable by 10^N resulting in
@@ -313,7 +312,8 @@ namespace convert
     }
 }
 
-static inline int digits10(uint128_t x)
+template <typename T>
+static inline int digits10(T x)
 {
     if (x < 10ULL)
         return 1;
@@ -346,8 +346,11 @@ static inline int digits10(uint128_t x)
     return 12 + digits10(x / 1000000000000ULL);
 }
 
-static inline char * writeUIntText(uint128_t x, char * p)
+template <typename T>
+static inline char * writeUIntText(T x, char * p)
 {
+    static_assert(is_unsigned_v<T>);
+
     int len = digits10(x);
     auto pp = p + len;
     while (x >= 100)
@@ -370,14 +373,28 @@ static inline char * writeLeadingMinus(char * pos)
     return pos + 1;
 }
 
-static inline char * writeSIntText(int128_t x, char * pos)
+template <typename T>
+static inline char * writeSIntText(T x, char * pos)
 {
-    static constexpr int128_t min_int128 = uint128_t(1) << 127;
+    static_assert(std::is_same_v<T, Int128> || std::is_same_v<T, Int256>);
 
-    if (unlikely(x == min_int128))
+    using UnsignedT = make_unsigned_t<T>;
+    static constexpr T min_int = UnsignedT(1) << (sizeof(T) * 8 - 1);
+
+    if (unlikely(x == min_int))
     {
-        memcpy(pos, "-170141183460469231731687303715884105728", 40);
+        if constexpr (std::is_same_v<T, Int128>)
-        return pos + 40;
+        {
+            const char * res = "-170141183460469231731687303715884105728";
+            memcpy(pos, res, strlen(res));
+            return pos + strlen(res);
+        }
+        else if constexpr (std::is_same_v<T, Int256>)
+        {
+            const char * res = "-57896044618658097711785492504343953926634992332820282019728792003956564819968";
+            memcpy(pos, res, strlen(res));
+            return pos + strlen(res);
+        }
     }
 
     if (x < 0)
@@ -385,7 +402,7 @@ static inline char * writeSIntText(int128_t x, char * pos)
         x = -x;
         pos = writeLeadingMinus(pos);
     }
-    return writeUIntText(static_cast<uint128_t>(x), pos);
+    return writeUIntText(UnsignedT(x), pos);
 }
 
 }
@@ -403,13 +420,25 @@ inline char * itoa(char8_t i, char * p)
 }
 
 template <>
-inline char * itoa<uint128_t>(uint128_t i, char * p)
+inline char * itoa(UInt128 i, char * p)
 {
     return impl::writeUIntText(i, p);
 }
 
 template <>
-inline char * itoa<int128_t>(int128_t i, char * p)
+inline char * itoa(Int128 i, char * p)
+{
+    return impl::writeSIntText(i, p);
+}
+
+template <>
+inline char * itoa(UInt256 i, char * p)
+{
+    return impl::writeUIntText(i, p);
+}
+
+template <>
+inline char * itoa(Int256 i, char * p)
 {
     return impl::writeSIntText(i, p);
 }

base/common/logger_useful.h

@@ -3,7 +3,6 @@
 /// Macros for convenient usage of Poco logger.
 
 #include <fmt/format.h>
-#include <fmt/ostream.h>
 #include <Poco/Logger.h>
 #include <Poco/Message.h>
 #include <Common/CurrentThread.h>

base/common/map.h

@@ -0,0 +1,52 @@
+#pragma once
+
+#include <type_traits>
+#include <boost/iterator/transform_iterator.hpp>
+
+namespace collections
+{
+
+/// \brief Strip type off top level reference and cv-qualifiers thus allowing storage in containers
+template <typename T>
+using unqualified_t = std::remove_cv_t<std::remove_reference_t<T>>;
+
+/** \brief Returns collection of the same container-type as the input collection,
+  *    with each element transformed by the application of `mapper`.
+  */
+template <template <typename...> class Collection, typename... Params, typename Mapper>
+auto map(const Collection<Params...> & collection, Mapper && mapper)
+{
+    using value_type = unqualified_t<decltype(mapper(*std::begin(collection)))>;
+
+    return Collection<value_type>(
+        boost::make_transform_iterator(std::begin(collection), std::forward<Mapper>(mapper)),
+        boost::make_transform_iterator(std::end(collection), std::forward<Mapper>(mapper)));
+}
+
+/** \brief Returns collection of specified container-type,
+  *    with each element transformed by the application of `mapper`.
+  *    Allows conversion between different container-types, e.g. std::vector to std::list
+  */
+template <template <typename...> class ResultCollection, typename Collection, typename Mapper>
+auto map(const Collection & collection, Mapper && mapper)
+{
+    using value_type = unqualified_t<decltype(mapper(*std::begin(collection)))>;
+
+    return ResultCollection<value_type>(
+        boost::make_transform_iterator(std::begin(collection), std::forward<Mapper>(mapper)),
+        boost::make_transform_iterator(std::end(collection), std::forward<Mapper>(mapper)));
+}
+
+/** \brief Returns collection of specified type,
+  *    with each element transformed by the application of `mapper`.
+  *    Allows leveraging implicit conversion between the result of applying `mapper` and R::value_type.
+  */
+template <typename ResultCollection, typename Collection, typename Mapper>
+auto map(const Collection & collection, Mapper && mapper)
+{
+    return ResultCollection(
+        boost::make_transform_iterator(std::begin(collection), std::forward<Mapper>(mapper)),
+        boost::make_transform_iterator(std::end(collection), std::forward<Mapper>(mapper)));
+}
+
+}

base/common/memory.h

@@ -1,57 +0,0 @@
-#pragma once
-
-#include <new>
-#include "defines.h"
-
-#if USE_JEMALLOC
-#    include <jemalloc/jemalloc.h>
-#endif
-
-#if !USE_JEMALLOC || JEMALLOC_VERSION_MAJOR < 4
-#    include <cstdlib>
-#endif
-
-
-namespace Memory
-{
-
-inline ALWAYS_INLINE void * newImpl(std::size_t size)
-{
-    auto * ptr = malloc(size);
-    if (likely(ptr != nullptr))
-        return ptr;
-
-    /// @note no std::get_new_handler logic implemented
-    throw std::bad_alloc{};
-}
-
-inline ALWAYS_INLINE void * newNoExept(std::size_t size) noexcept
-{
-    return malloc(size);
-}
-
-inline ALWAYS_INLINE void deleteImpl(void * ptr) noexcept
-{
-    free(ptr);
-}
-
-#if USE_JEMALLOC && JEMALLOC_VERSION_MAJOR >= 4
-
-inline ALWAYS_INLINE void deleteSized(void * ptr, std::size_t size) noexcept
-{
-    if (unlikely(ptr == nullptr))
-        return;
-
-    sdallocx(ptr, size, 0);
-}
-
-#else
-
-inline ALWAYS_INLINE void deleteSized(void * ptr, std::size_t size [[maybe_unused]]) noexcept
-{
-    free(ptr);
-}
-
-#endif
-
-}

base/common/phdr_cache.cpp

@@ -15,11 +15,11 @@
 #endif
 
 #define __msan_unpoison(X, Y) // NOLINT
-#if defined(__has_feature)
+#if defined(ch_has_feature)
-#   if __has_feature(memory_sanitizer)
+#    if ch_has_feature(memory_sanitizer)
-#       undef __msan_unpoison
+#        undef __msan_unpoison
-#       include <sanitizer/msan_interface.h>
+#        include <sanitizer/msan_interface.h>
-#   endif
+#    endif
 #endif
 
 #include <link.h>

base/common/range.h

@@ -4,9 +4,9 @@
 #include <boost/range/adaptor/transformed.hpp>
 #include <type_traits>
 
-
+namespace collections
-namespace ext
 {
+
 namespace internal
 {
     template <typename ResultType, typename CountingType, typename BeginType, typename EndType>
@@ -24,11 +24,11 @@ namespace internal
 /// For loop adaptor which is used to iterate through a half-closed interval [begin, end).
 /// The parameters `begin` and `end` can have any integral or enum types.
 template <typename BeginType,
-          typename EndType,
+        typename EndType,
-          typename = std::enable_if_t<
+        typename = std::enable_if_t<
-              (std::is_integral_v<BeginType> || std::is_enum_v<BeginType>) &&
+            (std::is_integral_v<BeginType> || std::is_enum_v<BeginType>) &&
-              (std::is_integral_v<EndType> || std::is_enum_v<EndType>) &&
+            (std::is_integral_v<EndType> || std::is_enum_v<EndType>) &&
-              (!std::is_enum_v<BeginType> || !std::is_enum_v<EndType> || std::is_same_v<BeginType, EndType>), void>>
+            (!std::is_enum_v<BeginType> || !std::is_enum_v<EndType> || std::is_same_v<BeginType, EndType>), void>>
 inline auto range(BeginType begin, EndType end)
 {
     if constexpr (std::is_integral_v<BeginType> && std::is_integral_v<EndType>)
@@ -51,7 +51,7 @@ inline auto range(BeginType begin, EndType end)
 /// The parameter `end` can have any integral or enum type.
 /// The same as range(0, end).
 template <typename Type,
-          typename = std::enable_if_t<std::is_integral_v<Type> || std::is_enum_v<Type>, void>>
+        typename = std::enable_if_t<std::is_integral_v<Type> || std::is_enum_v<Type>, void>>
 inline auto range(Type end)
 {
     if constexpr (std::is_integral_v<Type>)
@@ -59,4 +59,5 @@ inline auto range(Type end)
     else
         return internal::rangeImpl<Type, std::underlying_type_t<Type>>(0, end);
 }
+
 }

base/common/removeDuplicates.h

@@ -0,0 +1,24 @@
+#pragma once
+#include <vector>
+
+/// Removes duplicates from a container without changing the order of its elements.
+/// Keeps the last occurrence of each element.
+/// Should NOT be used for containers with a lot of elements because it has O(N^2) complexity.
+template <typename T>
+void removeDuplicatesKeepLast(std::vector<T> & vec)
+{
+    auto begin = vec.begin();
+    auto end = vec.end();
+    auto new_begin = end;
+    for (auto current = end; current != begin;)
+    {
+        --current;
+        if (std::find(new_begin, end, *current) == end)
+        {
+            --new_begin;
+            if (new_begin != current)
+                *new_begin = *current;
+        }
+    }
+    vec.erase(begin, new_begin);
+}

base/common/scope_guard.h

@@ -4,9 +4,6 @@
 #include <memory>
 #include <utility>
 
-
-namespace ext
-{
 template <class F>
 class [[nodiscard]] basic_scope_guard
 {
@@ -105,10 +102,9 @@ using scope_guard = basic_scope_guard<std::function<void(void)>>;
 
 template <class F>
 inline basic_scope_guard<F> make_scope_guard(F && function_) { return std::forward<F>(function_); }
-}
 
 #define SCOPE_EXIT_CONCAT(n, ...) \
-const auto scope_exit##n = ext::make_scope_guard([&] { __VA_ARGS__; })
+const auto scope_exit##n = make_scope_guard([&] { __VA_ARGS__; })
 #define SCOPE_EXIT_FWD(n, ...) SCOPE_EXIT_CONCAT(n, __VA_ARGS__)
 #define SCOPE_EXIT(...) SCOPE_EXIT_FWD(__LINE__, __VA_ARGS__)
 

base/common/scope_guard_safe.h

@@ -0,0 +1,68 @@
+#pragma once
+
+#include <common/scope_guard.h>
+#include <common/logger_useful.h>
+#include <Common/MemoryTracker.h>
+
+/// Same as SCOPE_EXIT() but block the MEMORY_LIMIT_EXCEEDED errors.
+///
+/// Typical example of SCOPE_EXIT_MEMORY() usage is when code under it may do
+/// some tiny allocations, that may fail under high memory pressure or/and low
+/// max_memory_usage (and related limits).
+///
+/// NOTE: it should be used with caution.
+#define SCOPE_EXIT_MEMORY(...) SCOPE_EXIT(                    \
+    MemoryTracker::LockExceptionInThread                      \
+                lock_memory_tracker(VariableContext::Global); \
+    __VA_ARGS__;                                              \
+)
+
+/// Same as SCOPE_EXIT() but try/catch/tryLogCurrentException any exceptions.
+///
+/// SCOPE_EXIT_SAFE() should be used in case the exception during the code
+/// under SCOPE_EXIT() is not "that fatal" and error message in log is enough.
+///
+/// Good example is calling CurrentThread::detachQueryIfNotDetached().
+///
+/// Anti-pattern is calling WriteBuffer::finalize() under SCOPE_EXIT_SAFE()
+/// (since finalize() can do final write and it is better to fail abnormally
+/// instead of ignoring write error).
+///
+/// NOTE: it should be used with double caution.
+#define SCOPE_EXIT_SAFE(...) SCOPE_EXIT(             \
+    try                                              \
+    {                                                \
+        __VA_ARGS__;                                 \
+    }                                                \
+    catch (...)                                      \
+    {                                                \
+        tryLogCurrentException(__PRETTY_FUNCTION__); \
+    }                                                \
+)
+
+/// Same as SCOPE_EXIT() but:
+/// - block the MEMORY_LIMIT_EXCEEDED errors,
+/// - try/catch/tryLogCurrentException any exceptions.
+///
+/// SCOPE_EXIT_MEMORY_SAFE() can be used when the error can be ignored, and in
+/// addition to SCOPE_EXIT_SAFE() it will also lock MEMORY_LIMIT_EXCEEDED to
+/// avoid such exceptions.
+///
+/// It does exists as a separate helper, since you do not need to lock
+/// MEMORY_LIMIT_EXCEEDED always (there are cases when code under SCOPE_EXIT does
+/// not do any allocations, while LockExceptionInThread increment atomic
+/// variable).
+///
+/// NOTE: it should be used with triple caution.
+#define SCOPE_EXIT_MEMORY_SAFE(...) SCOPE_EXIT(                   \
+    try                                                           \
+    {                                                             \
+        MemoryTracker::LockExceptionInThread                      \
+                    lock_memory_tracker(VariableContext::Global); \
+        __VA_ARGS__;                                              \
+    }                                                             \
+    catch (...)                                                   \
+    {                                                             \
+        tryLogCurrentException(__PRETTY_FUNCTION__);              \
+    }                                                             \
+)

base/common/setTerminalEcho.cpp

@@ -1,44 +1,28 @@
-// https://stackoverflow.com/questions/1413445/reading-a-password-from-stdcin
-
 #include <common/setTerminalEcho.h>
+#include <common/errnoToString.h>
 #include <stdexcept>
 #include <cstring>
 #include <string>
-
-#ifdef WIN32
-#include <windows.h>
-#else
 #include <termios.h>
 #include <unistd.h>
-#include <errno.h>
+
-#endif
 
 void setTerminalEcho(bool enable)
 {
-#ifdef WIN32
+    /// Obtain terminal attributes,
-    auto handle = GetStdHandle(STD_INPUT_HANDLE);
+    /// toggle the ECHO flag
-    DWORD mode;
+    /// and set them back.
-    if (!GetConsoleMode(handle, &mode))
-        throw std::runtime_error(std::string("setTerminalEcho failed get: ") + std::to_string(GetLastError()));
 
-    if (!enable)
+    struct termios tty{};
-        mode &= ~ENABLE_ECHO_INPUT;
-    else
-        mode |= ENABLE_ECHO_INPUT;
 
-    if (!SetConsoleMode(handle, mode))
+    if (0 != tcgetattr(STDIN_FILENO, &tty))
-        throw std::runtime_error(std::string("setTerminalEcho failed set: ") + std::to_string(GetLastError()));
+        throw std::runtime_error(std::string("setTerminalEcho failed get: ") + errnoToString(errno));
-#else
+
-    struct termios tty;
+    if (enable)
-    if (tcgetattr(STDIN_FILENO, &tty))
-        throw std::runtime_error(std::string("setTerminalEcho failed get: ") + strerror(errno));
-    if (!enable)
-        tty.c_lflag &= ~ECHO;
-    else
         tty.c_lflag |= ECHO;
+    else
+        tty.c_lflag &= ~ECHO;
 
-    auto ret = tcsetattr(STDIN_FILENO, TCSANOW, &tty);
+    if (0 != tcsetattr(STDIN_FILENO, TCSANOW, &tty))
-    if (ret)
+        throw std::runtime_error(std::string("setTerminalEcho failed set: ") + errnoToString(errno));
-        throw std::runtime_error(std::string("setTerminalEcho failed set: ") + strerror(errno));
-#endif
 }

base/common/shared_ptr_helper.h

@@ -2,8 +2,6 @@
 
 #include <memory>
 
-namespace ext
-{
 
 /** Allows to make std::shared_ptr from T with protected constructor.
   *
@@ -36,4 +34,3 @@ struct is_shared_ptr<std::shared_ptr<T>>
 
 template <typename T>
 inline constexpr bool is_shared_ptr_v = is_shared_ptr<T>::value;
-}

base/common/sort.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#if !defined(ARCADIA_BUILD)
+#    include <miniselect/floyd_rivest_select.h>  // Y_IGNORE
+#else
+#    include <algorithm>
+#endif
+
+template <class RandomIt>
+void nth_element(RandomIt first, RandomIt nth, RandomIt last)
+{
+#if !defined(ARCADIA_BUILD)
+    ::miniselect::floyd_rivest_select(first, nth, last);
+#else
+    ::std::nth_element(first, nth, last);
+#endif
+}
+
+template <class RandomIt>
+void partial_sort(RandomIt first, RandomIt middle, RandomIt last)
+{
+#if !defined(ARCADIA_BUILD)
+    ::miniselect::floyd_rivest_partial_sort(first, middle, last);
+#else
+    ::std::partial_sort(first, middle, last);
+#endif
+}
+
+template <class RandomIt, class Compare>
+void partial_sort(RandomIt first, RandomIt middle, RandomIt last, Compare compare)
+{
+#if !defined(ARCADIA_BUILD)
+    ::miniselect::floyd_rivest_partial_sort(first, middle, last, compare);
+#else
+    ::std::partial_sort(first, middle, last, compare);
+#endif
+}

base/common/strong_typedef.h

@@ -4,7 +4,8 @@
 #include <type_traits>
 #include <utility>
 
-template <class T, class Tag>
+
+template <typename T, typename Tag>
 struct StrongTypedef
 {
 private:
@@ -12,6 +13,7 @@ private:
     T t;
 
 public:
+    using UnderlyingType = T;
     template <class Enable = typename std::is_copy_constructible<T>::type>
     explicit StrongTypedef(const T & t_) : t(t_) {}
     template <class Enable = typename std::is_move_constructible<T>::type>
@@ -37,14 +39,16 @@ public:
 
     bool operator==(const Self & rhs) const { return t == rhs.t; }
     bool operator<(const Self & rhs) const { return t < rhs.t; }
+    bool operator>(const Self & rhs) const { return t > rhs.t; }
 
     T & toUnderType() { return t; }
     const T & toUnderType() const { return t; }
 };
 
+
 namespace std
 {
-    template <class T, class Tag>
+    template <typename T, typename Tag>
     struct hash<StrongTypedef<T, Tag>>
     {
         size_t operator()(const StrongTypedef<T, Tag> & x) const

base/common/tests/CMakeLists.txt

@@ -1,25 +1,2 @@
-include (${ClickHouse_SOURCE_DIR}/cmake/add_check.cmake)
-
-add_executable (date_lut2 date_lut2.cpp)
-add_executable (date_lut3 date_lut3.cpp)
-add_executable (date_lut_default_timezone date_lut_default_timezone.cpp)
-add_executable (local_date_time_comparison local_date_time_comparison.cpp)
-add_executable (realloc-perf allocator.cpp)
-
-set(PLATFORM_LIBS ${CMAKE_DL_LIBS})
-
-target_link_libraries (date_lut2 PRIVATE common ${PLATFORM_LIBS})
-target_link_libraries (date_lut3 PRIVATE common ${PLATFORM_LIBS})
-target_link_libraries (date_lut_default_timezone PRIVATE common ${PLATFORM_LIBS})
-target_link_libraries (local_date_time_comparison PRIVATE common)
-target_link_libraries (realloc-perf PRIVATE common)
-add_check(local_date_time_comparison)
-
-if(USE_GTEST)
-    add_executable(unit_tests_libcommon gtest_json_test.cpp gtest_strong_typedef.cpp gtest_find_symbols.cpp)
-    target_link_libraries(unit_tests_libcommon PRIVATE common ${GTEST_MAIN_LIBRARIES} ${GTEST_LIBRARIES})
-    add_check(unit_tests_libcommon)
-endif()
-
 add_executable (dump_variable dump_variable.cpp)
 target_link_libraries (dump_variable PRIVATE clickhouse_common_io)

base/common/tests/allocator.cpp

@@ -1,47 +0,0 @@
-#include <cstdlib>
-#include <cstring>
-#include <vector>
-#include <thread>
-
-
-void thread_func()
-{
-    for (size_t i = 0; i < 100; ++i)
-    {
-        size_t size = 4096;
-
-        void * buf = malloc(size);
-        if (!buf)
-            abort();
-        memset(buf, 0, size);
-
-        while (size < 1048576)
-        {
-            size_t next_size = size * 4;
-
-            void * new_buf = realloc(buf, next_size);
-            if (!new_buf)
-                abort();
-            buf = new_buf;
-
-            memset(reinterpret_cast<char*>(buf) + size, 0, next_size - size);
-            size = next_size;
-        }
-
-        free(buf);
-    }
-}
-
-
-int main(int, char **)
-{
-    std::vector<std::thread> threads(16);
-    for (size_t i = 0; i < 1000; ++i)
-    {
-        for (auto & thread : threads)
-            thread = std::thread(thread_func);
-        for (auto & thread : threads)
-            thread.join();
-    }
-    return 0;
-}

base/common/tests/date_lut2.cpp

@@ -1,53 +0,0 @@
-#include <iostream>
-#include <cstring>
-
-#include <common/DateLUT.h>
-
-
-static std::string toString(time_t Value)
-{
-    struct tm tm;
-    char buf[96];
-
-    localtime_r(&Value, &tm);
-    snprintf(buf, sizeof(buf), "%04d-%02d-%02d %02d:%02d:%02d",
-             tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec);
-
-    return buf;
-}
-
-static time_t orderedIdentifierToDate(unsigned value)
-{
-    struct tm tm;
-
-    memset(&tm, 0, sizeof(tm));
-
-    tm.tm_year = value / 10000 - 1900;
-    tm.tm_mon = (value % 10000) / 100 - 1;
-    tm.tm_mday = value % 100;
-    tm.tm_isdst = -1;
-
-    return mktime(&tm);
-}
-
-
-void loop(time_t begin, time_t end, int step)
-{
-    const auto & date_lut = DateLUT::instance();
-
-    for (time_t t = begin; t < end; t += step)
-        std::cout << toString(t)
-            << ", " << toString(date_lut.toTime(t))
-            << ", " << date_lut.toHour(t)
-            << std::endl;
-}
-
-
-int main(int, char **)
-{
-    loop(orderedIdentifierToDate(20101031), orderedIdentifierToDate(20101101), 15 * 60);
-    loop(orderedIdentifierToDate(20100328), orderedIdentifierToDate(20100330), 15 * 60);
-    loop(orderedIdentifierToDate(20141020), orderedIdentifierToDate(20141106), 15 * 60);
-
-    return 0;
-}

base/common/tests/date_lut3.cpp

@@ -1,62 +0,0 @@
-#include <iostream>
-#include <cstring>
-
-#include <Poco/Exception.h>
-
-#include <common/DateLUT.h>
-
-
-static std::string toString(time_t Value)
-{
-    struct tm tm;
-    char buf[96];
-
-    localtime_r(&Value, &tm);
-    snprintf(buf, sizeof(buf), "%04d-%02d-%02d %02d:%02d:%02d",
-             tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec);
-
-    return buf;
-}
-
-static time_t orderedIdentifierToDate(unsigned value)
-{
-    struct tm tm;
-
-    memset(&tm, 0, sizeof(tm));
-
-    tm.tm_year = value / 10000 - 1900;
-    tm.tm_mon = (value % 10000) / 100 - 1;
-    tm.tm_mday = value % 100;
-    tm.tm_isdst = -1;
-
-    return mktime(&tm);
-}
-
-
-void loop(time_t begin, time_t end, int step)
-{
-    const auto & date_lut = DateLUT::instance();
-
-    for (time_t t = begin; t < end; t += step)
-    {
-        time_t t2 = date_lut.makeDateTime(date_lut.toYear(t), date_lut.toMonth(t), date_lut.toDayOfMonth(t),
-            date_lut.toHour(t), date_lut.toMinute(t), date_lut.toSecond(t));
-
-        std::string s1 = toString(t);
-        std::string s2 = toString(t2);
-
-        std::cerr << s1 << ", " << s2 << std::endl;
-
-        if (s1 != s2)
-            throw Poco::Exception("Test failed.");
-    }
-}
-
-
-int main(int, char **)
-{
-    loop(orderedIdentifierToDate(20101031), orderedIdentifierToDate(20101101), 15 * 60);
-    loop(orderedIdentifierToDate(20100328), orderedIdentifierToDate(20100330), 15 * 60);
-
-    return 0;
-}

base/common/tests/date_lut_default_timezone.cpp

@@ -1,31 +0,0 @@
-#include <iostream>
-#include <common/DateLUT.h>
-#include <Poco/Exception.h>
-
-int main(int, char **)
-{
-    try
-    {
-        const auto & date_lut = DateLUT::instance();
-        std::cout << "Detected default timezone: `" << date_lut.getTimeZone() << "'" << std::endl;
-        time_t now = time(nullptr);
-        std::cout << "Current time: " << date_lut.timeToString(now)
-                  << ", UTC: " << DateLUT::instance("UTC").timeToString(now) << std::endl;
-    }
-    catch (const Poco::Exception & e)
-    {
-        std::cerr << e.displayText() << std::endl;
-        return 1;
-    }
-    catch (std::exception & e)
-    {
-        std::cerr << "std::exception: " << e.what() << std::endl;
-        return 2;
-    }
-    catch (...)
-    {
-        std::cerr << "Some exception" << std::endl;
-        return 3;
-    }
-    return 0;
-}

base/common/tests/gtest_json_test.cpp

@@ -1,656 +0,0 @@
-#include <vector>
-#include <string>
-#include <exception>
-#include <common/JSON.h>
-
-#include <boost/range/irange.hpp>
-
-using namespace std::literals::string_literals;
-
-#include <gtest/gtest.h>
-
-enum class ResultType
-{
-    Return,
-    Throw
-};
-
-struct GetStringTestRecord
-{
-    const char * input;
-    ResultType result_type;
-    const char * result;
-};
-
-TEST(JSONSuite, SimpleTest)
-{
-    std::vector<GetStringTestRecord> test_data =
-    {
-        { R"("name")", ResultType::Return, "name" },
-        { R"("Вафельница Vitek WX-1102 FL")", ResultType::Return, "Вафельница Vitek WX-1102 FL" },
-        { R"("brand")", ResultType::Return, "brand" },
-        { R"("184509")", ResultType::Return, "184509" },
-        { R"("category")", ResultType::Return, "category" },
-        { R"("Все для детей/Детская техника/Vitek")", ResultType::Return, "Все для детей/Детская техника/Vitek" },
-        { R"("variant")", ResultType::Return, "variant" },
-        { R"("В наличии")", ResultType::Return, "В наличии" },
-        { R"("price")", ResultType::Return, "price" },
-        { R"("2390.00")", ResultType::Return, "2390.00" },
-        { R"("list")", ResultType::Return, "list" },
-        { R"("Карточка")", ResultType::Return, "Карточка" },
-        { R"("position")", ResultType::Return, "position" },
-        { R"("detail")", ResultType::Return, "detail" },
-        { R"("actionField")", ResultType::Return, "actionField" },
-        { R"("list")", ResultType::Return, "list" },
-        { R"("http://www.techport.ru/q/?t=вафельница&sort=price&sdim=asc")", ResultType::Return, "http://www.techport.ru/q/?t=вафельница&sort=price&sdim=asc" },
-        { R"("action")", ResultType::Return, "action" },
-        { R"("detail")", ResultType::Return, "detail" },
-        { R"("products")", ResultType::Return, "products" },
-        { R"("name")", ResultType::Return, "name" },
-        { R"("Вафельница Vitek WX-1102 FL")", ResultType::Return, "Вафельница Vitek WX-1102 FL" },
-        { R"("id")", ResultType::Return, "id" },
-        { R"("184509")", ResultType::Return, "184509" },
-        { R"("price")", ResultType::Return, "price" },
-        { R"("2390.00")", ResultType::Return, "2390.00" },
-        { R"("brand")", ResultType::Return, "brand" },
-        { R"("Vitek")", ResultType::Return, "Vitek" },
-        { R"("category")", ResultType::Return, "category" },
-        { R"("Все для детей/Детская техника/Vitek")", ResultType::Return, "Все для детей/Детская техника/Vitek" },
-        { R"("variant")", ResultType::Return, "variant" },
-        { R"("В наличии")", ResultType::Return, "В наличии" },
-        { R"("ru")", ResultType::Return, "ru" },
-        { R"("experiments")", ResultType::Return, "experiments" },
-        { R"("lang")", ResultType::Return, "lang" },
-        { R"("ru")", ResultType::Return, "ru" },
-        { R"("los_portal")", ResultType::Return, "los_portal" },
-        { R"("los_level")", ResultType::Return, "los_level" },
-        { R"("none")", ResultType::Return, "none" },
-        { R"("isAuthorized")", ResultType::Return, "isAuthorized" },
-        { R"("isSubscriber")", ResultType::Return, "isSubscriber" },
-        { R"("postType")", ResultType::Return, "postType" },
-        { R"("Новости")", ResultType::Return, "Новости" },
-        { R"("experiments")", ResultType::Return, "experiments" },
-        { R"("lang")", ResultType::Return, "lang" },
-        { R"("ru")", ResultType::Return, "ru" },
-        { R"("los_portal")", ResultType::Return, "los_portal" },
-        { R"("los_level")", ResultType::Return, "los_level" },
-        { R"("none")", ResultType::Return, "none" },
-        { R"("lang")", ResultType::Return, "lang" },
-        { R"("ru")", ResultType::Return, "ru" },
-        { R"("Электроплита GEFEST Брест ЭПНД 5140-01 0001")", ResultType::Return, "Электроплита GEFEST Брест ЭПНД 5140-01 0001" },
-        { R"("price")", ResultType::Return, "price" },
-        { R"("currencyCode")", ResultType::Return, "currencyCode" },
-        { R"("RUB")", ResultType::Return, "RUB" },
-        { R"("lang")", ResultType::Return, "lang" },
-        { R"("ru")", ResultType::Return, "ru" },
-        { R"("experiments")", ResultType::Return, "experiments" },
-        { R"("lang")", ResultType::Return, "lang" },
-        { R"("ru")", ResultType::Return, "ru" },
-        { R"("los_portal")", ResultType::Return, "los_portal" },
-        { R"("los_level")", ResultType::Return, "los_level" },
-        { R"("none")", ResultType::Return, "none" },
-        { R"("trash_login")", ResultType::Return, "trash_login" },
-        { R"("novikoff")", ResultType::Return, "novikoff" },
-        { R"("trash_cat_link")", ResultType::Return, "trash_cat_link" },
-        { R"("progs")", ResultType::Return, "progs" },
-        { R"("trash_parent_link")", ResultType::Return, "trash_parent_link" },
-        { R"("content")", ResultType::Return, "content" },
-        { R"("trash_posted_parent")", ResultType::Return, "trash_posted_parent" },
-        { R"("content.01.2016")", ResultType::Return, "content.01.2016" },
-        { R"("trash_posted_cat")", ResultType::Return, "trash_posted_cat" },
-        { R"("progs.01.2016")", ResultType::Return, "progs.01.2016" },
-        { R"("trash_virus_count")", ResultType::Return, "trash_virus_count" },
-        { R"("trash_is_android")", ResultType::Return, "trash_is_android" },
-        { R"("trash_is_wp8")", ResultType::Return, "trash_is_wp8" },
-        { R"("trash_is_ios")", ResultType::Return, "trash_is_ios" },
-        { R"("trash_posted")", ResultType::Return, "trash_posted" },
-        { R"("01.2016")", ResultType::Return, "01.2016" },
-        { R"("experiments")", ResultType::Return, "experiments" },
-        { R"("lang")", ResultType::Return, "lang" },
-        { R"("ru")", ResultType::Return, "ru" },
-        { R"("los_portal")", ResultType::Return, "los_portal" },
-        { R"("los_level")", ResultType::Return, "los_level" },
-        { R"("none")", ResultType::Return, "none" },
-        { R"("merchantId")", ResultType::Return, "merchantId" },
-        { R"("13694_49246")", ResultType::Return, "13694_49246" },
-        { R"("cps-source")", ResultType::Return, "cps-source" },
-        { R"("wargaming")", ResultType::Return, "wargaming" },
-        { R"("cps_provider")", ResultType::Return, "cps_provider" },
-        { R"("default")", ResultType::Return, "default" },
-        { R"("errorReason")", ResultType::Return, "errorReason" },
-        { R"("no errors")", ResultType::Return, "no errors" },
-        { R"("scid")", ResultType::Return, "scid" },
-        { R"("isAuthPayment")", ResultType::Return, "isAuthPayment" },
-        { R"("lang")", ResultType::Return, "lang" },
-        { R"("ru")", ResultType::Return, "ru" },
-        { R"("rubric")", ResultType::Return, "rubric" },
-        { R"("")", ResultType::Return, "" },
-        { R"("rubric")", ResultType::Return, "rubric" },
-        { R"("Мир")", ResultType::Return, "Мир" },
-        { R"("lang")", ResultType::Return, "lang" },
-        { R"("ru")", ResultType::Return, "ru" },
-        { R"("experiments")", ResultType::Return, "experiments" },
-        { R"("lang")", ResultType::Return, "lang" },
-        { R"("ru")", ResultType::Return, "ru" },
-        { R"("los_portal")", ResultType::Return, "los_portal" },
-        { R"("los_level")", ResultType::Return, "los_level" },
-        { R"("none")", ResultType::Return, "none" },
-        { R"("lang")", ResultType::Return, "lang" },
-        { R"("ru")", ResultType::Return, "ru" },
-        { R"("__ym")", ResultType::Return, "__ym" },
-        { R"("ecommerce")", ResultType::Return, "ecommerce" },
-        { R"("impressions")", ResultType::Return, "impressions" },
-        { R"("id")", ResultType::Return, "id" },
-        { R"("863813")", ResultType::Return, "863813" },
-        { R"("name")", ResultType::Return, "name" },
-        { R"("Футболка детская 3D Happy, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Happy, возраст 1-2 года, трикотаж" },
-        { R"("category")", ResultType::Return, "category" },
-        { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" },
-        { R"("variant")", ResultType::Return, "variant" },
-        { R"("")", ResultType::Return, "" },
-        { R"("price")", ResultType::Return, "price" },
-        { R"("390.00")", ResultType::Return, "390.00" },
-        { R"("list")", ResultType::Return, "list" },
-        { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" },
-        { R"("position")", ResultType::Return, "position" },
-        { R"("brand")", ResultType::Return, "brand" },
-        { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" },
-        { R"("id")", ResultType::Return, "id" },
-        { R"("863839")", ResultType::Return, "863839" },
-        { R"("name")", ResultType::Return, "name" },
-        { R"("Футболка детская 3D Pretty kitten, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Pretty kitten, возраст 1-2 года, трикотаж" },
-        { R"("category")", ResultType::Return, "category" },
-        { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" },
-        { R"("variant")", ResultType::Return, "variant" },
-        { R"("")", ResultType::Return, "" },
-        { R"("price")", ResultType::Return, "price" },
-        { R"("390.00")", ResultType::Return, "390.00" },
-        { R"("list")", ResultType::Return, "list" },
-        { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" },
-        { R"("position")", ResultType::Return, "position" },
-        { R"("brand")", ResultType::Return, "brand" },
-        { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" },
-        { R"("id")", ResultType::Return, "id" },
-        { R"("863847")", ResultType::Return, "863847" },
-        { R"("name")", ResultType::Return, "name" },
-        { R"("Футболка детская 3D Little tiger, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Little tiger, возраст 1-2 года, трикотаж" },
-        { R"("category")", ResultType::Return, "category" },
-        { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" },
-        { R"("variant")", ResultType::Return, "variant" },
-        { R"("")", ResultType::Return, "" },
-        { R"("price")", ResultType::Return, "price" },
-        { R"("390.00")", ResultType::Return, "390.00" },
-        { R"("list")", ResultType::Return, "list" },
-        { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" },
-        { R"("position")", ResultType::Return, "position" },
-        { R"("brand")", ResultType::Return, "brand" },
-        { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" },
-        { R"("id")", ResultType::Return, "id" },
-        { R"("911480")", ResultType::Return, "911480" },
-        { R"("name")", ResultType::Return, "name" },
-        { R"("Футболка детская 3D Puppy, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Puppy, возраст 1-2 года, трикотаж" },
-        { R"("category")", ResultType::Return, "category" },
-        { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" },
-        { R"("variant")", ResultType::Return, "variant" },
-        { R"("")", ResultType::Return, "" },
-        { R"("price")", ResultType::Return, "price" },
-        { R"("390.00")", ResultType::Return, "390.00" },
-        { R"("list")", ResultType::Return, "list" },
-        { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" },
-        { R"("position")", ResultType::Return, "position" },
-        { R"("brand")", ResultType::Return, "brand" },
-        { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" },
-        { R"("id")", ResultType::Return, "id" },
-        { R"("911484")", ResultType::Return, "911484" },
-        { R"("name")", ResultType::Return, "name" },
-        { R"("Футболка детская 3D Little bears, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Little bears, возраст 1-2 года, трикотаж" },
-        { R"("category")", ResultType::Return, "category" },
-        { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" },
-        { R"("variant")", ResultType::Return, "variant" },
-        { R"("")", ResultType::Return, "" },
-        { R"("price")", ResultType::Return, "price" },
-        { R"("390.00")", ResultType::Return, "390.00" },
-        { R"("list")", ResultType::Return, "list" },
-        { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" },
-        { R"("position")", ResultType::Return, "position" },
-        { R"("brand")", ResultType::Return, "brand" },
-        { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" },
-        { R"("id")", ResultType::Return, "id" },
-        { R"("911489")", ResultType::Return, "911489" },
-        { R"("name")", ResultType::Return, "name" },
-        { R"("Футболка детская 3D Dolphin, возраст 2-4 года, трикотаж")", ResultType::Return, "Футболка детская 3D Dolphin, возраст 2-4 года, трикотаж" },
-        { R"("category")", ResultType::Return, "category" },
-        { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" },
-        { R"("variant")", ResultType::Return, "variant" },
-        { R"("")", ResultType::Return, "" },
-        { R"("price")", ResultType::Return, "price" },
-        { R"("390.00")", ResultType::Return, "390.00" },
-        { R"("list")", ResultType::Return, "list" },
-        { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" },
-        { R"("position")", ResultType::Return, "position" },
-        { R"("brand")", ResultType::Return, "brand" },
-        { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" },
-        { R"("id")", ResultType::Return, "id" },
-        { R"("911496")", ResultType::Return, "911496" },
-        { R"("name")", ResultType::Return, "name" },
-        { R"("Футболка детская 3D Pretty, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Pretty, возраст 1-2 года, трикотаж" },
-        { R"("category")", ResultType::Return, "category" },
-        { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" },
-        { R"("variant")", ResultType::Return, "variant" },
-        { R"("")", ResultType::Return, "" },
-        { R"("price")", ResultType::Return, "price" },
-        { R"("390.00")", ResultType::Return, "390.00" },
-        { R"("list")", ResultType::Return, "list" },
-        { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" },
-        { R"("position")", ResultType::Return, "position" },
-        { R"("brand")", ResultType::Return, "brand" },
-        { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" },
-        { R"("id")", ResultType::Return, "id" },
-        { R"("911504")", ResultType::Return, "911504" },
-        { R"("name")", ResultType::Return, "name" },
-        { R"("Футболка детская 3D Fairytale, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Fairytale, возраст 1-2 года, трикотаж" },
-        { R"("category")", ResultType::Return, "category" },
-        { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" },
-        { R"("variant")", ResultType::Return, "variant" },
-        { R"("")", ResultType::Return, "" },
-        { R"("price")", ResultType::Return, "price" },
-        { R"("390.00")", ResultType::Return, "390.00" },
-        { R"("list")", ResultType::Return, "list" },
-        { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" },
-        { R"("position")", ResultType::Return, "position" },
-        { R"("brand")", ResultType::Return, "brand" },
-        { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" },
-        { R"("id")", ResultType::Return, "id" },
-        { R"("911508")", ResultType::Return, "911508" },
-        { R"("name")", ResultType::Return, "name" },
-        { R"("Футболка детская 3D Kittens, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Kittens, возраст 1-2 года, трикотаж" },
-        { R"("category")", ResultType::Return, "category" },
-        { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" },
-        { R"("variant")", ResultType::Return, "variant" },
-        { R"("")", ResultType::Return, "" },
-        { R"("price")", ResultType::Return, "price" },
-        { R"("390.00")", ResultType::Return, "390.00" },
-        { R"("list")", ResultType::Return, "list" },
-        { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" },
-        { R"("position")", ResultType::Return, "position" },
-        { R"("brand")", ResultType::Return, "brand" },
-        { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" },
-        { R"("id")", ResultType::Return, "id" },
-        { R"("911512")", ResultType::Return, "911512" },
-        { R"("name")", ResultType::Return, "name" },
-        { R"("Футболка детская 3D Sunshine, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Sunshine, возраст 1-2 года, трикотаж" },
-        { R"("category")", ResultType::Return, "category" },
-        { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" },
-        { R"("variant")", ResultType::Return, "variant" },
-        { R"("")", ResultType::Return, "" },
-        { R"("price")", ResultType::Return, "price" },
-        { R"("390.00")", ResultType::Return, "390.00" },
-        { R"("list")", ResultType::Return, "list" },
-        { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" },
-        { R"("position")", ResultType::Return, "position" },
-        { R"("brand")", ResultType::Return, "brand" },
-        { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" },
-        { R"("id")", ResultType::Return, "id" },
-        { R"("911516")", ResultType::Return, "911516" },
-        { R"("name")", ResultType::Return, "name" },
-        { R"("Футболка детская 3D Dog in bag, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Dog in bag, возраст 1-2 года, трикотаж" },
-        { R"("category")", ResultType::Return, "category" },
-        { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" },
-        { R"("variant")", ResultType::Return, "variant" },
-        { R"("")", ResultType::Return, "" },
-        { R"("price")", ResultType::Return, "price" },
-        { R"("390.00")", ResultType::Return, "390.00" },
-        { R"("list")", ResultType::Return, "list" },
-        { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" },
-        { R"("position")", ResultType::Return, "position" },
-        { R"("brand")", ResultType::Return, "brand" },
-        { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" },
-        { R"("id")", ResultType::Return, "id" },
-        { R"("911520")", ResultType::Return, "911520" },
-        { R"("name")", ResultType::Return, "name" },
-        { R"("Футболка детская 3D Cute puppy, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Cute puppy, возраст 1-2 года, трикотаж" },
-        { R"("category")", ResultType::Return, "category" },
-        { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" },
-        { R"("variant")", ResultType::Return, "variant" },
-        { R"("")", ResultType::Return, "" },
-        { R"("price")", ResultType::Return, "price" },
-        { R"("390.00")", ResultType::Return, "390.00" },
-        { R"("list")", ResultType::Return, "list" },
-        { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" },
-        { R"("position")", ResultType::Return, "position" },
-        { R"("brand")", ResultType::Return, "brand" },
-        { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" },
-        { R"("id")", ResultType::Return, "id" },
-        { R"("911524")", ResultType::Return, "911524" },
-        { R"("name")", ResultType::Return, "name" },
-        { R"("Футболка детская 3D Rabbit, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Rabbit, возраст 1-2 года, трикотаж" },
-        { R"("category")", ResultType::Return, "category" },
-        { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" },
-        { R"("variant")", ResultType::Return, "variant" },
-        { R"("")", ResultType::Return, "" },
-        { R"("price")", ResultType::Return, "price" },
-        { R"("390.00")", ResultType::Return, "390.00" },
-        { R"("list")", ResultType::Return, "list" },
-        { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" },
-        { R"("position")", ResultType::Return, "position" },
-        { R"("brand")", ResultType::Return, "brand" },
-        { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" },
-        { R"("id")", ResultType::Return, "id" },
-        { R"("911528")", ResultType::Return, "911528" },
-        { R"("name")", ResultType::Return, "name" },
-        { R"("Футболка детская 3D Turtle, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Turtle, возраст 1-2 года, трикотаж" },
-        { R"("category")", ResultType::Return, "category" },
-        { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" },
-        { R"("variant")", ResultType::Return, "variant" },
-        { R"("")", ResultType::Return, "" },
-        { R"("price")", ResultType::Return, "price" },
-        { R"("390.00")", ResultType::Return, "390.00" },
-        { R"("list")", ResultType::Return, "list" },
-        { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" },
-        { R"("position")", ResultType::Return, "position" },
-        { R"("brand")", ResultType::Return, "brand" },
-        { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" },
-        { R"("id")", ResultType::Return, "id" },
-        { R"("888616")", ResultType::Return, "888616" },
-        { R"("name")", ResultType::Return, "name" },
-        { "\"3Д Футболка мужская \\\"Collorista\\\" Светлое завтра р-р XL(52-54), 100% хлопок, трикотаж\"", ResultType::Return, "3Д Футболка мужская \"Collorista\" Светлое завтра р-р XL(52-54), 100% хлопок, трикотаж" },
-        { R"("category")", ResultType::Return, "category" },
-        { R"("/Одежда и обувь/Мужская одежда/Футболки/")", ResultType::Return, "/Одежда и обувь/Мужская одежда/Футболки/" },
-        { R"("variant")", ResultType::Return, "variant" },
-        { R"("")", ResultType::Return, "" },
-        { R"("price")", ResultType::Return, "price" },
-        { R"("406.60")", ResultType::Return, "406.60" },
-        { R"("list")", ResultType::Return, "list" },
-        { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" },
-        { R"("position")", ResultType::Return, "position" },
-        { R"("brand")", ResultType::Return, "brand" },
-        { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" },
-        { R"("id")", ResultType::Return, "id" },
-        { R"("913361")", ResultType::Return, "913361" },
-        { R"("name")", ResultType::Return, "name" },
-        { R"("3Д Футболка детская World р-р 8-10, 100% хлопок, трикотаж")", ResultType::Return, "3Д Футболка детская World р-р 8-10, 100% хлопок, трикотаж" },
-        { R"("category")", ResultType::Return, "category" },
-        { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" },
-        { R"("variant")", ResultType::Return, "variant" },
-        { R"("")", ResultType::Return, "" },
-        { R"("price")", ResultType::Return, "price" },
-        { R"("470.00")", ResultType::Return, "470.00" },
-        { R"("list")", ResultType::Return, "list" },
-        { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" },
-        { R"("position")", ResultType::Return, "position" },
-        { R"("brand")", ResultType::Return, "brand" },
-        { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" },
-        { R"("id")", ResultType::Return, "id" },
-        { R"("913364")", ResultType::Return, "913364" },
-        { R"("name")", ResultType::Return, "name" },
-        { R"("3Д Футболка детская Force р-р 8-10, 100% хлопок, трикотаж")", ResultType::Return, "3Д Футболка детская Force р-р 8-10, 100% хлопок, трикотаж" },
-        { R"("category")", ResultType::Return, "category" },
-        { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" },
-        { R"("variant")", ResultType::Return, "variant" },
-        { R"("")", ResultType::Return, "" },
-        { R"("price")", ResultType::Return, "price" },
-        { R"("470.00")", ResultType::Return, "470.00" },
-        { R"("list")", ResultType::Return, "list" },
-        { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" },
-        { R"("position")", ResultType::Return, "position" },
-        { R"("brand")", ResultType::Return, "brand" },
-        { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" },
-        { R"("id")", ResultType::Return, "id" },
-        { R"("913367")", ResultType::Return, "913367" },
-        { R"("name")", ResultType::Return, "name" },
-        { R"("3Д Футболка детская Winter tale р-р 8-10, 100% хлопок, трикотаж")", ResultType::Return, "3Д Футболка детская Winter tale р-р 8-10, 100% хлопок, трикотаж" },
-        { R"("category")", ResultType::Return, "category" },
-        { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" },
-        { R"("variant")", ResultType::Return, "variant" },
-        { R"("")", ResultType::Return, "" },
-        { R"("price")", ResultType::Return, "price" },
-        { R"("470.00")", ResultType::Return, "470.00" },
-        { R"("list")", ResultType::Return, "list" },
-        { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" },
-        { R"("position")", ResultType::Return, "position" },
-        { R"("brand")", ResultType::Return, "brand" },
-        { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" },
-        { R"("id")", ResultType::Return, "id" },
-        { R"("913385")", ResultType::Return, "913385" },
-        { R"("name")", ResultType::Return, "name" },
-        { R"("3Д Футболка детская Moonshine р-р 8-10, 100% хлопок, трикотаж")", ResultType::Return, "3Д Футболка детская Moonshine р-р 8-10, 100% хлопок, трикотаж" },
-        { R"("category")", ResultType::Return, "category" },
-        { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" },
-        { R"("variant")", ResultType::Return, "variant" },
-        { R"("")", ResultType::Return, "" },
-        { R"("price")", ResultType::Return, "price" },
-        { R"("470.00")", ResultType::Return, "470.00" },
-        { R"("list")", ResultType::Return, "list" },
-        { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" },
-        { R"("position")", ResultType::Return, "position" },
-        { R"("brand")", ResultType::Return, "brand" },
-        { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" },
-        { R"("id")", ResultType::Return, "id" },
-        { R"("913391")", ResultType::Return, "913391" },
-        { R"("name")", ResultType::Return, "name" },
-        { R"("3Д Футболка детская Shaman р-р 8-10, 100% хлопок, трикотаж")", ResultType::Return, "3Д Футболка детская Shaman р-р 8-10, 100% хлопок, трикотаж" },
-        { R"("category")", ResultType::Return, "category" },
-        { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" },
-        { R"("variant")", ResultType::Return, "variant" },
-        { R"("")", ResultType::Return, "" },
-        { R"("price")", ResultType::Return, "price" },
-        { R"("470.00")", ResultType::Return, "470.00" },
-        { R"("list")", ResultType::Return, "list" },
-        { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" },
-        { R"("position")", ResultType::Return, "position" },
-        { R"("brand")", ResultType::Return, "brand" },
-        { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" },
-        { R"("usertype")", ResultType::Return, "usertype" },
-        { R"("visitor")", ResultType::Return, "visitor" },
-        { R"("lang")", ResultType::Return, "lang" },
-        { R"("ru")", ResultType::Return, "ru" },
-        { R"("__ym")", ResultType::Return, "__ym" },
-        { R"("ecommerce")", ResultType::Return, "ecommerce" },
-        { R"("impressions")", ResultType::Return, "impressions" },
-        { R"("experiments")", ResultType::Return, "experiments" },
-        { R"("lang")", ResultType::Return, "lang" },
-        { R"("ru")", ResultType::Return, "ru" },
-        { R"("los_portal")", ResultType::Return, "los_portal" },
-        { R"("los_level")", ResultType::Return, "los_level" },
-        { R"("none")", ResultType::Return, "none" },
-        { R"("experiments")", ResultType::Return, "experiments" },
-        { R"("lang")", ResultType::Return, "lang" },
-        { R"("ru")", ResultType::Return, "ru" },
-        { R"("los_portal")", ResultType::Return, "los_portal" },
-        { R"("los_level")", ResultType::Return, "los_level" },
-        { R"("none")", ResultType::Return, "none" },
-        { R"("experiments")", ResultType::Return, "experiments" },
-        { R"("lang")", ResultType::Return, "lang" },
-        { R"("ru")", ResultType::Return, "ru" },
-        { R"("los_portal")", ResultType::Return, "los_portal" },
-        { R"("los_level")", ResultType::Return, "los_level" },
-        { R"("none")", ResultType::Return, "none" },
-        { R"("experiments")", ResultType::Return, "experiments" },
-        { R"("lang")", ResultType::Return, "lang" },
-        { R"("ru")", ResultType::Return, "ru" },
-        { R"("los_portal")", ResultType::Return, "los_portal" },
-        { R"("los_level")", ResultType::Return, "los_level" },
-        { R"("none")", ResultType::Return, "none" },
-        { R"("experiments")", ResultType::Return, "experiments" },
-        { R"("lang")", ResultType::Return, "lang" },
-        { R"("ru")", ResultType::Return, "ru" },
-        { R"("los_portal")", ResultType::Return, "los_portal" },
-        { R"("los_level")", ResultType::Return, "los_level" },
-        { R"("none")", ResultType::Return, "none" },
-        { R"("__ym")", ResultType::Return, "__ym" },
-        { R"("ecommerce")", ResultType::Return, "ecommerce" },
-        { R"("currencyCode")", ResultType::Return, "currencyCode" },
-        { R"("RUR")", ResultType::Return, "RUR" },
-        { R"("impressions")", ResultType::Return, "impressions" },
-        { R"("name")", ResultType::Return, "name" },
-        { R"("Чайник электрический Mystery MEK-1627, белый")", ResultType::Return, "Чайник электрический Mystery MEK-1627, белый" },
-        { R"("brand")", ResultType::Return, "brand" },
-        { R"("Mystery")", ResultType::Return, "Mystery" },
-        { R"("id")", ResultType::Return, "id" },
-        { R"("187180")", ResultType::Return, "187180" },
-        { R"("category")", ResultType::Return, "category" },
-        { R"("Мелкая бытовая техника/Мелкие кухонные приборы/Чайники электрические/Mystery")", ResultType::Return, "Мелкая бытовая техника/Мелкие кухонные приборы/Чайники электрические/Mystery" },
-        { R"("variant")", ResultType::Return, "variant" },
-        { R"("В наличии")", ResultType::Return, "В наличии" },
-        { R"("price")", ResultType::Return, "price" },
-        { R"("1630.00")", ResultType::Return, "1630.00" },
-        { R"("list")", ResultType::Return, "list" },
-        { R"("Карточка")", ResultType::Return, "Карточка" },
-        { R"("position")", ResultType::Return, "position" },
-        { R"("detail")", ResultType::Return, "detail" },
-        { R"("actionField")", ResultType::Return, "actionField" },
-        { R"("list")", ResultType::Return, "list" },
-        { "\0\"", ResultType::Throw, "JSON: expected \", got \0" },
-        { "\"/igrushki/konstruktory\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/1290414/komplekt-zhenskiy-dzhemper-plusbryuki-m-254-09-malina-plustemno-siniy-\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/Творчество/Рисование/Инструменты и кра\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Строительство и ремонт/Силовая техника/Зарядные устройства для автомобильных аккумуляторов/Пуско-зарядные устр\xD0\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Строительство и ремонт/Силовая техника/Зарядные устройств\xD0\0t", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Строительство и ремонт/Силовая техника/Зарядные устройства для автомобиль\0k", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\0t", ResultType::Throw, "JSON: expected \", got \0" },
-        { "\"/Хозтовары/Хранение вещей и организа\xD1\0t", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/Хозтовары/Товары для стир\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"li\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/734859/samolet-radioupravlyaemyy-istrebitel-rabotaet-o\0k", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/kosmetika-i-parfyum/parfyumeriya/mu\0t", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/ko\0\x04", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "", ResultType::Throw, "JSON: begin >= end." },
-        { "\"/stroitelstvo-i-remont/stroit\0t", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/stroitelstvo-i-remont/stroitelnyy-instrument/av\0k", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/s\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/Строительство и ремонт/Строительный инструмент/Изм\0e", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/avto/soputstvuy\0l", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/str\0\xD0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Отвертка 2 в 1 \\\"TUNDRA basic\\\" 5х75 мм (+,-) \0\xFF", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/stroitelstvo-i-remont/stroitelnyy-instrument/avtoinstrumen\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Мелкая бытовая техника/Мелки\xD0\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Пряжа \\\"Бамбук стрейч\\0\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Карандаш чёрнографитны\xD0\0\xD0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/Творчество/Рукоделие, аппликации/Пряжа и шерсть для \xD0\0l", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/1071547/karandash-chernografitnyy-volshebstvo-nv-kruglyy-d-7-2mm-dl-176mm-plast-tuba/\0e", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"ca\0e", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"ca\0e", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/1165424/chipbord-vyrubnoy-dlya-skrapbukinga-malyshi-mikki-maus-disney-bebi\0t", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/posuda/kuhonnye-prinadlezhnosti-i-i\0d", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/Канцтовары/Ежедневники и блокн\xD0\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/kanctovary/ezhednevniki-i-blok\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Стакан \xD0\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Набор бумаги для скрапбукинга \\\"Мои первый годик\\\": Микки Маус, Дисней бэби, 12 листов 29.5 х 29.5 см, 160\0\x80", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"c\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Органайзер для хранения аксессуаров, \0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"quantity\00", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Сменный блок для тетрадей на кольцах А5, 160 листов клетка, офсет \xE2\x84\0=", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/Сувениры/Ф\xD0\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"\0\"", ResultType::Return, "\0" },
-        { "\"\0\x04", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"va\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"ca\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"В \0\x04", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/letnie-tovary/z\0\x04", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Посудомоечная машина Ha\0=", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Крупная бытов\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Полочная акустическая система Magnat Needl\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"brand\00", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"\0d", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"pos\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"c\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"var\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Телевизоры и видеотехника/Всё для домашних кинотеатр\0=", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Флеш-диск Transcend JetFlash 620 8GB (TS8GJF62\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Табурет Мег\0\xD0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"variant\0\x04", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Катал\xD0\0\"", ResultType::Return, "Катал\xD0\0" },
-        { "\"К\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Полочная акустическая система Magnat Needl\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"brand\00", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"\0d", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"pos\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"c\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"17\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/igrushki/razvivayusc\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Ключница \\\"\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/Игр\xD1\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/Игрушки/Игрушки для девочек/Игровые модули дл\xD1\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Крупная бытовая техника/Стиральные машины/С фронт\xD0\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\0 ", ResultType::Throw, "JSON: expected \", got \0" },
-        { "\"Светодиодная лента SMD3528, 5 м. IP33, 60LED, зеленый, 4,8W/мет\xD1\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Сантехника/Мебель для ванных комнат/Стол\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\0o", ResultType::Throw, "JSON: expected \", got \0" },
-        { "\"/igrushki/konstruktory\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/posuda/kuhonnye-prinadlezhnosti-i-instrumenty/kuhonnye-pr\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/1290414/komplekt-zhenskiy-dzhemper-plusbryuki-m-254-09-malina-plustemno-siniy-\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/Творчество/Рисование/Инструменты и кра\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Строительство и ремонт/Силовая техника/Зарядные устройства для автомобильных аккумуляторов/Пуско-зарядные устр\xD0\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Строительство и ремонт/Силовая техника/Зарядные устройств\xD0\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Строительство и ремонт/Силовая техника/Зарядные устройства для автомобиль\0d", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\0 ", ResultType::Throw, "JSON: expected \", got \0" },
-        { "\"/Хозтовары/Хранение вещей и организа\xD1\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/Хозтовары/Товары для стир\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"li\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/igrushki/igrus\0d", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/734859/samolet-radioupravlyaemyy-istrebitel-rabotaet-o\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/kosmetika-i-parfyum/parfyumeriya/mu\00", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/ko\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/avto/avtomobilnyy\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/stroitelstvo-i-remont/stroit\00", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/stroitelstvo-i-remont/stroitelnyy-instrument/av\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/s\0d", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/Строительство и ремонт/Строительный инструмент/Изм\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/avto/soputstvuy\0\"", ResultType::Return, "/avto/soputstvuy\0" },
-        { "\"/str\0k", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Отвертка 2 в 1 \\\"TUNDRA basic\\\" 5х75 мм (+,-) \0\xD0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/stroitelstvo-i-remont/stroitelnyy-instrument/avtoinstrumen\0=", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Чайник электрический Vitesse\0=", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Мелкая бытовая техника/Мелки\xD0\0\xD0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Пряжа \\\"Бамбук стрейч\\0о", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Карандаш чёрнографитны\xD0\0k", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/Творчество/Рукоделие, аппликации/Пряжа и шерсть для \xD0\0\"", ResultType::Return, "/Творчество/Рукоделие, аппликации/Пряжа и шерсть для \xD0\0" },
-        { "\"/1071547/karandash-chernografitnyy-volshebstvo-nv-kruglyy-d-7-2mm-dl-176mm-plast-tuba/\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"ca\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/Подаро\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Средство для прочис\xD1\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"i\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/p\0\"", ResultType::Return, "/p\0" },
-        { "\"/Сувениры/Магниты, н\xD0\0k", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Дерев\xD0\0=", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/prazdniki/svadba/svadebnaya-c\0\xD0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/Канцт\0d", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/Праздники/То\xD0\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"v\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/Косметика \xD0\0d", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/Спорт и отдых/Настольные игры/Покер, руле\xD1\0\xD0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"categ\0=", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/retailr\0k", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/retailrocket\0k", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Ежедневник недат А5 140л кл,ляссе,обл пв\0=", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/432809/ezhednevnik-organayzer-sredniy-s-remeshkom-na-knopke-v-oblozhke-kalkulyator-kalendar-do-\0\xD0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/1165424/chipbord-vyrubnoy-dlya-skrapbukinga-malyshi-mikki-maus-disney-bebi\0d", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/posuda/kuhonnye-prinadlezhnosti-i-i\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/Канцтовары/Ежедневники и блокн\xD0\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"/kanctovary/ezhednevniki-i-blok\00", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Стакан \xD0\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"Набор бумаги для скрапбукинга \\\"Мои первый годик\\\": Микки Маус, Дисней бэби, 12 листов 29.5 х 29.5 см, 160\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." },
-        { "\"c\0\"", ResultType::Return, "c\0" },
-    };
-
-    for (auto i : boost::irange(0, 1/*00000*/))
-    {
-        static_cast<void>(i);
-
-        for (auto & r : test_data)
-        {
-            try
-            {
-                JSON j(r.input, r.input + strlen(r.input));
-
-                ASSERT_EQ(j.getString(), r.result);
-                ASSERT_TRUE(r.result_type == ResultType::Return);
-            }
-            catch (JSONException & e)
-            {
-                ASSERT_TRUE(r.result_type == ResultType::Throw);
-                ASSERT_EQ(e.message(), r.result);
-            }
-        }
-    }
-}

base/common/tests/local_date_time_comparison.cpp

@@ -1,29 +0,0 @@
-#include <iostream>
-#include <stdexcept>
-
-#include <common/LocalDateTime.h>
-
-
-void fillStackWithGarbage()
-{
-    volatile uint64_t a = 0xAABBCCDDEEFF0011ULL;
-    volatile uint64_t b = 0x2233445566778899ULL;
-    std::cout << a + b << '\n';
-}
-
-void checkComparison()
-{
-    LocalDateTime a("2018-07-18 01:02:03");
-    LocalDateTime b("2018-07-18 01:02:03");
-
-    if (a != b)
-        throw std::runtime_error("Test failed");
-}
-
-
-int main(int, char **)
-{
-    fillStackWithGarbage();
-    checkComparison();
-    return 0;
-}

base/common/throwError.h

@@ -1,13 +1,15 @@
 #pragma once
+
 #include <stdexcept>
 
+
 /// Throw DB::Exception-like exception before its definition.
 /// DB::Exception derived from Poco::Exception derived from std::exception.
-/// DB::Exception generally cought as Poco::Exception. std::exception generally has other catch blocks and could lead to other outcomes.
+/// DB::Exception generally caught as Poco::Exception. std::exception generally has other catch blocks and could lead to other outcomes.
 /// DB::Exception is not defined yet. It'd better to throw Poco::Exception but we do not want to include any big header here, even <string>.
 /// So we throw some std::exception instead in the hope its catch block is the same as DB::Exception one.
 template <typename T>
-inline void throwError(const T & err)
+[[noreturn]] inline void throwError(const T & err)
 {
     throw std::runtime_error(err);
 }

base/common/time.h

@@ -2,7 +2,7 @@
 
 #include <time.h>
 
-#if defined (OS_DARWIN)
+#if defined (OS_DARWIN) || defined (OS_SUNOS)
 #    define CLOCK_MONOTONIC_COARSE CLOCK_MONOTONIC
 #elif defined (OS_FREEBSD)
 #    define CLOCK_MONOTONIC_COARSE CLOCK_MONOTONIC_FAST

base/common/types.h

@@ -8,12 +8,17 @@ using Int16 = int16_t;
 using Int32 = int32_t;
 using Int64 = int64_t;
 
-#if __cplusplus <= 201703L
+#ifndef __cpp_char8_t
 using char8_t = unsigned char;
 #endif
 
 /// This is needed for more strict aliasing. https://godbolt.org/z/xpJBSb https://stackoverflow.com/a/57453713
+#if !defined(PVS_STUDIO) /// But PVS-Studio does not treat it correctly.
 using UInt8 = char8_t;
+#else
+using UInt8 = uint8_t;
+#endif
+
 using UInt16 = uint16_t;
 using UInt32 = uint32_t;
 using UInt64 = uint64_t;

base/common/unit.h

@@ -0,0 +1,10 @@
+#pragma once
+#include <cstddef>
+
+constexpr size_t KiB = 1024;
+constexpr size_t MiB = 1024 * KiB;
+constexpr size_t GiB = 1024 * MiB;
+
+constexpr size_t operator"" _KiB(unsigned long long val) { return val * KiB; }
+constexpr size_t operator"" _MiB(unsigned long long val) { return val * MiB; }
+constexpr size_t operator"" _GiB(unsigned long long val) { return val * GiB; }

base/common/wide_integer.h

@@ -58,10 +58,11 @@ public:
     using signed_base_type = int64_t;
 
     // ctors
-    integer() = default;
+    constexpr integer() noexcept = default;
 
     template <typename T>
     constexpr integer(T rhs) noexcept;
+
     template <typename T>
     constexpr integer(std::initializer_list<T> il) noexcept;
 
@@ -108,10 +109,7 @@ public:
 
     constexpr explicit operator bool() const noexcept;
 
-    template <class T>
+    template <typename T, typename = std::enable_if_t<std::is_arithmetic_v<T>, T>>
-    using __integral_not_wide_integer_class = typename std::enable_if<std::is_arithmetic<T>::value, T>::type;
-
-    template <class T, class = __integral_not_wide_integer_class<T>>
     constexpr operator T() const noexcept;
 
     constexpr operator long double() const noexcept;
@@ -120,25 +118,27 @@ public:
 
     struct _impl;
 
+    base_type items[_impl::item_count];
+
 private:
     template <size_t Bits2, typename Signed2>
     friend class integer;
 
     friend class std::numeric_limits<integer<Bits, signed>>;
     friend class std::numeric_limits<integer<Bits, unsigned>>;
-
-    base_type items[_impl::item_count];
 };
 
 template <typename T>
 static constexpr bool ArithmeticConcept() noexcept;
+
 template <class T1, class T2>
-using __only_arithmetic = typename std::enable_if<ArithmeticConcept<T1>() && ArithmeticConcept<T2>()>::type;
+using _only_arithmetic = typename std::enable_if<ArithmeticConcept<T1>() && ArithmeticConcept<T2>()>::type;
 
 template <typename T>
 static constexpr bool IntegralConcept() noexcept;
+
 template <class T, class T2>
-using __only_integer = typename std::enable_if<IntegralConcept<T>() && IntegralConcept<T2>()>::type;
+using _only_integer = typename std::enable_if<IntegralConcept<T>() && IntegralConcept<T2>()>::type;
 
 // Unary operators
 template <size_t Bits, typename Signed>
@@ -154,54 +154,55 @@ constexpr integer<Bits, Signed> operator+(const integer<Bits, Signed> & lhs) noe
 template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
 std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
 operator*(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
-template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
+template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
 std::common_type_t<Arithmetic, Arithmetic2> constexpr operator*(const Arithmetic & rhs, const Arithmetic2 & lhs);
 
 template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
 std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
 operator/(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
-template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
+template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
 std::common_type_t<Arithmetic, Arithmetic2> constexpr operator/(const Arithmetic & rhs, const Arithmetic2 & lhs);
 
 template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
 std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
 operator+(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
-template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
+template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
 std::common_type_t<Arithmetic, Arithmetic2> constexpr operator+(const Arithmetic & rhs, const Arithmetic2 & lhs);
 
 template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
 std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
 operator-(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
-template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
+template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
 std::common_type_t<Arithmetic, Arithmetic2> constexpr operator-(const Arithmetic & rhs, const Arithmetic2 & lhs);
 
 template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
 std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
 operator%(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
-template <typename Integral, typename Integral2, class = __only_integer<Integral, Integral2>>
+template <typename Integral, typename Integral2, class = _only_integer<Integral, Integral2>>
 std::common_type_t<Integral, Integral2> constexpr operator%(const Integral & rhs, const Integral2 & lhs);
 
 template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
 std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
 operator&(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
-template <typename Integral, typename Integral2, class = __only_integer<Integral, Integral2>>
+template <typename Integral, typename Integral2, class = _only_integer<Integral, Integral2>>
 std::common_type_t<Integral, Integral2> constexpr operator&(const Integral & rhs, const Integral2 & lhs);
 
 template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
 std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
 operator|(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
-template <typename Integral, typename Integral2, class = __only_integer<Integral, Integral2>>
+template <typename Integral, typename Integral2, class = _only_integer<Integral, Integral2>>
 std::common_type_t<Integral, Integral2> constexpr operator|(const Integral & rhs, const Integral2 & lhs);
 
 template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
 std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
 operator^(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
-template <typename Integral, typename Integral2, class = __only_integer<Integral, Integral2>>
+template <typename Integral, typename Integral2, class = _only_integer<Integral, Integral2>>
 std::common_type_t<Integral, Integral2> constexpr operator^(const Integral & rhs, const Integral2 & lhs);
 
 // TODO: Integral
 template <size_t Bits, typename Signed>
 constexpr integer<Bits, Signed> operator<<(const integer<Bits, Signed> & lhs, int n) noexcept;
+
 template <size_t Bits, typename Signed>
 constexpr integer<Bits, Signed> operator>>(const integer<Bits, Signed> & lhs, int n) noexcept;
 
@@ -218,32 +219,32 @@ constexpr integer<Bits, Signed> operator>>(const integer<Bits, Signed> & lhs, In
 
 template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
 constexpr bool operator<(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
-template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
+template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
 constexpr bool operator<(const Arithmetic & rhs, const Arithmetic2 & lhs);
 
 template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
 constexpr bool operator>(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
-template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
+template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
 constexpr bool operator>(const Arithmetic & rhs, const Arithmetic2 & lhs);
 
 template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
 constexpr bool operator<=(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
-template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
+template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
 constexpr bool operator<=(const Arithmetic & rhs, const Arithmetic2 & lhs);
 
 template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
 constexpr bool operator>=(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
-template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
+template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
 constexpr bool operator>=(const Arithmetic & rhs, const Arithmetic2 & lhs);
 
 template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
 constexpr bool operator==(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
-template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
+template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
 constexpr bool operator==(const Arithmetic & rhs, const Arithmetic2 & lhs);
 
 template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
 constexpr bool operator!=(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
-template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
+template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
 constexpr bool operator!=(const Arithmetic & rhs, const Arithmetic2 & lhs);
 
 }

base/common/wide_integer_impl.h

@@ -6,6 +6,13 @@
 
 #include "throwError.h"
 
+#include <cmath>
+#include <cfloat>
+#include <cassert>
+#include <tuple>
+#include <limits>
+
+
 namespace wide
 {
 
@@ -33,6 +40,18 @@ static constexpr bool IntegralConcept() noexcept
     return std::is_integral_v<T> || IsWideInteger<T>::value;
 }
 
+template <typename T>
+class IsTupleLike
+{
+    template <typename U>
+    static auto check(U * p) -> decltype(std::tuple_size<U>::value, int());
+    template <typename>
+    static void check(...);
+
+public:
+    static constexpr const bool value = !std::is_void<decltype(check<T>(nullptr))>::value;
+};
+
 }
 
 namespace std
@@ -76,7 +95,7 @@ public:
             res.items[T::_impl::big(0)] = std::numeric_limits<typename wide::integer<Bits, Signed>::signed_base_type>::min();
             return res;
         }
-        return 0;
+        return wide::integer<Bits, Signed>(0);
     }
 
     static constexpr wide::integer<Bits, Signed> max() noexcept
@@ -146,7 +165,7 @@ namespace wide
 template <size_t Bits, typename Signed>
 struct integer<Bits, Signed>::_impl
 {
-    static constexpr size_t _Bits = Bits;
+    static constexpr size_t _bits = Bits;
     static constexpr const unsigned byte_count = Bits / 8;
     static constexpr const unsigned item_count = byte_count / sizeof(base_type);
     static constexpr const unsigned base_bits = sizeof(base_type) * 8;
@@ -171,7 +190,7 @@ struct integer<Bits, Signed>::_impl
     constexpr static bool is_negative(const integer<B, T> & n) noexcept
     {
         if constexpr (std::is_same_v<T, signed>)
-            return static_cast<signed_base_type>(n.items[big(0)]) < 0;
+            return static_cast<signed_base_type>(n.items[integer<B, T>::_impl::big(0)]) < 0;
         else
             return false;
     }
@@ -188,62 +207,114 @@ struct integer<Bits, Signed>::_impl
     template <size_t B, class S>
     constexpr static integer<B, S> make_positive(const integer<B, S> & n) noexcept
     {
-        return is_negative(n) ? operator_unary_minus(n) : n;
+        return is_negative(n) ? integer<B, S>(operator_unary_minus(n)) : n;
     }
 
     template <typename T>
-    constexpr static auto to_Integral(T f) noexcept
+    __attribute__((no_sanitize("undefined"))) constexpr static auto to_Integral(T f) noexcept
     {
-        if constexpr (std::is_same_v<T, __int128>)
+        if constexpr (std::is_signed_v<T>)
-            return f;
-        else if constexpr (std::is_signed_v<T>)
             return static_cast<int64_t>(f);
         else
             return static_cast<uint64_t>(f);
     }
 
     template <typename Integral>
-    constexpr static void wide_integer_from_bultin(integer<Bits, Signed> & self, Integral rhs) noexcept
+    constexpr static void wide_integer_from_builtin(integer<Bits, Signed> & self, Integral rhs) noexcept
     {
-        self.items[0] = _impl::to_Integral(rhs);
+        static_assert(sizeof(Integral) <= sizeof(base_type));
-        if constexpr (std::is_same_v<Integral, __int128>)
-            self.items[1] = rhs >> base_bits;
 
-        constexpr const unsigned start = (sizeof(Integral) == 16) ? 2 : 1;
+        self.items[0] = _impl::to_Integral(rhs);
 
         if constexpr (std::is_signed_v<Integral>)
         {
             if (rhs < 0)
             {
-                for (unsigned i = start; i < item_count; ++i)
+                for (size_t i = 1; i < item_count; ++i)
                     self.items[i] = -1;
                 return;
             }
         }
 
-        for (unsigned i = start; i < item_count; ++i)
+        for (size_t i = 1; i < item_count; ++i)
             self.items[i] = 0;
     }
 
-    constexpr static void wide_integer_from_bultin(integer<Bits, Signed> & self, double rhs) noexcept
+    template <typename TupleLike, size_t i = 0>
+    constexpr static void wide_integer_from_tuple_like(integer<Bits, Signed> & self, const TupleLike & tuple) noexcept
     {
-        if ((rhs > 0 && rhs < std::numeric_limits<uint64_t>::max()) || (rhs < 0 && rhs > std::numeric_limits<int64_t>::min()))
+        if constexpr (i < item_count)
         {
-            self = to_Integral(rhs);
+            if constexpr (i < std::tuple_size_v<TupleLike>)
+                self.items[i] = std::get<i>(tuple);
+            else
+                self.items[i] = 0;
+            wide_integer_from_tuple_like<TupleLike, i + 1>(self, tuple);
+        }
+    }
+
+    /**
+     * N.B. t is constructed from double, so max(t) = max(double) ~ 2^310
+     * the recursive call happens when t / 2^64 > 2^64, so there won't be more than 5 of them.
+     *
+     * t = a1 * max_int + b1,   a1 > max_int, b1 < max_int
+     * a1 = a2 * max_int + b2,  a2 > max_int, b2 < max_int
+     * a_(n - 1) = a_n * max_int + b2, a_n <= max_int <- base case.
+     */
+    template <class T>
+    constexpr static void set_multiplier(integer<Bits, Signed> & self, T t) noexcept
+    {
+        constexpr uint64_t max_int = std::numeric_limits<uint64_t>::max();
+
+        /// Implementation specific behaviour on overflow (if we don't check here, stack overflow will triggered in bigint_cast).
+        if (!std::isfinite(t))
+        {
+            self = 0;
             return;
         }
 
-        long double r = rhs;
+        const T alpha = t / static_cast<T>(max_int);
-        if (r < 0)
-            r = -r;
 
-        size_t count = r / std::numeric_limits<uint64_t>::max();
+        if (alpha <= static_cast<T>(max_int))
-        self = count;
+            self = static_cast<uint64_t>(alpha);
-        self *= std::numeric_limits<uint64_t>::max();
+        else // max(double) / 2^64 will surely contain less than 52 precision bits, so speed up computations.
-        long double to_diff = count;
+            set_multiplier<double>(self, alpha);
-        to_diff *= std::numeric_limits<uint64_t>::max();
 
-        self += to_Integral(r - to_diff);
+        self *= max_int;
+        self += static_cast<uint64_t>(t - floor(alpha) * static_cast<T>(max_int)); // += b_i
+    }
+
+    constexpr static void wide_integer_from_builtin(integer<Bits, Signed> & self, double rhs) noexcept
+    {
+        constexpr int64_t max_int = std::numeric_limits<int64_t>::max();
+        constexpr int64_t min_int = std::numeric_limits<int64_t>::lowest();
+
+        /// There are values in int64 that have more than 53 significant bits (in terms of double
+        /// representation). Such values, being promoted to double, are rounded up or down. If they are rounded up,
+        /// the result may not fit in 64 bits.
+        /// The example of such a number is 9.22337e+18.
+        /// As to_Integral does a static_cast to int64_t, it may result in UB.
+        /// The necessary check here is that long double has enough significant (mantissa) bits to store the
+        /// int64_t max value precisely.
+
+        // TODO Be compatible with Apple aarch64
+#if not (defined(__APPLE__) && defined(__aarch64__))
+        static_assert(LDBL_MANT_DIG >= 64,
+            "On your system long double has less than 64 precision bits, "
+            "which may result in UB when initializing double from int64_t");
+#endif
+
+        if (rhs > static_cast<long double>(min_int) && rhs < static_cast<long double>(max_int))
+        {
+            self = static_cast<int64_t>(rhs);
+            return;
+        }
+
+        const long double rhs_long_double = (static_cast<long double>(rhs) < 0)
+            ? -static_cast<long double>(rhs)
+            : rhs;
+
+        set_multiplier(self, rhs_long_double);
 
         if (rhs < 0)
             self = -self;
@@ -337,13 +408,13 @@ struct integer<Bits, Signed>::_impl
             if (bit_shift)
                 lhs.items[big(items_shift)] |= std::numeric_limits<base_type>::max() << (base_bits - bit_shift);
 
-            for (unsigned i = item_count - items_shift; i < items_shift; ++i)
+            for (unsigned i = 0; i < items_shift; ++i)
-                lhs.items[little(i)] = std::numeric_limits<base_type>::max();
+                lhs.items[big(i)] = std::numeric_limits<base_type>::max();
         }
         else
         {
-            for (unsigned i = item_count - items_shift; i < items_shift; ++i)
+            for (unsigned i = 0; i < items_shift; ++i)
-                lhs.items[little(i)] = 0;
+                lhs.items[big(i)] = 0;
         }
 
         return lhs;
@@ -351,23 +422,23 @@ struct integer<Bits, Signed>::_impl
 
 private:
     template <typename T>
-    constexpr static base_type get_item(const T & x, unsigned number)
+    constexpr static base_type get_item(const T & x, unsigned idx)
     {
         if constexpr (IsWideInteger<T>::value)
         {
-            if (number < T::_impl::item_count)
+            if (idx < T::_impl::item_count)
-                return x.items[number];
+                return x.items[idx];
             return 0;
         }
         else
         {
             if constexpr (sizeof(T) <= sizeof(base_type))
             {
-                if (!number)
+                if (0 == idx)
                     return x;
             }
-            else if (number * sizeof(base_type) < sizeof(T))
+            else if (idx * sizeof(base_type) < sizeof(T))
-                return x >> (number * base_bits); // & std::numeric_limits<base_type>::max()
+                return x >> (idx * base_bits); // & std::numeric_limits<base_type>::max()
             return 0;
         }
     }
@@ -393,7 +464,7 @@ private:
 
         for (unsigned i = 1; i < item_count; ++i)
         {
-            if (underflows[i-1])
+            if (underflows[i - 1])
             {
                 base_type & res_item = res.items[little(i)];
                 if (res_item == 0)
@@ -426,7 +497,7 @@ private:
 
         for (unsigned i = 1; i < item_count; ++i)
         {
-            if (overflows[i-1])
+            if (overflows[i - 1])
             {
                 base_type & res_item = res.items[little(i)];
                 ++res_item;
@@ -486,6 +557,17 @@ private:
             res.items[little(2)] = r12 >> 64;
             return res;
         }
+        else if constexpr (Bits == 128 && sizeof(base_type) == 8)
+        {
+            using CompilerUInt128 = unsigned __int128;
+            CompilerUInt128 a = (CompilerUInt128(lhs.items[1]) << 64) + lhs.items[0];
+            CompilerUInt128 b = (CompilerUInt128(rhs.items[1]) << 64) + rhs.items[0];
+            CompilerUInt128 c = a * b;
+            integer<Bits, Signed> res;
+            res.items[0] = c;
+            res.items[1] = c >> 64;
+            return res;
+        }
         else
         {
             integer<Bits, Signed> res{};
@@ -558,8 +640,8 @@ public:
         else
         {
             static_assert(IsWideInteger<T>::value);
-            return std::common_type_t<integer<Bits, Signed>, integer<T::_impl::_Bits, Signed>>::_impl::operator_plus(
+            return std::common_type_t<integer<Bits, Signed>, integer<T::_impl::_bits, Signed>>::_impl::operator_plus(
-                integer<T::_impl::_Bits, Signed>(lhs), rhs);
+                integer<T::_impl::_bits, Signed>(lhs), rhs);
         }
     }
 
@@ -576,8 +658,8 @@ public:
         else
         {
             static_assert(IsWideInteger<T>::value);
-            return std::common_type_t<integer<Bits, Signed>, integer<T::_impl::_Bits, Signed>>::_impl::operator_minus(
+            return std::common_type_t<integer<Bits, Signed>, integer<T::_impl::_bits, Signed>>::_impl::operator_minus(
-                integer<T::_impl::_Bits, Signed>(lhs), rhs);
+                integer<T::_impl::_bits, Signed>(lhs), rhs);
         }
     }
 
@@ -611,7 +693,7 @@ public:
     }
 
     template <typename T>
-    constexpr static bool operator_more(const integer<Bits, Signed> & lhs, const T & rhs) noexcept
+    constexpr static bool operator_greater(const integer<Bits, Signed> & lhs, const T & rhs) noexcept
     {
         if constexpr (should_keep_size<T>())
         {
@@ -631,7 +713,7 @@ public:
         else
         {
             static_assert(IsWideInteger<T>::value);
-            return std::common_type_t<integer<Bits, Signed>, T>::_impl::operator_more(T(lhs), rhs);
+            return std::common_type_t<integer<Bits, Signed>, T>::_impl::operator_greater(T(lhs), rhs);
         }
     }
 
@@ -718,7 +800,6 @@ public:
         }
     }
 
-private:
     template <typename T>
     constexpr static bool is_zero(const T & x)
     {
@@ -735,46 +816,65 @@ private:
     }
 
     /// returns quotient as result and remainder in numerator.
-    template <typename T>
+    template <size_t Bits2>
-    constexpr static T divide(T & numerator, T && denominator)
+    constexpr static integer<Bits2, unsigned> divide(integer<Bits2, unsigned> & numerator, integer<Bits2, unsigned> denominator)
     {
-        if (is_zero(denominator))
+        static_assert(std::is_unsigned_v<Signed>);
-            throwError("divide by zero");
 
-        T & n = numerator;
+        if constexpr (Bits == 128 && sizeof(base_type) == 8)
-        T & d = denominator;
-        T x = 1;
-        T quotient = 0;
-
-        while (!operator_more(d, n) && operator_eq(operator_amp(shift_right(d, base_bits * item_count - 1), 1), 0))
         {
-            x = shift_left(x, 1);
+            using CompilerUInt128 = unsigned __int128;
-            d = shift_left(d, 1);
+
+            CompilerUInt128 a = (CompilerUInt128(numerator.items[1]) << 64) + numerator.items[0];
+            CompilerUInt128 b = (CompilerUInt128(denominator.items[1]) << 64) + denominator.items[0];
+            CompilerUInt128 c = a / b;
+
+            integer<Bits, Signed> res;
+            res.items[0] = c;
+            res.items[1] = c >> 64;
+
+            CompilerUInt128 remainder = a - b * c;
+            numerator.items[0] = remainder;
+            numerator.items[1] = remainder >> 64;
+
+            return res;
         }
 
-        while (!operator_eq(x, 0))
+        if (is_zero(denominator))
+            throwError("Division by zero");
+
+        integer<Bits2, unsigned> x = 1;
+        integer<Bits2, unsigned> quotient = 0;
+
+        while (!operator_greater(denominator, numerator) && is_zero(operator_amp(shift_right(denominator, Bits2 - 1), 1)))
         {
-            if (!operator_more(d, n))
+            x = shift_left(x, 1);
+            denominator = shift_left(denominator, 1);
+        }
+
+        while (!is_zero(x))
+        {
+            if (!operator_greater(denominator, numerator))
             {
-                n = operator_minus(n, d);
+                numerator = operator_minus(numerator, denominator);
                 quotient = operator_pipe(quotient, x);
             }
 
             x = shift_right(x, 1);
-            d = shift_right(d, 1);
+            denominator = shift_right(denominator, 1);
         }
 
         return quotient;
     }
 
-public:
     template <typename T>
     constexpr static auto operator_slash(const integer<Bits, Signed> & lhs, const T & rhs)
     {
         if constexpr (should_keep_size<T>())
         {
-            integer<Bits, Signed> numerator = make_positive(lhs);
+            integer<Bits, unsigned> numerator = make_positive(lhs);
-            integer<Bits, Signed> quotient = divide(numerator, make_positive(integer<Bits, Signed>(rhs)));
+            integer<Bits, unsigned> denominator = make_positive(integer<Bits, Signed>(rhs));
+            integer<Bits, unsigned> quotient = integer<Bits, unsigned>::_impl::divide(numerator, std::move(denominator));
 
             if (std::is_same_v<Signed, signed> && is_negative(rhs) != is_negative(lhs))
                 quotient = operator_unary_minus(quotient);
@@ -783,7 +883,7 @@ public:
         else
         {
             static_assert(IsWideInteger<T>::value);
-            return std::common_type_t<integer<Bits, Signed>, integer<T::_impl::_Bits, Signed>>::operator_slash(T(lhs), rhs);
+            return std::common_type_t<integer<Bits, Signed>, integer<T::_impl::_bits, Signed>>::operator_slash(T(lhs), rhs);
         }
     }
 
@@ -792,8 +892,9 @@ public:
     {
         if constexpr (should_keep_size<T>())
         {
-            integer<Bits, Signed> remainder = make_positive(lhs);
+            integer<Bits, unsigned> remainder = make_positive(lhs);
-            divide(remainder, make_positive(integer<Bits, Signed>(rhs)));
+            integer<Bits, unsigned> denominator = make_positive(integer<Bits, Signed>(rhs));
+            integer<Bits, unsigned>::_impl::divide(remainder, std::move(denominator));
 
             if (std::is_same_v<Signed, signed> && is_negative(lhs))
                 remainder = operator_unary_minus(remainder);
@@ -802,7 +903,7 @@ public:
         else
         {
             static_assert(IsWideInteger<T>::value);
-            return std::common_type_t<integer<Bits, Signed>, integer<T::_impl::_Bits, Signed>>::operator_percent(T(lhs), rhs);
+            return std::common_type_t<integer<Bits, Signed>, integer<T::_impl::_bits, Signed>>::operator_percent(T(lhs), rhs);
         }
     }
 
@@ -859,7 +960,7 @@ public:
                     ++c;
                 }
                 else
-                    throwError("invalid char from");
+                    throwError("Invalid char from");
             }
         }
         else
@@ -867,7 +968,7 @@ public:
             while (*c)
             {
                 if (*c < '0' || *c > '9')
-                    throwError("invalid char from");
+                    throwError("Invalid char from");
 
                 res = multiply(res, 10U);
                 res = plus(res, *c - '0');
@@ -891,8 +992,10 @@ constexpr integer<Bits, Signed>::integer(T rhs) noexcept
 {
     if constexpr (IsWideInteger<T>::value)
         _impl::wide_integer_from_wide_integer(*this, rhs);
+    else if  constexpr (IsTupleLike<T>::value)
+        _impl::wide_integer_from_tuple_like(*this, rhs);
     else
-        _impl::wide_integer_from_bultin(*this, rhs);
+        _impl::wide_integer_from_builtin(*this, rhs);
 }
 
 template <size_t Bits, typename Signed>
@@ -904,11 +1007,22 @@ constexpr integer<Bits, Signed>::integer(std::initializer_list<T> il) noexcept
     {
         if constexpr (IsWideInteger<T>::value)
             _impl::wide_integer_from_wide_integer(*this, *il.begin());
+        else if  constexpr (IsTupleLike<T>::value)
+            _impl::wide_integer_from_tuple_like(*this, *il.begin());
         else
-            _impl::wide_integer_from_bultin(*this, *il.begin());
+            _impl::wide_integer_from_builtin(*this, *il.begin());
+    }
+    else if (il.size() == 0)
+    {
+        _impl::wide_integer_from_builtin(*this, 0);
     }
     else
-        _impl::wide_integer_from_bultin(*this, 0);
+    {
+        auto it = il.begin();
+        for (size_t i = 0; i < _impl::item_count; ++i)
+            if (it < il.end())
+                items[i] = *it;
+    }
 }
 
 template <size_t Bits, typename Signed>
@@ -923,7 +1037,10 @@ template <size_t Bits, typename Signed>
 template <typename T>
 constexpr integer<Bits, Signed> & integer<Bits, Signed>::operator=(T rhs) noexcept
 {
-    _impl::wide_integer_from_bultin(*this, rhs);
+    if  constexpr (IsTupleLike<T>::value)
+        _impl::wide_integer_from_tuple_like(*this, rhs);
+    else
+        _impl::wide_integer_from_builtin(*this, rhs);
     return *this;
 }
 
@@ -1006,7 +1123,7 @@ constexpr integer<Bits, Signed> & integer<Bits, Signed>::operator>>=(int n) noex
 {
     if (static_cast<size_t>(n) >= Bits)
     {
-        if (is_negative(*this))
+        if (_impl::is_negative(*this))
             *this = -1;
         else
             *this = 0;
@@ -1056,16 +1173,17 @@ template <size_t Bits, typename Signed>
 template <class T, class>
 constexpr integer<Bits, Signed>::operator T() const noexcept
 {
-    if constexpr (std::is_same_v<T, __int128>)
+    static_assert(std::numeric_limits<T>::is_integer);
-    {
+
-        static_assert(Bits >= 128);
+    /// NOTE: memcpy will suffice, but unfortunately, this function is constexpr.
-        return (__int128(items[1]) << 64) | items[0];
+
-    }
+    using UnsignedT = std::make_unsigned_t<T>;
-    else
+
-    {
+    UnsignedT res{};
-        static_assert(std::numeric_limits<T>::is_integer);
+    for (unsigned i = 0; i < _impl::item_count && i < (sizeof(T) + sizeof(base_type) - 1) / sizeof(base_type); ++i)
-        return items[0];
+        res += UnsignedT(items[i]) << (sizeof(base_type) * 8 * i);
-    }
+
+    return res;
 }
 
 template <size_t Bits, typename Signed>
@@ -1229,7 +1347,7 @@ template <size_t Bits, typename Signed>
 constexpr integer<Bits, Signed> operator<<(const integer<Bits, Signed> & lhs, int n) noexcept
 {
     if (static_cast<size_t>(n) >= Bits)
-        return 0;
+        return integer<Bits, Signed>(0);
     if (n <= 0)
         return lhs;
     return integer<Bits, Signed>::_impl::shift_left(lhs, n);
@@ -1238,7 +1356,7 @@ template <size_t Bits, typename Signed>
 constexpr integer<Bits, Signed> operator>>(const integer<Bits, Signed> & lhs, int n) noexcept
 {
     if (static_cast<size_t>(n) >= Bits)
-        return 0;
+        return integer<Bits, Signed>(0);
     if (n <= 0)
         return lhs;
     return integer<Bits, Signed>::_impl::shift_right(lhs, n);
@@ -1258,7 +1376,7 @@ constexpr bool operator<(const Arithmetic & lhs, const Arithmetic2 & rhs)
 template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
 constexpr bool operator>(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs)
 {
-    return std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>>::_impl::operator_more(lhs, rhs);
+    return std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>>::_impl::operator_greater(lhs, rhs);
 }
 template <typename Arithmetic, typename Arithmetic2, class>
 constexpr bool operator>(const Arithmetic & lhs, const Arithmetic2 & rhs)
@@ -1281,7 +1399,7 @@ constexpr bool operator<=(const Arithmetic & lhs, const Arithmetic2 & rhs)
 template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
 constexpr bool operator>=(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs)
 {
-    return std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>>::_impl::operator_more(lhs, rhs)
+    return std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>>::_impl::operator_greater(lhs, rhs)
         || std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>>::_impl::operator_eq(lhs, rhs);
 }
 template <typename Arithmetic, typename Arithmetic2, class>

base/common/wide_integer_to_string.h

@@ -1,9 +1,12 @@
 #pragma once
 
 #include <string>
+#include <ostream>
+#include <fmt/format.h>
 
 #include "wide_integer.h"
 
+
 namespace wide
 {
 
@@ -33,3 +36,34 @@ inline std::string to_string(const integer<Bits, Signed> & n)
 }
 
 }
+
+
+template <size_t Bits, typename Signed>
+std::ostream & operator<<(std::ostream & out, const wide::integer<Bits, Signed> & value)
+{
+    return out << to_string(value);
+}
+
+
+/// See https://fmt.dev/latest/api.html#formatting-user-defined-types
+template <size_t Bits, typename Signed>
+struct fmt::formatter<wide::integer<Bits, Signed>>
+{
+    constexpr auto parse(format_parse_context & ctx)
+    {
+        auto it = ctx.begin();
+        auto end = ctx.end();
+
+        /// Only support {}.
+        if (it != end && *it != '}')
+            throw format_error("invalid format");
+
+        return it;
+    }
+
+    template <typename FormatContext>
+    auto format(const wide::integer<Bits, Signed> & value, FormatContext & ctx)
+    {
+        return format_to(ctx.out(), "{}", to_string(value));
+    }
+};

base/common/ya.make

@@ -1,9 +1,10 @@
 # This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it.
+OWNER(g:clickhouse)
+
 LIBRARY()
 
 ADDINCL(
     GLOBAL clickhouse/base
-    GLOBAL contrib/libs/cctz/include
 )
 
 CFLAGS (GLOBAL -DARCADIA_BUILD)
@@ -22,7 +23,7 @@ ELSEIF (OS_LINUX)
 ENDIF ()
 
 PEERDIR(
-    contrib/libs/cctz/src
+    contrib/libs/cctz
     contrib/libs/cxxsupp/libcxx-filesystem
     contrib/libs/poco/Net
     contrib/libs/poco/Util
@@ -46,6 +47,7 @@ SRCS(
     errnoToString.cpp
     getFQDNOrHostName.cpp
     getMemoryAmount.cpp
+    getPageSize.cpp
     getResource.cpp
     getThreadId.cpp
     mremap.cpp

base/common/ya.make.in

@@ -1,8 +1,9 @@
+OWNER(g:clickhouse)
+
 LIBRARY()
 
 ADDINCL(
     GLOBAL clickhouse/base
-    GLOBAL contrib/libs/cctz/include
 )
 
 CFLAGS (GLOBAL -DARCADIA_BUILD)
@@ -21,7 +22,7 @@ ELSEIF (OS_LINUX)
 ENDIF ()
 
 PEERDIR(
-    contrib/libs/cctz/src
+    contrib/libs/cctz
     contrib/libs/cxxsupp/libcxx-filesystem
     contrib/libs/poco/Net
     contrib/libs/poco/Util
@@ -34,7 +35,7 @@ PEERDIR(
 CFLAGS(-g0)
 
 SRCS(
-<? find . -name '*.cpp' | grep -v -F tests/ | grep -v -F Replxx | grep -v -F Readline | sed 's/^\.\//    /' | sort ?>
+<? find . -name '*.cpp' | grep -v -F tests/ | grep -v -F examples | grep -v -F Replxx | grep -v -F Readline | sed 's/^\.\//    /' | sort ?>
 )
 
 END()