Merge branch 'master' into ADQM-822

.clang-format

@@ -74,6 +74,7 @@ ConstructorInitializerIndentWidth: 4
 ContinuationIndentWidth: 4
 DerivePointerAlignment: false
 DisableFormat: false
+IndentRequiresClause: false
 IndentWidth: 4
 IndentWrappedFunctionNames: false
 MacroBlockBegin: ''

.github/workflows/pull_request.yml

@@ -46,7 +46,12 @@ jobs:
       - name: Python unit tests
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
+          echo "Testing the main ci directory"
           python3 -m unittest discover -s . -p '*_test.py'
+          for dir in *_lambda/; do
+            echo "Testing $dir"
+            python3 -m unittest discover -s "$dir" -p '*_test.py'
+          done
   DockerHubPushAarch64:
     needs: CheckLabels
     runs-on: [self-hosted, style-checker-aarch64]

.gitmodules

@@ -19,7 +19,7 @@
 	url = https://github.com/google/googletest
 [submodule "contrib/capnproto"]
 	path = contrib/capnproto
-	url = https://github.com/capnproto/capnproto
+	url = https://github.com/ClickHouse/capnproto
 [submodule "contrib/double-conversion"]
 	path = contrib/double-conversion
 	url = https://github.com/google/double-conversion
@@ -35,10 +35,9 @@
 [submodule "contrib/unixodbc"]
 	path = contrib/unixodbc
 	url = https://github.com/ClickHouse/UnixODBC
-[submodule "contrib/protobuf"]
+[submodule "contrib/google-protobuf"]
-	path = contrib/protobuf
+	path = contrib/google-protobuf
-	url = https://github.com/ClickHouse/protobuf
+	url = https://github.com/ClickHouse/google-protobuf.git
-	branch = v3.13.0.1
 [submodule "contrib/boost"]
 	path = contrib/boost
 	url = https://github.com/ClickHouse/boost
@@ -267,10 +266,7 @@
 	url = https://github.com/ClickHouse/nats.c
 [submodule "contrib/vectorscan"]
 	path = contrib/vectorscan
-	url = https://github.com/ClickHouse/vectorscan.git
+	url = https://github.com/VectorCamp/vectorscan.git
-[submodule "contrib/c-ares"]
-	path = contrib/c-ares
-	url = https://github.com/ClickHouse/c-ares
 [submodule "contrib/llvm-project"]
 	path = contrib/llvm-project
 	url = https://github.com/ClickHouse/llvm-project
@@ -344,3 +340,6 @@
 [submodule "contrib/isa-l"]
 	path = contrib/isa-l
 	url = https://github.com/ClickHouse/isa-l.git
+[submodule "contrib/c-ares"]
+	path = contrib/c-ares
+	url = https://github.com/c-ares/c-ares.git

.yamllint

@@ -6,8 +6,10 @@ rules:
         level: warning
         indent-sequences: consistent
     line-length:
-        # there are some bash -c "", so this is OK
+        # there are:
-        max: 300
+        # - bash -c "", so this is OK
+        # - yaml in tests
+        max: 1000
         level: warning
     comments:
         min-spaces-from-content: 1

CHANGELOG.md

@@ -1,4 +1,5 @@
 ### Table of Contents
+**[ClickHouse release v23.5, 2023-06-08](#235)**<br/>
 **[ClickHouse release v23.4, 2023-04-26](#234)**<br/>
 **[ClickHouse release v23.3 LTS, 2023-03-30](#233)**<br/>
 **[ClickHouse release v23.2, 2023-02-23](#232)**<br/>
@@ -7,6 +8,258 @@
 
 # 2023 Changelog
 
+### <a id="235"></a> ClickHouse release 23.5, 2023-06-08
+
+#### Upgrade Notes
+* Compress marks and primary key by default. It significantly reduces the cold query time. Upgrade notes: the support for compressed marks and primary key has been added in version 22.9. If you turned on compressed marks or primary key or installed version 23.5 or newer, which has compressed marks or primary key on by default, you will not be able to downgrade to version 22.8 or earlier. You can also explicitly disable compressed marks or primary keys by specifying the `compress_marks` and `compress_primary_key` settings in the `<merge_tree>` section of the server configuration file. **Upgrade notes:** If you upgrade from versions prior to 22.9, you should either upgrade all replicas at once or disable the compression before upgrade, or upgrade through an intermediate version, where the compressed marks are supported but not enabled by default, such as 23.3. [#42587](https://github.com/ClickHouse/ClickHouse/pull/42587) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Make local object storage work consistently with s3 object storage, fix problem with append (closes [#48465](https://github.com/ClickHouse/ClickHouse/issues/48465)), make it configurable as independent storage. The change is backward incompatible because the cache on top of local object storage is not compatible to previous versions. [#48791](https://github.com/ClickHouse/ClickHouse/pull/48791) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* The experimental feature "in-memory data parts" is removed. The data format is still supported, but the settings are no-op, and compact or wide parts will be used instead. This closes [#45409](https://github.com/ClickHouse/ClickHouse/issues/45409). [#49429](https://github.com/ClickHouse/ClickHouse/pull/49429) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Changed default values of settings `parallelize_output_from_storages` and `input_format_parquet_preserve_order`. This allows ClickHouse to reorder rows when reading from files (e.g. CSV or Parquet), greatly improving performance in many cases. To restore the old behavior of preserving order, use `parallelize_output_from_storages = 0`, `input_format_parquet_preserve_order = 1`. [#49479](https://github.com/ClickHouse/ClickHouse/pull/49479) ([Michael Kolupaev](https://github.com/al13n321)).
+* Make projections production-ready. Add the `optimize_use_projections` setting to control whether the projections will be selected for SELECT queries. The setting `allow_experimental_projection_optimization` is obsolete and does nothing. [#49719](https://github.com/ClickHouse/ClickHouse/pull/49719) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Mark `joinGet` as non-deterministic (so as `dictGet`). It allows using them in mutations without an extra setting. [#49843](https://github.com/ClickHouse/ClickHouse/pull/49843) ([Azat Khuzhin](https://github.com/azat)).
+* Revert the "`groupArray` returns cannot be nullable" change (due to binary compatibility breakage for `groupArray`/`groupArrayLast`/`groupArraySample` over `Nullable` types, which likely will lead to `TOO_LARGE_ARRAY_SIZE` or `CANNOT_READ_ALL_DATA`). [#49971](https://github.com/ClickHouse/ClickHouse/pull/49971) ([Azat Khuzhin](https://github.com/azat)).
+* Setting `enable_memory_bound_merging_of_aggregation_results` is enabled by default. If you update from version prior to 22.12, we recommend to set this flag to `false` until update is finished. [#50319](https://github.com/ClickHouse/ClickHouse/pull/50319) ([Nikita Taranov](https://github.com/nickitat)).
+
+#### New Feature
+* Added storage engine AzureBlobStorage and azureBlobStorage table function. The supported set of features is very similar to storage/table function S3 [#50604] (https://github.com/ClickHouse/ClickHouse/pull/50604) ([alesapin](https://github.com/alesapin)) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni).
+* Added native ClickHouse Keeper CLI Client, it is available as `clickhouse keeper-client` [#47414](https://github.com/ClickHouse/ClickHouse/pull/47414) ([pufit](https://github.com/pufit)).
+* Add `urlCluster` table function. Refactor all *Cluster table functions to reduce code duplication. Make schema inference work for all possible *Cluster function signatures and for named collections. Closes [#38499](https://github.com/ClickHouse/ClickHouse/issues/38499). [#45427](https://github.com/ClickHouse/ClickHouse/pull/45427) ([attack204](https://github.com/attack204)), Pavel Kruglov.
+* The query cache can now be used for production workloads. [#47977](https://github.com/ClickHouse/ClickHouse/pull/47977) ([Robert Schulze](https://github.com/rschu1ze)). The query cache can now support queries with totals and extremes modifier. [#48853](https://github.com/ClickHouse/ClickHouse/pull/48853) ([Robert Schulze](https://github.com/rschu1ze)). Make `allow_experimental_query_cache` setting as obsolete for backward-compatibility. It was removed in https://github.com/ClickHouse/ClickHouse/pull/47977. [#49934](https://github.com/ClickHouse/ClickHouse/pull/49934) ([Timur Solodovnikov](https://github.com/tsolodov)).
+* Geographical data types (`Point`, `Ring`, `Polygon`, and `MultiPolygon`) are production-ready. [#50022](https://github.com/ClickHouse/ClickHouse/pull/50022) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Add schema inference to PostgreSQL, MySQL, MeiliSearch, and SQLite table engines. Closes [#49972](https://github.com/ClickHouse/ClickHouse/issues/49972). [#50000](https://github.com/ClickHouse/ClickHouse/pull/50000) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Password type in queries like `CREATE USER u IDENTIFIED BY 'p'` will be automatically set according to the setting `default_password_type` in the `config.xml` on the server. Closes [#42915](https://github.com/ClickHouse/ClickHouse/issues/42915). [#44674](https://github.com/ClickHouse/ClickHouse/pull/44674) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Add bcrypt password authentication type. Closes [#34599](https://github.com/ClickHouse/ClickHouse/issues/34599). [#44905](https://github.com/ClickHouse/ClickHouse/pull/44905) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Introduces new keyword `INTO OUTFILE 'file.txt' APPEND`. [#48880](https://github.com/ClickHouse/ClickHouse/pull/48880) ([alekar](https://github.com/alekar)).
+* Added `system.zookeeper_connection` table that shows information about Keeper connections. [#45245](https://github.com/ClickHouse/ClickHouse/pull/45245) ([mateng915](https://github.com/mateng0915)).
+* Add new function `generateRandomStructure` that generates random table structure. It can be used in combination with table function `generateRandom`. [#47409](https://github.com/ClickHouse/ClickHouse/pull/47409) ([Kruglov Pavel](https://github.com/Avogar)).
+* Allow the use of `CASE` without an `ELSE` branch and extended `transform` to deal with more types. Also fix some issues that made transform() return incorrect results when decimal types were mixed with other numeric types. [#48300](https://github.com/ClickHouse/ClickHouse/pull/48300) ([Salvatore Mesoraca](https://github.com/aiven-sal)). This closes #2655. This closes #9596. This closes #38666.
+* Added [server-side encryption using KMS keys](https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingKMSEncryption.html) with S3 tables, and the `header` setting with S3 disks. Closes [#48723](https://github.com/ClickHouse/ClickHouse/issues/48723). [#48724](https://github.com/ClickHouse/ClickHouse/pull/48724) ([Johann Gan](https://github.com/johanngan)).
+* Add MemoryTracker for the background tasks (merges and mutation). Introduces `merges_mutations_memory_usage_soft_limit` and `merges_mutations_memory_usage_to_ram_ratio` settings that represent the soft memory limit for merges and mutations. If this limit is reached ClickHouse won't schedule new merge or mutation tasks. Also `MergesMutationsMemoryTracking` metric is introduced to allow observing current memory usage of background tasks. Resubmit [#46089](https://github.com/ClickHouse/ClickHouse/issues/46089). Closes [#48774](https://github.com/ClickHouse/ClickHouse/issues/48774). [#48787](https://github.com/ClickHouse/ClickHouse/pull/48787) ([Dmitry Novik](https://github.com/novikd)).
+* Function `dotProduct` work for array. [#49050](https://github.com/ClickHouse/ClickHouse/pull/49050) ([FFFFFFFHHHHHHH](https://github.com/FFFFFFFHHHHHHH)).
+* Support statement `SHOW INDEX` to improve compatibility with MySQL. [#49158](https://github.com/ClickHouse/ClickHouse/pull/49158) ([Robert Schulze](https://github.com/rschu1ze)).
+* Add virtual column `_file` and `_path` support to table function `url`. - Impove error message for table function `url`. - resolves [#49231](https://github.com/ClickHouse/ClickHouse/issues/49231) - resolves [#49232](https://github.com/ClickHouse/ClickHouse/issues/49232). [#49356](https://github.com/ClickHouse/ClickHouse/pull/49356) ([Ziyi Tan](https://github.com/Ziy1-Tan)).
+* Adding the `grants` field in the users.xml file, which allows specifying grants for users. [#49381](https://github.com/ClickHouse/ClickHouse/pull/49381) ([pufit](https://github.com/pufit)).
+* Support full/right join by using grace hash join algorithm. [#49483](https://github.com/ClickHouse/ClickHouse/pull/49483) ([lgbo](https://github.com/lgbo-ustc)).
+* `WITH FILL` modifier groups filling by sorting prefix. Controlled by `use_with_fill_by_sorting_prefix` setting (enabled by default). Related to [#33203](https://github.com/ClickHouse/ClickHouse/issues/33203)#issuecomment-1418736794. [#49503](https://github.com/ClickHouse/ClickHouse/pull/49503) ([Igor Nikonov](https://github.com/devcrafter)).
+* Clickhouse-client now accepts queries after "--multiquery" when "--query" (or "-q") is absent. example: clickhouse-client --multiquery "select 1; select 2;". [#49870](https://github.com/ClickHouse/ClickHouse/pull/49870) ([Alexey Gerasimchuk](https://github.com/Demilivor)).
+* Add separate `handshake_timeout` for receiving Hello packet from replica. Closes [#48854](https://github.com/ClickHouse/ClickHouse/issues/48854). [#49948](https://github.com/ClickHouse/ClickHouse/pull/49948) ([Kruglov Pavel](https://github.com/Avogar)).
+* Added a function "space" which repeats a space as many times as specified. [#50103](https://github.com/ClickHouse/ClickHouse/pull/50103) ([Robert Schulze](https://github.com/rschu1ze)).
+* Added --input_format_csv_trim_whitespaces option. [#50215](https://github.com/ClickHouse/ClickHouse/pull/50215) ([Alexey Gerasimchuk](https://github.com/Demilivor)).
+* Allow the `dictGetAll` function for regexp tree dictionaries to return values from multiple matches as arrays. Closes [#50254](https://github.com/ClickHouse/ClickHouse/issues/50254). [#50255](https://github.com/ClickHouse/ClickHouse/pull/50255) ([Johann Gan](https://github.com/johanngan)).
+* Added `toLastDayOfWeek` function to round a date or a date with time up to the nearest Saturday or Sunday. [#50315](https://github.com/ClickHouse/ClickHouse/pull/50315) ([Victor Krasnov](https://github.com/sirvickr)).
+* Ability to ignore a skip index by specifying `ignore_data_skipping_indices`. [#50329](https://github.com/ClickHouse/ClickHouse/pull/50329) ([Boris Kuschel](https://github.com/bkuschel)).
+* Add `system.user_processes` table and `SHOW USER PROCESSES` query to show memory info and ProfileEvents on user level. [#50492](https://github.com/ClickHouse/ClickHouse/pull/50492) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Add server and format settings `display_secrets_in_show_and_select` for displaying secrets of tables, databases, table functions, and dictionaries. Add privilege `displaySecretsInShowAndSelect` controlling which users can view secrets. [#46528](https://github.com/ClickHouse/ClickHouse/pull/46528) ([Mike Kot](https://github.com/myrrc)).
+* Allow to set up a ROW POLICY for all tables that belong to a DATABASE. [#47640](https://github.com/ClickHouse/ClickHouse/pull/47640) ([Ilya Golshtein](https://github.com/ilejn)).
+
+#### Performance Improvement
+* Compress marks and primary key by default. It significantly reduces the cold query time. Upgrade notes: the support for compressed marks and primary key has been added in version 22.9. If you turned on compressed marks or primary key or installed version 23.5 or newer, which has compressed marks or primary key on by default, you will not be able to downgrade to version 22.8 or earlier. You can also explicitly disable compressed marks or primary keys by specifying the `compress_marks` and `compress_primary_key` settings in the `<merge_tree>` section of the server configuration file. [#42587](https://github.com/ClickHouse/ClickHouse/pull/42587) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* New setting s3_max_inflight_parts_for_one_file sets the limit of concurrently loaded parts with multipart upload request in scope of one file. [#49961](https://github.com/ClickHouse/ClickHouse/pull/49961) ([Sema Checherinda](https://github.com/CheSema)).
+* When reading from multiple files reduce parallel parsing threads for each file. Resolves [#42192](https://github.com/ClickHouse/ClickHouse/issues/42192). [#46661](https://github.com/ClickHouse/ClickHouse/pull/46661) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Use aggregate projection only if it reads fewer granules than normal reading. It should help in case if query hits the PK of the table, but not the projection. Fixes [#49150](https://github.com/ClickHouse/ClickHouse/issues/49150). [#49417](https://github.com/ClickHouse/ClickHouse/pull/49417) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Do not store blocks in `ANY` hash join if nothing is inserted. [#48633](https://github.com/ClickHouse/ClickHouse/pull/48633) ([vdimir](https://github.com/vdimir)).
+* Fixes aggregate combinator `-If` when JIT compiled, and enable JIT compilation for aggregate functions. Closes [#48120](https://github.com/ClickHouse/ClickHouse/issues/48120). [#49083](https://github.com/ClickHouse/ClickHouse/pull/49083) ([Igor Nikonov](https://github.com/devcrafter)).
+* For reading from remote tables we use smaller tasks (instead of reading the whole part) to make tasks stealing work * task size is determined by size of columns to read * always use 1mb buffers for reading from s3 * boundaries of cache segments aligned to 1mb so they have decent size even with small tasks. it also should prevent fragmentation. [#49287](https://github.com/ClickHouse/ClickHouse/pull/49287) ([Nikita Taranov](https://github.com/nickitat)).
+* Introduced settings: - `merge_max_block_size_bytes` to limit the amount of memory used for background operations. - `vertical_merge_algorithm_min_bytes_to_activate` to add another condition to activate vertical merges. [#49313](https://github.com/ClickHouse/ClickHouse/pull/49313) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Default size of a read buffer for reading from local filesystem changed to a slightly better value. Also two new settings are introduced: `max_read_buffer_size_local_fs` and `max_read_buffer_size_remote_fs`. [#49321](https://github.com/ClickHouse/ClickHouse/pull/49321) ([Nikita Taranov](https://github.com/nickitat)).
+* Improve memory usage and speed of `SPARSE_HASHED`/`HASHED` dictionaries (e.g. `SPARSE_HASHED` now eats 2.6x less memory, and is ~2x faster). [#49380](https://github.com/ClickHouse/ClickHouse/pull/49380) ([Azat Khuzhin](https://github.com/azat)).
+* Optimize the `system.query_log` and `system.query_thread_log` tables by applying `LowCardinality` when appropriate. The queries over these tables will be faster. [#49530](https://github.com/ClickHouse/ClickHouse/pull/49530) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Better performance when reading local `Parquet` files (through parallel reading). [#49539](https://github.com/ClickHouse/ClickHouse/pull/49539) ([Michael Kolupaev](https://github.com/al13n321)).
+* Improve the performance of `RIGHT/FULL JOIN` by up to 2 times in certain scenarios, especially when joining a small left table with a large right table. [#49585](https://github.com/ClickHouse/ClickHouse/pull/49585) ([lgbo](https://github.com/lgbo-ustc)).
+* Improve performance of BLAKE3 by 11% by enabling LTO for Rust. [#49600](https://github.com/ClickHouse/ClickHouse/pull/49600) ([Azat Khuzhin](https://github.com/azat)). Now it is on par with C++.
+* Optimize the structure of the `system.opentelemetry_span_log`. Use `LowCardinality` where appropriate. Although this table is generally stupid (it is using the Map data type even for common attributes), it will be slightly better. [#49647](https://github.com/ClickHouse/ClickHouse/pull/49647) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Try to reserve hash table's size in `grace_hash` join. [#49816](https://github.com/ClickHouse/ClickHouse/pull/49816) ([lgbo](https://github.com/lgbo-ustc)).
+* Parallel merge of `uniqExactIf` states. Closes [#49885](https://github.com/ClickHouse/ClickHouse/issues/49885). [#50285](https://github.com/ClickHouse/ClickHouse/pull/50285) ([flynn](https://github.com/ucasfl)).
+* Keeper improvement: add `CheckNotExists` request to Keeper, which allows to improve the performance of Replicated tables. [#48897](https://github.com/ClickHouse/ClickHouse/pull/48897) ([Antonio Andelic](https://github.com/antonio2368)).
+* Keeper performance improvements: avoid serializing same request twice while processing. Cache deserialization results of large requests. Controlled by new coordination setting `min_request_size_for_cache`. [#49004](https://github.com/ClickHouse/ClickHouse/pull/49004) ([Antonio Andelic](https://github.com/antonio2368)).
+* Reduced number of `List` ZooKeeper requests when selecting parts to merge and a lot of partitions do not have anything to merge. [#49637](https://github.com/ClickHouse/ClickHouse/pull/49637) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Rework locking in the FS cache [#44985](https://github.com/ClickHouse/ClickHouse/pull/44985) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Disable pure parallel replicas if trivial count optimization is possible. [#50594](https://github.com/ClickHouse/ClickHouse/pull/50594) ([Raúl Marín](https://github.com/Algunenano)).
+* Don't send head request for all keys in Iceberg schema inference, only for keys that are used for reaing data. [#50203](https://github.com/ClickHouse/ClickHouse/pull/50203) ([Kruglov Pavel](https://github.com/Avogar)).
+* Setting `enable_memory_bound_merging_of_aggregation_results` is enabled by default. [#50319](https://github.com/ClickHouse/ClickHouse/pull/50319) ([Nikita Taranov](https://github.com/nickitat)).
+
+#### Experimental Feature
+* `DEFLATE_QPL` codec lower the minimum simd version to SSE 4.2. [doc change in qpl](https://github.com/intel/qpl/commit/3f8f5cea27739f5261e8fd577dc233ffe88bf679) - Intel® QPL relies on a run-time kernels dispatcher and cpuid check to choose the best available implementation(sse/avx2/avx512) - restructured cmakefile for qpl build in clickhouse to align with latest upstream qpl. [#49811](https://github.com/ClickHouse/ClickHouse/pull/49811) ([jasperzhu](https://github.com/jinjunzh)).
+* Add initial support to do JOINs with pure parallel replicas. [#49544](https://github.com/ClickHouse/ClickHouse/pull/49544) ([Raúl Marín](https://github.com/Algunenano)).
+* More parallelism on `Outdated` parts removal with "zero-copy replication". [#49630](https://github.com/ClickHouse/ClickHouse/pull/49630) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Parallel Replicas: 1) Fixed an error `NOT_FOUND_COLUMN_IN_BLOCK` in case of using parallel replicas with non-replicated storage with disabled setting `parallel_replicas_for_non_replicated_merge_tree` 2) Now `allow_experimental_parallel_reading_from_replicas` have 3 possible values - 0, 1 and 2. 0 - disabled, 1 - enabled, silently disable them in case of failure (in case of FINAL or JOIN), 2 - enabled, throw an expection in case of failure. 3) If FINAL modifier is used in SELECT query and parallel replicas are enabled, ClickHouse will try to disable them if `allow_experimental_parallel_reading_from_replicas` is set to 1 and throw an exception otherwise. [#50195](https://github.com/ClickHouse/ClickHouse/pull/50195) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* When parallel replicas are enabled they will always skip unavailable servers (the behavior is controlled by the setting `skip_unavailable_shards`, enabled by default and can be only disabled). This closes: [#48565](https://github.com/ClickHouse/ClickHouse/issues/48565). [#50293](https://github.com/ClickHouse/ClickHouse/pull/50293) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+
+#### Improvement
+* The `BACKUP` command will not decrypt data from encrypted disks while making a backup. Instead the data will be stored in a backup in encrypted form. Such backups can be restored only to an encrypted disk with the same (or extended) list of encryption keys. [#48896](https://github.com/ClickHouse/ClickHouse/pull/48896) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Added possibility to use temporary tables in FROM part of ATTACH PARTITION FROM and REPLACE PARTITION FROM. [#49436](https://github.com/ClickHouse/ClickHouse/pull/49436) ([Roman Vasin](https://github.com/rvasin)).
+* Added setting `async_insert` for `MergeTree` tables. It has the same meaning as query-level setting `async_insert` and enables asynchronous inserts for specific table. Note: it doesn't take effect for insert queries from `clickhouse-client`, use query-level setting in that case. [#49122](https://github.com/ClickHouse/ClickHouse/pull/49122) ([Anton Popov](https://github.com/CurtizJ)).
+* Add support for size suffixes in quota creation statement parameters. [#49087](https://github.com/ClickHouse/ClickHouse/pull/49087) ([Eridanus](https://github.com/Eridanus117)).
+* Extend `first_value` and `last_value` to accept NULL. [#46467](https://github.com/ClickHouse/ClickHouse/pull/46467) ([lgbo](https://github.com/lgbo-ustc)).
+* Add alias `str_to_map` and `mapFromString` for `extractKeyValuePairs`. closes https://github.com/clickhouse/clickhouse/issues/47185. [#49466](https://github.com/ClickHouse/ClickHouse/pull/49466) ([flynn](https://github.com/ucasfl)).
+* Add support for CGroup version 2 for asynchronous metrics about the memory usage and availability. This closes [#37983](https://github.com/ClickHouse/ClickHouse/issues/37983). [#45999](https://github.com/ClickHouse/ClickHouse/pull/45999) ([sichenzhao](https://github.com/sichenzhao)).
+* Cluster table functions should always skip unavailable shards. close [#46314](https://github.com/ClickHouse/ClickHouse/issues/46314). [#46765](https://github.com/ClickHouse/ClickHouse/pull/46765) ([zk_kiger](https://github.com/zk-kiger)).
+* Allow CSV file to contain empty columns in its header. [#47496](https://github.com/ClickHouse/ClickHouse/pull/47496) ([你不要过来啊](https://github.com/iiiuwioajdks)).
+* Add Google Cloud Storage S3 compatible table function `gcs`. Like the `oss` and `cosn` functions, it is just an alias over the `s3` table function, and it does not bring any new features. [#47815](https://github.com/ClickHouse/ClickHouse/pull/47815) ([Kuba Kaflik](https://github.com/jkaflik)).
+* Add ability to use strict parts size for S3 (compatibility with CloudFlare R2 S3 Storage). [#48492](https://github.com/ClickHouse/ClickHouse/pull/48492) ([Azat Khuzhin](https://github.com/azat)).
+* Added new columns with info about `Replicated` database replicas to `system.clusters`: `database_shard_name`, `database_replica_name`, `is_active`. Added an optional `FROM SHARD` clause to `SYSTEM DROP DATABASE REPLICA` query. [#48548](https://github.com/ClickHouse/ClickHouse/pull/48548) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Add a new column `zookeeper_name` in system.replicas, to indicate on which (auxiliary) zookeeper cluster the replicated table's metadata is stored. [#48549](https://github.com/ClickHouse/ClickHouse/pull/48549) ([cangyin](https://github.com/cangyin)).
+* `IN` operator support the comparison of `Date` and `Date32`. Closes [#48736](https://github.com/ClickHouse/ClickHouse/issues/48736). [#48806](https://github.com/ClickHouse/ClickHouse/pull/48806) ([flynn](https://github.com/ucasfl)).
+* Support for erasure codes in `HDFS`, author: @M1eyu2018, @tomscut. [#48833](https://github.com/ClickHouse/ClickHouse/pull/48833) ([M1eyu](https://github.com/M1eyu2018)).
+* Implement SYSTEM DROP REPLICA from auxillary ZooKeeper clusters, may be close [#48931](https://github.com/ClickHouse/ClickHouse/issues/48931). [#48932](https://github.com/ClickHouse/ClickHouse/pull/48932) ([wangxiaobo](https://github.com/wzb5212)).
+* Add Array data type to MongoDB. Closes [#48598](https://github.com/ClickHouse/ClickHouse/issues/48598). [#48983](https://github.com/ClickHouse/ClickHouse/pull/48983) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Support storing `Interval` data types in tables. [#49085](https://github.com/ClickHouse/ClickHouse/pull/49085) ([larryluogit](https://github.com/larryluogit)).
+* Allow using `ntile` window function without explicit window frame definition: `ntile(3) OVER (ORDER BY a)`, close [#46763](https://github.com/ClickHouse/ClickHouse/issues/46763). [#49093](https://github.com/ClickHouse/ClickHouse/pull/49093) ([vdimir](https://github.com/vdimir)).
+* Added settings (`number_of_mutations_to_delay`, `number_of_mutations_to_throw`) to delay or throw `ALTER` queries that create mutations (`ALTER UPDATE`, `ALTER DELETE`, `ALTER MODIFY COLUMN`, ...) in case when table already has a lot of unfinished mutations. [#49117](https://github.com/ClickHouse/ClickHouse/pull/49117) ([Anton Popov](https://github.com/CurtizJ)).
+* Catch exception from `create_directories` in filesystem cache. [#49203](https://github.com/ClickHouse/ClickHouse/pull/49203) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Copies embedded examples to a new field `example` in `system.functions` to supplement the field `description`. [#49222](https://github.com/ClickHouse/ClickHouse/pull/49222) ([Dan Roscigno](https://github.com/DanRoscigno)).
+* Enable connection options for the MongoDB dictionary. Example: ``` xml <source> <mongodb> <host>localhost</host> <port>27017</port> <user></user> <password></password> <db>test</db> <collection>dictionary_source</collection> <options>ssl=true</options> </mongodb> </source> ``` ### Documentation entry for user-facing changes. [#49225](https://github.com/ClickHouse/ClickHouse/pull/49225) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
+* Added an alias `asymptotic` for `asymp` computational method for `kolmogorovSmirnovTest`. Improved documentation. [#49286](https://github.com/ClickHouse/ClickHouse/pull/49286) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Aggregation function groupBitAnd/Or/Xor now work on signed integer data. This makes them consistent with the behavior of scalar functions bitAnd/Or/Xor. [#49292](https://github.com/ClickHouse/ClickHouse/pull/49292) ([exmy](https://github.com/exmy)).
+* Split function-documentation into more fine-granular fields. [#49300](https://github.com/ClickHouse/ClickHouse/pull/49300) ([Robert Schulze](https://github.com/rschu1ze)).
+* Use multiple threads shared between all tables within a server to load outdated data parts. The the size of the pool and its queue is controlled by `max_outdated_parts_loading_thread_pool_size` and `outdated_part_loading_thread_pool_queue_size` settings. [#49317](https://github.com/ClickHouse/ClickHouse/pull/49317) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Don't overestimate the size of processed data for `LowCardinality` columns when they share dictionaries between blocks. This closes [#49322](https://github.com/ClickHouse/ClickHouse/issues/49322). See also [#48745](https://github.com/ClickHouse/ClickHouse/issues/48745). [#49323](https://github.com/ClickHouse/ClickHouse/pull/49323) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Parquet writer now uses reasonable row group size when invoked through `OUTFILE`. [#49325](https://github.com/ClickHouse/ClickHouse/pull/49325) ([Michael Kolupaev](https://github.com/al13n321)).
+* Allow restricted keywords like `ARRAY` as an alias if the alias is quoted. Closes [#49324](https://github.com/ClickHouse/ClickHouse/issues/49324). [#49360](https://github.com/ClickHouse/ClickHouse/pull/49360) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Data parts loading and deletion jobs were moved to shared server-wide pools instead of per-table pools. Pools sizes are controlled via settings `max_active_parts_loading_thread_pool_size`, `max_outdated_parts_loading_thread_pool_size` and `max_parts_cleaning_thread_pool_size` in top-level config. Table-level settings `max_part_loading_threads` and `max_part_removal_threads` became obsolete. [#49474](https://github.com/ClickHouse/ClickHouse/pull/49474) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Allow `?password=pass` in URL of the Play UI. Password is replaced in browser history. [#49505](https://github.com/ClickHouse/ClickHouse/pull/49505) ([Mike Kot](https://github.com/myrrc)).
+* Allow reading zero-size objects from remote filesystems. (because empty files are not backup'd, so we might end up with zero blobs in metadata file). Closes [#49480](https://github.com/ClickHouse/ClickHouse/issues/49480). [#49519](https://github.com/ClickHouse/ClickHouse/pull/49519) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Attach thread MemoryTracker to `total_memory_tracker` after `ThreadGroup` detached. [#49527](https://github.com/ClickHouse/ClickHouse/pull/49527) ([Dmitry Novik](https://github.com/novikd)).
+* Fix parameterized views when a query parameter is used multiple times in the query. [#49556](https://github.com/ClickHouse/ClickHouse/pull/49556) ([Azat Khuzhin](https://github.com/azat)).
+* Release memory allocated for the last sent ProfileEvents snapshot in the context of a query. Followup [#47564](https://github.com/ClickHouse/ClickHouse/issues/47564). [#49561](https://github.com/ClickHouse/ClickHouse/pull/49561) ([Dmitry Novik](https://github.com/novikd)).
+* Function "makeDate" now provides a MySQL-compatible overload (year & day of the year argument). [#49603](https://github.com/ClickHouse/ClickHouse/pull/49603) ([Robert Schulze](https://github.com/rschu1ze)).
+* Support `dictionary` table function for `RegExpTreeDictionary`. [#49666](https://github.com/ClickHouse/ClickHouse/pull/49666) ([Han Fei](https://github.com/hanfei1991)).
+* Added weighted fair IO scheduling policy. Added dynamic resource manager, which allows IO scheduling hierarchy to be updated in runtime w/o server restarts. [#49671](https://github.com/ClickHouse/ClickHouse/pull/49671) ([Sergei Trifonov](https://github.com/serxa)).
+* Add compose request after multipart upload to GCS. This enables the usage of copy operation on objects uploaded with the multipart upload. It's recommended to set `s3_strict_upload_part_size` to some value because compose request can fail on objects created with parts of different sizes. [#49693](https://github.com/ClickHouse/ClickHouse/pull/49693) ([Antonio Andelic](https://github.com/antonio2368)).
+* For the `extractKeyValuePairs` function: improve the "best-effort" parsing logic to accept `key_value_delimiter` as a valid part of the value. This also simplifies branching and might even speed up things a bit. [#49760](https://github.com/ClickHouse/ClickHouse/pull/49760) ([Arthur Passos](https://github.com/arthurpassos)).
+* Add `initial_query_id` field for system.processors_profile_log [#49777](https://github.com/ClickHouse/ClickHouse/pull/49777) ([helifu](https://github.com/helifu)).
+* System log tables can now have custom sorting keys. [#49778](https://github.com/ClickHouse/ClickHouse/pull/49778) ([helifu](https://github.com/helifu)).
+* A new field `partitions` to `system.query_log` is used to indicate which partitions are participating in the calculation. [#49779](https://github.com/ClickHouse/ClickHouse/pull/49779) ([helifu](https://github.com/helifu)).
+* Added `enable_the_endpoint_id_with_zookeeper_name_prefix` setting for `ReplicatedMergeTree` (disabled by default). When enabled, it adds ZooKeeper cluster name to table's interserver communication endpoint. It avoids `Duplicate interserver IO endpoint` errors when having replicated tables with the same path, but different auxiliary ZooKeepers. [#49780](https://github.com/ClickHouse/ClickHouse/pull/49780) ([helifu](https://github.com/helifu)).
+* Add query parameters to `clickhouse-local`. Closes [#46561](https://github.com/ClickHouse/ClickHouse/issues/46561). [#49785](https://github.com/ClickHouse/ClickHouse/pull/49785) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Allow loading dictionaries and functions from YAML by default. In previous versions, it required editing the `dictionaries_config` or `user_defined_executable_functions_config` in the configuration file, as they expected `*.xml` files. [#49812](https://github.com/ClickHouse/ClickHouse/pull/49812) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* The Kafka table engine now allows to use alias columns. [#49824](https://github.com/ClickHouse/ClickHouse/pull/49824) ([Aleksandr Musorin](https://github.com/AVMusorin)).
+* Add setting to limit the max number of pairs produced by `extractKeyValuePairs`, a safeguard to avoid using way too much memory. [#49836](https://github.com/ClickHouse/ClickHouse/pull/49836) ([Arthur Passos](https://github.com/arthurpassos)).
+* Add support for (an unusual) case where the arguments in the `IN` operator are single-element tuples. [#49844](https://github.com/ClickHouse/ClickHouse/pull/49844) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
+* `bitHammingDistance` function support `String` and `FixedString` data type. Closes [#48827](https://github.com/ClickHouse/ClickHouse/issues/48827). [#49858](https://github.com/ClickHouse/ClickHouse/pull/49858) ([flynn](https://github.com/ucasfl)).
+* Fix timeout resetting errors in the client on OS X. [#49863](https://github.com/ClickHouse/ClickHouse/pull/49863) ([alekar](https://github.com/alekar)).
+* Add support for big integers, such as UInt128, Int128, UInt256, and Int256 in the function `bitCount`. This enables Hamming distance over large bit masks for AI applications. [#49867](https://github.com/ClickHouse/ClickHouse/pull/49867) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fingerprints to be used instead of key IDs in encrypted disks. This simplifies the configuration of encrypted disks. [#49882](https://github.com/ClickHouse/ClickHouse/pull/49882) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Add UUID data type to PostgreSQL. Closes [#49739](https://github.com/ClickHouse/ClickHouse/issues/49739). [#49894](https://github.com/ClickHouse/ClickHouse/pull/49894) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Function `toUnixTimestamp` now accepts `Date` and `Date32` arguments. [#49989](https://github.com/ClickHouse/ClickHouse/pull/49989) ([Victor Krasnov](https://github.com/sirvickr)).
+* Charge only server memory for dictionaries. [#49995](https://github.com/ClickHouse/ClickHouse/pull/49995) ([Azat Khuzhin](https://github.com/azat)).
+* The server will allow using the `SQL_*` settings such as `SQL_AUTO_IS_NULL` as no-ops for MySQL compatibility. This closes [#49927](https://github.com/ClickHouse/ClickHouse/issues/49927). [#50013](https://github.com/ClickHouse/ClickHouse/pull/50013) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Preserve initial_query_id for ON CLUSTER queries, which is useful for introspection (under `distributed_ddl_entry_format_version=5`). [#50015](https://github.com/ClickHouse/ClickHouse/pull/50015) ([Azat Khuzhin](https://github.com/azat)).
+* Preserve backward incompatibility for renamed settings by using aliases (`allow_experimental_projection_optimization` for `optimize_use_projections`, `allow_experimental_lightweight_delete` for `enable_lightweight_delete`). [#50044](https://github.com/ClickHouse/ClickHouse/pull/50044) ([Azat Khuzhin](https://github.com/azat)).
+* Support passing FQDN through setting my_hostname to register cluster node in keeper. Add setting of invisible to support multi compute groups. A compute group as a cluster, is invisible to other compute groups. [#50186](https://github.com/ClickHouse/ClickHouse/pull/50186) ([Yangkuan Liu](https://github.com/LiuYangkuan)).
+* Fix PostgreSQL reading all the data even though `LIMIT n` could be specified. [#50187](https://github.com/ClickHouse/ClickHouse/pull/50187) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Add new profile events for queries with subqueries (`QueriesWithSubqueries`/`SelectQueriesWithSubqueries`/`InsertQueriesWithSubqueries`). [#50204](https://github.com/ClickHouse/ClickHouse/pull/50204) ([Azat Khuzhin](https://github.com/azat)).
+* Adding the roles field in the users.xml file, which allows specifying roles with grants via a config file. [#50278](https://github.com/ClickHouse/ClickHouse/pull/50278) ([pufit](https://github.com/pufit)).
+* Report `CGroupCpuCfsPeriod` and `CGroupCpuCfsQuota` in AsynchronousMetrics. - Respect cgroup v2 memory limits during server startup. [#50379](https://github.com/ClickHouse/ClickHouse/pull/50379) ([alekar](https://github.com/alekar)).
+* Add a signal handler for SIGQUIT to work the same way as SIGINT. Closes [#50298](https://github.com/ClickHouse/ClickHouse/issues/50298). [#50435](https://github.com/ClickHouse/ClickHouse/pull/50435) ([Nikolay Degterinsky](https://github.com/evillique)).
+* In case JSON parse fails due to the large size of the object output the last position to allow debugging. [#50474](https://github.com/ClickHouse/ClickHouse/pull/50474) ([Valentin Alexeev](https://github.com/valentinalexeev)).
+* Support decimals with not fixed size. Closes [#49130](https://github.com/ClickHouse/ClickHouse/issues/49130). [#50586](https://github.com/ClickHouse/ClickHouse/pull/50586) ([Kruglov Pavel](https://github.com/Avogar)).
+
+#### Build/Testing/Packaging Improvement
+* New and improved `keeper-bench`. Everything can be customized from YAML/XML file: - request generator - each type of request generator can have a specific set of fields - multi requests can be generated just by doing the same under `multi` key - for each request or subrequest in multi a `weight` field can be defined to control distribution - define trees that need to be setup for a test run - hosts can be defined with all timeouts customizable and it's possible to control how many sessions to generate for each host - integers defined with `min_value` and `max_value` fields are random number generators. [#48547](https://github.com/ClickHouse/ClickHouse/pull/48547) ([Antonio Andelic](https://github.com/antonio2368)).
+* Io_uring is not supported on macos, don't choose it when running tests on local to avoid occassional failures. [#49250](https://github.com/ClickHouse/ClickHouse/pull/49250) ([Frank Chen](https://github.com/FrankChen021)).
+* Support named fault injection for testing. [#49361](https://github.com/ClickHouse/ClickHouse/pull/49361) ([Han Fei](https://github.com/hanfei1991)).
+* Allow running ClickHouse in the OS where the `prctl` (process control) syscall is not available, such as AWS Lambda. [#49538](https://github.com/ClickHouse/ClickHouse/pull/49538) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fixed the issue of build conflict between contrib/isa-l and isa-l in qpl [49296](https://github.com/ClickHouse/ClickHouse/issues/49296). [#49584](https://github.com/ClickHouse/ClickHouse/pull/49584) ([jasperzhu](https://github.com/jinjunzh)).
+* Utilities are now only build if explicitly requested ("-DENABLE_UTILS=1") instead of by default, this reduces link times in typical development builds. [#49620](https://github.com/ClickHouse/ClickHouse/pull/49620) ([Robert Schulze](https://github.com/rschu1ze)).
+* Pull build description of idxd-config into a separate CMake file to avoid accidental removal in future. [#49651](https://github.com/ClickHouse/ClickHouse/pull/49651) ([jasperzhu](https://github.com/jinjunzh)).
+* Add CI check with an enabled analyzer in the master. Follow-up [#49562](https://github.com/ClickHouse/ClickHouse/issues/49562). [#49668](https://github.com/ClickHouse/ClickHouse/pull/49668) ([Dmitry Novik](https://github.com/novikd)).
+* Switch to LLVM/clang 16. [#49678](https://github.com/ClickHouse/ClickHouse/pull/49678) ([Azat Khuzhin](https://github.com/azat)).
+* Allow building ClickHouse with clang-17. [#49851](https://github.com/ClickHouse/ClickHouse/pull/49851) ([Alexey Milovidov](https://github.com/alexey-milovidov)). [#50410](https://github.com/ClickHouse/ClickHouse/pull/50410) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* ClickHouse is now easier to be integrated into other cmake projects. [#49991](https://github.com/ClickHouse/ClickHouse/pull/49991) ([Amos Bird](https://github.com/amosbird)). (Which is strongly discouraged - Alexey Milovidov).
+* Fix strange additional QEMU logging after [#47151](https://github.com/ClickHouse/ClickHouse/issues/47151), see https://s3.amazonaws.com/clickhouse-test-reports/50078/a4743996ee4f3583884d07bcd6501df0cfdaa346/stateless_tests__release__databasereplicated__[3_4].html. [#50442](https://github.com/ClickHouse/ClickHouse/pull/50442) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* ClickHouse can work on Linux RISC-V 6.1.22. This closes [#50456](https://github.com/ClickHouse/ClickHouse/issues/50456). [#50457](https://github.com/ClickHouse/ClickHouse/pull/50457) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Bump internal protobuf to v3.18 (fixes bogus CVE-2022-1941). [#50400](https://github.com/ClickHouse/ClickHouse/pull/50400) ([Robert Schulze](https://github.com/rschu1ze)).
+* Bump internal libxml2 to v2.10.4 (fixes bogus CVE-2023-28484 and bogus CVE-2023-29469). [#50402](https://github.com/ClickHouse/ClickHouse/pull/50402) ([Robert Schulze](https://github.com/rschu1ze)).
+* Bump c-ares to v1.19.1 (bogus CVE-2023-32067, bogus CVE-2023-31130, bogus CVE-2023-31147). [#50403](https://github.com/ClickHouse/ClickHouse/pull/50403) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix bogus CVE-2022-2469 in libgsasl. [#50404](https://github.com/ClickHouse/ClickHouse/pull/50404) ([Robert Schulze](https://github.com/rschu1ze)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+
+* ActionsDAG: fix wrong optimization [#47584](https://github.com/ClickHouse/ClickHouse/pull/47584) ([Salvatore Mesoraca](https://github.com/aiven-sal)).
+* Correctly handle concurrent snapshots in Keeper [#48466](https://github.com/ClickHouse/ClickHouse/pull/48466) ([Antonio Andelic](https://github.com/antonio2368)).
+* MergeTreeMarksLoader holds DataPart instead of DataPartStorage [#48515](https://github.com/ClickHouse/ClickHouse/pull/48515) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Sequence state fix [#48603](https://github.com/ClickHouse/ClickHouse/pull/48603) ([Ilya Golshtein](https://github.com/ilejn)).
+* Back/Restore concurrency check on previous fails [#48726](https://github.com/ClickHouse/ClickHouse/pull/48726) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Fix Attaching a table with non-existent ZK path does not increase the ReadonlyReplica metric [#48954](https://github.com/ClickHouse/ClickHouse/pull/48954) ([wangxiaobo](https://github.com/wzb5212)).
+* Fix possible terminate called for uncaught exception in some places [#49112](https://github.com/ClickHouse/ClickHouse/pull/49112) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix key not found error for queries with multiple StorageJoin [#49137](https://github.com/ClickHouse/ClickHouse/pull/49137) ([vdimir](https://github.com/vdimir)).
+* Fix wrong query result when using nullable primary key [#49172](https://github.com/ClickHouse/ClickHouse/pull/49172) ([Duc Canh Le](https://github.com/canhld94)).
+* Fix reinterpretAs*() on big endian machines [#49198](https://github.com/ClickHouse/ClickHouse/pull/49198) ([Suzy Wang](https://github.com/SuzyWangIBMer)).
+* (Experimental zero-copy replication) Lock zero copy parts more atomically [#49211](https://github.com/ClickHouse/ClickHouse/pull/49211) ([alesapin](https://github.com/alesapin)).
+* Fix race on Outdated parts loading [#49223](https://github.com/ClickHouse/ClickHouse/pull/49223) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fix all key value is null and group use rollup return wrong answer [#49282](https://github.com/ClickHouse/ClickHouse/pull/49282) ([Shuai li](https://github.com/loneylee)).
+* Fix calculating load_factor for HASHED dictionaries with SHARDS [#49319](https://github.com/ClickHouse/ClickHouse/pull/49319) ([Azat Khuzhin](https://github.com/azat)).
+* Disallow configuring compression CODECs for alias columns [#49363](https://github.com/ClickHouse/ClickHouse/pull/49363) ([Timur Solodovnikov](https://github.com/tsolodov)).
+* Fix bug in removal of existing part directory [#49365](https://github.com/ClickHouse/ClickHouse/pull/49365) ([alesapin](https://github.com/alesapin)).
+* Properly fix GCS when HMAC is used [#49390](https://github.com/ClickHouse/ClickHouse/pull/49390) ([Antonio Andelic](https://github.com/antonio2368)).
+* Fix fuzz bug when subquery set is not built when reading from remote() [#49425](https://github.com/ClickHouse/ClickHouse/pull/49425) ([Alexander Gololobov](https://github.com/davenger)).
+* Invert `shutdown_wait_unfinished_queries` [#49427](https://github.com/ClickHouse/ClickHouse/pull/49427) ([Konstantin Bogdanov](https://github.com/thevar1able)).
+* (Experimental zero-copy replication) Fix another zero copy bug [#49473](https://github.com/ClickHouse/ClickHouse/pull/49473) ([alesapin](https://github.com/alesapin)).
+* Fix postgres database setting [#49481](https://github.com/ClickHouse/ClickHouse/pull/49481) ([Mal Curtis](https://github.com/snikch)).
+* Correctly handle `s3Cluster` arguments [#49490](https://github.com/ClickHouse/ClickHouse/pull/49490) ([Antonio Andelic](https://github.com/antonio2368)).
+* Fix bug in TraceCollector destructor. [#49508](https://github.com/ClickHouse/ClickHouse/pull/49508) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Fix AsynchronousReadIndirectBufferFromRemoteFS breaking on short seeks [#49525](https://github.com/ClickHouse/ClickHouse/pull/49525) ([Michael Kolupaev](https://github.com/al13n321)).
+* Fix dictionaries loading order [#49560](https://github.com/ClickHouse/ClickHouse/pull/49560) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Forbid the change of data type of Object('json') column [#49563](https://github.com/ClickHouse/ClickHouse/pull/49563) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Fix stress test (Logical error: Expected 7134 >= 11030) [#49623](https://github.com/ClickHouse/ClickHouse/pull/49623) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix bug in DISTINCT [#49628](https://github.com/ClickHouse/ClickHouse/pull/49628) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix: DISTINCT in order with zero values in non-sorted columns [#49636](https://github.com/ClickHouse/ClickHouse/pull/49636) ([Igor Nikonov](https://github.com/devcrafter)).
+* Fix one-off error in big integers found by UBSan with fuzzer [#49645](https://github.com/ClickHouse/ClickHouse/pull/49645) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix reading from sparse columns after restart [#49660](https://github.com/ClickHouse/ClickHouse/pull/49660) ([Anton Popov](https://github.com/CurtizJ)).
+* Fix assert in SpanHolder::finish() with fibers [#49673](https://github.com/ClickHouse/ClickHouse/pull/49673) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix short circuit functions and mutations with sparse arguments [#49716](https://github.com/ClickHouse/ClickHouse/pull/49716) ([Anton Popov](https://github.com/CurtizJ)).
+* Fix writing appended files to incremental backups [#49725](https://github.com/ClickHouse/ClickHouse/pull/49725) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Fix "There is no physical column _row_exists in table" error occurring during lightweight delete mutation on a table with Object column. [#49737](https://github.com/ClickHouse/ClickHouse/pull/49737) ([Alexander Gololobov](https://github.com/davenger)).
+* Fix msan issue in randomStringUTF8(uneven number) [#49750](https://github.com/ClickHouse/ClickHouse/pull/49750) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix aggregate function kolmogorovSmirnovTest [#49768](https://github.com/ClickHouse/ClickHouse/pull/49768) ([FFFFFFFHHHHHHH](https://github.com/FFFFFFFHHHHHHH)).
+* Fix settings aliases in native protocol [#49776](https://github.com/ClickHouse/ClickHouse/pull/49776) ([Azat Khuzhin](https://github.com/azat)).
+* Fix `arrayMap` with array of tuples with single argument [#49789](https://github.com/ClickHouse/ClickHouse/pull/49789) ([Anton Popov](https://github.com/CurtizJ)).
+* Fix per-query IO/BACKUPs throttling settings [#49797](https://github.com/ClickHouse/ClickHouse/pull/49797) ([Azat Khuzhin](https://github.com/azat)).
+* Fix setting NULL in profile definition [#49831](https://github.com/ClickHouse/ClickHouse/pull/49831) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Fix a bug with projections and the aggregate_functions_null_for_empty setting (for query_plan_optimize_projection) [#49873](https://github.com/ClickHouse/ClickHouse/pull/49873) ([Amos Bird](https://github.com/amosbird)).
+* Fix processing pending batch for Distributed async INSERT after restart [#49884](https://github.com/ClickHouse/ClickHouse/pull/49884) ([Azat Khuzhin](https://github.com/azat)).
+* Fix assertion in CacheMetadata::doCleanup [#49914](https://github.com/ClickHouse/ClickHouse/pull/49914) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* fix `is_prefix` in OptimizeRegularExpression [#49919](https://github.com/ClickHouse/ClickHouse/pull/49919) ([Han Fei](https://github.com/hanfei1991)).
+* Fix metrics `WriteBufferFromS3Bytes`, `WriteBufferFromS3Microseconds` and `WriteBufferFromS3RequestsErrors` [#49930](https://github.com/ClickHouse/ClickHouse/pull/49930) ([Aleksandr Musorin](https://github.com/AVMusorin)).
+* Fix IPv6 encoding in protobuf [#49933](https://github.com/ClickHouse/ClickHouse/pull/49933) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Fix possible Logical error on bad Nullable parsing for text formats [#49960](https://github.com/ClickHouse/ClickHouse/pull/49960) ([Kruglov Pavel](https://github.com/Avogar)).
+* Add setting output_format_parquet_compliant_nested_types to produce more compatible Parquet files [#50001](https://github.com/ClickHouse/ClickHouse/pull/50001) ([Michael Kolupaev](https://github.com/al13n321)).
+* Fix logical error in stress test "Not enough space to add ..." [#50021](https://github.com/ClickHouse/ClickHouse/pull/50021) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Avoid deadlock when starting table in attach thread of `ReplicatedMergeTree` [#50026](https://github.com/ClickHouse/ClickHouse/pull/50026) ([Antonio Andelic](https://github.com/antonio2368)).
+* Fix assert in SpanHolder::finish() with fibers attempt 2 [#50034](https://github.com/ClickHouse/ClickHouse/pull/50034) ([Kruglov Pavel](https://github.com/Avogar)).
+* Add proper escaping for DDL OpenTelemetry context serialization [#50045](https://github.com/ClickHouse/ClickHouse/pull/50045) ([Azat Khuzhin](https://github.com/azat)).
+* Fix reporting broken projection parts [#50052](https://github.com/ClickHouse/ClickHouse/pull/50052) ([Amos Bird](https://github.com/amosbird)).
+* JIT compilation not equals NaN fix [#50056](https://github.com/ClickHouse/ClickHouse/pull/50056) ([Maksim Kita](https://github.com/kitaisreal)).
+* Fix crashing in case of Replicated database without arguments [#50058](https://github.com/ClickHouse/ClickHouse/pull/50058) ([Azat Khuzhin](https://github.com/azat)).
+* Fix crash with `multiIf` and constant condition and nullable arguments [#50123](https://github.com/ClickHouse/ClickHouse/pull/50123) ([Anton Popov](https://github.com/CurtizJ)).
+* Fix invalid index analysis for date related keys [#50153](https://github.com/ClickHouse/ClickHouse/pull/50153) ([Amos Bird](https://github.com/amosbird)).
+* do not allow modify order by when there are no order by cols [#50154](https://github.com/ClickHouse/ClickHouse/pull/50154) ([Han Fei](https://github.com/hanfei1991)).
+* Fix broken index analysis when binary operator contains a null constant argument [#50177](https://github.com/ClickHouse/ClickHouse/pull/50177) ([Amos Bird](https://github.com/amosbird)).
+* clickhouse-client: disallow usage of `--query` and `--queries-file` at the same time [#50210](https://github.com/ClickHouse/ClickHouse/pull/50210) ([Alexey Gerasimchuk](https://github.com/Demilivor)).
+* Fix UB for INTO OUTFILE extensions (APPEND / AND STDOUT) and WATCH EVENTS [#50216](https://github.com/ClickHouse/ClickHouse/pull/50216) ([Azat Khuzhin](https://github.com/azat)).
+* Fix skipping spaces at end of row in CustomSeparatedIgnoreSpaces format [#50224](https://github.com/ClickHouse/ClickHouse/pull/50224) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix iceberg metadata parsing [#50232](https://github.com/ClickHouse/ClickHouse/pull/50232) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix nested distributed SELECT in WITH clause [#50234](https://github.com/ClickHouse/ClickHouse/pull/50234) ([Azat Khuzhin](https://github.com/azat)).
+* Fix msan issue in keyed siphash [#50245](https://github.com/ClickHouse/ClickHouse/pull/50245) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix bugs in Poco sockets in non-blocking mode, use true non-blocking sockets [#50252](https://github.com/ClickHouse/ClickHouse/pull/50252) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix checksum calculation for backup entries [#50264](https://github.com/ClickHouse/ClickHouse/pull/50264) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Comparison functions NaN fix [#50287](https://github.com/ClickHouse/ClickHouse/pull/50287) ([Maksim Kita](https://github.com/kitaisreal)).
+* JIT aggregation nullable key fix [#50291](https://github.com/ClickHouse/ClickHouse/pull/50291) ([Maksim Kita](https://github.com/kitaisreal)).
+* Fix clickhouse-local crashing when writing empty Arrow or Parquet output [#50328](https://github.com/ClickHouse/ClickHouse/pull/50328) ([Michael Kolupaev](https://github.com/al13n321)).
+* Fix crash when Pool::Entry::disconnect() is called [#50334](https://github.com/ClickHouse/ClickHouse/pull/50334) ([Val Doroshchuk](https://github.com/valbok)).
+* Improved fetch part by holding directory lock longer [#50339](https://github.com/ClickHouse/ClickHouse/pull/50339) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Fix bitShift* functions with both constant arguments [#50343](https://github.com/ClickHouse/ClickHouse/pull/50343) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix Keeper deadlock on exception when preprocessing requests. [#50387](https://github.com/ClickHouse/ClickHouse/pull/50387) ([frinkr](https://github.com/frinkr)).
+* Fix hashing of const integer values [#50421](https://github.com/ClickHouse/ClickHouse/pull/50421) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix merge_tree_min_rows_for_seek/merge_tree_min_bytes_for_seek for data skipping indexes [#50432](https://github.com/ClickHouse/ClickHouse/pull/50432) ([Azat Khuzhin](https://github.com/azat)).
+* Limit the number of in-flight tasks for loading outdated parts [#50450](https://github.com/ClickHouse/ClickHouse/pull/50450) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Keeper fix: apply uncommitted state after snapshot install [#50483](https://github.com/ClickHouse/ClickHouse/pull/50483) ([Antonio Andelic](https://github.com/antonio2368)).
+* Fix incorrect constant folding [#50536](https://github.com/ClickHouse/ClickHouse/pull/50536) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix logical error in stress test (Not enough space to add ...) [#50583](https://github.com/ClickHouse/ClickHouse/pull/50583) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix converting Null to LowCardinality(Nullable) in values table function [#50637](https://github.com/ClickHouse/ClickHouse/pull/50637) ([Kruglov Pavel](https://github.com/Avogar)).
+* Revert invalid RegExpTreeDictionary optimization [#50642](https://github.com/ClickHouse/ClickHouse/pull/50642) ([Johann Gan](https://github.com/johanngan)).
+
 ### <a id="234"></a> ClickHouse release 23.4, 2023-04-26
 
 #### Backward Incompatible Change

CMakeLists.txt

@@ -102,6 +102,17 @@ if (ENABLE_FUZZING)
     set (ENABLE_PROTOBUF 1)
 endif()
 
+option (ENABLE_WOBOQ_CODEBROWSER "Build for woboq codebrowser" OFF)
+
+if (ENABLE_WOBOQ_CODEBROWSER)
+    set (ENABLE_EMBEDDED_COMPILER 0)
+    set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-poison-system-directories")
+    # woboq codebrowser uses clang tooling, and they could add default system
+    # clang includes, and later clang will warn for those added by itself
+    # includes.
+    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-poison-system-directories")
+endif()
+
 # Global libraries
 # See:
 # - default_libs.cmake
@@ -259,8 +270,8 @@ endif ()
 option (ENABLE_BUILD_PATH_MAPPING "Enable remapping of file source paths in debug info, predefined preprocessor macros, and __builtin_FILE(). It's used to generate reproducible builds. See https://reproducible-builds.org/docs/build-path" ${ENABLE_BUILD_PATH_MAPPING_DEFAULT})
 
 if (ENABLE_BUILD_PATH_MAPPING)
-    set (COMPILER_FLAGS "${COMPILER_FLAGS} -ffile-prefix-map=${CMAKE_SOURCE_DIR}=.")
+    set (COMPILER_FLAGS "${COMPILER_FLAGS} -ffile-prefix-map=${PROJECT_SOURCE_DIR}=.")
-    set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} -ffile-prefix-map=${CMAKE_SOURCE_DIR}=.")
+    set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} -ffile-prefix-map=${PROJECT_SOURCE_DIR}=.")
 endif ()
 
 option (ENABLE_BUILD_PROFILING "Enable profiling of build time" OFF)
@@ -557,7 +568,7 @@ if (NATIVE_BUILD_TARGETS
 )
     message (STATUS "Building native targets...")
 
-    set (NATIVE_BUILD_DIR "${CMAKE_BINARY_DIR}/native")
+    set (NATIVE_BUILD_DIR "${PROJECT_BINARY_DIR}/native")
 
     execute_process(
         COMMAND ${CMAKE_COMMAND} -E make_directory "${NATIVE_BUILD_DIR}"
@@ -571,7 +582,7 @@ if (NATIVE_BUILD_TARGETS
             # Avoid overriding .cargo/config.toml with native toolchain.
             "-DENABLE_RUST=OFF"
             "-DENABLE_CLICKHOUSE_SELF_EXTRACTING=${ENABLE_CLICKHOUSE_SELF_EXTRACTING}"
-        ${CMAKE_SOURCE_DIR}
+        ${PROJECT_SOURCE_DIR}
         WORKING_DIRECTORY "${NATIVE_BUILD_DIR}"
         COMMAND_ECHO STDOUT)
 

README.md

@@ -22,12 +22,10 @@ curl https://clickhouse.com/ | sh
 
 ## Upcoming Events
 
-* [**v23.5 Release Webinar**](https://clickhouse.com/company/events/v23-5-release-webinar?utm_source=github&utm_medium=social&utm_campaign=release-webinar-2023-05) - May 31 - 23.5 is rapidly approaching. Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release.
+* [**v23.5 Release Webinar**](https://clickhouse.com/company/events/v23-5-release-webinar?utm_source=github&utm_medium=social&utm_campaign=release-webinar-2023-05) - Jun 8 - 23.5 is rapidly approaching. Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release.
-* [**ClickHouse Meetup in Berlin**](https://www.meetup.com/clickhouse-berlin-user-group/events/292892466) - May 16 
+* [**ClickHouse Meetup in Bangalore**](https://www.meetup.com/clickhouse-bangalore-user-group/events/293740066/) - Jun 7
-* [**ClickHouse Meetup in Barcelona**](https://www.meetup.com/clickhouse-barcelona-user-group/events/292892669) - May 25 
-* [**ClickHouse Meetup in London**](https://www.meetup.com/clickhouse-london-user-group/events/292892824) - May 25 
 * [**ClickHouse Meetup in San Francisco**](https://www.meetup.com/clickhouse-silicon-valley-meetup-group/events/293426725/) - Jun 7 
-* [**ClickHouse Meetup in Stockholm**](https://www.meetup.com/clickhouse-berlin-user-group/events/292892466) - Jun 13 
+
 
 Also, keep an eye out for upcoming meetups in Amsterdam, Boston, NYC, Beijing, and Toronto. Somewhere else you want us to be? Please feel free to reach out to tyler <at> clickhouse <dot> com.
 

SECURITY.md

@@ -13,9 +13,10 @@ The following versions of ClickHouse server are currently being supported with s
 
 | Version | Supported |
 |:-|:-|
+| 23.5 | ✔️ |
 | 23.4 | ✔️ |
 | 23.3 | ✔️ |
-| 23.2 | ✔️ |
+| 23.2 | ❌ |
 | 23.1 | ❌ |
 | 22.12 | ❌ |
 | 22.11 | ❌ |

base/base/StringRef.h

@@ -3,6 +3,7 @@
 #include <cassert>
 #include <stdexcept> // for std::logic_error
 #include <string>
+#include <type_traits>
 #include <vector>
 #include <functional>
 #include <iosfwd>
@@ -326,5 +327,16 @@ namespace ZeroTraits
     inline void set(StringRef & x) { x.size = 0; }
 }
 
+namespace PackedZeroTraits
+{
+    template <typename Second, template <typename, typename> class PackedPairNoInit>
+    inline bool check(const PackedPairNoInit<StringRef, Second> p)
+    { return 0 == p.key.size; }
+
+    template <typename Second, template <typename, typename> class PackedPairNoInit>
+    inline void set(PackedPairNoInit<StringRef, Second> & p)
+    { p.key.size = 0; }
+}
+
 
 std::ostream & operator<<(std::ostream & os, const StringRef & str);

base/base/getMemoryAmount.cpp

@@ -28,14 +28,28 @@ uint64_t getMemoryAmountOrZero()
 
 #if defined(OS_LINUX)
     // Try to lookup at the Cgroup limit
-    std::ifstream cgroup_limit("/sys/fs/cgroup/memory/memory.limit_in_bytes");
+
-    if (cgroup_limit.is_open())
+    // CGroups v2
+    std::ifstream cgroupv2_limit("/sys/fs/cgroup/memory.max");
+    if (cgroupv2_limit.is_open())
     {
-        uint64_t memory_limit = 0; // in case of read error
+        uint64_t memory_limit = 0;
-        cgroup_limit >> memory_limit;
+        cgroupv2_limit >> memory_limit;
         if (memory_limit > 0 && memory_limit < memory_amount)
             memory_amount = memory_limit;
     }
+    else
+    {
+        // CGroups v1
+        std::ifstream cgroup_limit("/sys/fs/cgroup/memory/memory.limit_in_bytes");
+        if (cgroup_limit.is_open())
+        {
+            uint64_t memory_limit = 0; // in case of read error
+            cgroup_limit >> memory_limit;
+            if (memory_limit > 0 && memory_limit < memory_amount)
+                memory_amount = memory_limit;
+        }
+    }
 #endif
 
     return memory_amount;

base/poco/Foundation/include/Poco/BinaryReader.h

@@ -117,6 +117,9 @@ public:
     void readRaw(char * buffer, std::streamsize length);
     /// Reads length bytes of raw data into buffer.
 
+    void readCString(std::string& value);
+		/// Reads zero-terminated C-string into value.
+
     void readBOM();
     /// Reads a byte-order mark from the stream and configures
     /// the reader for the encountered byte order.

base/poco/Foundation/include/Poco/BinaryWriter.h

@@ -56,6 +56,8 @@ public:
         LITTLE_ENDIAN_BYTE_ORDER = 3 /// little-endian byte-order
     };
 
+    static const std::streamsize	DEFAULT_MAX_CSTR_LENGTH { 1024 };
+
     BinaryWriter(std::ostream & ostr, StreamByteOrder byteOrder = NATIVE_BYTE_ORDER);
     /// Creates the BinaryWriter.
 
@@ -131,6 +133,9 @@ public:
     void writeRaw(const char * buffer, std::streamsize length);
     /// Writes length raw bytes from the given buffer to the stream.
 
+    void writeCString(const char* cString, std::streamsize maxLength = DEFAULT_MAX_CSTR_LENGTH);
+		/// Writes zero-terminated C-string.
+
     void writeBOM();
     /// Writes a byte-order mark to the stream. A byte order mark is
     /// a 16-bit integer with a value of 0xFEFF, written in host byte-order.

base/poco/Foundation/src/BinaryReader.cpp

@@ -274,6 +274,31 @@ void BinaryReader::readRaw(char* buffer, std::streamsize length)
 }
 
 
+void BinaryReader::readCString(std::string& value)
+{
+	value.clear();
+	if (!_istr.good())
+	{
+		return;
+	}
+	value.reserve(256);
+	while (true)
+	{
+		char c;
+		_istr.get(c);
+		if (!_istr.good())
+		{
+			break;
+		}
+		if (c == '\0')
+		{
+			break;
+		}
+		value += c;
+	}
+}
+
+
 void BinaryReader::readBOM()
 {
 	UInt16 bom;

base/poco/Foundation/src/BinaryWriter.cpp

@@ -271,7 +271,7 @@ BinaryWriter& BinaryWriter::operator << (const std::string& value)
 BinaryWriter& BinaryWriter::operator << (const char* value)
 {
 	poco_check_ptr (value);
-	
+
 	if (_pTextConverter)
 	{
 		std::string converted;
@@ -332,6 +332,15 @@ void BinaryWriter::writeRaw(const char* buffer, std::streamsize length)
 }
 
 
+void BinaryWriter::writeCString(const char* cString, std::streamsize maxLength)
+{
+	const std::size_t len = ::strnlen(cString, maxLength);
+	writeRaw(cString, len);
+	static const char zero = '\0';
+	_ostr.write(&zero, sizeof(zero));
+}
+
+
 void BinaryWriter::writeBOM()
 {
 	UInt16 value = 0xFEFF;

base/poco/MongoDB/CMakeLists.txt

@@ -13,3 +13,4 @@ target_compile_options (_poco_mongodb
 
 target_include_directories (_poco_mongodb SYSTEM PUBLIC "include")
 target_link_libraries (_poco_mongodb PUBLIC Poco::Net)
+

base/poco/MongoDB/include/Poco/MongoDB/Array.h

@@ -33,7 +33,7 @@ namespace MongoDB
     /// This class represents a BSON Array.
     {
     public:
-        typedef SharedPtr<Array> Ptr;
+        using Ptr = SharedPtr<Array>;
 
         Array();
         /// Creates an empty Array.
@@ -41,8 +41,31 @@ namespace MongoDB
         virtual ~Array();
         /// Destroys the Array.
 
+        // Document template functions available for backward compatibility
+        using Document::add;
+        using Document::get;
+
         template <typename T>
-        T get(int pos) const
+        Document & add(T value)
+        /// Creates an element with the name from the current pos and value and
+        /// adds it to the array document.
+        ///
+        /// The active document is returned to allow chaining of the add methods.
+        {
+            return Document::add<T>(Poco::NumberFormatter::format(size()), value);
+        }
+
+        Document & add(const char * value)
+        /// Creates an element with a name from the current pos and value and
+        /// adds it to the array document.
+        ///
+        /// The active document is returned to allow chaining of the add methods.
+        {
+            return Document::add(Poco::NumberFormatter::format(size()), value);
+        }
+
+        template <typename T>
+        T get(std::size_t pos) const
         /// Returns the element at the given index and tries to convert
         /// it to the template type. If the element is not found, a
         /// Poco::NotFoundException will be thrown. If the element cannot be
@@ -52,7 +75,7 @@ namespace MongoDB
         }
 
         template <typename T>
-        T get(int pos, const T & deflt) const
+        T get(std::size_t pos, const T & deflt) const
         /// Returns the element at the given index and tries to convert
         /// it to the template type. If the element is not found, or
         /// has the wrong type, the deflt argument will be returned.
@@ -60,12 +83,12 @@ namespace MongoDB
             return Document::get<T>(Poco::NumberFormatter::format(pos), deflt);
         }
 
-        Element::Ptr get(int pos) const;
+        Element::Ptr get(std::size_t pos) const;
         /// Returns the element at the given index.
         /// An empty element will be returned if the element is not found.
 
         template <typename T>
-        bool isType(int pos) const
+        bool isType(std::size_t pos) const
         /// Returns true if the type of the element equals the TypeId of ElementTrait,
         /// otherwise false.
         {
@@ -74,6 +97,9 @@ namespace MongoDB
 
         std::string toString(int indent = 0) const;
         /// Returns a string representation of the Array.
+
+    private:
+        friend void BSONReader::read<Array::Ptr>(Array::Ptr & to);
     };
 
 

base/poco/MongoDB/include/Poco/MongoDB/Binary.h

@@ -40,7 +40,7 @@ namespace MongoDB
     /// A Binary stores its data in a Poco::Buffer<unsigned char>.
     {
     public:
-        typedef SharedPtr<Binary> Ptr;
+        using Ptr = SharedPtr<Binary>;
 
         Binary();
         /// Creates an empty Binary with subtype 0.

base/poco/MongoDB/include/Poco/MongoDB/Connection.h

@@ -18,6 +18,7 @@
 #define MongoDB_Connection_INCLUDED
 
 
+#include "Poco/MongoDB/OpMsgMessage.h"
 #include "Poco/MongoDB/RequestMessage.h"
 #include "Poco/MongoDB/ResponseMessage.h"
 #include "Poco/Mutex.h"
@@ -39,7 +40,7 @@ namespace MongoDB
     /// for more information on the wire protocol.
     {
     public:
-        typedef Poco::SharedPtr<Connection> Ptr;
+        using Ptr = Poco::SharedPtr<Connection>;
 
         class MongoDB_API SocketFactory
         {
@@ -90,7 +91,7 @@ namespace MongoDB
 
         Poco::Net::SocketAddress address() const;
         /// Returns the address of the MongoDB server.
-        
+
         const std::string & uri() const;
         /// Returns the uri on which the connection was made.
 
@@ -145,6 +146,21 @@ namespace MongoDB
         /// Use this when a response is expected: only a "query" or "getmore"
         /// request will return a response.
 
+        void sendRequest(OpMsgMessage & request, OpMsgMessage & response);
+        /// Sends a request to the MongoDB server and receives the response
+        /// using newer wire protocol with OP_MSG.
+
+        void sendRequest(OpMsgMessage & request);
+        /// Sends an unacknowledged request to the MongoDB server using newer
+        /// wire protocol with OP_MSG.
+        /// No response is sent by the server.
+
+        void readResponse(OpMsgMessage & response);
+        /// Reads additional response data when previous message's flag moreToCome
+        /// indicates that server will send more data.
+        /// NOTE: See comments in OpMsgCursor code.
+
+
     protected:
         void connect();
 
@@ -164,7 +180,7 @@ namespace MongoDB
     }
     inline const std::string & Connection::uri() const
     {
-    	return _uri;
+        return _uri;
     }
 
 

base/poco/MongoDB/include/Poco/MongoDB/Cursor.h

@@ -40,6 +40,9 @@ namespace MongoDB
         Cursor(const std::string & fullCollectionName, QueryRequest::Flags flags = QueryRequest::QUERY_DEFAULT);
         /// Creates a Cursor for the given database and collection ("database.collection"), using the specified flags.
 
+        Cursor(const Document & aggregationResponse);
+        /// Creates a Cursor for the given aggregation query response.
+
         virtual ~Cursor();
         /// Destroys the Cursor.
 

base/poco/MongoDB/include/Poco/MongoDB/Database.h

@@ -26,6 +26,8 @@
 #include "Poco/MongoDB/QueryRequest.h"
 #include "Poco/MongoDB/UpdateRequest.h"
 
+#include "Poco/MongoDB/OpMsgCursor.h"
+#include "Poco/MongoDB/OpMsgMessage.h"
 
 namespace Poco
 {
@@ -45,6 +47,9 @@ namespace MongoDB
         virtual ~Database();
         /// Destroys the Database.
 
+        const std::string & name() const;
+        /// Database name
+
         bool authenticate(
             Connection & connection,
             const std::string & username,
@@ -62,34 +67,49 @@ namespace MongoDB
         /// May throw a Poco::ProtocolException if authentication fails for a reason other than
         /// invalid credentials.
 
+        Document::Ptr queryBuildInfo(Connection & connection) const;
+        /// Queries server build info (all wire protocols)
+
+        Document::Ptr queryServerHello(Connection & connection) const;
+        /// Queries hello response from server (all wire protocols)
+
         Int64 count(Connection & connection, const std::string & collectionName) const;
-        /// Sends a count request for the given collection to MongoDB.
+        /// Sends a count request for the given collection to MongoDB. (old wire protocol)
         ///
         /// If the command fails, -1 is returned.
 
         Poco::SharedPtr<Poco::MongoDB::QueryRequest> createCommand() const;
-        /// Creates a QueryRequest for a command.
+        /// Creates a QueryRequest for a command. (old wire protocol)
 
         Poco::SharedPtr<Poco::MongoDB::QueryRequest> createCountRequest(const std::string & collectionName) const;
         /// Creates a QueryRequest to count the given collection.
-        /// The collectionname must not contain the database name.
+        /// The collectionname must not contain the database name. (old wire protocol)
 
         Poco::SharedPtr<Poco::MongoDB::DeleteRequest> createDeleteRequest(const std::string & collectionName) const;
         /// Creates a DeleteRequest to delete documents in the given collection.
-        /// The collectionname must not contain the database name.
+        /// The collectionname must not contain the database name. (old wire protocol)
 
         Poco::SharedPtr<Poco::MongoDB::InsertRequest> createInsertRequest(const std::string & collectionName) const;
         /// Creates an InsertRequest to insert new documents in the given collection.
-        /// The collectionname must not contain the database name.
+        /// The collectionname must not contain the database name. (old wire protocol)
 
         Poco::SharedPtr<Poco::MongoDB::QueryRequest> createQueryRequest(const std::string & collectionName) const;
-        /// Creates a QueryRequest.
+        /// Creates a QueryRequest. (old wire protocol)
         /// The collectionname must not contain the database name.
 
         Poco::SharedPtr<Poco::MongoDB::UpdateRequest> createUpdateRequest(const std::string & collectionName) const;
-        /// Creates an UpdateRequest.
+        /// Creates an UpdateRequest. (old wire protocol)
         /// The collectionname must not contain the database name.
 
+        Poco::SharedPtr<Poco::MongoDB::OpMsgMessage> createOpMsgMessage(const std::string & collectionName) const;
+        /// Creates OpMsgMessage. (new wire protocol)
+
+        Poco::SharedPtr<Poco::MongoDB::OpMsgMessage> createOpMsgMessage() const;
+        /// Creates OpMsgMessage for database commands that do not require collection as an argument. (new wire protocol)
+
+        Poco::SharedPtr<Poco::MongoDB::OpMsgCursor> createOpMsgCursor(const std::string & collectionName) const;
+        /// Creates OpMsgCursor. (new wire protocol)
+
         Poco::MongoDB::Document::Ptr ensureIndex(
             Connection & connection,
             const std::string & collection,
@@ -100,14 +120,16 @@ namespace MongoDB
             int version = 0,
             int ttl = 0);
         /// Creates an index. The document returned is the result of a getLastError call.
-        /// For more info look at the ensureIndex information on the MongoDB website.
+        /// For more info look at the ensureIndex information on the MongoDB website. (old wire protocol)
 
         Document::Ptr getLastErrorDoc(Connection & connection) const;
         /// Sends the getLastError command to the database and returns the error document.
+        /// (old wire protocol)
 
         std::string getLastError(Connection & connection) const;
         /// Sends the getLastError command to the database and returns the err element
         /// from the error document. When err is null, an empty string is returned.
+        /// (old wire protocol)
 
         static const std::string AUTH_MONGODB_CR;
         /// Default authentication mechanism prior to MongoDB 3.0.
@@ -115,6 +137,27 @@ namespace MongoDB
         static const std::string AUTH_SCRAM_SHA1;
         /// Default authentication mechanism for MongoDB 3.0.
 
+        enum WireVersion
+        /// Wire version as reported by the command hello.
+        /// See details in MongoDB github, repository specifications.
+        /// @see queryServerHello
+        {
+            VER_26 = 1,
+            VER_26_2 = 2,
+            VER_30 = 3,
+            VER_32 = 4,
+            VER_34 = 5,
+            VER_36 = 6, ///< First wire version that supports OP_MSG
+            VER_40 = 7,
+            VER_42 = 8,
+            VER_44 = 9,
+            VER_50 = 13,
+            VER_51 = 14, ///< First wire version that supports only OP_MSG
+            VER_52 = 15,
+            VER_53 = 16,
+            VER_60 = 17
+        };
+
     protected:
         bool authCR(Connection & connection, const std::string & username, const std::string & password);
         bool authSCRAM(Connection & connection, const std::string & username, const std::string & password);
@@ -127,6 +170,12 @@ namespace MongoDB
     //
     // inlines
     //
+    inline const std::string & Database::name() const
+    {
+        return _dbname;
+    }
+
+
     inline Poco::SharedPtr<Poco::MongoDB::QueryRequest> Database::createCommand() const
     {
         Poco::SharedPtr<Poco::MongoDB::QueryRequest> cmd = createQueryRequest("$cmd");
@@ -158,6 +207,24 @@ namespace MongoDB
         return new Poco::MongoDB::UpdateRequest(_dbname + '.' + collectionName);
     }
 
+    // -- New wire protocol commands
+
+    inline Poco::SharedPtr<Poco::MongoDB::OpMsgMessage> Database::createOpMsgMessage(const std::string & collectionName) const
+    {
+        return new Poco::MongoDB::OpMsgMessage(_dbname, collectionName);
+    }
+
+    inline Poco::SharedPtr<Poco::MongoDB::OpMsgMessage> Database::createOpMsgMessage() const
+    {
+        // Collection name for database commands is not needed.
+        return createOpMsgMessage("");
+    }
+
+    inline Poco::SharedPtr<Poco::MongoDB::OpMsgCursor> Database::createOpMsgCursor(const std::string & collectionName) const
+    {
+        return new Poco::MongoDB::OpMsgCursor(_dbname, collectionName);
+    }
+
 
 }
 } // namespace Poco::MongoDB

base/poco/MongoDB/include/Poco/MongoDB/Document.h

@@ -31,6 +31,7 @@ namespace Poco
 namespace MongoDB
 {
 
+    class Array;
 
     class ElementFindByName
     {
@@ -48,8 +49,8 @@ namespace MongoDB
     /// Represents a MongoDB (BSON) document.
     {
     public:
-        typedef SharedPtr<Document> Ptr;
+        using Ptr = SharedPtr<Document>;
-        typedef std::vector<Document::Ptr> Vector;
+        using Vector = std::vector<Document::Ptr>;
 
         Document();
         /// Creates an empty Document.
@@ -86,6 +87,10 @@ namespace MongoDB
         /// Unlike the other add methods, this method returns
         /// a reference to the new document.
 
+        Array & addNewArray(const std::string & name);
+        /// Create a new array and add it to this document.
+        /// Method returns a reference to the new array.
+
         void clear();
         /// Removes all elements from the document.
 
@@ -95,7 +100,7 @@ namespace MongoDB
         bool empty() const;
         /// Returns true if the document doesn't contain any documents.
 
-        bool exists(const std::string & name);
+        bool exists(const std::string & name) const;
         /// Returns true if the document has an element with the given name.
 
         template <typename T>
@@ -158,6 +163,9 @@ namespace MongoDB
         /// return an Int64. When the element is not found, a
         /// Poco::NotFoundException will be thrown.
 
+        bool remove(const std::string & name);
+        /// Removes an element from the document.
+
         template <typename T>
         bool isType(const std::string & name) const
         /// Returns true when the type of the element equals the TypeId of ElementTrait.
@@ -227,12 +235,23 @@ namespace MongoDB
     }
 
 
-    inline bool Document::exists(const std::string & name)
+    inline bool Document::exists(const std::string & name) const
     {
         return std::find_if(_elements.begin(), _elements.end(), ElementFindByName(name)) != _elements.end();
     }
 
 
+    inline bool Document::remove(const std::string & name)
+    {
+        auto it = std::find_if(_elements.begin(), _elements.end(), ElementFindByName(name));
+        if (it == _elements.end())
+            return false;
+
+        _elements.erase(it);
+        return true;
+    }
+
+
     inline std::size_t Document::size() const
     {
         return _elements.size();

base/poco/MongoDB/include/Poco/MongoDB/Element.h

@@ -45,7 +45,7 @@ namespace MongoDB
     /// Represents an Element of a Document or an Array.
     {
     public:
-        typedef Poco::SharedPtr<Element> Ptr;
+        using Ptr = Poco::SharedPtr<Element>;
 
         explicit Element(const std::string & name);
         /// Creates the Element with the given name.
@@ -80,7 +80,7 @@ namespace MongoDB
     }
 
 
-    typedef std::list<Element::Ptr> ElementSet;
+    using ElementSet = std::list<Element::Ptr>;
 
 
     template <typename T>
@@ -266,7 +266,7 @@ namespace MongoDB
     }
 
 
-    typedef Nullable<unsigned char> NullValue;
+    using NullValue = Nullable<unsigned char>;
 
 
     // BSON Null Value

base/poco/MongoDB/include/Poco/MongoDB/JavaScriptCode.h

@@ -35,7 +35,7 @@ namespace MongoDB
     /// Represents JavaScript type in BSON.
     {
     public:
-        typedef SharedPtr<JavaScriptCode> Ptr;
+        using Ptr = SharedPtr<JavaScriptCode>;
 
         JavaScriptCode();
         /// Creates an empty JavaScriptCode object.

base/poco/MongoDB/include/Poco/MongoDB/MessageHeader.h

@@ -28,6 +28,9 @@ namespace MongoDB
 {
 
 
+    class Message; // Required to disambiguate friend declaration in MessageHeader.
+
+
     class MongoDB_API MessageHeader
     /// Represents the message header which is always prepended to a
     /// MongoDB request or response message.
@@ -37,14 +40,18 @@ namespace MongoDB
 
         enum OpCode
         {
+            // Opcodes deprecated in MongoDB 5.0
             OP_REPLY = 1,
-            OP_MSG = 1000,
             OP_UPDATE = 2001,
             OP_INSERT = 2002,
             OP_QUERY = 2004,
             OP_GET_MORE = 2005,
             OP_DELETE = 2006,
-            OP_KILL_CURSORS = 2007
+            OP_KILL_CURSORS = 2007,
+
+            /// Opcodes supported in MongoDB 5.1 and later
+            OP_COMPRESSED = 2012,
+            OP_MSG = 2013
         };
 
         explicit MessageHeader(OpCode);

base/poco/MongoDB/include/Poco/MongoDB/MongoDB.h

@@ -33,6 +33,13 @@
 //
 
 
+#if defined(_WIN32) && defined(POCO_DLL)
+#    if defined(MongoDB_EXPORTS)
+#        define MongoDB_API __declspec(dllexport)
+#    else
+#        define MongoDB_API __declspec(dllimport)
+#    endif
+#endif
 
 
 #if !defined(MongoDB_API)
@@ -47,6 +54,11 @@
 //
 // Automatically link MongoDB library.
 //
+#if defined(_MSC_VER)
+#    if !defined(POCO_NO_AUTOMATIC_LIBS) && !defined(MongoDB_EXPORTS)
+#        pragma comment(lib, "PocoMongoDB" POCO_LIB_SUFFIX)
+#    endif
+#endif
 
 
 #endif // MongoDBMongoDB_INCLUDED

base/poco/MongoDB/include/Poco/MongoDB/ObjectId.h

@@ -44,7 +44,7 @@ namespace MongoDB
     /// as its value.
     {
     public:
-        typedef SharedPtr<ObjectId> Ptr;
+        using Ptr = SharedPtr<ObjectId>;
 
         explicit ObjectId(const std::string & id);
         /// Creates an ObjectId from a string.

base/poco/MongoDB/include/Poco/MongoDB/OpMsgCursor.h

@@ -0,0 +1,96 @@
+//
+// OpMsgCursor.h
+//
+// Library: MongoDB
+// Package: MongoDB
+// Module:  OpMsgCursor
+//
+// Definition of the OpMsgCursor class.
+//
+// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
+// and Contributors.
+//
+// SPDX-License-Identifier:	BSL-1.0
+//
+
+
+#ifndef MongoDB_OpMsgCursor_INCLUDED
+#define MongoDB_OpMsgCursor_INCLUDED
+
+
+#include "Poco/MongoDB/Connection.h"
+#include "Poco/MongoDB/MongoDB.h"
+#include "Poco/MongoDB/OpMsgMessage.h"
+
+namespace Poco
+{
+namespace MongoDB
+{
+
+
+    class MongoDB_API OpMsgCursor : public Document
+    /// OpMsgCursor is an helper class for querying multiple documents using OpMsgMessage.
+    {
+    public:
+        OpMsgCursor(const std::string & dbname, const std::string & collectionName);
+        /// Creates a OpMsgCursor for the given database and collection.
+
+        virtual ~OpMsgCursor();
+        /// Destroys the OpMsgCursor.
+
+        void setEmptyFirstBatch(bool empty);
+        /// Empty first batch is used to get error response faster with little server processing
+
+        bool emptyFirstBatch() const;
+
+        void setBatchSize(Int32 batchSize);
+        /// Set non-default batch size
+
+        Int32 batchSize() const;
+        /// Current batch size (zero or negative number indicates default batch size)
+
+        Int64 cursorID() const;
+
+        OpMsgMessage & next(Connection & connection);
+        /// Tries to get the next documents. As long as response message has a
+        /// cursor ID next can be called to retrieve the next bunch of documents.
+        ///
+        /// The cursor must be killed (see kill()) when not all documents are needed.
+
+        OpMsgMessage & query();
+        /// Returns the associated query.
+
+        void kill(Connection & connection);
+        /// Kills the cursor and reset it so that it can be reused.
+
+    private:
+        OpMsgMessage _query;
+        OpMsgMessage _response;
+
+        bool _emptyFirstBatch{false};
+        Int32 _batchSize{-1};
+        /// Batch size used in the cursor. Zero or negative value means that default shall be used.
+
+        Int64 _cursorID{0};
+    };
+
+
+    //
+    // inlines
+    //
+    inline OpMsgMessage & OpMsgCursor::query()
+    {
+        return _query;
+    }
+
+    inline Int64 OpMsgCursor::cursorID() const
+    {
+        return _cursorID;
+    }
+
+
+}
+} // namespace Poco::MongoDB
+
+
+#endif // MongoDB_OpMsgCursor_INCLUDED

base/poco/MongoDB/include/Poco/MongoDB/OpMsgMessage.h

@@ -0,0 +1,163 @@
+//
+// OpMsgMessage.h
+//
+// Library: MongoDB
+// Package: MongoDB
+// Module:  OpMsgMessage
+//
+// Definition of the OpMsgMessage class.
+//
+// Copyright (c) 2022, Applied Informatics Software Engineering GmbH.
+// and Contributors.
+//
+// SPDX-License-Identifier:	BSL-1.0
+//
+
+
+#ifndef MongoDB_OpMsgMessage_INCLUDED
+#define MongoDB_OpMsgMessage_INCLUDED
+
+
+#include "Poco/MongoDB/Document.h"
+#include "Poco/MongoDB/Message.h"
+#include "Poco/MongoDB/MongoDB.h"
+
+#include <string>
+
+namespace Poco
+{
+namespace MongoDB
+{
+
+
+    class MongoDB_API OpMsgMessage : public Message
+    /// This class represents a request/response (OP_MSG) to send requests and receive responses to/from MongoDB.
+    {
+    public:
+        // Constants for most often used MongoDB commands that can be sent using OP_MSG
+        // For complete list see: https://www.mongodb.com/docs/manual/reference/command/
+
+        // Query and write
+        static const std::string CMD_INSERT;
+        static const std::string CMD_DELETE;
+        static const std::string CMD_UPDATE;
+        static const std::string CMD_FIND;
+        static const std::string CMD_FIND_AND_MODIFY;
+        static const std::string CMD_GET_MORE;
+
+        // Aggregation
+        static const std::string CMD_AGGREGATE;
+        static const std::string CMD_COUNT;
+        static const std::string CMD_DISTINCT;
+        static const std::string CMD_MAP_REDUCE;
+
+        // Replication and administration
+        static const std::string CMD_HELLO;
+        static const std::string CMD_REPL_SET_GET_STATUS;
+        static const std::string CMD_REPL_SET_GET_CONFIG;
+
+        static const std::string CMD_CREATE;
+        static const std::string CMD_CREATE_INDEXES;
+        static const std::string CMD_DROP;
+        static const std::string CMD_DROP_DATABASE;
+        static const std::string CMD_KILL_CURSORS;
+        static const std::string CMD_LIST_DATABASES;
+        static const std::string CMD_LIST_INDEXES;
+
+        // Diagnostic
+        static const std::string CMD_BUILD_INFO;
+        static const std::string CMD_COLL_STATS;
+        static const std::string CMD_DB_STATS;
+        static const std::string CMD_HOST_INFO;
+
+
+        enum Flags : UInt32
+        {
+            MSG_FLAGS_DEFAULT = 0,
+
+            MSG_CHECKSUM_PRESENT = (1 << 0),
+
+            MSG_MORE_TO_COME = (1 << 1),
+            /// Sender will send another message and is not prepared for overlapping messages
+
+            MSG_EXHAUST_ALLOWED = (1 << 16)
+            /// Client is prepared for multiple replies (using the moreToCome bit) to this request
+        };
+
+        OpMsgMessage();
+        /// Creates an OpMsgMessage for response.
+
+        OpMsgMessage(const std::string & databaseName, const std::string & collectionName, UInt32 flags = MSG_FLAGS_DEFAULT);
+        /// Creates an OpMsgMessage for requests.
+
+        virtual ~OpMsgMessage();
+
+        const std::string & databaseName() const;
+
+        const std::string & collectionName() const;
+
+        void setCommandName(const std::string & command);
+        /// Sets the command name and clears the command document
+
+        void setCursor(Poco::Int64 cursorID, Poco::Int32 batchSize = -1);
+        /// Sets the command "getMore" for the cursor id with batch size (if it is not negative).
+
+        const std::string & commandName() const;
+        /// Current command name.
+
+        void setAcknowledgedRequest(bool ack);
+        /// Set false to create request that does not return response.
+        /// It has effect only for commands that write or delete documents.
+        /// Default is true (request returns acknowledge response).
+
+        bool acknowledgedRequest() const;
+
+        UInt32 flags() const;
+
+        Document & body();
+        /// Access to body document.
+        /// Additional query arguments shall be added after setting the command name.
+
+        const Document & body() const;
+
+        Document::Vector & documents();
+        /// Documents prepared for request or retrieved in response.
+
+        const Document::Vector & documents() const;
+        /// Documents prepared for request or retrieved in response.
+
+        bool responseOk() const;
+        /// Reads "ok" status from the response message.
+
+        void clear();
+        /// Clears the message.
+
+        void send(std::ostream & ostr);
+        /// Writes the request to stream.
+
+        void read(std::istream & istr);
+        /// Reads the response from the stream.
+
+    private:
+        enum PayloadType : UInt8
+        {
+            PAYLOAD_TYPE_0 = 0,
+            PAYLOAD_TYPE_1 = 1
+        };
+
+        std::string _databaseName;
+        std::string _collectionName;
+        UInt32 _flags{MSG_FLAGS_DEFAULT};
+        std::string _commandName;
+        bool _acknowledged{true};
+
+        Document _body;
+        Document::Vector _documents;
+    };
+
+
+}
+} // namespace Poco::MongoDB
+
+
+#endif // MongoDB_OpMsgMessage_INCLUDED

base/poco/MongoDB/include/Poco/MongoDB/PoolableConnectionFactory.h

@@ -94,7 +94,23 @@ namespace MongoDB
 
         operator Connection::Ptr() { return _connection; }
 
+#if defined(POCO_ENABLE_CPP11)
+        // Disable copy to prevent unwanted release of resources: C++11 way
+        PooledConnection(const PooledConnection &) = delete;
+        PooledConnection & operator=(const PooledConnection &) = delete;
+
+        // Enable move semantics
+        PooledConnection(PooledConnection && other) = default;
+        PooledConnection & operator=(PooledConnection &&) = default;
+#endif
+
     private:
+#if !defined(POCO_ENABLE_CPP11)
+        // Disable copy to prevent unwanted release of resources: pre C++11 way
+        PooledConnection(const PooledConnection &);
+        PooledConnection & operator=(const PooledConnection &);
+#endif
+
         Poco::ObjectPool<Connection, Connection::Ptr> & _pool;
         Connection::Ptr _connection;
     };

base/poco/MongoDB/include/Poco/MongoDB/RegularExpression.h

@@ -33,7 +33,7 @@ namespace MongoDB
     /// Represents a regular expression in BSON format.
     {
     public:
-        typedef SharedPtr<RegularExpression> Ptr;
+        using Ptr = SharedPtr<RegularExpression>;
 
         RegularExpression();
         /// Creates an empty RegularExpression.

base/poco/MongoDB/include/Poco/MongoDB/ResponseMessage.h

@@ -38,6 +38,9 @@ namespace MongoDB
         ResponseMessage();
         /// Creates an empty ResponseMessage.
 
+        ResponseMessage(const Int64 & cursorID);
+        /// Creates an ResponseMessage for existing cursor ID.
+
         virtual ~ResponseMessage();
         /// Destroys the ResponseMessage.
 

base/poco/MongoDB/src/Array.cpp

@@ -20,7 +20,7 @@ namespace Poco {
 namespace MongoDB {
 
 
-Array::Array(): 
+Array::Array():
 	Document()
 {
 }
@@ -31,7 +31,7 @@ Array::~Array()
 }
 
 
-Element::Ptr Array::get(int pos) const
+Element::Ptr Array::get(std::size_t pos) const
 {
 	std::string name = Poco::NumberFormatter::format(pos);
 	return Document::get(name);

base/poco/MongoDB/src/Connection.cpp

@@ -319,4 +319,30 @@ void Connection::sendRequest(RequestMessage& request, ResponseMessage& response)
 }
 
 
+void Connection::sendRequest(OpMsgMessage& request, OpMsgMessage& response)
+{
+	Poco::Net::SocketOutputStream sos(_socket);
+	request.send(sos);
+
+	response.clear();
+	readResponse(response);
+}
+
+
+void Connection::sendRequest(OpMsgMessage& request)
+{
+	request.setAcknowledgedRequest(false);
+	Poco::Net::SocketOutputStream sos(_socket);
+	request.send(sos);
+}
+
+
+void Connection::readResponse(OpMsgMessage& response)
+{
+	Poco::Net::SocketInputStream sis(_socket);
+	response.read(sis);
+}
+
+
+
 } } // Poco::MongoDB

base/poco/MongoDB/src/Cursor.cpp

@@ -33,6 +33,12 @@ Cursor::Cursor(const std::string& fullCollectionName, QueryRequest::Flags flags)
 }
 
 
+Cursor::Cursor(const Document& aggregationResponse) :
+    _query(aggregationResponse.get<Poco::MongoDB::Document::Ptr>("cursor")->get<std::string>("ns")),
+    _response(aggregationResponse.get<Poco::MongoDB::Document::Ptr>("cursor")->get<Int64>("id"))
+{
+}
+
 Cursor::~Cursor()
 {
 	try

base/poco/MongoDB/src/Database.cpp

@@ -334,6 +334,50 @@ bool Database::authSCRAM(Connection& connection, const std::string& username, co
 }
 
 
+Document::Ptr Database::queryBuildInfo(Connection& connection) const
+{
+	// build info can be issued on "config" system database
+	Poco::SharedPtr<Poco::MongoDB::QueryRequest> request = createCommand();
+	request->selector().add("buildInfo", 1);
+
+	Poco::MongoDB::ResponseMessage response;
+	connection.sendRequest(*request, response);
+
+	Document::Ptr buildInfo;
+	if ( response.documents().size() > 0 )
+	{
+		buildInfo = response.documents()[0];
+	}
+	else
+	{
+		throw Poco::ProtocolException("Didn't get a response from the buildinfo command");
+	}
+	return buildInfo;
+}
+
+
+Document::Ptr Database::queryServerHello(Connection& connection) const
+{
+	// hello can be issued on "config" system database
+	Poco::SharedPtr<Poco::MongoDB::QueryRequest> request = createCommand();
+	request->selector().add("hello", 1);
+
+	Poco::MongoDB::ResponseMessage response;
+	connection.sendRequest(*request, response);
+
+	Document::Ptr hello;
+	if ( response.documents().size() > 0 )
+	{
+		hello = response.documents()[0];
+	}
+	else
+	{
+		throw Poco::ProtocolException("Didn't get a response from the hello command");
+	}
+	return hello;
+}
+
+
 Int64 Database::count(Connection& connection, const std::string& collectionName) const
 {
 	Poco::SharedPtr<Poco::MongoDB::QueryRequest> countRequest = createCountRequest(collectionName);
@@ -390,7 +434,7 @@ Document::Ptr Database::getLastErrorDoc(Connection& connection) const
 {
 	Document::Ptr errorDoc;
 
-	Poco::SharedPtr<Poco::MongoDB::QueryRequest> request = createQueryRequest("$cmd");
+	Poco::SharedPtr<Poco::MongoDB::QueryRequest> request = createCommand();
 	request->setNumberToReturn(1);
 	request->selector().add("getLastError", 1);
 
@@ -420,7 +464,7 @@ std::string Database::getLastError(Connection& connection) const
 
 Poco::SharedPtr<Poco::MongoDB::QueryRequest> Database::createCountRequest(const std::string& collectionName) const
 {
-	Poco::SharedPtr<Poco::MongoDB::QueryRequest> request = createQueryRequest("$cmd");
+	Poco::SharedPtr<Poco::MongoDB::QueryRequest> request = createCommand();
 	request->setNumberToReturn(1);
 	request->selector().add("count", collectionName);
 	return request;

base/poco/MongoDB/src/DeleteRequest.cpp

@@ -20,8 +20,8 @@ namespace MongoDB {
 
 
 DeleteRequest::DeleteRequest(const std::string& collectionName, DeleteRequest::Flags flags):
-	RequestMessage(MessageHeader::OP_DELETE), 
+	RequestMessage(MessageHeader::OP_DELETE),
-	_flags(flags), 
+	_flags(flags),
 	_fullCollectionName(collectionName),
 	_selector()
 {

base/poco/MongoDB/src/Document.cpp

@@ -35,6 +35,14 @@ Document::~Document()
 }
 
 
+Array& Document::addNewArray(const std::string& name)
+{
+	Array::Ptr newArray = new Array();
+	add(name, newArray);
+	return *newArray;
+}
+
+
 Element::Ptr Document::get(const std::string& name) const
 {
 	Element::Ptr element;
@@ -84,7 +92,7 @@ void Document::read(BinaryReader& reader)
 	while (type != '\0')
 	{
 		Element::Ptr element;
-		
+
 		std::string name = BSONReader(reader).readCString();
 
 		switch (type)
@@ -198,7 +206,7 @@ void Document::write(BinaryWriter& writer)
 	else
 	{
 		std::stringstream sstream;
-		Poco::BinaryWriter tempWriter(sstream);
+		Poco::BinaryWriter tempWriter(sstream, BinaryWriter::LITTLE_ENDIAN_BYTE_ORDER);
 		for (ElementSet::iterator it = _elements.begin(); it != _elements.end(); ++it)
 		{
 			tempWriter << static_cast<unsigned char>((*it)->type());
@@ -207,7 +215,7 @@ void Document::write(BinaryWriter& writer)
 			element->write(tempWriter);
 		}
 		tempWriter.flush();
-		
+
 		Poco::Int32 len = static_cast<Poco::Int32>(5 + sstream.tellp()); /* 5 = sizeof(len) + 0-byte */
 		writer << len;
 		writer.writeRaw(sstream.str());

base/poco/MongoDB/src/Element.cpp

@@ -24,7 +24,7 @@ Element::Element(const std::string& name) : _name(name)
 }
 
 
-Element::~Element() 
+Element::~Element()
 {
 }
 

base/poco/MongoDB/src/GetMoreRequest.cpp

@@ -21,7 +21,7 @@ namespace MongoDB {
 
 
 GetMoreRequest::GetMoreRequest(const std::string& collectionName, Int64 cursorID):
-	RequestMessage(MessageHeader::OP_GET_MORE), 
+	RequestMessage(MessageHeader::OP_GET_MORE),
 	_fullCollectionName(collectionName),
 	_numberToReturn(100),
 	_cursorID(cursorID)

base/poco/MongoDB/src/InsertRequest.cpp

@@ -20,7 +20,7 @@ namespace MongoDB {
 
 
 InsertRequest::InsertRequest(const std::string& collectionName, Flags flags):
-	RequestMessage(MessageHeader::OP_INSERT), 
+	RequestMessage(MessageHeader::OP_INSERT),
 	_flags(flags),
 	_fullCollectionName(collectionName)
 {

base/poco/MongoDB/src/KillCursorsRequest.cpp

@@ -37,7 +37,7 @@ void KillCursorsRequest::buildRequest(BinaryWriter& writer)
 	for (std::vector<Int64>::iterator it = _cursors.begin(); it != _cursors.end(); ++it)
 	{
 		writer << *it;
-	}		
+	}
 }
 
 

base/poco/MongoDB/src/Message.cpp

@@ -19,7 +19,7 @@ namespace Poco {
 namespace MongoDB {
 
 
-Message::Message(MessageHeader::OpCode opcode): 
+Message::Message(MessageHeader::OpCode opcode):
 	_header(opcode)
 {
 }

base/poco/MongoDB/src/MessageHeader.cpp

@@ -20,10 +20,10 @@ namespace Poco {
 namespace MongoDB {
 
 
-MessageHeader::MessageHeader(OpCode opCode): 
+MessageHeader::MessageHeader(OpCode opCode):
-	_messageLength(0), 
+	_messageLength(0),
-	_requestID(0), 
+	_requestID(0),
-	_responseTo(0), 
+	_responseTo(0),
 	_opCode(opCode)
 {
 }
@@ -42,7 +42,7 @@ void MessageHeader::read(BinaryReader& reader)
 
 	Int32 opCode;
 	reader >> opCode;
-	_opCode = (OpCode) opCode;
+	_opCode = static_cast<OpCode>(opCode);
 
 	if (!reader.good())
 	{
@@ -56,7 +56,7 @@ void MessageHeader::write(BinaryWriter& writer)
 	writer << _messageLength;
 	writer << _requestID;
 	writer << _responseTo;
-	writer << (Int32) _opCode;
+	writer << static_cast<Int32>(_opCode);
 }
 
 

base/poco/MongoDB/src/ObjectId.cpp

@@ -32,7 +32,7 @@ ObjectId::ObjectId(const std::string& id)
 	poco_assert_dbg(id.size() == 24);
 
     const char* p = id.c_str();
-    for (std::size_t i = 0; i < 12; ++i) 
+    for (std::size_t i = 0; i < 12; ++i)
     {
 		_id[i] = fromHex(p);
 		p += 2;

base/poco/MongoDB/src/OpMsgCursor.cpp

@@ -0,0 +1,187 @@
+//
+// OpMsgCursor.cpp
+//
+// Library: MongoDB
+// Package: MongoDB
+// Module:  OpMsgCursor
+//
+// Copyright (c) 2022, Applied Informatics Software Engineering GmbH.
+// and Contributors.
+//
+// SPDX-License-Identifier:	BSL-1.0
+//
+
+
+#include "Poco/MongoDB/OpMsgCursor.h"
+#include "Poco/MongoDB/Array.h"
+
+//
+// NOTE:
+//
+// MongoDB specification indicates that the flag MSG_EXHAUST_ALLOWED shall be
+// used in the request when the receiver is ready to receive multiple messages
+// without sending additional requests in between. Sender (MongoDB) indicates
+// that more messages follow with flag MSG_MORE_TO_COME.
+//
+// It seems that this does not work properly. MSG_MORE_TO_COME is set and reading
+// next messages sometimes works, however often the data is missing in response
+// or the message header contains wrong message length and reading blocks.
+// Opcode in the header is correct.
+//
+// Using MSG_EXHAUST_ALLOWED is therefore currently disabled.
+//
+// It seems that related JIRA ticket is:
+//
+// https://jira.mongodb.org/browse/SERVER-57297
+//
+// https://github.com/mongodb/specifications/blob/master/source/message/OP_MSG.rst
+//
+
+#define MONGODB_EXHAUST_ALLOWED_WORKS	false
+
+namespace Poco {
+namespace MongoDB {
+
+
+static const std::string keyCursor		{"cursor"};
+static const std::string keyFirstBatch	{"firstBatch"};
+static const std::string keyNextBatch	{"nextBatch"};
+
+static Poco::Int64 cursorIdFromResponse(const MongoDB::Document& doc);
+
+
+OpMsgCursor::OpMsgCursor(const std::string& db, const std::string& collection):
+#if MONGODB_EXHAUST_ALLOWED_WORKS
+	_query(db, collection, OpMsgMessage::MSG_EXHAUST_ALLOWED)
+#else
+	_query(db, collection)
+#endif
+{
+}
+
+OpMsgCursor::~OpMsgCursor()
+{
+	try
+	{
+		poco_assert_dbg(_cursorID == 0);
+	}
+	catch (...)
+	{
+	}
+}
+
+
+void OpMsgCursor::setEmptyFirstBatch(bool empty)
+{
+	_emptyFirstBatch = empty;
+}
+
+
+bool OpMsgCursor::emptyFirstBatch() const
+{
+	return _emptyFirstBatch;
+}
+
+
+void OpMsgCursor::setBatchSize(Int32 batchSize)
+{
+	_batchSize = batchSize;
+}
+
+
+Int32 OpMsgCursor::batchSize() const
+{
+	return _batchSize;
+}
+
+
+OpMsgMessage& OpMsgCursor::next(Connection& connection)
+{
+	if (_cursorID == 0)
+	{
+		_response.clear();
+
+		if (_emptyFirstBatch || _batchSize > 0)
+		{
+			Int32 bsize = _emptyFirstBatch ? 0 : _batchSize;
+			if (_query.commandName() == OpMsgMessage::CMD_FIND)
+			{
+				_query.body().add("batchSize", bsize);
+			}
+			else if (_query.commandName() == OpMsgMessage::CMD_AGGREGATE)
+			{
+				auto& cursorDoc = _query.body().addNewDocument("cursor");
+				cursorDoc.add("batchSize", bsize);
+			}
+		}
+
+		connection.sendRequest(_query, _response);
+
+		const auto& rdoc = _response.body();
+		_cursorID = cursorIdFromResponse(rdoc);
+	}
+	else
+	{
+#if MONGODB_EXHAUST_ALLOWED_WORKS
+		std::cout << "Response flags: " << _response.flags() << std::endl;
+		if (_response.flags() & OpMsgMessage::MSG_MORE_TO_COME)
+		{
+			std::cout << "More to come. Reading more response: " << std::endl;
+			_response.clear();
+			connection.readResponse(_response);
+		}
+		else
+#endif		
+		{
+			_response.clear();
+			_query.setCursor(_cursorID, _batchSize);
+			connection.sendRequest(_query, _response);
+		}
+	}
+
+	const auto& rdoc = _response.body();
+	_cursorID = cursorIdFromResponse(rdoc);
+
+	return _response;
+}
+
+
+void OpMsgCursor::kill(Connection& connection)
+{
+	_response.clear();
+	if (_cursorID != 0)
+	{
+		_query.setCommandName(OpMsgMessage::CMD_KILL_CURSORS);
+
+		MongoDB::Array::Ptr cursors = new MongoDB::Array();
+		cursors->add<Poco::Int64>(_cursorID);
+		_query.body().add("cursors", cursors);
+
+		connection.sendRequest(_query, _response);
+
+		const auto killed = _response.body().get<MongoDB::Array::Ptr>("cursorsKilled", nullptr);
+		if (!killed || killed->size() != 1 || killed->get<Poco::Int64>(0, -1) != _cursorID)
+		{
+			throw Poco::ProtocolException("Cursor not killed as expected: " + std::to_string(_cursorID));
+		}
+
+		_cursorID = 0;
+		_query.clear();
+		_response.clear();
+	}
+}
+
+
+Poco::Int64 cursorIdFromResponse(const MongoDB::Document& doc)
+{
+	Poco::Int64 id {0};
+	auto cursorDoc = doc.get<Document::Ptr>(keyCursor, nullptr);
+	if(cursorDoc)
+	{
+		id = cursorDoc->get<Poco::Int64>("id", 0);
+	}
+	return id;
+}
+
+
+} } // Namespace Poco::MongoDB

base/poco/MongoDB/src/OpMsgMessage.cpp

@@ -0,0 +1,412 @@
+//
+// OpMsgMessage.cpp
+//
+// Library: MongoDB
+// Package: MongoDB
+// Module:  OpMsgMessage
+//
+// Copyright (c) 2022, Applied Informatics Software Engineering GmbH.
+// and Contributors.
+//
+// SPDX-License-Identifier:	BSL-1.0
+//
+
+#include "Poco/MongoDB/OpMsgMessage.h"
+#include "Poco/MongoDB/MessageHeader.h"
+#include "Poco/MongoDB/Array.h"
+#include "Poco/StreamCopier.h"
+#include "Poco/Logger.h"
+
+#define POCO_MONGODB_DUMP	false
+
+namespace Poco {
+namespace MongoDB {
+
+// Query and write
+const std::string OpMsgMessage::CMD_INSERT { "insert" };
+const std::string OpMsgMessage::CMD_DELETE { "delete" };
+const std::string OpMsgMessage::CMD_UPDATE { "update" };
+const std::string OpMsgMessage::CMD_FIND { "find" };
+const std::string OpMsgMessage::CMD_FIND_AND_MODIFY { "findAndModify" };
+const std::string OpMsgMessage::CMD_GET_MORE { "getMore" };
+
+// Aggregation
+const std::string OpMsgMessage::CMD_AGGREGATE { "aggregate" };
+const std::string OpMsgMessage::CMD_COUNT { "count" };
+const std::string OpMsgMessage::CMD_DISTINCT { "distinct" };
+const std::string OpMsgMessage::CMD_MAP_REDUCE { "mapReduce" };
+
+// Replication and administration 
+const std::string OpMsgMessage::CMD_HELLO { "hello" };
+const std::string OpMsgMessage::CMD_REPL_SET_GET_STATUS { "replSetGetStatus" };
+const std::string OpMsgMessage::CMD_REPL_SET_GET_CONFIG { "replSetGetConfig" };
+
+const std::string OpMsgMessage::CMD_CREATE { "create" };
+const std::string OpMsgMessage::CMD_CREATE_INDEXES { "createIndexes" };
+const std::string OpMsgMessage::CMD_DROP { "drop" };
+const std::string OpMsgMessage::CMD_DROP_DATABASE { "dropDatabase" };
+const std::string OpMsgMessage::CMD_KILL_CURSORS { "killCursors" };
+const std::string OpMsgMessage::CMD_LIST_DATABASES { "listDatabases" };
+const std::string OpMsgMessage::CMD_LIST_INDEXES { "listIndexes" };
+
+// Diagnostic
+const std::string OpMsgMessage::CMD_BUILD_INFO { "buildInfo" };
+const std::string OpMsgMessage::CMD_COLL_STATS { "collStats" };
+const std::string OpMsgMessage::CMD_DB_STATS { "dbStats" };
+const std::string OpMsgMessage::CMD_HOST_INFO { "hostInfo" };
+
+
+static const std::string& commandIdentifier(const std::string& command);
+	/// Commands have different names for the payload that is sent in a separate section
+
+
+static const std::string keyCursor		{"cursor"};
+static const std::string keyFirstBatch	{"firstBatch"};
+static const std::string keyNextBatch	{"nextBatch"};
+
+
+OpMsgMessage::OpMsgMessage() :
+	Message(MessageHeader::OP_MSG)
+{
+}
+
+
+OpMsgMessage::OpMsgMessage(const std::string& databaseName, const std::string& collectionName, UInt32 flags) :
+	Message(MessageHeader::OP_MSG),
+	_databaseName(databaseName),
+	_collectionName(collectionName),
+	_flags(flags)
+{
+}
+
+
+OpMsgMessage::~OpMsgMessage()
+{
+}
+
+const std::string& OpMsgMessage::databaseName() const
+{
+	return _databaseName;
+}
+
+
+const std::string& OpMsgMessage::collectionName() const
+{
+	return _collectionName;
+}
+
+
+void OpMsgMessage::setCommandName(const std::string& command)
+{
+	_commandName = command;
+	_body.clear();
+
+	// IMPORTANT: Command name must be first
+	if (_collectionName.empty())
+	{
+		// Collection is not specified. It is assumed that this particular command does 
+		// not need it.
+		_body.add(_commandName, Int32(1));
+	}
+	else
+	{
+		_body.add(_commandName, _collectionName);
+	}
+	_body.add("$db", _databaseName);
+}
+
+
+void OpMsgMessage::setCursor(Poco::Int64 cursorID, Poco::Int32 batchSize)
+{
+	_commandName = OpMsgMessage::CMD_GET_MORE;
+	_body.clear();
+
+	// IMPORTANT: Command name must be first
+	_body.add(_commandName, cursorID);
+	_body.add("$db", _databaseName);
+	_body.add("collection", _collectionName);
+	if (batchSize > 0)
+	{
+		_body.add("batchSize", batchSize);
+	}
+}
+
+
+const std::string& OpMsgMessage::commandName() const
+{
+	return _commandName;
+}
+
+
+void OpMsgMessage::setAcknowledgedRequest(bool ack)
+{
+	const auto& id = commandIdentifier(_commandName);
+	if (id.empty())
+		return;
+
+	_acknowledged = ack;
+
+	auto writeConcern = _body.get<Document::Ptr>("writeConcern", nullptr);
+	if (writeConcern)
+		writeConcern->remove("w");
+
+	if (ack)
+	{
+		_flags = _flags & (~MSG_MORE_TO_COME);
+	}
+	else
+	{
+		_flags = _flags | MSG_MORE_TO_COME;
+		if (!writeConcern)
+			_body.addNewDocument("writeConcern").add("w", 0);
+		else
+			writeConcern->add("w", 0);
+	}
+
+}
+
+
+bool OpMsgMessage::acknowledgedRequest() const
+{
+	return _acknowledged;
+}
+
+
+UInt32 OpMsgMessage::flags() const
+{
+	return _flags;
+}
+
+
+Document& OpMsgMessage::body()
+{
+	return _body;
+}
+
+
+const Document& OpMsgMessage::body() const
+{
+	return _body;
+}
+
+
+Document::Vector& OpMsgMessage::documents()
+{
+	return _documents;
+}
+
+
+const Document::Vector& OpMsgMessage::documents() const
+{
+	return _documents;
+}
+
+
+bool OpMsgMessage::responseOk() const
+{
+	Poco::Int64 ok {false};
+	if (_body.exists("ok"))
+	{
+		ok = _body.getInteger("ok");
+	}
+	return (ok != 0);
+}
+
+
+void OpMsgMessage::clear()
+{
+	_flags = MSG_FLAGS_DEFAULT;
+	_commandName.clear();
+	_body.clear();
+	_documents.clear();
+}
+
+
+void OpMsgMessage::send(std::ostream& ostr)
+{
+	BinaryWriter socketWriter(ostr, BinaryWriter::LITTLE_ENDIAN_BYTE_ORDER);
+
+	// Serialise the body
+	std::stringstream ss;
+	BinaryWriter writer(ss, BinaryWriter::LITTLE_ENDIAN_BYTE_ORDER);
+	writer << _flags;
+
+	writer << PAYLOAD_TYPE_0;
+	_body.write(writer);
+
+	if (!_documents.empty())
+	{
+		// Serialise attached documents
+
+		std::stringstream ssdoc;
+		BinaryWriter wdoc(ssdoc, BinaryWriter::LITTLE_ENDIAN_BYTE_ORDER);
+		for (auto& doc: _documents)
+		{
+			doc->write(wdoc);
+		}
+		wdoc.flush();
+
+		const std::string& identifier = commandIdentifier(_commandName);
+		const Poco::Int32 size = static_cast<Poco::Int32>(sizeof(size) + identifier.size() + 1 + ssdoc.tellp());
+		writer << PAYLOAD_TYPE_1;
+		writer << size;
+		writer.writeCString(identifier.c_str());
+		StreamCopier::copyStream(ssdoc, ss);
+	}
+	writer.flush();
+
+#if POCO_MONGODB_DUMP
+	const std::string section = ss.str();
+	std::string dump;
+	Logger::formatDump(dump, section.data(), section.length());
+	std::cout << dump << std::endl;
+#endif
+
+	messageLength(static_cast<Poco::Int32>(ss.tellp()));
+
+	_header.write(socketWriter);
+	StreamCopier::copyStream(ss, ostr);
+
+	ostr.flush();
+}
+
+
+void OpMsgMessage::read(std::istream& istr)
+{
+	std::string message;
+	{
+		BinaryReader reader(istr, BinaryReader::LITTLE_ENDIAN_BYTE_ORDER);
+		_header.read(reader);
+
+		poco_assert_dbg(_header.opCode() == _header.OP_MSG);
+
+		const std::streamsize remainingSize {_header.getMessageLength() - _header.MSG_HEADER_SIZE };
+		message.reserve(remainingSize);
+
+#if POCO_MONGODB_DUMP
+		std::cout
+			<< "Message hdr: " << _header.getMessageLength() << " " << remainingSize << " "
+			<< _header.opCode() << " " << _header.getRequestID() << " " << _header.responseTo()
+			<< std::endl;
+#endif
+		
+		reader.readRaw(remainingSize, message);
+
+#if POCO_MONGODB_DUMP
+		std::string dump;
+		Logger::formatDump(dump, message.data(), message.length());
+		std::cout << dump << std::endl;
+#endif
+	}
+	// Read complete message and then interpret it.
+
+	std::istringstream msgss(message);
+	BinaryReader reader(msgss, BinaryReader::LITTLE_ENDIAN_BYTE_ORDER);
+
+	Poco::UInt8 payloadType {0xFF};
+
+	reader >> _flags;
+	reader >> payloadType;
+	poco_assert_dbg(payloadType == PAYLOAD_TYPE_0);
+
+	_body.read(reader);
+
+	// Read next sections from the buffer
+	while (msgss.good())
+	{
+		// NOTE: Not tested yet with database, because it returns everything in the body.
+		// Does MongoDB ever return documents as Payload type 1?
+		reader >> payloadType;
+		if (!msgss.good())
+		{
+			break;
+		}
+		poco_assert_dbg(payloadType == PAYLOAD_TYPE_1);
+#if POCO_MONGODB_DUMP
+		std::cout << "section payload: " << payloadType << std::endl;
+#endif
+
+		Poco::Int32 sectionSize {0};
+		reader >> sectionSize;
+		poco_assert_dbg(sectionSize > 0);
+
+#if POCO_MONGODB_DUMP
+		std::cout << "section size: " << sectionSize << std::endl;
+#endif
+		std::streamoff offset = sectionSize - sizeof(sectionSize);
+		std::streampos endOfSection = msgss.tellg() + offset;
+
+		std::string identifier;
+		reader.readCString(identifier);
+#if POCO_MONGODB_DUMP
+		std::cout << "section identifier: " << identifier << std::endl;
+#endif
+
+		// Loop to read documents from this section.
+		while (msgss.tellg() < endOfSection)
+		{
+#if POCO_MONGODB_DUMP
+			std::cout << "section doc: " << msgss.tellg() << " " << endOfSection << std::endl;
+#endif
+			Document::Ptr doc = new Document();
+			doc->read(reader);
+			_documents.push_back(doc);
+			if (msgss.tellg() < 0)
+			{
+				break;
+			}
+		}
+	}
+
+	// Extract documents from the cursor batch if they are there.
+	MongoDB::Array::Ptr batch;
+	auto curDoc = _body.get<MongoDB::Document::Ptr>(keyCursor, nullptr);
+	if (curDoc)
+	{
+		batch = curDoc->get<MongoDB::Array::Ptr>(keyFirstBatch, nullptr);
+		if (!batch)
+		{
+			batch = curDoc->get<MongoDB::Array::Ptr>(keyNextBatch, nullptr);
+		}
+	}
+	if (batch)
+	{
+		for(std::size_t i = 0; i < batch->size(); i++)
+		{
+			const auto& d = batch->get<MongoDB::Document::Ptr>(i, nullptr);
+			if (d)
+			{
+				_documents.push_back(d);
+			}
+		}
+	}
+
+}
+
+const std::string& commandIdentifier(const std::string& command)
+{
+	// Names of identifiers for commands that send bulk documents in the request
+	// The identifier is set in the section type 1.
+	static std::map<std::string, std::string> identifiers {
+		{ OpMsgMessage::CMD_INSERT, "documents" },
+		{ OpMsgMessage::CMD_DELETE, "deletes" },
+		{ OpMsgMessage::CMD_UPDATE, "updates" },
+
+		// Not sure if create index can send document section
+		{ OpMsgMessage::CMD_CREATE_INDEXES, "indexes" }
+	};
+
+	const auto i = identifiers.find(command);
+	if (i != identifiers.end())
+	{
+		return i->second;
+	}
+
+	// This likely means that documents are incorrectly set for a command
+	// that does not send list of documents in section type 1.
+	static const std::string emptyIdentifier;
+	return emptyIdentifier;
+}
+
+
+} } // namespace Poco::MongoDB

base/poco/MongoDB/src/QueryRequest.cpp

@@ -20,10 +20,10 @@ namespace MongoDB {
 
 
 QueryRequest::QueryRequest(const std::string& collectionName, QueryRequest::Flags flags):
-	RequestMessage(MessageHeader::OP_QUERY), 
+	RequestMessage(MessageHeader::OP_QUERY),
-	_flags(flags), 
+	_flags(flags),
 	_fullCollectionName(collectionName),
-	_numberToSkip(0), 
+	_numberToSkip(0),
 	_numberToReturn(100),
 	_selector(),
 	_returnFieldSelector()

base/poco/MongoDB/src/RegularExpression.cpp

@@ -25,8 +25,8 @@ RegularExpression::RegularExpression()
 }
 
 
-RegularExpression::RegularExpression(const std::string& pattern, const std::string& options): 
+RegularExpression::RegularExpression(const std::string& pattern, const std::string& options):
-	_pattern(pattern), 
+	_pattern(pattern),
 	_options(options)
 {
 }

base/poco/MongoDB/src/ReplicaSet.cpp

@@ -21,7 +21,7 @@ namespace Poco {
 namespace MongoDB {
 
 
-ReplicaSet::ReplicaSet(const std::vector<Net::SocketAddress> &addresses): 
+ReplicaSet::ReplicaSet(const std::vector<Net::SocketAddress> &addresses):
 	_addresses(addresses)
 {
 }
@@ -81,8 +81,8 @@ Connection::Ptr ReplicaSet::isMaster(const Net::SocketAddress& address)
 	{
 		conn = 0;
 	}
-	
+
-	return 0; 
+	return 0;
 }
 
 

base/poco/MongoDB/src/RequestMessage.cpp

@@ -21,7 +21,7 @@ namespace Poco {
 namespace MongoDB {
 
 
-RequestMessage::RequestMessage(MessageHeader::OpCode opcode): 
+RequestMessage::RequestMessage(MessageHeader::OpCode opcode):
 	Message(opcode)
 {
 }
@@ -35,7 +35,7 @@ RequestMessage::~RequestMessage()
 void RequestMessage::send(std::ostream& ostr)
 {
 	std::stringstream ss;
-	BinaryWriter requestWriter(ss);
+	BinaryWriter requestWriter(ss, BinaryWriter::LITTLE_ENDIAN_BYTE_ORDER);
 	buildRequest(requestWriter);
 	requestWriter.flush();
 

base/poco/MongoDB/src/ResponseMessage.cpp

@@ -21,10 +21,20 @@ namespace MongoDB {
 
 
 ResponseMessage::ResponseMessage():
-	Message(MessageHeader::OP_REPLY), 
+	Message(MessageHeader::OP_REPLY),
-	_responseFlags(0), 
+	_responseFlags(0),
-	_cursorID(0), 
+	_cursorID(0),
-	_startingFrom(0), 
+	_startingFrom(0),
+	_numberReturned(0)
+{
+}
+
+
+ResponseMessage::ResponseMessage(const Int64& cursorID):
+	Message(MessageHeader::OP_REPLY),
+	_responseFlags(0),
+	_cursorID(cursorID),
+	_startingFrom(0),
 	_numberReturned(0)
 {
 }
@@ -50,7 +60,7 @@ void ResponseMessage::read(std::istream& istr)
 	clear();
 
 	BinaryReader reader(istr, BinaryReader::LITTLE_ENDIAN_BYTE_ORDER);
-	
+
 	_header.read(reader);
 
 	reader >> _responseFlags;

base/poco/MongoDB/src/UpdateRequest.cpp

@@ -20,7 +20,7 @@ namespace MongoDB {
 
 
 UpdateRequest::UpdateRequest(const std::string& collectionName, UpdateRequest::Flags flags):
-	RequestMessage(MessageHeader::OP_UPDATE), 
+	RequestMessage(MessageHeader::OP_UPDATE),
 	_flags(flags),
 	_fullCollectionName(collectionName),
 	_selector(),

base/poco/Net/include/Poco/Net/HTTPClientSession.h

@@ -127,6 +127,9 @@ namespace Net
 
         void setResolvedHost(std::string resolved_host) { _resolved_host.swap(resolved_host); }
 
+        std::string getResolvedHost() const { return _resolved_host; }
+        /// Returns the resolved IP address of the target HTTP server.
+
         Poco::UInt16 getPort() const;
         /// Returns the port number of the target HTTP server.
 

base/poco/Net/src/SocketImpl.cpp

@@ -274,7 +274,9 @@ void SocketImpl::shutdown()
 
 int SocketImpl::sendBytes(const void* buffer, int length, int flags)
 {
-	if (_isBrokenTimeout)
+    bool blocking = _blocking && (flags & MSG_DONTWAIT) == 0;
+
+	if (_isBrokenTimeout && blocking)
 	{
 		if (_sndTimeout.totalMicroseconds() != 0)
 		{
@@ -289,11 +291,13 @@ int SocketImpl::sendBytes(const void* buffer, int length, int flags)
 		if (_sockfd == POCO_INVALID_SOCKET) throw InvalidSocketException();
 		rc = ::send(_sockfd, reinterpret_cast<const char*>(buffer), length, flags);
 	}
-	while (_blocking && rc < 0 && lastError() == POCO_EINTR);
+	while (blocking && rc < 0 && lastError() == POCO_EINTR);
 	if (rc < 0)
 	{
 		int err = lastError();
-		if (err == POCO_EAGAIN || err == POCO_ETIMEDOUT)
+		if ((err == POCO_EAGAIN || err == POCO_EWOULDBLOCK) && !blocking)
+			;
+		else if (err == POCO_EAGAIN || err == POCO_ETIMEDOUT)
 			throw TimeoutException();
 		else
 			error(err);

base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureSocketImpl.h

@@ -183,6 +183,16 @@ namespace Net
         /// Returns true iff a reused session was negotiated during
         /// the handshake.
 
+        virtual void setBlocking(bool flag);
+        /// Sets the socket in blocking mode if flag is true,
+        /// disables blocking mode if flag is false.
+
+        virtual bool getBlocking() const;
+        /// Returns the blocking mode of the socket.
+        /// This method will only work if the blocking modes of
+        /// the socket are changed via the setBlocking method!
+
+
     protected:
         void acceptSSL();
         /// Assume per-object mutex is locked.

base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureStreamSocketImpl.h

@@ -201,6 +201,16 @@ namespace Net
         /// Returns true iff a reused session was negotiated during
         /// the handshake.
 
+        virtual void setBlocking(bool flag);
+        /// Sets the socket in blocking mode if flag is true,
+        /// disables blocking mode if flag is false.
+
+        virtual bool getBlocking() const;
+        /// Returns the blocking mode of the socket.
+        /// This method will only work if the blocking modes of
+        /// the socket are changed via the setBlocking method!
+
+
     protected:
         void acceptSSL();
         /// Performs a SSL server-side handshake.

base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp

@@ -629,5 +629,15 @@ bool SecureSocketImpl::sessionWasReused()
 		return false;
 }
 
+void SecureSocketImpl::setBlocking(bool flag)
+{
+    _pSocket->setBlocking(flag);
+}
+
+bool SecureSocketImpl::getBlocking() const
+{
+    return _pSocket->getBlocking();
+}
+
 
 } } // namespace Poco::Net

base/poco/NetSSL_OpenSSL/src/SecureStreamSocketImpl.cpp

@@ -237,5 +237,15 @@ int SecureStreamSocketImpl::completeHandshake()
 	return _impl.completeHandshake();
 }
 
+bool SecureStreamSocketImpl::getBlocking() const
+{
+    return _impl.getBlocking();
+}
+
+void SecureStreamSocketImpl::setBlocking(bool flag)
+{
+    _impl.setBlocking(flag);
+}
+
 
 } } // namespace Poco::Net

cmake/add_check.cmake

@@ -5,11 +5,11 @@ if (NOT TARGET check)
     if (CMAKE_CONFIGURATION_TYPES)
         add_custom_target (check COMMAND ${CMAKE_CTEST_COMMAND}
             --force-new-ctest-process --output-on-failure --build-config "$<CONFIGURATION>"
-            WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+            WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
     else ()
         add_custom_target (check COMMAND ${CMAKE_CTEST_COMMAND}
             --force-new-ctest-process --output-on-failure
-            WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+            WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
     endif ()
 endif ()
 

cmake/autogenerated_versions.txt

@@ -2,11 +2,11 @@
 
 # NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION,
 # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes.
-SET(VERSION_REVISION 54474)
+SET(VERSION_REVISION 54475)
 SET(VERSION_MAJOR 23)
-SET(VERSION_MINOR 5)
+SET(VERSION_MINOR 6)
 SET(VERSION_PATCH 1)
-SET(VERSION_GITHASH 3920eb987f7ed837ada5de8907284adf123f0583)
+SET(VERSION_GITHASH 2fec796e73efda10a538a03af3205ce8ffa1b2de)
-SET(VERSION_DESCRIBE v23.5.1.1-testing)
+SET(VERSION_DESCRIBE v23.6.1.1-testing)
-SET(VERSION_STRING 23.5.1.1)
+SET(VERSION_STRING 23.6.1.1)
 # end of autochange

cmake/git.cmake

@@ -5,14 +5,14 @@ if (Git_FOUND)
   # Commit hash + whether the building workspace was dirty or not
   execute_process(COMMAND
     "${GIT_EXECUTABLE}" rev-parse HEAD
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
     OUTPUT_VARIABLE GIT_HASH
     ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
 
   # Branch name
   execute_process(COMMAND
     "${GIT_EXECUTABLE}" rev-parse --abbrev-ref HEAD
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
     OUTPUT_VARIABLE GIT_BRANCH
     ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
 
@@ -20,14 +20,14 @@ if (Git_FOUND)
   SET(ENV{TZ} "UTC")
   execute_process(COMMAND
     "${GIT_EXECUTABLE}" log -1 --format=%ad --date=iso-local
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
     OUTPUT_VARIABLE GIT_DATE
     ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
 
   # Subject of the commit
   execute_process(COMMAND
     "${GIT_EXECUTABLE}" log -1 --format=%s
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
     OUTPUT_VARIABLE GIT_COMMIT_SUBJECT
     ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
 
@@ -35,7 +35,7 @@ if (Git_FOUND)
 
   execute_process(
     COMMAND ${GIT_EXECUTABLE} status
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_STRIP_TRAILING_WHITESPACE)
+    WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} OUTPUT_STRIP_TRAILING_WHITESPACE)
 else()
   message(STATUS "Git could not be found.")
 endif()

cmake/print_flags.cmake

@@ -7,6 +7,6 @@ message (STATUS "compiler CXX = ${CMAKE_CXX_COMPILER} ${FULL_CXX_FLAGS}")
 message (STATUS "LINKER_FLAGS = ${FULL_EXE_LINKER_FLAGS}")
 
 # Reproducible builds
-string (REPLACE "${CMAKE_SOURCE_DIR}" "." FULL_C_FLAGS_NORMALIZED "${FULL_C_FLAGS}")
+string (REPLACE "${PROJECT_SOURCE_DIR}" "." FULL_C_FLAGS_NORMALIZED "${FULL_C_FLAGS}")
-string (REPLACE "${CMAKE_SOURCE_DIR}" "." FULL_CXX_FLAGS_NORMALIZED "${FULL_CXX_FLAGS}")
+string (REPLACE "${PROJECT_SOURCE_DIR}" "." FULL_CXX_FLAGS_NORMALIZED "${FULL_CXX_FLAGS}")
-string (REPLACE "${CMAKE_SOURCE_DIR}" "." FULL_EXE_LINKER_FLAGS_NORMALIZED "${FULL_EXE_LINKER_FLAGS}")
+string (REPLACE "${PROJECT_SOURCE_DIR}" "." FULL_EXE_LINKER_FLAGS_NORMALIZED "${FULL_EXE_LINKER_FLAGS}")

cmake/sanitize.cmake

@@ -8,6 +8,9 @@ option (SANITIZE "Enable one of the code sanitizers" "")
 
 set (SAN_FLAGS "${SAN_FLAGS} -g -fno-omit-frame-pointer -DSANITIZER")
 
+# It's possible to pass an ignore list to sanitizers (-fsanitize-ignorelist). Intentionally not doing this because
+# 1. out-of-source suppressions are awkward 2. it seems ignore lists don't work after the Clang v16 upgrade (#49829)
+
 if (SANITIZE)
     if (SANITIZE STREQUAL "address")
         set (ASAN_FLAGS "-fsanitize=address -fsanitize-address-use-after-scope")
@@ -29,14 +32,14 @@ if (SANITIZE)
 
         # Linking can fail due to relocation overflows (see #49145), caused by too big object files / libraries.
         # Work around this with position-independent builds (-fPIC and -fpie), this is slightly slower than non-PIC/PIE but that's okay.
-        set (MSAN_FLAGS "-fsanitize=memory -fsanitize-memory-use-after-dtor -fsanitize-memory-track-origins -fno-optimize-sibling-calls -fPIC -fpie -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/tests/msan_suppressions.txt")
+        set (MSAN_FLAGS "-fsanitize=memory -fsanitize-memory-use-after-dtor -fsanitize-memory-track-origins -fno-optimize-sibling-calls -fPIC -fpie")
         set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${MSAN_FLAGS}")
         set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} ${MSAN_FLAGS}")
 
     elseif (SANITIZE STREQUAL "thread")
         set (TSAN_FLAGS "-fsanitize=thread")
         if (COMPILER_CLANG)
-            set (TSAN_FLAGS "${TSAN_FLAGS} -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/tests/tsan_suppressions.txt")
+            set (TSAN_FLAGS "${TSAN_FLAGS} -fsanitize-blacklist=${PROJECT_SOURCE_DIR}/tests/tsan_suppressions.txt")
         endif()
 
         set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${TSAN_FLAGS}")
@@ -54,7 +57,7 @@ if (SANITIZE)
             set(UBSAN_FLAGS "${UBSAN_FLAGS} -fno-sanitize=unsigned-integer-overflow")
         endif()
         if (COMPILER_CLANG)
-            set (UBSAN_FLAGS "${UBSAN_FLAGS} -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/tests/ubsan_suppressions.txt")
+            set (UBSAN_FLAGS "${UBSAN_FLAGS} -fsanitize-blacklist=${PROJECT_SOURCE_DIR}/tests/ubsan_suppressions.txt")
         endif()
 
         set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${UBSAN_FLAGS}")

cmake/toolchain/darwin-x86_64/README.txt

@@ -1,2 +1,2 @@
-wget https://github.com/phracker/MacOSX-SDKs/releases/download/10.15/MacOSX10.15.sdk.tar.xz
+wget https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX11.0.sdk.tar.xz
-tar xJf MacOSX10.15.sdk.tar.xz --strip-components=1
+tar xJf MacOSX11.0.sdk.tar.xz --strip-components=1

cmake/version.cmake

@@ -1,4 +1,4 @@
-include(${CMAKE_SOURCE_DIR}/cmake/autogenerated_versions.txt)
+include(${PROJECT_SOURCE_DIR}/cmake/autogenerated_versions.txt)
 
 set(VERSION_EXTRA "" CACHE STRING "")
 set(VERSION_TWEAK "" CACHE STRING "")

contrib/CMakeLists.txt

@@ -88,7 +88,7 @@ add_contrib (thrift-cmake thrift)
 # parquet/arrow/orc
 add_contrib (arrow-cmake arrow) # requires: snappy, thrift, double-conversion
 add_contrib (avro-cmake avro) # requires: snappy
-add_contrib (protobuf-cmake protobuf)
+add_contrib (google-protobuf-cmake google-protobuf)
 add_contrib (openldap-cmake openldap)
 add_contrib (grpc-cmake grpc)
 add_contrib (msgpack-c-cmake msgpack-c)
@@ -156,7 +156,7 @@ add_contrib (libgsasl-cmake libgsasl) # requires krb5
 add_contrib (librdkafka-cmake librdkafka) # requires: libgsasl
 add_contrib (nats-io-cmake nats-io)
 add_contrib (isa-l-cmake isa-l)
-add_contrib (libhdfs3-cmake libhdfs3) # requires: protobuf, krb5, isa-l
+add_contrib (libhdfs3-cmake libhdfs3) # requires: google-protobuf, krb5, isa-l
 add_contrib (hive-metastore-cmake hive-metastore) # requires: thrift/avro/arrow/libhdfs3
 add_contrib (cppkafka-cmake cppkafka)
 add_contrib (libpqxx-cmake libpqxx)

contrib/NuRaft

@@ -1 +1 @@
-Subproject commit b56784be1aec568fb72aff47f281097c017623cb
+Subproject commit 491eaf592d950e0e37accbe8b3f217e068c9fecf

contrib/avro-cmake/CMakeLists.txt

@@ -6,7 +6,7 @@ if (NOT ENABLE_AVRO)
     return()
 endif()
 
-set(AVROCPP_ROOT_DIR "${CMAKE_SOURCE_DIR}/contrib/avro/lang/c++")
+set(AVROCPP_ROOT_DIR "${PROJECT_SOURCE_DIR}/contrib/avro/lang/c++")
 set(AVROCPP_INCLUDE_DIR "${AVROCPP_ROOT_DIR}/api")
 set(AVROCPP_SOURCE_DIR "${AVROCPP_ROOT_DIR}/impl")
 

contrib/aws

@@ -1 +1 @@
-Subproject commit ecccfc026a42b30023289410a67024d561f4bf3e
+Subproject commit ca02358dcc7ce3ab733dd4cbcc32734eecfa4ee3

contrib/aws-c-auth

@@ -1 +1 @@
-Subproject commit 30df6c407e2df43bd244e2c34c9b4a4b87372bfb
+Subproject commit 97133a2b5dbca1ccdf88cd6f44f39d0531d27d12

contrib/aws-c-common

@@ -1 +1 @@
-Subproject commit 324fd1d973ccb25c813aa747bf1759cfde5121c5
+Subproject commit 45dcb2849c891dba2100b270b4676765c92949ff

contrib/aws-c-event-stream

@@ -1 +1 @@
-Subproject commit 39bfa94a14b7126bf0c1330286ef8db452d87e66
+Subproject commit 2f9b60c42f90840ec11822acda3d8cdfa97a773d

contrib/aws-c-http

@@ -1 +1 @@
-Subproject commit 2c5a2a7d5556600b9782ffa6c9d7e09964df1abc
+Subproject commit dd34461987947672444d0bc872c5a733dfdb9711

contrib/aws-c-io

@@ -1 +1 @@
-Subproject commit 5d32c453560d0823df521a686bf7fbacde7f9be3
+Subproject commit d58ed4f272b1cb4f89ac9196526ceebe5f2b0d89

contrib/aws-c-mqtt

@@ -1 +1 @@
-Subproject commit 882c689561a3db1466330ccfe3b63637e0a575d3
+Subproject commit 33c3455cec82b16feb940e12006cefd7b3ef4194

contrib/aws-c-s3

@@ -1 +1 @@
-Subproject commit a41255ece72a7c887bba7f9d998ca3e14f4c8a1b
+Subproject commit d7bfe602d6925948f1fff95784e3613cca6a3900

contrib/aws-c-sdkutils

@@ -1 +1 @@
-Subproject commit 25bf5cf225f977c3accc6a05a0a7a181ef2a4a30
+Subproject commit 208a701fa01e99c7c8cc3dcebc8317da71362972

contrib/aws-checksums

@@ -1 +1 @@
-Subproject commit 48e7c0e01479232f225c8044d76c84e74192889d
+Subproject commit ad53be196a25bbefa3700a01187fdce573a7d2d0

contrib/aws-cmake/CMakeLists.txt

@@ -52,8 +52,8 @@ endif()
 
 # Directories.
 SET(AWS_SDK_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws")
-SET(AWS_SDK_CORE_DIR "${AWS_SDK_DIR}/aws-cpp-sdk-core")
+SET(AWS_SDK_CORE_DIR "${AWS_SDK_DIR}/src/aws-cpp-sdk-core")
-SET(AWS_SDK_S3_DIR "${AWS_SDK_DIR}/aws-cpp-sdk-s3")
+SET(AWS_SDK_S3_DIR "${AWS_SDK_DIR}/generated/src/aws-cpp-sdk-s3")
 
 SET(AWS_AUTH_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-auth")
 SET(AWS_CAL_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-cal")
@@ -118,7 +118,7 @@ configure_file("${AWS_SDK_CORE_DIR}/include/aws/core/SDKConfig.h.in"
 list(APPEND AWS_PUBLIC_COMPILE_DEFS "-DAWS_SDK_VERSION_MAJOR=1")
 list(APPEND AWS_PUBLIC_COMPILE_DEFS "-DAWS_SDK_VERSION_MINOR=10")
 list(APPEND AWS_PUBLIC_COMPILE_DEFS "-DAWS_SDK_VERSION_PATCH=36")
-    
+
 list(APPEND AWS_SOURCES ${AWS_SDK_CORE_SRC} ${AWS_SDK_CORE_NET_SRC} ${AWS_SDK_CORE_PLATFORM_SRC})
 
 list(APPEND AWS_PUBLIC_INCLUDES

contrib/aws-crt-cpp

@@ -1 +1 @@
-Subproject commit ec0bea288f451d884c0d80d534bc5c66241c39a4
+Subproject commit 8a301b7e842f1daed478090c869207300972379f

contrib/aws-s2n-tls

@@ -1 +1 @@
-Subproject commit 0f1ba9e5c4a67cb3898de0c0b4f911d4194dc8de
+Subproject commit 71f4794b7580cf780eb4aca77d69eded5d3c7bb4

contrib/azure-cmake/CMakeLists.txt

@@ -1,6 +1,6 @@
 option (ENABLE_AZURE_BLOB_STORAGE "Enable Azure blob storage" ${ENABLE_LIBRARIES})
 
-if (NOT ENABLE_AZURE_BLOB_STORAGE OR BUILD_STANDALONE_KEEPER OR OS_FREEBSD)
+if (NOT ENABLE_AZURE_BLOB_STORAGE OR BUILD_STANDALONE_KEEPER OR OS_FREEBSD OR (NOT ARCH_AMD64))
     message(STATUS "Not using Azure blob storage")
     return()
 endif()

contrib/boost

@@ -1 +1 @@
-Subproject commit 8fe7b3326ef482ee6ecdf5a4f698f2b8c2780f98
+Subproject commit aec12eea7fc762721ae16943d1361340c66c9c17

contrib/c-ares

@@ -1 +1 @@
-Subproject commit afee6748b0b99acf4509d42fa37ac8422262f91b
+Subproject commit 6360e96b5cf8e5980c887ce58ef727e53d77243a

contrib/c-ares-cmake/CMakeLists.txt

@@ -48,6 +48,7 @@ SET(SRCS
     "${LIBRARY_DIR}/src/lib/ares_platform.c"
     "${LIBRARY_DIR}/src/lib/ares_process.c"
     "${LIBRARY_DIR}/src/lib/ares_query.c"
+    "${LIBRARY_DIR}/src/lib/ares_rand.c"
     "${LIBRARY_DIR}/src/lib/ares_search.c"
     "${LIBRARY_DIR}/src/lib/ares_send.c"
     "${LIBRARY_DIR}/src/lib/ares_strcasecmp.c"

contrib/capnproto

@@ -1 +1 @@
-Subproject commit dc8b50b999777bcb23c89bb5907c785c3f654441
+Subproject commit 976209a6d18074804f60d18ef99b6a809d27dadf

contrib/cassandra-cmake/CMakeLists.txt

@@ -18,7 +18,7 @@ endif()
 # Need to use C++17 since the compilation is not possible with C++20 currently.
 set (CMAKE_CXX_STANDARD 17)
 
-set(CASS_ROOT_DIR ${CMAKE_SOURCE_DIR}/contrib/cassandra)
+set(CASS_ROOT_DIR ${PROJECT_SOURCE_DIR}/contrib/cassandra)
 set(CASS_SRC_DIR "${CASS_ROOT_DIR}/src")
 set(CASS_INCLUDE_DIR "${CASS_ROOT_DIR}/include")
 

contrib/cctz-cmake/CMakeLists.txt

@@ -26,7 +26,7 @@ endif ()
 # StorageSystemTimeZones.generated.cpp is autogenerated each time during a build
 # data in this file will be used to populate the system.time_zones table, this is specific to OS_LINUX
 # as the library that's built using embedded tzdata is also specific to OS_LINUX
-set(SYSTEM_STORAGE_TZ_FILE "${CMAKE_BINARY_DIR}/src/Storages/System/StorageSystemTimeZones.generated.cpp")
+set(SYSTEM_STORAGE_TZ_FILE "${PROJECT_BINARY_DIR}/src/Storages/System/StorageSystemTimeZones.generated.cpp")
 # remove existing copies so that its generated fresh on each build.
 file(REMOVE ${SYSTEM_STORAGE_TZ_FILE})
 

contrib/google-protobuf

@@ -0,0 +1 @@
+Subproject commit c47efe2d8f6a60022b49ecd6cc23660687c8598f

contrib/google-protobuf-cmake/CMakeLists.txt

@@ -5,7 +5,7 @@ if(NOT ENABLE_PROTOBUF)
   return()
 endif()
 
-set(Protobuf_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/protobuf/src")
+set(Protobuf_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/google-protobuf/src")
 if(OS_FREEBSD AND SANITIZE STREQUAL "address")
   # ../contrib/protobuf/src/google/protobuf/arena_impl.h:45:10: fatal error: 'sanitizer/asan_interface.h' file not found
   # #include <sanitizer/asan_interface.h>
@@ -17,8 +17,8 @@ if(OS_FREEBSD AND SANITIZE STREQUAL "address")
   endif()
 endif()
 
-set(protobuf_source_dir "${ClickHouse_SOURCE_DIR}/contrib/protobuf")
+set(protobuf_source_dir "${ClickHouse_SOURCE_DIR}/contrib/google-protobuf")
-set(protobuf_binary_dir "${ClickHouse_BINARY_DIR}/contrib/protobuf")
+set(protobuf_binary_dir "${ClickHouse_BINARY_DIR}/contrib/google-protobuf")
 
 
 add_definitions(-DGOOGLE_PROTOBUF_CMAKE_BUILD)
@@ -35,7 +35,6 @@ set(libprotobuf_lite_files
   ${protobuf_source_dir}/src/google/protobuf/arena.cc
   ${protobuf_source_dir}/src/google/protobuf/arenastring.cc
   ${protobuf_source_dir}/src/google/protobuf/extension_set.cc
-  ${protobuf_source_dir}/src/google/protobuf/field_access_listener.cc
   ${protobuf_source_dir}/src/google/protobuf/generated_enum_util.cc
   ${protobuf_source_dir}/src/google/protobuf/generated_message_table_driven_lite.cc
   ${protobuf_source_dir}/src/google/protobuf/generated_message_util.cc
@@ -86,6 +85,7 @@ set(libprotobuf_files
   ${protobuf_source_dir}/src/google/protobuf/empty.pb.cc
   ${protobuf_source_dir}/src/google/protobuf/extension_set_heavy.cc
   ${protobuf_source_dir}/src/google/protobuf/field_mask.pb.cc
+  ${protobuf_source_dir}/src/google/protobuf/generated_message_bases.cc
   ${protobuf_source_dir}/src/google/protobuf/generated_message_reflection.cc
   ${protobuf_source_dir}/src/google/protobuf/generated_message_table_driven.cc
   ${protobuf_source_dir}/src/google/protobuf/io/gzip_stream.cc
@@ -316,7 +316,7 @@ else ()
     add_dependencies(protoc "${PROTOC_BUILD_DIR}/protoc")
 endif ()
 
-include("${ClickHouse_SOURCE_DIR}/contrib/protobuf-cmake/protobuf_generate.cmake")
+include("${ClickHouse_SOURCE_DIR}/contrib/google-protobuf-cmake/protobuf_generate.cmake")
 
 add_library(_protobuf INTERFACE)
 target_link_libraries(_protobuf INTERFACE _libprotobuf)

contrib/libgsasl

@@ -1 +1 @@
-Subproject commit f4e7bf0bb068030d57266f87ccac4c8c012fb5c4
+Subproject commit 0fb79e7609ae5a5e015a41d24bcbadd48f8f5469

contrib/libpqxx

@@ -1 +1 @@
-Subproject commit a4e834839270a8c1f7ff1db351ba85afced3f0e2
+Subproject commit bdd6540fb95ff56c813691ceb5da5a3266cf235d

contrib/libuv-cmake/CMakeLists.txt

@@ -1,7 +1,7 @@
 # This file is a modified version of contrib/libuv/CMakeLists.txt
 
-set (SOURCE_DIR "${CMAKE_SOURCE_DIR}/contrib/libuv")
+set (SOURCE_DIR "${PROJECT_SOURCE_DIR}/contrib/libuv")
-set (BINARY_DIR "${CMAKE_BINARY_DIR}/contrib/libuv")
+set (BINARY_DIR "${PROJECT_BINARY_DIR}/contrib/libuv")
 
 set(uv_sources
     src/fs-poll.c

contrib/libxml2

@@ -1 +1 @@
-Subproject commit f507d167f1755b7eaea09fb1a44d29aab828b6d1
+Subproject commit 223cb03a5d27b1b2393b266a8657443d046139d6

contrib/lz4

@@ -1 +1 @@
-Subproject commit 4c9431e9af596af0556e5da0ae99305bafb2b10b
+Subproject commit e82198428c8061372d5adef1f9bfff4203f6081e

contrib/lz4-cmake/CMakeLists.txt

@@ -12,6 +12,7 @@ add_library (_lz4 ${SRCS})
 add_library (ch_contrib::lz4 ALIAS _lz4)
 
 target_compile_definitions (_lz4 PUBLIC LZ4_DISABLE_DEPRECATE_WARNINGS=1)
+target_compile_definitions (_lz4 PUBLIC LZ4_FAST_DEC_LOOP=1)
 if (SANITIZE STREQUAL "undefined")
     target_compile_options (_lz4 PRIVATE -fno-sanitize=undefined)
 endif ()

contrib/mariadb-connector-c-cmake/CMakeLists.txt

@@ -15,7 +15,7 @@ endif()
 
 # This is the LGPL libmariadb project.
 
-set(CC_SOURCE_DIR ${CMAKE_SOURCE_DIR}/contrib/mariadb-connector-c)
+set(CC_SOURCE_DIR ${PROJECT_SOURCE_DIR}/contrib/mariadb-connector-c)
 set(CC_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
 
 set(WITH_SSL ON)

contrib/protobuf

@@ -1 +0,0 @@
-Subproject commit 6bb70196c5360268d9f021bb7936fb0b551724c2

contrib/qpl-cmake/CMakeLists.txt

@@ -25,6 +25,9 @@ message(STATUS "Intel QPL version: ${QPL_VERSION}")
 # Generate 8 library targets: middle_layer_lib, isal, isal_asm, qplcore_px, qplcore_avx512, qplcore_sw_dispatcher, core_iaa, middle_layer_lib.
 # Output ch_contrib::qpl by linking with 8 library targets.
 
+# The qpl submodule comes with its own version of isal. It contains code which does not exist in upstream isal. It would be nice to link
+# only upstream isal (ch_contrib::isal) but at this point we can't.
+
 include("${QPL_PROJECT_DIR}/cmake/CompileOptions.cmake")
 
 # check nasm compiler
@@ -308,7 +311,7 @@ target_include_directories(middle_layer_lib
 target_compile_definitions(middle_layer_lib PUBLIC -DQPL_LIB)
 
 # [SUBDIR]c_api
-file(GLOB_RECURSE QPL_C_API_SRC 
+file(GLOB_RECURSE QPL_C_API_SRC
         ${QPL_SRC_DIR}/c_api/*.c
         ${QPL_SRC_DIR}/c_api/*.cpp)