diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 960e24d693c..4a3880543c4 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -168,35 +168,6 @@ jobs: docker kill "$(docker ps -q)" ||: docker rm -f "$(docker ps -a -q)" ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - PVSCheck: - needs: [DockerHubPush, FastTest] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/pvs_check - REPO_COPY=${{runner.temp}}/pvs_check/ClickHouse - EOF - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - with: - submodules: 'true' - - name: PVS Check - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 pvs_check.py - - name: Cleanup - if: always() - run: | - docker kill "$(docker ps -q)" ||: - docker rm -f "$(docker ps -a -q)" ||: - sudo rm -fr "$TEMP_PATH" CompatibilityCheck: needs: [BuilderDebRelease] runs-on: [self-hosted, style-checker] @@ -3096,7 +3067,6 @@ jobs: - PerformanceComparison1 - PerformanceComparison2 - PerformanceComparison3 - - PVSCheck - UnitTestsAsan - UnitTestsTsan - UnitTestsMsan diff --git a/.gitmodules b/.gitmodules index 91f4ddb2007..2a1859d5d26 100644 --- a/.gitmodules +++ b/.gitmodules @@ -38,7 +38,7 @@ url = https://github.com/ClickHouse-Extras/mariadb-connector-c.git [submodule "contrib/jemalloc"] path = contrib/jemalloc - url = https://github.com/ClickHouse-Extras/jemalloc.git + url = https://github.com/jemalloc/jemalloc.git [submodule "contrib/unixodbc"] path = contrib/unixodbc url = https://github.com/ClickHouse-Extras/UnixODBC.git diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e4ea95c08c..5bbadb2115d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,179 @@ +### ClickHouse release v22.2, 2022-02-17 + +#### Upgrade Notes + +* Applying data skipping indexes for queries with FINAL may produce incorrect result. In this release we disabled data skipping indexes by default for queries with FINAL (a new setting `use_skip_indexes_if_final` is introduced and disabled by default). [#34243](https://github.com/ClickHouse/ClickHouse/pull/34243) ([Azat Khuzhin](https://github.com/azat)). + +#### New Feature + +* Projections are production ready. Set `allow_experimental_projection_optimization` by default and deprecate this setting. [#34456](https://github.com/ClickHouse/ClickHouse/pull/34456) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* An option to create a new files on insert for `File`/`S3`/`HDFS` engines. Allow to overwrite a file in `HDFS`. Throw an exception in attempt to overwrite a file in `S3` by default. Throw an exception in attempt to append data to file in formats that have a suffix (and thus don't support appends, like `Parquet`, `ORC`). Closes [#31640](https://github.com/ClickHouse/ClickHouse/issues/31640) Closes [#31622](https://github.com/ClickHouse/ClickHouse/issues/31622) Closes [#23862](https://github.com/ClickHouse/ClickHouse/issues/23862) Closes [#15022](https://github.com/ClickHouse/ClickHouse/issues/15022) Closes [#16674](https://github.com/ClickHouse/ClickHouse/issues/16674). [#33302](https://github.com/ClickHouse/ClickHouse/pull/33302) ([Kruglov Pavel](https://github.com/Avogar)). +* Add a setting that allows a user to provide own deduplication semantic in `MergeTree`/`ReplicatedMergeTree` If provided, it's used instead of data digest to generate block ID. So, for example, by providing a unique value for the setting in each INSERT statement, the user can avoid the same inserted data being deduplicated. This closes: [#7461](https://github.com/ClickHouse/ClickHouse/issues/7461). [#32304](https://github.com/ClickHouse/ClickHouse/pull/32304) ([Igor Nikonov](https://github.com/devcrafter)). +* Add support of `DEFAULT` keyword for INSERT statements. Closes [#6331](https://github.com/ClickHouse/ClickHouse/issues/6331). [#33141](https://github.com/ClickHouse/ClickHouse/pull/33141) ([Andrii Buriachevskyi](https://github.com/1over)). +* `EPHEMERAL` column specifier is added to `CREATE TABLE` query. Closes [#9436](https://github.com/ClickHouse/ClickHouse/issues/9436). [#34424](https://github.com/ClickHouse/ClickHouse/pull/34424) ([yakov-olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Support `IF EXISTS` clause for `TTL expr TO [DISK|VOLUME] [IF EXISTS] 'xxx'` feature. Parts will be moved to disk or volume only if it exists on replica, so `MOVE TTL` rules will be able to behave differently on replicas according to the existing storage policies. Resolves [#34455](https://github.com/ClickHouse/ClickHouse/issues/34455). [#34504](https://github.com/ClickHouse/ClickHouse/pull/34504) ([Anton Popov](https://github.com/CurtizJ)). +* Allow set default table engine and to create tables without specifying ENGINE. [#34187](https://github.com/ClickHouse/ClickHouse/pull/34187) ([Ilya Yatsishin](https://github.com/qoega)). +* Add table function `format(format_name, data)`. [#34125](https://github.com/ClickHouse/ClickHouse/pull/34125) ([Kruglov Pavel](https://github.com/Avogar)). +* Detect format in `clickhouse-local` by file name even in the case when it is passed to stdin. [#33829](https://github.com/ClickHouse/ClickHouse/pull/33829) ([Kruglov Pavel](https://github.com/Avogar)). +* Add schema inference for `values` table function. Closes [#33811](https://github.com/ClickHouse/ClickHouse/issues/33811). [#34017](https://github.com/ClickHouse/ClickHouse/pull/34017) ([Kruglov Pavel](https://github.com/Avogar)). +* Dynamic reload of server TLS certificates on config reload. Closes [#15764](https://github.com/ClickHouse/ClickHouse/issues/15764). [#15765](https://github.com/ClickHouse/ClickHouse/pull/15765) ([johnskopis](https://github.com/johnskopis)). [#31257](https://github.com/ClickHouse/ClickHouse/pull/31257) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Now ReplicatedMergeTree can recover data when some of its disks are broken. [#13544](https://github.com/ClickHouse/ClickHouse/pull/13544) ([Amos Bird](https://github.com/amosbird)). +* Fault-tolerant connections in clickhouse-client: `clickhouse-client ... --host host1 --host host2 --port port2 --host host3 --port port --host host4`. [#34490](https://github.com/ClickHouse/ClickHouse/pull/34490) ([Kruglov Pavel](https://github.com/Avogar)). [#33824](https://github.com/ClickHouse/ClickHouse/pull/33824) ([Filippov Denis](https://github.com/DF5HSE)). +* Add `DEGREES` and `RADIANS` functions for MySQL compatibility. [#33769](https://github.com/ClickHouse/ClickHouse/pull/33769) ([Bharat Nallan](https://github.com/bharatnc)). +* Add `h3ToCenterChild` function. [#33313](https://github.com/ClickHouse/ClickHouse/pull/33313) ([Bharat Nallan](https://github.com/bharatnc)). Add new h3 miscellaneous functions: `edgeLengthKm`,`exactEdgeLengthKm`,`exactEdgeLengthM`,`exactEdgeLengthRads`,`numHexagons`. [#33621](https://github.com/ClickHouse/ClickHouse/pull/33621) ([Bharat Nallan](https://github.com/bharatnc)). +* Add function `bitSlice` to extract bit subsequences from String/FixedString. [#33360](https://github.com/ClickHouse/ClickHouse/pull/33360) ([RogerYK](https://github.com/RogerYK)). +* Implemented `meanZTest` aggregate function. [#33354](https://github.com/ClickHouse/ClickHouse/pull/33354) ([achimbab](https://github.com/achimbab)). +* Add confidence intervals to T-tests aggregate functions. [#33260](https://github.com/ClickHouse/ClickHouse/pull/33260) ([achimbab](https://github.com/achimbab)). +* Add function `addressToLineWithInlines`. Close [#26211](https://github.com/ClickHouse/ClickHouse/issues/26211). [#33467](https://github.com/ClickHouse/ClickHouse/pull/33467) ([SuperDJY](https://github.com/cmsxbc)). +* Added `#!` and `# ` as a recognised start of a single line comment. Closes [#34138](https://github.com/ClickHouse/ClickHouse/issues/34138). [#34230](https://github.com/ClickHouse/ClickHouse/pull/34230) ([Aaron Katz](https://github.com/aaronstephenkatz)). + +#### Experimental Feature + +* Functions for text classification: language and charset detection. See [#23271](https://github.com/ClickHouse/ClickHouse/issues/23271). [#33314](https://github.com/ClickHouse/ClickHouse/pull/33314) ([Nikolay Degterinsky](https://github.com/evillique)). +* Add memory overcommit to `MemoryTracker`. Added `guaranteed` settings for memory limits which represent soft memory limits. In case when hard memory limit is reached, `MemoryTracker` tries to cancel the most overcommited query. New setting `memory_usage_overcommit_max_wait_microseconds` specifies how long queries may wait another query to stop. Closes [#28375](https://github.com/ClickHouse/ClickHouse/issues/28375). [#31182](https://github.com/ClickHouse/ClickHouse/pull/31182) ([Dmitry Novik](https://github.com/novikd)). +* Enable stream to table join in WindowView. [#33729](https://github.com/ClickHouse/ClickHouse/pull/33729) ([vxider](https://github.com/Vxider)). +* Support `SET`, `YEAR`, `TIME` and `GEOMETRY` data types in `MaterializedMySQL` (experimental feature). Fixes [#18091](https://github.com/ClickHouse/ClickHouse/issues/18091), [#21536](https://github.com/ClickHouse/ClickHouse/issues/21536), [#26361](https://github.com/ClickHouse/ClickHouse/issues/26361). [#33429](https://github.com/ClickHouse/ClickHouse/pull/33429) ([zzsmdfj](https://github.com/zzsmdfj)). +* Fix various issues when projection is enabled by default. Each issue is described in separate commit. This is for [#33678](https://github.com/ClickHouse/ClickHouse/issues/33678) . This fixes [#34273](https://github.com/ClickHouse/ClickHouse/issues/34273). [#34305](https://github.com/ClickHouse/ClickHouse/pull/34305) ([Amos Bird](https://github.com/amosbird)). + +#### Performance Improvement + +* Support `optimize_read_in_order` if prefix of sorting key is already sorted. E.g. if we have sorting key `ORDER BY (a, b)` in table and query with `WHERE a = const ORDER BY b` clauses, now it will be applied reading in order of sorting key instead of full sort. [#32748](https://github.com/ClickHouse/ClickHouse/pull/32748) ([Anton Popov](https://github.com/CurtizJ)). +* Improve performance of partitioned insert into table functions `URL`, `S3`, `File`, `HDFS`. Closes [#34348](https://github.com/ClickHouse/ClickHouse/issues/34348). [#34510](https://github.com/ClickHouse/ClickHouse/pull/34510) ([Maksim Kita](https://github.com/kitaisreal)). +* Multiple performance improvements of clickhouse-keeper. [#34484](https://github.com/ClickHouse/ClickHouse/pull/34484) [#34587](https://github.com/ClickHouse/ClickHouse/pull/34587) ([zhanglistar](https://github.com/zhanglistar)). +* `FlatDictionary` improve performance of dictionary data load. [#33871](https://github.com/ClickHouse/ClickHouse/pull/33871) ([Maksim Kita](https://github.com/kitaisreal)). +* Improve performance of `mapPopulateSeries` function. Closes [#33944](https://github.com/ClickHouse/ClickHouse/issues/33944). [#34318](https://github.com/ClickHouse/ClickHouse/pull/34318) ([Maksim Kita](https://github.com/kitaisreal)). +* `_file` and `_path` virtual columns (in file-like table engines) are made `LowCardinality` - it will make queries for multiple files faster. Closes [#34300](https://github.com/ClickHouse/ClickHouse/issues/34300). [#34317](https://github.com/ClickHouse/ClickHouse/pull/34317) ([flynn](https://github.com/ucasfl)). +* Speed up loading of data parts. It was not parallelized before: the setting `part_loading_threads` did not have effect. See [#4699](https://github.com/ClickHouse/ClickHouse/issues/4699). [#34310](https://github.com/ClickHouse/ClickHouse/pull/34310) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Improve performance of `LineAsString` format. This closes [#34303](https://github.com/ClickHouse/ClickHouse/issues/34303). [#34306](https://github.com/ClickHouse/ClickHouse/pull/34306) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Optimize `quantilesExact{Low,High}` to use `nth_element` instead of `sort`. [#34287](https://github.com/ClickHouse/ClickHouse/pull/34287) ([Danila Kutenin](https://github.com/danlark1)). +* Slightly improve performance of `Regexp` format. [#34202](https://github.com/ClickHouse/ClickHouse/pull/34202) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Minor improvement for analysis of scalar subqueries. [#34128](https://github.com/ClickHouse/ClickHouse/pull/34128) ([Federico Rodriguez](https://github.com/fedrod)). +* Make ORDER BY tuple almost as fast as ORDER BY columns. We have special optimizations for multiple column ORDER BY: https://github.com/ClickHouse/ClickHouse/pull/10831 . It's beneficial to also apply to tuple columns. [#34060](https://github.com/ClickHouse/ClickHouse/pull/34060) ([Amos Bird](https://github.com/amosbird)). +* Rework and reintroduce the scalar subqueries cache to Materialized Views execution. [#33958](https://github.com/ClickHouse/ClickHouse/pull/33958) ([Raúl Marín](https://github.com/Algunenano)). +* Slightly improve performance of `ORDER BY` by adding x86-64 AVX-512 support for `memcmpSmall` functions to accelerate memory comparison. It works only if you compile ClickHouse by yourself. [#33706](https://github.com/ClickHouse/ClickHouse/pull/33706) ([hanqf-git](https://github.com/hanqf-git)). +* Improve `range_hashed` dictionary performance if for key there are a lot of intervals. Fixes [#23821](https://github.com/ClickHouse/ClickHouse/issues/23821). [#33516](https://github.com/ClickHouse/ClickHouse/pull/33516) ([Maksim Kita](https://github.com/kitaisreal)). +* For inserts and merges into S3, write files in parallel whenever possible (TODO: check if it's merged). [#33291](https://github.com/ClickHouse/ClickHouse/pull/33291) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Improve `clickhouse-keeper` performance and fix several memory leaks in NuRaft library. [#33329](https://github.com/ClickHouse/ClickHouse/pull/33329) ([alesapin](https://github.com/alesapin)). + +#### Improvement + +* Support asynchronous inserts in `clickhouse-client` for queries with inlined data. [#34267](https://github.com/ClickHouse/ClickHouse/pull/34267) ([Anton Popov](https://github.com/CurtizJ)). +* Functions `dictGet`, `dictHas` implicitly cast key argument to dictionary key structure, if they are different. [#33672](https://github.com/ClickHouse/ClickHouse/pull/33672) ([Maksim Kita](https://github.com/kitaisreal)). +* Improvements for `range_hashed` dictionaries. Improve performance of load time if there are multiple attributes. Allow to create a dictionary without attributes. Added option to specify strategy when intervals `start` and `end` have `Nullable` type `convert_null_range_bound_to_open` by default is `true`. Closes [#29791](https://github.com/ClickHouse/ClickHouse/issues/29791). Allow to specify `Float`, `Decimal`, `DateTime64`, `Int128`, `Int256`, `UInt128`, `UInt256` as range types. `RangeHashedDictionary` added support for range values that extend `Int64` type. Closes [#28322](https://github.com/ClickHouse/ClickHouse/issues/28322). Added option `range_lookup_strategy` to specify range lookup type `min`, `max` by default is `min` . Closes [#21647](https://github.com/ClickHouse/ClickHouse/issues/21647). Fixed allocated bytes calculations. Fixed type name in `system.dictionaries` in case of `ComplexKeyHashedDictionary`. [#33927](https://github.com/ClickHouse/ClickHouse/pull/33927) ([Maksim Kita](https://github.com/kitaisreal)). +* `flat`, `hashed`, `hashed_array` dictionaries now support creating with empty attributes, with support of reading the keys and using `dictHas`. Fixes [#33820](https://github.com/ClickHouse/ClickHouse/issues/33820). [#33918](https://github.com/ClickHouse/ClickHouse/pull/33918) ([Maksim Kita](https://github.com/kitaisreal)). +* Added support for `DateTime64` data type in dictionaries. [#33914](https://github.com/ClickHouse/ClickHouse/pull/33914) ([Maksim Kita](https://github.com/kitaisreal)). +* Allow to write `s3(url, access_key_id, secret_access_key)` (autodetect of data format and table structure, but with explicit credentials). [#34503](https://github.com/ClickHouse/ClickHouse/pull/34503) ([Kruglov Pavel](https://github.com/Avogar)). +* Added sending of the output format back to client like it's done in HTTP protocol as suggested in [#34362](https://github.com/ClickHouse/ClickHouse/issues/34362). Closes [#34362](https://github.com/ClickHouse/ClickHouse/issues/34362). [#34499](https://github.com/ClickHouse/ClickHouse/pull/34499) ([Vitaly Baranov](https://github.com/vitlibar)). +* Send ProfileEvents statistics in case of INSERT SELECT query (to display query metrics in `clickhouse-client` for this type of queries). [#34498](https://github.com/ClickHouse/ClickHouse/pull/34498) ([Dmitry Novik](https://github.com/novikd)). +* Recognize `.jsonl` extension for JSONEachRow format. [#34496](https://github.com/ClickHouse/ClickHouse/pull/34496) ([Kruglov Pavel](https://github.com/Avogar)). +* Improve schema inference in clickhouse-local. Allow to write just `clickhouse-local -q "select * from table" < data.format`. [#34495](https://github.com/ClickHouse/ClickHouse/pull/34495) ([Kruglov Pavel](https://github.com/Avogar)). +* Privileges CREATE/ALTER/DROP ROW POLICY now can be granted on a table or on `database.*` as well as globally `*.*`. [#34489](https://github.com/ClickHouse/ClickHouse/pull/34489) ([Vitaly Baranov](https://github.com/vitlibar)). +* Allow to export arbitrary large files to `s3`. Add two new settings: `s3_upload_part_size_multiply_factor` and `s3_upload_part_size_multiply_parts_count_threshold`. Now each time `s3_upload_part_size_multiply_parts_count_threshold` uploaded to S3 from a single query `s3_min_upload_part_size` multiplied by `s3_upload_part_size_multiply_factor`. Fixes [#34244](https://github.com/ClickHouse/ClickHouse/issues/34244). [#34422](https://github.com/ClickHouse/ClickHouse/pull/34422) ([alesapin](https://github.com/alesapin)). +* Allow to skip not found (404) URLs for globs when using URL storage / table function. Also closes [#34359](https://github.com/ClickHouse/ClickHouse/issues/34359). [#34392](https://github.com/ClickHouse/ClickHouse/pull/34392) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Default input and output formats for `clickhouse-local` that can be overriden by --input-format and --output-format. Close [#30631](https://github.com/ClickHouse/ClickHouse/issues/30631). [#34352](https://github.com/ClickHouse/ClickHouse/pull/34352) ([李扬](https://github.com/taiyang-li)). +* Add options for `clickhouse-format`. Which close [#30528](https://github.com/ClickHouse/ClickHouse/issues/30528) - `max_query_size` - `max_parser_depth`. [#34349](https://github.com/ClickHouse/ClickHouse/pull/34349) ([李扬](https://github.com/taiyang-li)). +* Better handling of pre-inputs before client start. This is for [#34308](https://github.com/ClickHouse/ClickHouse/issues/34308). [#34336](https://github.com/ClickHouse/ClickHouse/pull/34336) ([Amos Bird](https://github.com/amosbird)). +* `REGEXP_MATCHES` and `REGEXP_REPLACE` function aliases for compatibility with PostgreSQL. Close [#30885](https://github.com/ClickHouse/ClickHouse/issues/30885). [#34334](https://github.com/ClickHouse/ClickHouse/pull/34334) ([李扬](https://github.com/taiyang-li)). +* Some servers expect a User-Agent header in their HTTP requests. A `User-Agent` header entry has been added to HTTP requests of the form: User-Agent: ClickHouse/VERSION_STRING. [#34330](https://github.com/ClickHouse/ClickHouse/pull/34330) ([Saad Ur Rahman](https://github.com/surahman)). +* Cancel merges before acquiring table lock for `TRUNCATE` query to avoid `DEADLOCK_AVOIDED` error in some cases. Fixes [#34302](https://github.com/ClickHouse/ClickHouse/issues/34302). [#34304](https://github.com/ClickHouse/ClickHouse/pull/34304) ([tavplubix](https://github.com/tavplubix)). +* Change severity of the "Cancelled merging parts" message in logs, because it's not an error. This closes [#34148](https://github.com/ClickHouse/ClickHouse/issues/34148). [#34232](https://github.com/ClickHouse/ClickHouse/pull/34232) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Add ability to compose PostgreSQL-style cast operator `::` with expressions using `[]` and `.` operators (array and tuple indexing). [#34229](https://github.com/ClickHouse/ClickHouse/pull/34229) ([Nikolay Degterinsky](https://github.com/evillique)). +* Recognize `YYYYMMDD-hhmmss` format in `parseDateTimeBestEffort` function. This closes [#34206](https://github.com/ClickHouse/ClickHouse/issues/34206). [#34208](https://github.com/ClickHouse/ClickHouse/pull/34208) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Allow carriage return in the middle of the line while parsing by `Regexp` format. This closes [#34200](https://github.com/ClickHouse/ClickHouse/issues/34200). [#34205](https://github.com/ClickHouse/ClickHouse/pull/34205) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Allow to parse dictionary's `PRIMARY KEY` as `PRIMARY KEY (id, value)`; previously supported only `PRIMARY KEY id, value`. Closes [#34135](https://github.com/ClickHouse/ClickHouse/issues/34135). [#34141](https://github.com/ClickHouse/ClickHouse/pull/34141) ([Maksim Kita](https://github.com/kitaisreal)). +* An optional argument for `splitByChar` to limit the number of resulting elements. close [#34081](https://github.com/ClickHouse/ClickHouse/issues/34081). [#34140](https://github.com/ClickHouse/ClickHouse/pull/34140) ([李扬](https://github.com/taiyang-li)). +* Improving the experience of multiple line editing for clickhouse-client. This is a follow-up of [#31123](https://github.com/ClickHouse/ClickHouse/pull/31123). [#34114](https://github.com/ClickHouse/ClickHouse/pull/34114) ([Amos Bird](https://github.com/amosbird)). +* Add `UUID` suport in `MsgPack` input/output format. [#34065](https://github.com/ClickHouse/ClickHouse/pull/34065) ([Kruglov Pavel](https://github.com/Avogar)). +* Tracing context (for OpenTelemetry) is now propagated from GRPC client metadata (this change is relevant for GRPC client-server protocol). [#34064](https://github.com/ClickHouse/ClickHouse/pull/34064) ([andremarianiello](https://github.com/andremarianiello)). +* Supports all types of `SYSTEM` queries with `ON CLUSTER` clause. [#34005](https://github.com/ClickHouse/ClickHouse/pull/34005) ([小路](https://github.com/nicelulu)). +* Improve memory accounting for queries that are using less than `max_untracker_memory`. [#34001](https://github.com/ClickHouse/ClickHouse/pull/34001) ([Azat Khuzhin](https://github.com/azat)). +* Fixed UTF-8 string case-insensitive search when lowercase and uppercase characters are represented by different number of bytes. Example is `ẞ` and `ß`. This closes [#7334](https://github.com/ClickHouse/ClickHouse/issues/7334). [#33992](https://github.com/ClickHouse/ClickHouse/pull/33992) ([Harry Lee](https://github.com/HarryLeeIBM)). +* Detect format and schema from stdin in `clickhouse-local`. [#33960](https://github.com/ClickHouse/ClickHouse/pull/33960) ([Kruglov Pavel](https://github.com/Avogar)). +* Correctly handle the case of misconfiguration when multiple disks are using the same path on the filesystem. [#29072](https://github.com/ClickHouse/ClickHouse/issues/29072). [#33905](https://github.com/ClickHouse/ClickHouse/pull/33905) ([zhongyuankai](https://github.com/zhongyuankai)). +* Try every resolved IP address while getting S3 proxy. S3 proxies are rarely used, mostly in Yandex Cloud. [#33862](https://github.com/ClickHouse/ClickHouse/pull/33862) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Support EXPLAIN AST CREATE FUNCTION query `EXPLAIN AST CREATE FUNCTION mycast AS (n) -> cast(n as String)` will return `EXPLAIN AST CREATE FUNCTION mycast AS n -> CAST(n, 'String')`. [#33819](https://github.com/ClickHouse/ClickHouse/pull/33819) ([李扬](https://github.com/taiyang-li)). +* Added support for cast from `Map(Key, Value)` to `Array(Tuple(Key, Value))`. [#33794](https://github.com/ClickHouse/ClickHouse/pull/33794) ([Maksim Kita](https://github.com/kitaisreal)). +* Add some improvements and fixes for `Bool` data type. Fixes [#33244](https://github.com/ClickHouse/ClickHouse/issues/33244). [#33737](https://github.com/ClickHouse/ClickHouse/pull/33737) ([Kruglov Pavel](https://github.com/Avogar)). +* Parse and store OpenTelemetry trace-id in big-endian order. [#33723](https://github.com/ClickHouse/ClickHouse/pull/33723) ([Frank Chen](https://github.com/FrankChen021)). +* Improvement for `fromUnixTimestamp64` family functions.. They now accept any integer value that can be converted to `Int64`. This closes: [#14648](https://github.com/ClickHouse/ClickHouse/issues/14648). [#33505](https://github.com/ClickHouse/ClickHouse/pull/33505) ([Andrey Zvonov](https://github.com/zvonand)). +* Reimplement `_shard_num` from constants (see [#7624](https://github.com/ClickHouse/ClickHouse/issues/7624)) with `shardNum()` function (seee [#27020](https://github.com/ClickHouse/ClickHouse/issues/27020)), to avoid possible issues (like those that had been found in [#16947](https://github.com/ClickHouse/ClickHouse/issues/16947)). [#33392](https://github.com/ClickHouse/ClickHouse/pull/33392) ([Azat Khuzhin](https://github.com/azat)). +* Enable binary arithmetic (plus, minus, multiply, division, least, greatest) between Decimal and Float. [#33355](https://github.com/ClickHouse/ClickHouse/pull/33355) ([flynn](https://github.com/ucasfl)). +* Respect cgroups limits in max_threads autodetection. [#33342](https://github.com/ClickHouse/ClickHouse/pull/33342) ([JaySon](https://github.com/JaySon-Huang)). +* Add new clickhouse-keeper setting `min_session_timeout_ms`. Now clickhouse-keeper will determine client session timeout according to `min_session_timeout_ms` and `session_timeout_ms` settings. [#33288](https://github.com/ClickHouse/ClickHouse/pull/33288) ([JackyWoo](https://github.com/JackyWoo)). +* Added `UUID` data type support for functions `hex` and `bin`. [#32170](https://github.com/ClickHouse/ClickHouse/pull/32170) ([Frank Chen](https://github.com/FrankChen021)). +* Fix reading of subcolumns with dots in their names. In particular fixed reading of `Nested` columns, if their element names contain dots (e.g ```Nested(`keys.name` String, `keys.id` UInt64, values UInt64)```). [#34228](https://github.com/ClickHouse/ClickHouse/pull/34228) ([Anton Popov](https://github.com/CurtizJ)). +* Fixes `parallel_view_processing = 0` not working when inserting into a table using `VALUES`. - Fixes `view_duration_ms` in the `query_views_log` not being set correctly for materialized views. [#34067](https://github.com/ClickHouse/ClickHouse/pull/34067) ([Raúl Marín](https://github.com/Algunenano)). +* Fix parsing tables structure from ZooKeeper: now metadata from ZooKeeper compared with local metadata in canonical form. It helps when canonical function names can change between ClickHouse versions. [#33933](https://github.com/ClickHouse/ClickHouse/pull/33933) ([sunny](https://github.com/sunny19930321)). +* Properly escape some characters for interaction with LDAP. [#33401](https://github.com/ClickHouse/ClickHouse/pull/33401) ([IlyaTsoi](https://github.com/IlyaTsoi)). + +#### Build/Testing/Packaging Improvement + +* Remove unbundled build support. [#33690](https://github.com/ClickHouse/ClickHouse/pull/33690) ([Azat Khuzhin](https://github.com/azat)). +* Ensure that tests don't depend on the result of non-stable sorting of equal elements. Added equal items ranges randomization in debug after sort to prevent issues when we rely on equal items sort order. [#34393](https://github.com/ClickHouse/ClickHouse/pull/34393) ([Maksim Kita](https://github.com/kitaisreal)). +* Add verbosity to a style check. [#34289](https://github.com/ClickHouse/ClickHouse/pull/34289) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Remove `clickhouse-test` debian package because it's obsolete. [#33948](https://github.com/ClickHouse/ClickHouse/pull/33948) ([Ilya Yatsishin](https://github.com/qoega)). +* Multiple improvements for build system to remove the possibility of occasionally using packages from the OS and to enforce hermetic builds. [#33695](https://github.com/ClickHouse/ClickHouse/pull/33695) ([Amos Bird](https://github.com/amosbird)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Fixed the assertion in case of using `allow_experimental_parallel_reading_from_replicas` with `max_parallel_replicas` equals to 1. This fixes [#34525](https://github.com/ClickHouse/ClickHouse/issues/34525). [#34613](https://github.com/ClickHouse/ClickHouse/pull/34613) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix rare bug while reading of empty arrays, which could lead to `Data compressed with different methods` error. It can reproduce if you have mostly empty arrays, but not always. And reading is performed in backward direction with ORDER BY ... DESC. This error is extremely unlikely to happen. [#34327](https://github.com/ClickHouse/ClickHouse/pull/34327) ([Anton Popov](https://github.com/CurtizJ)). +* Fix wrong result of `round`/`roundBankers` if integer values of small types are rounded. Closes [#33267](https://github.com/ClickHouse/ClickHouse/issues/33267). [#34562](https://github.com/ClickHouse/ClickHouse/pull/34562) ([李扬](https://github.com/taiyang-li)). +* Sometimes query cancellation did not work immediately when we were reading multiple files from s3 or HDFS. Fixes [#34301](https://github.com/ClickHouse/ClickHouse/issues/34301) Relates to [#34397](https://github.com/ClickHouse/ClickHouse/issues/34397). [#34539](https://github.com/ClickHouse/ClickHouse/pull/34539) ([Dmitry Novik](https://github.com/novikd)). +* Fix exception `Chunk should have AggregatedChunkInfo in MergingAggregatedTransform` (in case of `optimize_aggregation_in_order = 1` and `distributed_aggregation_memory_efficient = 0`). Fixes [#34526](https://github.com/ClickHouse/ClickHouse/issues/34526). [#34532](https://github.com/ClickHouse/ClickHouse/pull/34532) ([Anton Popov](https://github.com/CurtizJ)). +* Fix comparison between integers and floats in index analysis. Previously it could lead to skipping some granules for reading by mistake. Fixes [#34493](https://github.com/ClickHouse/ClickHouse/issues/34493). [#34528](https://github.com/ClickHouse/ClickHouse/pull/34528) ([Anton Popov](https://github.com/CurtizJ)). +* Fix compression support in URL engine. [#34524](https://github.com/ClickHouse/ClickHouse/pull/34524) ([Frank Chen](https://github.com/FrankChen021)). +* Fix possible error 'file_size: Operation not supported' in files' schema autodetection. [#34479](https://github.com/ClickHouse/ClickHouse/pull/34479) ([Kruglov Pavel](https://github.com/Avogar)). +* Fixes possible race with table deletion. [#34416](https://github.com/ClickHouse/ClickHouse/pull/34416) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix possible error `Cannot convert column Function to mask` in short circuit function evaluation. Closes [#34171](https://github.com/ClickHouse/ClickHouse/issues/34171). [#34415](https://github.com/ClickHouse/ClickHouse/pull/34415) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix potential crash when doing schema inference from url source. Closes [#34147](https://github.com/ClickHouse/ClickHouse/issues/34147). [#34405](https://github.com/ClickHouse/ClickHouse/pull/34405) ([Kruglov Pavel](https://github.com/Avogar)). +* For UDFs access permissions were checked for database level instead of global level as it should be. Closes [#34281](https://github.com/ClickHouse/ClickHouse/issues/34281). [#34404](https://github.com/ClickHouse/ClickHouse/pull/34404) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix wrong engine syntax in result of `SHOW CREATE DATABASE` query for databases with engine `Memory`. This closes [#34335](https://github.com/ClickHouse/ClickHouse/issues/34335). [#34345](https://github.com/ClickHouse/ClickHouse/pull/34345) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fixed a couple of extremely rare race conditions that might lead to broken state of replication queue and "intersecting parts" error. [#34297](https://github.com/ClickHouse/ClickHouse/pull/34297) ([tavplubix](https://github.com/tavplubix)). +* Fix progress bar width. It was incorrectly rounded to integer number of characters. [#34275](https://github.com/ClickHouse/ClickHouse/pull/34275) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix current_user/current_address client information fields for inter-server communication (before this patch current_user/current_address will be preserved from the previous query). [#34263](https://github.com/ClickHouse/ClickHouse/pull/34263) ([Azat Khuzhin](https://github.com/azat)). +* Fix memory leak in case of some Exception during query processing with `optimize_aggregation_in_order=1`. [#34234](https://github.com/ClickHouse/ClickHouse/pull/34234) ([Azat Khuzhin](https://github.com/azat)). +* Fix metric `Query`, which shows the number of executing queries. In last several releases it was always 0. [#34224](https://github.com/ClickHouse/ClickHouse/pull/34224) ([Anton Popov](https://github.com/CurtizJ)). +* Fix schema inference for table runction `s3`. [#34186](https://github.com/ClickHouse/ClickHouse/pull/34186) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix rare and benign race condition in `HDFS`, `S3` and `URL` storage engines which can lead to additional connections. [#34172](https://github.com/ClickHouse/ClickHouse/pull/34172) ([alesapin](https://github.com/alesapin)). +* Fix bug which can rarely lead to error "Cannot read all data" while reading LowCardinality columns of MergeTree table engines family which stores data on remote file system like S3 (virtual filesystem over s3 is an experimental feature that is not ready for production). [#34139](https://github.com/ClickHouse/ClickHouse/pull/34139) ([alesapin](https://github.com/alesapin)). +* Fix inserts to distributed tables in case of a change of native protocol. The last change was in the version 22.1, so there may be some failures of inserts to distributed tables after upgrade to that version. [#34132](https://github.com/ClickHouse/ClickHouse/pull/34132) ([Anton Popov](https://github.com/CurtizJ)). +* Fix possible data race in `File` table engine that was introduced in [#33960](https://github.com/ClickHouse/ClickHouse/pull/33960). Closes [#34111](https://github.com/ClickHouse/ClickHouse/issues/34111). [#34113](https://github.com/ClickHouse/ClickHouse/pull/34113) ([Kruglov Pavel](https://github.com/Avogar)). +* Fixed minor race condition that might cause "intersecting parts" error in extremely rare cases after ZooKeeper connection loss. [#34096](https://github.com/ClickHouse/ClickHouse/pull/34096) ([tavplubix](https://github.com/tavplubix)). +* Fix asynchronous inserts with `Native` format. [#34068](https://github.com/ClickHouse/ClickHouse/pull/34068) ([Anton Popov](https://github.com/CurtizJ)). +* Fix bug which lead to inability for server to start when both replicated access storage and keeper (embedded in clickhouse-server) are used. Introduced two settings for keeper socket timeout instead of settings from default user: `keeper_server.socket_receive_timeout_sec` and `keeper_server.socket_send_timeout_sec`. Fixes [#33973](https://github.com/ClickHouse/ClickHouse/issues/33973). [#33988](https://github.com/ClickHouse/ClickHouse/pull/33988) ([alesapin](https://github.com/alesapin)). +* Fix segfault while parsing ORC file with corrupted footer. Closes [#33797](https://github.com/ClickHouse/ClickHouse/issues/33797). [#33984](https://github.com/ClickHouse/ClickHouse/pull/33984) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix parsing IPv6 from query parameter (prepared statements) and fix IPv6 to string conversion. Closes [#33928](https://github.com/ClickHouse/ClickHouse/issues/33928). [#33971](https://github.com/ClickHouse/ClickHouse/pull/33971) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix crash while reading of nested tuples. Fixes [#33838](https://github.com/ClickHouse/ClickHouse/issues/33838). [#33956](https://github.com/ClickHouse/ClickHouse/pull/33956) ([Anton Popov](https://github.com/CurtizJ)). +* Fix usage of functions `array` and `tuple` with literal arguments in distributed queries. Previously it could lead to `Not found columns` exception. [#33938](https://github.com/ClickHouse/ClickHouse/pull/33938) ([Anton Popov](https://github.com/CurtizJ)). +* Aggregate function combinator `-If` did not correctly process `Nullable` filter argument. This closes [#27073](https://github.com/ClickHouse/ClickHouse/issues/27073). [#33920](https://github.com/ClickHouse/ClickHouse/pull/33920) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix potential race condition when doing remote disk read (virtual filesystem over s3 is an experimental feature that is not ready for production). [#33912](https://github.com/ClickHouse/ClickHouse/pull/33912) ([Amos Bird](https://github.com/amosbird)). +* Fix crash if SQL UDF is created with lambda with non identifier arguments. Closes [#33866](https://github.com/ClickHouse/ClickHouse/issues/33866). [#33868](https://github.com/ClickHouse/ClickHouse/pull/33868) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix usage of sparse columns (which can be enabled by experimental setting `ratio_of_defaults_for_sparse_serialization`). [#33849](https://github.com/ClickHouse/ClickHouse/pull/33849) ([Anton Popov](https://github.com/CurtizJ)). +* Fixed `replica is not readonly` logical error on `SYSTEM RESTORE REPLICA` query when replica is actually readonly. Fixes [#33806](https://github.com/ClickHouse/ClickHouse/issues/33806). [#33847](https://github.com/ClickHouse/ClickHouse/pull/33847) ([tavplubix](https://github.com/tavplubix)). +* Fix memory leak in `clickhouse-keeper` in case of compression is used (default). [#33840](https://github.com/ClickHouse/ClickHouse/pull/33840) ([Azat Khuzhin](https://github.com/azat)). +* Fix index analysis with no common types available. [#33833](https://github.com/ClickHouse/ClickHouse/pull/33833) ([Amos Bird](https://github.com/amosbird)). +* Fix schema inference for `JSONEachRow` and `JSONCompactEachRow`. [#33830](https://github.com/ClickHouse/ClickHouse/pull/33830) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix usage of external dictionaries with `redis` source and large number of keys. [#33804](https://github.com/ClickHouse/ClickHouse/pull/33804) ([Anton Popov](https://github.com/CurtizJ)). +* Fix bug in client that led to 'Connection reset by peer' in server. Closes [#33309](https://github.com/ClickHouse/ClickHouse/issues/33309). [#33790](https://github.com/ClickHouse/ClickHouse/pull/33790) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix parsing query INSERT INTO ... VALUES SETTINGS ... (...), ... [#33776](https://github.com/ClickHouse/ClickHouse/pull/33776) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix bug of check table when creating data part with wide format and projection. [#33774](https://github.com/ClickHouse/ClickHouse/pull/33774) ([李扬](https://github.com/taiyang-li)). +* Fix tiny race between count() and INSERT/merges/... in MergeTree (it is possible to return incorrect number of rows for SELECT with optimize_trivial_count_query). [#33753](https://github.com/ClickHouse/ClickHouse/pull/33753) ([Azat Khuzhin](https://github.com/azat)). +* Throw exception when directory listing request has failed in storage HDFS. [#33724](https://github.com/ClickHouse/ClickHouse/pull/33724) ([LiuNeng](https://github.com/liuneng1994)). +* Fix mutation when table contains projections. This fixes [#33010](https://github.com/ClickHouse/ClickHouse/issues/33010). This fixes [#33275](https://github.com/ClickHouse/ClickHouse/issues/33275). [#33679](https://github.com/ClickHouse/ClickHouse/pull/33679) ([Amos Bird](https://github.com/amosbird)). +* Correctly determine current database if `CREATE TEMPORARY TABLE AS SELECT` is queried inside a named HTTP session. This is a very rare use case. This closes [#8340](https://github.com/ClickHouse/ClickHouse/issues/8340). [#33676](https://github.com/ClickHouse/ClickHouse/pull/33676) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Allow some queries with sorting, LIMIT BY, ARRAY JOIN and lambda functions. This closes [#7462](https://github.com/ClickHouse/ClickHouse/issues/7462). [#33675](https://github.com/ClickHouse/ClickHouse/pull/33675) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix bug in "zero copy replication" (a feature that is under development and should not be used in production) which lead to data duplication in case of TTL move. Fixes [#33643](https://github.com/ClickHouse/ClickHouse/issues/33643). [#33642](https://github.com/ClickHouse/ClickHouse/pull/33642) ([alesapin](https://github.com/alesapin)). +* Fix `Chunk should have AggregatedChunkInfo in GroupingAggregatedTransform` (in case of `optimize_aggregation_in_order = 1`). [#33637](https://github.com/ClickHouse/ClickHouse/pull/33637) ([Azat Khuzhin](https://github.com/azat)). +* Fix error `Bad cast from type ... to DB::DataTypeArray` which may happen when table has `Nested` column with dots in name, and default value is generated for it (e.g. during insert, when column is not listed). Continuation of [#28762](https://github.com/ClickHouse/ClickHouse/issues/28762). [#33588](https://github.com/ClickHouse/ClickHouse/pull/33588) ([Alexey Pavlenko](https://github.com/alexeypavlenko)). +* Export into `lz4` files has been fixed. Closes [#31421](https://github.com/ClickHouse/ClickHouse/issues/31421). [#31862](https://github.com/ClickHouse/ClickHouse/pull/31862) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix potential crash if `group_by_overflow_mode` was set to `any` (approximate GROUP BY) and aggregation was performed by single column of type `LowCardinality`. [#34506](https://github.com/ClickHouse/ClickHouse/pull/34506) ([DR](https://github.com/freedomDR)). +* Fix inserting to temporary tables via gRPC client-server protocol. Fixes [#34347](https://github.com/ClickHouse/ClickHouse/issues/34347), issue `#2`. [#34364](https://github.com/ClickHouse/ClickHouse/pull/34364) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix issue [#19429](https://github.com/ClickHouse/ClickHouse/issues/19429). [#34225](https://github.com/ClickHouse/ClickHouse/pull/34225) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix issue [#18206](https://github.com/ClickHouse/ClickHouse/issues/18206). [#33977](https://github.com/ClickHouse/ClickHouse/pull/33977) ([Vitaly Baranov](https://github.com/vitlibar)). +* This PR allows using multiple LDAP storages in the same list of user directories. It worked earlier but was broken because LDAP tests are disabled (they are part of the testflows tests). [#33574](https://github.com/ClickHouse/ClickHouse/pull/33574) ([Vitaly Baranov](https://github.com/vitlibar)). + + ### ClickHouse release v22.1, 2022-01-18 #### Upgrade Notes diff --git a/CMakeLists.txt b/CMakeLists.txt index f27e9cdbea4..c0b6604c8f9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,8 +62,8 @@ set(CMAKE_DEBUG_POSTFIX "d" CACHE STRING "Generate debug library name with a pos # For more info see https://cmake.org/cmake/help/latest/prop_gbl/USE_FOLDERS.html set_property(GLOBAL PROPERTY USE_FOLDERS ON) -# Check that submodules are present only if source was downloaded with git -if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git" AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/boost/boost") +# Check that submodules are present +if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/sysroot/README.md") message (FATAL_ERROR "Submodules are not initialized. Run\n\tgit submodule update --init --recursive") endif () @@ -340,16 +340,19 @@ if (WITH_COVERAGE AND COMPILER_GCC) set(WITHOUT_COVERAGE "-fno-profile-arcs -fno-test-coverage") endif() -set(COMPILER_FLAGS "${COMPILER_FLAGS}") +set (COMPILER_FLAGS "${COMPILER_FLAGS}") + +# Our built-in unwinder only supports DWARF version up to 4. +set (DEBUG_INFO_FLAGS "-g -gdwarf-4") set (CMAKE_BUILD_COLOR_MAKEFILE ON) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS} ${PLATFORM_EXTRA_CXX_FLAG} ${COMMON_WARNING_FLAGS} ${CXX_WARNING_FLAGS}") -set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_CXX_FLAGS_ADD}") -set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_CXX_FLAGS_ADD}") +set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_CXX_FLAGS_ADD}") +set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} -fno-inline ${CMAKE_CXX_FLAGS_ADD}") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} ${COMMON_WARNING_FLAGS} ${CMAKE_C_FLAGS_ADD}") -set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_C_FLAGS_ADD}") -set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_C_FLAGS_ADD}") +set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_C_FLAGS_ADD}") +set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} -fno-inline ${CMAKE_C_FLAGS_ADD}") if (COMPILER_CLANG) if (OS_DARWIN) diff --git a/base/loggers/Loggers.cpp b/base/loggers/Loggers.cpp index 3c350c834e5..2f2eadea28f 100644 --- a/base/loggers/Loggers.cpp +++ b/base/loggers/Loggers.cpp @@ -247,11 +247,8 @@ void Loggers::updateLevels(Poco::Util::AbstractConfiguration & config, Poco::Log if (log_level > max_log_level) max_log_level = log_level; - const auto log_path = config.getString("logger.log", ""); - if (!log_path.empty()) + if (log_file) split->setLevel("log", log_level); - else - split->setLevel("log", 0); // Set level to console bool is_daemon = config.getBool("application.runAsDaemon", false); @@ -263,15 +260,13 @@ void Loggers::updateLevels(Poco::Util::AbstractConfiguration & config, Poco::Log split->setLevel("console", 0); // Set level to errorlog - int errorlog_level = 0; - const auto errorlog_path = config.getString("logger.errorlog", ""); - if (!errorlog_path.empty()) + if (error_log_file) { - errorlog_level = Poco::Logger::parseLevel(config.getString("logger.errorlog_level", "notice")); + int errorlog_level = Poco::Logger::parseLevel(config.getString("logger.errorlog_level", "notice")); if (errorlog_level > max_log_level) max_log_level = errorlog_level; + split->setLevel("errorlog", errorlog_level); } - split->setLevel("errorlog", errorlog_level); // Set level to syslog int syslog_level = 0; diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 39aa6c88c4e..8de2de1e3bb 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54459) +SET(VERSION_REVISION 54460) SET(VERSION_MAJOR 22) -SET(VERSION_MINOR 2) +SET(VERSION_MINOR 3) SET(VERSION_PATCH 1) -SET(VERSION_GITHASH dfe64a2789bbf51046bb6b5476f874f7b59d124c) -SET(VERSION_DESCRIBE v22.2.1.1-prestable) -SET(VERSION_STRING 22.2.1.1) +SET(VERSION_GITHASH 75366fc95e510b7ac76759ef670702ae5f488a51) +SET(VERSION_DESCRIBE v22.3.1.1-testing) +SET(VERSION_STRING 22.3.1.1) # end of autochange diff --git a/contrib/jemalloc b/contrib/jemalloc index a1404807211..ca709c3139f 160000 --- a/contrib/jemalloc +++ b/contrib/jemalloc @@ -1 +1 @@ -Subproject commit a1404807211b1612539f840b3dcb1bf38d1a269e +Subproject commit ca709c3139f77f4c00a903cdee46d71e9028f6c6 diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index b3845c7d56b..41b042df95b 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -53,43 +53,75 @@ set (SRCS "${LIBRARY_DIR}/src/background_thread.c" "${LIBRARY_DIR}/src/base.c" "${LIBRARY_DIR}/src/bin.c" + "${LIBRARY_DIR}/src/bin_info.c" "${LIBRARY_DIR}/src/bitmap.c" + "${LIBRARY_DIR}/src/buf_writer.c" + "${LIBRARY_DIR}/src/cache_bin.c" "${LIBRARY_DIR}/src/ckh.c" + "${LIBRARY_DIR}/src/counter.c" "${LIBRARY_DIR}/src/ctl.c" + "${LIBRARY_DIR}/src/decay.c" "${LIBRARY_DIR}/src/div.c" + "${LIBRARY_DIR}/src/ecache.c" + "${LIBRARY_DIR}/src/edata.c" + "${LIBRARY_DIR}/src/edata_cache.c" + "${LIBRARY_DIR}/src/ehooks.c" + "${LIBRARY_DIR}/src/emap.c" + "${LIBRARY_DIR}/src/eset.c" + "${LIBRARY_DIR}/src/exp_grow.c" "${LIBRARY_DIR}/src/extent.c" "${LIBRARY_DIR}/src/extent_dss.c" "${LIBRARY_DIR}/src/extent_mmap.c" - "${LIBRARY_DIR}/src/hash.c" + "${LIBRARY_DIR}/src/fxp.c" "${LIBRARY_DIR}/src/hook.c" + "${LIBRARY_DIR}/src/hpa.c" + "${LIBRARY_DIR}/src/hpa_hooks.c" + "${LIBRARY_DIR}/src/hpdata.c" + "${LIBRARY_DIR}/src/inspect.c" "${LIBRARY_DIR}/src/jemalloc.c" "${LIBRARY_DIR}/src/large.c" "${LIBRARY_DIR}/src/log.c" "${LIBRARY_DIR}/src/malloc_io.c" "${LIBRARY_DIR}/src/mutex.c" - "${LIBRARY_DIR}/src/mutex_pool.c" "${LIBRARY_DIR}/src/nstime.c" + "${LIBRARY_DIR}/src/pa.c" + "${LIBRARY_DIR}/src/pac.c" + "${LIBRARY_DIR}/src/pa_extra.c" "${LIBRARY_DIR}/src/pages.c" - "${LIBRARY_DIR}/src/prng.c" + "${LIBRARY_DIR}/src/pai.c" + "${LIBRARY_DIR}/src/peak_event.c" "${LIBRARY_DIR}/src/prof.c" + "${LIBRARY_DIR}/src/prof_data.c" + "${LIBRARY_DIR}/src/prof_log.c" + "${LIBRARY_DIR}/src/prof_recent.c" + "${LIBRARY_DIR}/src/prof_stats.c" + "${LIBRARY_DIR}/src/prof_sys.c" + "${LIBRARY_DIR}/src/psset.c" "${LIBRARY_DIR}/src/rtree.c" + "${LIBRARY_DIR}/src/safety_check.c" + "${LIBRARY_DIR}/src/san_bump.c" + "${LIBRARY_DIR}/src/san.c" "${LIBRARY_DIR}/src/sc.c" + "${LIBRARY_DIR}/src/sec.c" "${LIBRARY_DIR}/src/stats.c" "${LIBRARY_DIR}/src/sz.c" "${LIBRARY_DIR}/src/tcache.c" "${LIBRARY_DIR}/src/test_hooks.c" + "${LIBRARY_DIR}/src/thread_event.c" "${LIBRARY_DIR}/src/ticker.c" "${LIBRARY_DIR}/src/tsd.c" "${LIBRARY_DIR}/src/witness.c" - "${LIBRARY_DIR}/src/safety_check.c" ) if (OS_DARWIN) list(APPEND SRCS "${LIBRARY_DIR}/src/zone.c") endif () add_library(_jemalloc ${SRCS}) +# First include jemalloc-cmake files, to override anything that jemalloc has. +# (for example if you were trying to build jemalloc inside contrib/jemalloc you +# will have some files that may be out of date) +target_include_directories(_jemalloc PUBLIC include) target_include_directories(_jemalloc PRIVATE "${LIBRARY_DIR}/include") -target_include_directories(_jemalloc SYSTEM PUBLIC include) set (JEMALLOC_INCLUDE_PREFIX) # OS_ diff --git a/contrib/jemalloc-cmake/README b/contrib/jemalloc-cmake/README index 0af9c4f0e45..8d27e7844c5 100644 --- a/contrib/jemalloc-cmake/README +++ b/contrib/jemalloc-cmake/README @@ -1 +1,6 @@ It allows to integrate JEMalloc into CMake project. + +- Remove JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF because it's non standard. +- Added JEMALLOC_CONFIG_MALLOC_CONF substitution +- Add musl support (USE_MUSL) +- Also note, that darwin build requires JEMALLOC_PREFIX, while others don not diff --git a/contrib/jemalloc-cmake/include/jemalloc/internal/jemalloc_preamble.h b/contrib/jemalloc-cmake/include/jemalloc/internal/jemalloc_preamble.h index e5e34925b55..6169b385f46 100644 --- a/contrib/jemalloc-cmake/include/jemalloc/internal/jemalloc_preamble.h +++ b/contrib/jemalloc-cmake/include/jemalloc/internal/jemalloc_preamble.h @@ -4,8 +4,14 @@ #include "jemalloc_internal_defs.h" #include "jemalloc/internal/jemalloc_internal_decls.h" -#ifdef JEMALLOC_UTRACE +#if defined(JEMALLOC_UTRACE) || defined(JEMALLOC_UTRACE_LABEL) #include +# if defined(JEMALLOC_UTRACE) +# define UTRACE_CALL(p, l) utrace(p, l) +# else +# define UTRACE_CALL(p, l) utrace("jemalloc_process", p, l) +# define JEMALLOC_UTRACE +# endif #endif #define JEMALLOC_NO_DEMANGLE @@ -180,6 +186,35 @@ static const bool config_opt_safety_checks = #endif ; +/* + * Extra debugging of sized deallocations too onerous to be included in the + * general safety checks. + */ +static const bool config_opt_size_checks = +#if defined(JEMALLOC_OPT_SIZE_CHECKS) || defined(JEMALLOC_DEBUG) + true +#else + false +#endif + ; + +static const bool config_uaf_detection = +#if defined(JEMALLOC_UAF_DETECTION) || defined(JEMALLOC_DEBUG) + true +#else + false +#endif + ; + +/* Whether or not the C++ extensions are enabled. */ +static const bool config_enable_cxx = +#ifdef JEMALLOC_ENABLE_CXX + true +#else + false +#endif +; + #if defined(_WIN32) || defined(JEMALLOC_HAVE_SCHED_GETCPU) /* Currently percpu_arena depends on sched_getcpu. */ #define JEMALLOC_PERCPU_ARENA @@ -209,5 +244,20 @@ static const bool have_background_thread = false #endif ; +static const bool config_high_res_timer = +#ifdef JEMALLOC_HAVE_CLOCK_REALTIME + true +#else + false +#endif + ; + +static const bool have_memcntl = +#ifdef JEMALLOC_HAVE_MEMCNTL + true +#else + false +#endif + ; #endif /* JEMALLOC_PREAMBLE_H */ diff --git a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_defs.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_defs.h index 0aa4033f859..47569b16a88 100644 --- a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_defs.h +++ b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_defs.h @@ -9,11 +9,17 @@ #define JEMALLOC_HAVE_ATTR_FORMAT_ARG /* Defined if format(gnu_printf, ...) attribute is supported. */ -#define JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF +/* #undef JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF */ /* Defined if format(printf, ...) attribute is supported. */ #define JEMALLOC_HAVE_ATTR_FORMAT_PRINTF +/* Defined if fallthrough attribute is supported. */ +#define JEMALLOC_HAVE_ATTR_FALLTHROUGH + +/* Defined if cold attribute is supported. */ +#define JEMALLOC_HAVE_ATTR_COLD + /* * Define overrides for non-standard allocator-related functions if they are * present on the system. diff --git a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_macros.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_macros.h index 34235894285..a594a945351 100644 --- a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_macros.h +++ b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_macros.h @@ -4,13 +4,13 @@ #include #include -#define JEMALLOC_VERSION "5.2.1-0-gea6b3e973b477b8061e0076bb257dbd7f3faa756" +#define JEMALLOC_VERSION "5.3-RC" #define JEMALLOC_VERSION_MAJOR 5 -#define JEMALLOC_VERSION_MINOR 2 -#define JEMALLOC_VERSION_BUGFIX 1 +#define JEMALLOC_VERSION_MINOR 3 +#define JEMALLOC_VERSION_BUGFIX 0 #define JEMALLOC_VERSION_NREV 0 -#define JEMALLOC_VERSION_GID "ea6b3e973b477b8061e0076bb257dbd7f3faa756" -#define JEMALLOC_VERSION_GID_IDENT ea6b3e973b477b8061e0076bb257dbd7f3faa756 +#define JEMALLOC_VERSION_GID "ca709c3139f77f4c00a903cdee46d71e9028f6c6" +#define JEMALLOC_VERSION_GID_IDENT ca709c3139f77f4c00a903cdee46d71e9028f6c6 #define MALLOCX_LG_ALIGN(la) ((int)(la)) #if LG_SIZEOF_PTR == 2 @@ -71,6 +71,7 @@ # endif # define JEMALLOC_FORMAT_ARG(i) # define JEMALLOC_FORMAT_PRINTF(s, i) +# define JEMALLOC_FALLTHROUGH # define JEMALLOC_NOINLINE __declspec(noinline) # ifdef __cplusplus # define JEMALLOC_NOTHROW __declspec(nothrow) @@ -84,6 +85,7 @@ # else # define JEMALLOC_ALLOCATOR # endif +# define JEMALLOC_COLD #elif defined(JEMALLOC_HAVE_ATTR) # define JEMALLOC_ATTR(s) __attribute__((s)) # define JEMALLOC_ALIGNED(s) JEMALLOC_ATTR(aligned(s)) @@ -109,11 +111,21 @@ # else # define JEMALLOC_FORMAT_PRINTF(s, i) # endif +# ifdef JEMALLOC_HAVE_ATTR_FALLTHROUGH +# define JEMALLOC_FALLTHROUGH JEMALLOC_ATTR(fallthrough) +# else +# define JEMALLOC_FALLTHROUGH +# endif # define JEMALLOC_NOINLINE JEMALLOC_ATTR(noinline) # define JEMALLOC_NOTHROW JEMALLOC_ATTR(nothrow) # define JEMALLOC_SECTION(s) JEMALLOC_ATTR(section(s)) # define JEMALLOC_RESTRICT_RETURN # define JEMALLOC_ALLOCATOR +# ifdef JEMALLOC_HAVE_ATTR_COLD +# define JEMALLOC_COLD JEMALLOC_ATTR(__cold__) +# else +# define JEMALLOC_COLD +# endif #else # define JEMALLOC_ATTR(s) # define JEMALLOC_ALIGNED(s) @@ -121,9 +133,17 @@ # define JEMALLOC_ALLOC_SIZE2(s1, s2) # define JEMALLOC_EXPORT # define JEMALLOC_FORMAT_PRINTF(s, i) +# define JEMALLOC_FALLTHROUGH # define JEMALLOC_NOINLINE # define JEMALLOC_NOTHROW # define JEMALLOC_SECTION(s) # define JEMALLOC_RESTRICT_RETURN # define JEMALLOC_ALLOCATOR +# define JEMALLOC_COLD +#endif + +#if (defined(__APPLE__) || defined(__FreeBSD__)) && !defined(JEMALLOC_NO_RENAME) +# define JEMALLOC_SYS_NOTHROW +#else +# define JEMALLOC_SYS_NOTHROW JEMALLOC_NOTHROW #endif diff --git a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_protos.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_protos.h index 2e35e7b6249..e1e5b0575f3 100644 --- a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_protos.h +++ b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_protos.h @@ -4,6 +4,9 @@ # undef JEMALLOC_NOTHROW # define JEMALLOC_NOTHROW +# undef JEMALLOC_SYS_NOTHROW +# define JEMALLOC_SYS_NOTHROW + # undef JEMALLOC_CXX_THROW # define JEMALLOC_CXX_THROW #endif @@ -18,21 +21,22 @@ extern JEMALLOC_EXPORT void (*je_malloc_message)(void *cbopaque, const char *s); JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN - void JEMALLOC_NOTHROW *je_malloc(size_t size) + void JEMALLOC_SYS_NOTHROW *je_malloc(size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1); JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN - void JEMALLOC_NOTHROW *je_calloc(size_t num, size_t size) + void JEMALLOC_SYS_NOTHROW *je_calloc(size_t num, size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE2(1, 2); -JEMALLOC_EXPORT int JEMALLOC_NOTHROW je_posix_memalign(void **memptr, - size_t alignment, size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(nonnull(1)); +JEMALLOC_EXPORT int JEMALLOC_SYS_NOTHROW je_posix_memalign( + void **memptr, size_t alignment, size_t size) JEMALLOC_CXX_THROW + JEMALLOC_ATTR(nonnull(1)); JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN - void JEMALLOC_NOTHROW *je_aligned_alloc(size_t alignment, + void JEMALLOC_SYS_NOTHROW *je_aligned_alloc(size_t alignment, size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(2); JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN - void JEMALLOC_NOTHROW *je_realloc(void *ptr, size_t size) + void JEMALLOC_SYS_NOTHROW *je_realloc(void *ptr, size_t size) JEMALLOC_CXX_THROW JEMALLOC_ALLOC_SIZE(2); -JEMALLOC_EXPORT void JEMALLOC_NOTHROW je_free(void *ptr) +JEMALLOC_EXPORT void JEMALLOC_SYS_NOTHROW je_free(void *ptr) JEMALLOC_CXX_THROW; JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN @@ -62,15 +66,19 @@ JEMALLOC_EXPORT void JEMALLOC_NOTHROW je_malloc_stats_print( const char *opts); JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_malloc_usable_size( JEMALLOC_USABLE_SIZE_CONST void *ptr) JEMALLOC_CXX_THROW; +#ifdef JEMALLOC_HAVE_MALLOC_SIZE +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_malloc_size( + const void *ptr); +#endif #ifdef JEMALLOC_OVERRIDE_MEMALIGN JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN - void JEMALLOC_NOTHROW *je_memalign(size_t alignment, size_t size) + void JEMALLOC_SYS_NOTHROW *je_memalign(size_t alignment, size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc); #endif #ifdef JEMALLOC_OVERRIDE_VALLOC JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN - void JEMALLOC_NOTHROW *je_valloc(size_t size) JEMALLOC_CXX_THROW + void JEMALLOC_SYS_NOTHROW *je_valloc(size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc); #endif diff --git a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_protos_jet.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_protos_jet.h new file mode 100644 index 00000000000..72182727a6f --- /dev/null +++ b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_protos_jet.h @@ -0,0 +1,71 @@ +/* + * The jet_ prefix on the following public symbol declarations is an artifact + * of namespace management, and should be omitted in application code unless + * JEMALLOC_NO_DEMANGLE is defined (see jemalloc_mangle@install_suffix@.h). + */ +extern JEMALLOC_EXPORT const char *jet_malloc_conf; +extern JEMALLOC_EXPORT void (*jet_malloc_message)(void *cbopaque, + const char *s); + +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *jet_malloc(size_t size) + JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1); +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *jet_calloc(size_t num, size_t size) + JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE2(1, 2); +JEMALLOC_EXPORT int JEMALLOC_SYS_NOTHROW jet_posix_memalign( + void **memptr, size_t alignment, size_t size) JEMALLOC_CXX_THROW + JEMALLOC_ATTR(nonnull(1)); +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *jet_aligned_alloc(size_t alignment, + size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) + JEMALLOC_ALLOC_SIZE(2); +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *jet_realloc(void *ptr, size_t size) + JEMALLOC_CXX_THROW JEMALLOC_ALLOC_SIZE(2); +JEMALLOC_EXPORT void JEMALLOC_SYS_NOTHROW jet_free(void *ptr) + JEMALLOC_CXX_THROW; + +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_NOTHROW *jet_mallocx(size_t size, int flags) + JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1); +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_NOTHROW *jet_rallocx(void *ptr, size_t size, + int flags) JEMALLOC_ALLOC_SIZE(2); +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW jet_xallocx(void *ptr, size_t size, + size_t extra, int flags); +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW jet_sallocx(const void *ptr, + int flags) JEMALLOC_ATTR(pure); +JEMALLOC_EXPORT void JEMALLOC_NOTHROW jet_dallocx(void *ptr, int flags); +JEMALLOC_EXPORT void JEMALLOC_NOTHROW jet_sdallocx(void *ptr, size_t size, + int flags); +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW jet_nallocx(size_t size, int flags) + JEMALLOC_ATTR(pure); + +JEMALLOC_EXPORT int JEMALLOC_NOTHROW jet_mallctl(const char *name, + void *oldp, size_t *oldlenp, void *newp, size_t newlen); +JEMALLOC_EXPORT int JEMALLOC_NOTHROW jet_mallctlnametomib(const char *name, + size_t *mibp, size_t *miblenp); +JEMALLOC_EXPORT int JEMALLOC_NOTHROW jet_mallctlbymib(const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen); +JEMALLOC_EXPORT void JEMALLOC_NOTHROW jet_malloc_stats_print( + void (*write_cb)(void *, const char *), void *jet_cbopaque, + const char *opts); +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW jet_malloc_usable_size( + JEMALLOC_USABLE_SIZE_CONST void *ptr) JEMALLOC_CXX_THROW; +#ifdef JEMALLOC_HAVE_MALLOC_SIZE +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW jet_malloc_size( + const void *ptr); +#endif + +#ifdef JEMALLOC_OVERRIDE_MEMALIGN +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *jet_memalign(size_t alignment, size_t size) + JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc); +#endif + +#ifdef JEMALLOC_OVERRIDE_VALLOC +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *jet_valloc(size_t size) JEMALLOC_CXX_THROW + JEMALLOC_ATTR(malloc); +#endif diff --git a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_rename.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_rename.h index 2e94f7a0cc3..9bdbcf49e61 100644 --- a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_rename.h +++ b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_rename.h @@ -13,11 +13,12 @@ # define je_mallctlnametomib mallctlnametomib # define je_malloc malloc # define je_malloc_conf malloc_conf +# define je_malloc_conf_2_conf_harder malloc_conf_2_conf_harder # define je_malloc_message malloc_message # define je_malloc_stats_print malloc_stats_print # define je_malloc_usable_size malloc_usable_size # define je_mallocx mallocx -# define je_smallocx_ea6b3e973b477b8061e0076bb257dbd7f3faa756 smallocx_ea6b3e973b477b8061e0076bb257dbd7f3faa756 +# define je_smallocx_ca709c3139f77f4c00a903cdee46d71e9028f6c6 smallocx_ca709c3139f77f4c00a903cdee46d71e9028f6c6 # define je_nallocx nallocx # define je_posix_memalign posix_memalign # define je_rallocx rallocx diff --git a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_typedefs.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_typedefs.h index 5f94f16f937..1a58874306e 100644 --- a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_typedefs.h +++ b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_typedefs.h @@ -65,13 +65,13 @@ typedef bool (extent_merge_t)(extent_hooks_t *, void *, size_t, void *, size_t, bool, unsigned); struct extent_hooks_s { - extent_alloc_t *alloc; - extent_dalloc_t *dalloc; - extent_destroy_t *destroy; - extent_commit_t *commit; - extent_decommit_t *decommit; - extent_purge_t *purge_lazy; - extent_purge_t *purge_forced; - extent_split_t *split; - extent_merge_t *merge; + extent_alloc_t *alloc; + extent_dalloc_t *dalloc; + extent_destroy_t *destroy; + extent_commit_t *commit; + extent_decommit_t *decommit; + extent_purge_t *purge_lazy; + extent_purge_t *purge_forced; + extent_split_t *split; + extent_merge_t *merge; }; diff --git a/contrib/jemalloc-cmake/include_darwin_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_darwin_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in index 5c0407db24a..8ad95c51560 100644 --- a/contrib/jemalloc-cmake/include_darwin_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_darwin_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -45,17 +45,17 @@ #define LG_VADDR 64 /* Defined if C11 atomics are available. */ -#define JEMALLOC_C11_ATOMICS 1 +#define JEMALLOC_C11_ATOMICS /* Defined if GCC __atomic atomics are available. */ -#define JEMALLOC_GCC_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_ATOMIC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS /* Defined if GCC __sync atomics are available. */ -#define JEMALLOC_GCC_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_SYNC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_U8_SYNC_ATOMICS /* * Defined if __builtin_clz() and __builtin_clzl() are available. @@ -86,6 +86,12 @@ /* Defined if pthread_setname_np(3) is available. */ /* #undef JEMALLOC_HAVE_PTHREAD_SETNAME_NP */ +/* Defined if pthread_getname_np(3) is available. */ +#define JEMALLOC_HAVE_PTHREAD_GETNAME_NP + +/* Defined if pthread_get_name_np(3) is available. */ +/* #undef JEMALLOC_HAVE_PTHREAD_GET_NAME_NP */ + /* * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. */ @@ -99,7 +105,12 @@ /* * Defined if mach_absolute_time() is available. */ -#define JEMALLOC_HAVE_MACH_ABSOLUTE_TIME 1 +#define JEMALLOC_HAVE_MACH_ABSOLUTE_TIME + +/* + * Defined if clock_gettime(CLOCK_REALTIME, ...) is available. + */ +#define JEMALLOC_HAVE_CLOCK_REALTIME /* * Defined if _malloc_thread_cleanup() exists. At least in the case of @@ -163,6 +174,9 @@ /* Support utrace(2)-based tracing. */ /* #undef JEMALLOC_UTRACE */ +/* Support utrace(2)-based tracing (label based signature). */ +/* #undef JEMALLOC_UTRACE_LABEL */ + /* Support optional abort() on OOM. */ /* #undef JEMALLOC_XMALLOC */ @@ -178,6 +192,9 @@ /* One page is 2^LG_PAGE bytes. */ #define LG_PAGE 14 +/* Maximum number of regions in a slab. */ +/* #undef CONFIG_LG_SLAB_MAXREGS */ + /* * One huge page is 2^LG_HUGEPAGE bytes. Note that this is defined even if the * system does not explicitly support huge pages; system calls that require @@ -291,17 +308,46 @@ */ /* #undef JEMALLOC_MADVISE_DONTDUMP */ +/* + * Defined if MADV_[NO]CORE is supported as an argument to madvise. + */ +/* #undef JEMALLOC_MADVISE_NOCORE */ + +/* Defined if mprotect(2) is available. */ +#define JEMALLOC_HAVE_MPROTECT + /* * Defined if transparent huge pages (THPs) are supported via the * MADV_[NO]HUGEPAGE arguments to madvise(2), and THP support is enabled. */ /* #undef JEMALLOC_THP */ +/* Defined if posix_madvise is available. */ +/* #undef JEMALLOC_HAVE_POSIX_MADVISE */ + +/* + * Method for purging unused pages using posix_madvise. + * + * posix_madvise(..., POSIX_MADV_DONTNEED) + */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS */ + +/* + * Defined if memcntl page admin call is supported + */ +/* #undef JEMALLOC_HAVE_MEMCNTL */ + +/* + * Defined if malloc_size is supported + */ +#define JEMALLOC_HAVE_MALLOC_SIZE + /* Define if operating system has alloca.h header. */ /* #undef JEMALLOC_HAS_ALLOCA_H */ /* C99 restrict keyword supported. */ -#define JEMALLOC_HAS_RESTRICT 1 +#define JEMALLOC_HAS_RESTRICT /* For use by hash code. */ /* #undef JEMALLOC_BIG_ENDIAN */ @@ -351,7 +397,7 @@ /* #undef JEMALLOC_EXPORT */ /* config.malloc_conf options string. */ -#define JEMALLOC_CONFIG_MALLOC_CONF "" +#define JEMALLOC_CONFIG_MALLOC_CONF "@JEMALLOC_CONFIG_MALLOC_CONF@" /* If defined, jemalloc takes the malloc/free/etc. symbol names. */ /* #undef JEMALLOC_IS_MALLOC */ @@ -364,4 +410,16 @@ /* Performs additional safety checks when defined. */ /* #undef JEMALLOC_OPT_SAFETY_CHECKS */ +/* Is C++ support being built? */ +/* #undef JEMALLOC_ENABLE_CXX */ + +/* Performs additional size checks when defined. */ +/* #undef JEMALLOC_OPT_SIZE_CHECKS */ + +/* Allows sampled junk and stash for checking use-after-free when defined. */ +/* #undef JEMALLOC_UAF_DETECTION */ + +/* Darwin VM_MAKE_TAG support */ +#define JEMALLOC_HAVE_VM_MAKE_TAG + #endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/contrib/jemalloc-cmake/include_darwin_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_darwin_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in index 11fa5c4d727..8671da5db69 100644 --- a/contrib/jemalloc-cmake/include_darwin_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_darwin_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -45,17 +45,17 @@ #define LG_VADDR 48 /* Defined if C11 atomics are available. */ -#define JEMALLOC_C11_ATOMICS 1 +#define JEMALLOC_C11_ATOMICS /* Defined if GCC __atomic atomics are available. */ -#define JEMALLOC_GCC_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_ATOMIC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS /* Defined if GCC __sync atomics are available. */ -#define JEMALLOC_GCC_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_SYNC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_U8_SYNC_ATOMICS /* * Defined if __builtin_clz() and __builtin_clzl() are available. @@ -86,6 +86,12 @@ /* Defined if pthread_setname_np(3) is available. */ /* #undef JEMALLOC_HAVE_PTHREAD_SETNAME_NP */ +/* Defined if pthread_getname_np(3) is available. */ +#define JEMALLOC_HAVE_PTHREAD_GETNAME_NP + +/* Defined if pthread_get_name_np(3) is available. */ +/* #undef JEMALLOC_HAVE_PTHREAD_GET_NAME_NP */ + /* * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. */ @@ -99,12 +105,12 @@ /* * Defined if mach_absolute_time() is available. */ -#define JEMALLOC_HAVE_MACH_ABSOLUTE_TIME 1 +#define JEMALLOC_HAVE_MACH_ABSOLUTE_TIME /* * Defined if clock_gettime(CLOCK_REALTIME, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_REALTIME 1 +#define JEMALLOC_HAVE_CLOCK_REALTIME /* * Defined if _malloc_thread_cleanup() exists. At least in the case of @@ -168,6 +174,9 @@ /* Support utrace(2)-based tracing. */ /* #undef JEMALLOC_UTRACE */ +/* Support utrace(2)-based tracing (label based signature). */ +/* #undef JEMALLOC_UTRACE_LABEL */ + /* Support optional abort() on OOM. */ /* #undef JEMALLOC_XMALLOC */ @@ -183,6 +192,9 @@ /* One page is 2^LG_PAGE bytes. */ #define LG_PAGE 12 +/* Maximum number of regions in a slab. */ +/* #undef CONFIG_LG_SLAB_MAXREGS */ + /* * One huge page is 2^LG_HUGEPAGE bytes. Note that this is defined even if the * system does not explicitly support huge pages; system calls that require @@ -296,17 +308,46 @@ */ /* #undef JEMALLOC_MADVISE_DONTDUMP */ +/* + * Defined if MADV_[NO]CORE is supported as an argument to madvise. + */ +/* #undef JEMALLOC_MADVISE_NOCORE */ + +/* Defined if mprotect(2) is available. */ +#define JEMALLOC_HAVE_MPROTECT + /* * Defined if transparent huge pages (THPs) are supported via the * MADV_[NO]HUGEPAGE arguments to madvise(2), and THP support is enabled. */ /* #undef JEMALLOC_THP */ +/* Defined if posix_madvise is available. */ +/* #undef JEMALLOC_HAVE_POSIX_MADVISE */ + +/* + * Method for purging unused pages using posix_madvise. + * + * posix_madvise(..., POSIX_MADV_DONTNEED) + */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS */ + +/* + * Defined if memcntl page admin call is supported + */ +/* #undef JEMALLOC_HAVE_MEMCNTL */ + +/* + * Defined if malloc_size is supported + */ +#define JEMALLOC_HAVE_MALLOC_SIZE + /* Define if operating system has alloca.h header. */ /* #undef JEMALLOC_HAS_ALLOCA_H */ /* C99 restrict keyword supported. */ -#define JEMALLOC_HAS_RESTRICT 1 +#define JEMALLOC_HAS_RESTRICT /* For use by hash code. */ /* #undef JEMALLOC_BIG_ENDIAN */ @@ -369,4 +410,16 @@ /* Performs additional safety checks when defined. */ /* #undef JEMALLOC_OPT_SAFETY_CHECKS */ +/* Is C++ support being built? */ +/* #undef JEMALLOC_ENABLE_CXX */ + +/* Performs additional size checks when defined. */ +/* #undef JEMALLOC_OPT_SIZE_CHECKS */ + +/* Allows sampled junk and stash for checking use-after-free when defined. */ +/* #undef JEMALLOC_UAF_DETECTION */ + +/* Darwin VM_MAKE_TAG support */ +#define JEMALLOC_HAVE_VM_MAKE_TAG + #endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/contrib/jemalloc-cmake/include_freebsd_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_freebsd_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in index 3db0e14b268..0f61417d65f 100644 --- a/contrib/jemalloc-cmake/include_freebsd_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_freebsd_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -45,17 +45,17 @@ #define LG_VADDR 48 /* Defined if C11 atomics are available. */ -#define JEMALLOC_C11_ATOMICS 1 +#define JEMALLOC_C11_ATOMICS /* Defined if GCC __atomic atomics are available. */ -#define JEMALLOC_GCC_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_ATOMIC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS /* Defined if GCC __sync atomics are available. */ -#define JEMALLOC_GCC_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_SYNC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_U8_SYNC_ATOMICS /* * Defined if __builtin_clz() and __builtin_clzl() are available. @@ -83,9 +83,16 @@ /* Defined if pthread_atfork(3) is available. */ #define JEMALLOC_HAVE_PTHREAD_ATFORK +/* Only since 12.1-STABLE */ /* Defined if pthread_setname_np(3) is available. */ -// Only since 12.1-STABLE -// #define JEMALLOC_HAVE_PTHREAD_SETNAME_NP +/* #undef JEMALLOC_HAVE_PTHREAD_SETNAME_NP */ + +/* Only since 12.1-STABLE */ +/* Defined if pthread_getname_np(3) is available. */ +/* #undef JEMALLOC_HAVE_PTHREAD_GETNAME_NP */ + +/* Defined if pthread_get_name_np(3) is available. */ +#define JEMALLOC_HAVE_PTHREAD_GET_NAME_NP /* * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. @@ -95,7 +102,7 @@ /* * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_MONOTONIC 1 +#define JEMALLOC_HAVE_CLOCK_MONOTONIC /* * Defined if mach_absolute_time() is available. @@ -105,7 +112,7 @@ /* * Defined if clock_gettime(CLOCK_REALTIME, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_REALTIME 1 +#define JEMALLOC_HAVE_CLOCK_REALTIME /* * Defined if _malloc_thread_cleanup() exists. At least in the case of @@ -128,7 +135,7 @@ * _pthread_mutex_init_calloc_cb(), in which case the function is used in order * to avoid recursive allocation during mutex initialization. */ -#define JEMALLOC_MUTEX_INIT_CB 1 +#define JEMALLOC_MUTEX_INIT_CB /* Non-empty if the tls_model attribute is supported. */ #define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) @@ -161,7 +168,7 @@ * JEMALLOC_DSS enables use of sbrk(2) to allocate extents from the data storage * segment (DSS). */ -/* #undef JEMALLOC_DSS */ +#define JEMALLOC_DSS /* Support memory filling (junk/zero). */ #define JEMALLOC_FILL @@ -169,6 +176,9 @@ /* Support utrace(2)-based tracing. */ /* #undef JEMALLOC_UTRACE */ +/* Support utrace(2)-based tracing (label based signature). */ +/* #undef JEMALLOC_UTRACE_LABEL */ + /* Support optional abort() on OOM. */ /* #undef JEMALLOC_XMALLOC */ @@ -184,6 +194,9 @@ /* One page is 2^LG_PAGE bytes. */ #define LG_PAGE 16 +/* Maximum number of regions in a slab. */ +/* #undef CONFIG_LG_SLAB_MAXREGS */ + /* * One huge page is 2^LG_HUGEPAGE bytes. Note that this is defined even if the * system does not explicitly support huge pages; system calls that require @@ -297,17 +310,46 @@ */ /* #undef JEMALLOC_MADVISE_DONTDUMP */ +/* + * Defined if MADV_[NO]CORE is supported as an argument to madvise. + */ +#define JEMALLOC_MADVISE_NOCORE + +/* Defined if mprotect(2) is available. */ +#define JEMALLOC_HAVE_MPROTECT + /* * Defined if transparent huge pages (THPs) are supported via the * MADV_[NO]HUGEPAGE arguments to madvise(2), and THP support is enabled. */ /* #undef JEMALLOC_THP */ +/* Defined if posix_madvise is available. */ +/* #undef JEMALLOC_HAVE_POSIX_MADVISE */ + +/* + * Method for purging unused pages using posix_madvise. + * + * posix_madvise(..., POSIX_MADV_DONTNEED) + */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS */ + +/* + * Defined if memcntl page admin call is supported + */ +/* #undef JEMALLOC_HAVE_MEMCNTL */ + +/* + * Defined if malloc_size is supported + */ +/* #undef JEMALLOC_HAVE_MALLOC_SIZE */ + /* Define if operating system has alloca.h header. */ /* #undef JEMALLOC_HAS_ALLOCA_H */ /* C99 restrict keyword supported. */ -#define JEMALLOC_HAS_RESTRICT 1 +#define JEMALLOC_HAS_RESTRICT /* For use by hash code. */ /* #undef JEMALLOC_BIG_ENDIAN */ @@ -348,7 +390,7 @@ /* * If defined, all the features necessary for background threads are present. */ -#define JEMALLOC_BACKGROUND_THREAD 1 +#define JEMALLOC_BACKGROUND_THREAD /* * If defined, jemalloc symbols are not exported (doesn't work when @@ -360,7 +402,7 @@ #define JEMALLOC_CONFIG_MALLOC_CONF "@JEMALLOC_CONFIG_MALLOC_CONF@" /* If defined, jemalloc takes the malloc/free/etc. symbol names. */ -#define JEMALLOC_IS_MALLOC 1 +#define JEMALLOC_IS_MALLOC /* * Defined if strerror_r returns char * if _GNU_SOURCE is defined. @@ -370,4 +412,16 @@ /* Performs additional safety checks when defined. */ /* #undef JEMALLOC_OPT_SAFETY_CHECKS */ +/* Is C++ support being built? */ +/* #undef JEMALLOC_ENABLE_CXX */ + +/* Performs additional size checks when defined. */ +/* #undef JEMALLOC_OPT_SIZE_CHECKS */ + +/* Allows sampled junk and stash for checking use-after-free when defined. */ +/* #undef JEMALLOC_UAF_DETECTION */ + +/* Darwin VM_MAKE_TAG support */ +/* #undef JEMALLOC_HAVE_VM_MAKE_TAG */ + #endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/contrib/jemalloc-cmake/include_freebsd_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_freebsd_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in index dbf55f3f6e0..32cad025f5f 100644 --- a/contrib/jemalloc-cmake/include_freebsd_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_freebsd_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -45,17 +45,17 @@ #define LG_VADDR 48 /* Defined if C11 atomics are available. */ -#define JEMALLOC_C11_ATOMICS 1 +#define JEMALLOC_C11_ATOMICS /* Defined if GCC __atomic atomics are available. */ -#define JEMALLOC_GCC_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_ATOMIC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS /* Defined if GCC __sync atomics are available. */ -#define JEMALLOC_GCC_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_SYNC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_U8_SYNC_ATOMICS /* * Defined if __builtin_clz() and __builtin_clzl() are available. @@ -83,9 +83,16 @@ /* Defined if pthread_atfork(3) is available. */ #define JEMALLOC_HAVE_PTHREAD_ATFORK +/* Only since 12.1-STABLE */ /* Defined if pthread_setname_np(3) is available. */ -// Only since 12.1-STABLE -// #define JEMALLOC_HAVE_PTHREAD_SETNAME_NP +/* #undef JEMALLOC_HAVE_PTHREAD_SETNAME_NP */ + +/* Only since 12.1-STABLE */ +/* Defined if pthread_getname_np(3) is available. */ +/* #undef JEMALLOC_HAVE_PTHREAD_GETNAME_NP */ + +/* Defined if pthread_get_name_np(3) is available. */ +#define JEMALLOC_HAVE_PTHREAD_GET_NAME_NP /* * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. @@ -95,7 +102,7 @@ /* * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_MONOTONIC 1 +#define JEMALLOC_HAVE_CLOCK_MONOTONIC /* * Defined if mach_absolute_time() is available. @@ -105,7 +112,7 @@ /* * Defined if clock_gettime(CLOCK_REALTIME, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_REALTIME 1 +#define JEMALLOC_HAVE_CLOCK_REALTIME /* * Defined if _malloc_thread_cleanup() exists. At least in the case of @@ -128,7 +135,7 @@ * _pthread_mutex_init_calloc_cb(), in which case the function is used in order * to avoid recursive allocation during mutex initialization. */ -#define JEMALLOC_MUTEX_INIT_CB 1 +#define JEMALLOC_MUTEX_INIT_CB /* Non-empty if the tls_model attribute is supported. */ #define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) @@ -169,6 +176,9 @@ /* Support utrace(2)-based tracing. */ /* #undef JEMALLOC_UTRACE */ +/* Support utrace(2)-based tracing (label based signature). */ +/* #undef JEMALLOC_UTRACE_LABEL */ + /* Support optional abort() on OOM. */ /* #undef JEMALLOC_XMALLOC */ @@ -184,6 +194,9 @@ /* One page is 2^LG_PAGE bytes. */ #define LG_PAGE 12 +/* Maximum number of regions in a slab. */ +/* #undef CONFIG_LG_SLAB_MAXREGS */ + /* * One huge page is 2^LG_HUGEPAGE bytes. Note that this is defined even if the * system does not explicitly support huge pages; system calls that require @@ -297,17 +310,46 @@ */ /* #undef JEMALLOC_MADVISE_DONTDUMP */ +/* + * Defined if MADV_[NO]CORE is supported as an argument to madvise. + */ +#define JEMALLOC_MADVISE_NOCORE + +/* Defined if mprotect(2) is available. */ +#define JEMALLOC_HAVE_MPROTECT + /* * Defined if transparent huge pages (THPs) are supported via the * MADV_[NO]HUGEPAGE arguments to madvise(2), and THP support is enabled. */ /* #undef JEMALLOC_THP */ +/* Defined if posix_madvise is available. */ +/* #undef JEMALLOC_HAVE_POSIX_MADVISE */ + +/* + * Method for purging unused pages using posix_madvise. + * + * posix_madvise(..., POSIX_MADV_DONTNEED) + */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS */ + +/* + * Defined if memcntl page admin call is supported + */ +/* #undef JEMALLOC_HAVE_MEMCNTL */ + +/* + * Defined if malloc_size is supported + */ +/* #undef JEMALLOC_HAVE_MALLOC_SIZE */ + /* Define if operating system has alloca.h header. */ /* #undef JEMALLOC_HAS_ALLOCA_H */ /* C99 restrict keyword supported. */ -#define JEMALLOC_HAS_RESTRICT 1 +#define JEMALLOC_HAS_RESTRICT /* For use by hash code. */ /* #undef JEMALLOC_BIG_ENDIAN */ @@ -348,7 +390,7 @@ /* * If defined, all the features necessary for background threads are present. */ -#define JEMALLOC_BACKGROUND_THREAD 1 +#define JEMALLOC_BACKGROUND_THREAD /* * If defined, jemalloc symbols are not exported (doesn't work when @@ -360,7 +402,7 @@ #define JEMALLOC_CONFIG_MALLOC_CONF "@JEMALLOC_CONFIG_MALLOC_CONF@" /* If defined, jemalloc takes the malloc/free/etc. symbol names. */ -#define JEMALLOC_IS_MALLOC 1 +#define JEMALLOC_IS_MALLOC /* * Defined if strerror_r returns char * if _GNU_SOURCE is defined. @@ -370,4 +412,16 @@ /* Performs additional safety checks when defined. */ /* #undef JEMALLOC_OPT_SAFETY_CHECKS */ +/* Is C++ support being built? */ +/* #undef JEMALLOC_ENABLE_CXX */ + +/* Performs additional size checks when defined. */ +/* #undef JEMALLOC_OPT_SIZE_CHECKS */ + +/* Allows sampled junk and stash for checking use-after-free when defined. */ +/* #undef JEMALLOC_UAF_DETECTION */ + +/* Darwin VM_MAKE_TAG support */ +/* #undef JEMALLOC_HAVE_VM_MAKE_TAG */ + #endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/contrib/jemalloc-cmake/include_linux_aarch64/README b/contrib/jemalloc-cmake/include_linux_aarch64/README deleted file mode 100644 index 3cecf7fa36d..00000000000 --- a/contrib/jemalloc-cmake/include_linux_aarch64/README +++ /dev/null @@ -1,8 +0,0 @@ -Here are pre-generated files from jemalloc on Linux aarch64. -You can obtain these files by running ./autogen.sh inside jemalloc source directory. - -Added #define GNU_SOURCE -Added JEMALLOC_OVERRIDE___POSIX_MEMALIGN because why not. -Removed JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF because it's non standard. -Removed JEMALLOC_PURGE_MADVISE_FREE because it's available only from Linux 4.5. -Added JEMALLOC_CONFIG_MALLOC_CONF substitution diff --git a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in index 5e0135cc0d0..ad535e6d773 100644 --- a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -13,12 +13,14 @@ * Define overrides for non-standard allocator-related functions if they are * present on the system. */ -#define JEMALLOC_OVERRIDE___LIBC_CALLOC -#define JEMALLOC_OVERRIDE___LIBC_FREE -#define JEMALLOC_OVERRIDE___LIBC_MALLOC -#define JEMALLOC_OVERRIDE___LIBC_MEMALIGN -#define JEMALLOC_OVERRIDE___LIBC_REALLOC -#define JEMALLOC_OVERRIDE___LIBC_VALLOC +#if !defined(USE_MUSL) + #define JEMALLOC_OVERRIDE___LIBC_CALLOC + #define JEMALLOC_OVERRIDE___LIBC_FREE + #define JEMALLOC_OVERRIDE___LIBC_MALLOC + #define JEMALLOC_OVERRIDE___LIBC_MEMALIGN + #define JEMALLOC_OVERRIDE___LIBC_REALLOC + #define JEMALLOC_OVERRIDE___LIBC_VALLOC +#endif /* #undef JEMALLOC_OVERRIDE___POSIX_MEMALIGN */ /* @@ -45,17 +47,17 @@ #define LG_VADDR 48 /* Defined if C11 atomics are available. */ -#define JEMALLOC_C11_ATOMICS 1 +#define JEMALLOC_C11_ATOMICS /* Defined if GCC __atomic atomics are available. */ -#define JEMALLOC_GCC_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_ATOMIC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS /* Defined if GCC __sync atomics are available. */ -#define JEMALLOC_GCC_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_SYNC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_U8_SYNC_ATOMICS /* * Defined if __builtin_clz() and __builtin_clzl() are available. @@ -73,7 +75,7 @@ /* * Defined if secure_getenv(3) is available. */ -// #define JEMALLOC_HAVE_SECURE_GETENV +/* #undef JEMALLOC_HAVE_SECURE_GETENV */ /* * Defined if issetugid(2) is available. @@ -86,21 +88,32 @@ /* Defined if pthread_setname_np(3) is available. */ #define JEMALLOC_HAVE_PTHREAD_SETNAME_NP +/* Defined if pthread_getname_np(3) is available. */ +#define JEMALLOC_HAVE_PTHREAD_GETNAME_NP + +/* Defined if pthread_get_name_np(3) is available. */ +/* #undef JEMALLOC_HAVE_PTHREAD_GET_NAME_NP */ + /* * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE 1 +#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE /* * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_MONOTONIC 1 +#define JEMALLOC_HAVE_CLOCK_MONOTONIC /* * Defined if mach_absolute_time() is available. */ /* #undef JEMALLOC_HAVE_MACH_ABSOLUTE_TIME */ +/* + * Defined if clock_gettime(CLOCK_REALTIME, ...) is available. + */ +#define JEMALLOC_HAVE_CLOCK_REALTIME + /* * Defined if _malloc_thread_cleanup() exists. At least in the case of * FreeBSD, pthread_key_create() allocates, which if used during malloc @@ -163,6 +176,9 @@ /* Support utrace(2)-based tracing. */ /* #undef JEMALLOC_UTRACE */ +/* Support utrace(2)-based tracing (label based signature). */ +/* #undef JEMALLOC_UTRACE_LABEL */ + /* Support optional abort() on OOM. */ /* #undef JEMALLOC_XMALLOC */ @@ -178,6 +194,9 @@ /* One page is 2^LG_PAGE bytes. */ #define LG_PAGE 16 +/* Maximum number of regions in a slab. */ +/* #undef CONFIG_LG_SLAB_MAXREGS */ + /* * One huge page is 2^LG_HUGEPAGE bytes. Note that this is defined even if the * system does not explicitly support huge pages; system calls that require @@ -291,17 +310,46 @@ */ #define JEMALLOC_MADVISE_DONTDUMP +/* + * Defined if MADV_[NO]CORE is supported as an argument to madvise. + */ +/* #undef JEMALLOC_MADVISE_NOCORE */ + +/* Defined if mprotect(2) is available. */ +#define JEMALLOC_HAVE_MPROTECT + /* * Defined if transparent huge pages (THPs) are supported via the * MADV_[NO]HUGEPAGE arguments to madvise(2), and THP support is enabled. */ /* #undef JEMALLOC_THP */ +/* Defined if posix_madvise is available. */ +/* #undef JEMALLOC_HAVE_POSIX_MADVISE */ + +/* + * Method for purging unused pages using posix_madvise. + * + * posix_madvise(..., POSIX_MADV_DONTNEED) + */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS */ + +/* + * Defined if memcntl page admin call is supported + */ +/* #undef JEMALLOC_HAVE_MEMCNTL */ + +/* + * Defined if malloc_size is supported + */ +/* #undef JEMALLOC_HAVE_MALLOC_SIZE */ + /* Define if operating system has alloca.h header. */ -#define JEMALLOC_HAS_ALLOCA_H 1 +#define JEMALLOC_HAS_ALLOCA_H /* C99 restrict keyword supported. */ -#define JEMALLOC_HAS_RESTRICT 1 +#define JEMALLOC_HAS_RESTRICT /* For use by hash code. */ /* #undef JEMALLOC_BIG_ENDIAN */ @@ -342,7 +390,7 @@ /* * If defined, all the features necessary for background threads are present. */ -#define JEMALLOC_BACKGROUND_THREAD 1 +#define JEMALLOC_BACKGROUND_THREAD /* * If defined, jemalloc symbols are not exported (doesn't work when @@ -354,7 +402,7 @@ #define JEMALLOC_CONFIG_MALLOC_CONF "@JEMALLOC_CONFIG_MALLOC_CONF@" /* If defined, jemalloc takes the malloc/free/etc. symbol names. */ -#define JEMALLOC_IS_MALLOC 1 +#define JEMALLOC_IS_MALLOC /* * Defined if strerror_r returns char * if _GNU_SOURCE is defined. @@ -364,4 +412,16 @@ /* Performs additional safety checks when defined. */ /* #undef JEMALLOC_OPT_SAFETY_CHECKS */ +/* Is C++ support being built? */ +/* #undef JEMALLOC_ENABLE_CXX */ + +/* Performs additional size checks when defined. */ +/* #undef JEMALLOC_OPT_SIZE_CHECKS */ + +/* Allows sampled junk and stash for checking use-after-free when defined. */ +/* #undef JEMALLOC_UAF_DETECTION */ + +/* Darwin VM_MAKE_TAG support */ +/* #undef JEMALLOC_HAVE_VM_MAKE_TAG */ + #endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/contrib/jemalloc-cmake/include_linux_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in index 97d0d4d8471..12890f80ef1 100644 --- a/contrib/jemalloc-cmake/include_linux_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in @@ -13,12 +13,14 @@ * Define overrides for non-standard allocator-related functions if they are * present on the system. */ -#define JEMALLOC_OVERRIDE___LIBC_CALLOC -#define JEMALLOC_OVERRIDE___LIBC_FREE -#define JEMALLOC_OVERRIDE___LIBC_MALLOC -#define JEMALLOC_OVERRIDE___LIBC_MEMALIGN -#define JEMALLOC_OVERRIDE___LIBC_REALLOC -#define JEMALLOC_OVERRIDE___LIBC_VALLOC +#if !defined(USE_MUSL) + #define JEMALLOC_OVERRIDE___LIBC_CALLOC + #define JEMALLOC_OVERRIDE___LIBC_FREE + #define JEMALLOC_OVERRIDE___LIBC_MALLOC + #define JEMALLOC_OVERRIDE___LIBC_MEMALIGN + #define JEMALLOC_OVERRIDE___LIBC_REALLOC + #define JEMALLOC_OVERRIDE___LIBC_VALLOC +#endif /* #undef JEMALLOC_OVERRIDE___POSIX_MEMALIGN */ /* @@ -45,17 +47,17 @@ #define LG_VADDR 64 /* Defined if C11 atomics are available. */ -#define JEMALLOC_C11_ATOMICS 1 +#define JEMALLOC_C11_ATOMICS /* Defined if GCC __atomic atomics are available. */ -#define JEMALLOC_GCC_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_ATOMIC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS /* Defined if GCC __sync atomics are available. */ -#define JEMALLOC_GCC_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_SYNC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_U8_SYNC_ATOMICS /* * Defined if __builtin_clz() and __builtin_clzl() are available. @@ -73,7 +75,7 @@ /* * Defined if secure_getenv(3) is available. */ -// #define JEMALLOC_HAVE_SECURE_GETENV +/* #undef JEMALLOC_HAVE_SECURE_GETENV */ /* * Defined if issetugid(2) is available. @@ -86,21 +88,32 @@ /* Defined if pthread_setname_np(3) is available. */ #define JEMALLOC_HAVE_PTHREAD_SETNAME_NP +/* Defined if pthread_getname_np(3) is available. */ +#define JEMALLOC_HAVE_PTHREAD_GETNAME_NP + +/* Defined if pthread_get_name_np(3) is available. */ +/* #undef JEMALLOC_HAVE_PTHREAD_GET_NAME_NP */ + /* * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE 1 +#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE /* * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_MONOTONIC 1 +#define JEMALLOC_HAVE_CLOCK_MONOTONIC /* * Defined if mach_absolute_time() is available. */ /* #undef JEMALLOC_HAVE_MACH_ABSOLUTE_TIME */ +/* + * Defined if clock_gettime(CLOCK_REALTIME, ...) is available. + */ +#define JEMALLOC_HAVE_CLOCK_REALTIME + /* * Defined if _malloc_thread_cleanup() exists. At least in the case of * FreeBSD, pthread_key_create() allocates, which if used during malloc @@ -163,6 +176,9 @@ /* Support utrace(2)-based tracing. */ /* #undef JEMALLOC_UTRACE */ +/* Support utrace(2)-based tracing (label based signature). */ +/* #undef JEMALLOC_UTRACE_LABEL */ + /* Support optional abort() on OOM. */ /* #undef JEMALLOC_XMALLOC */ @@ -178,6 +194,9 @@ /* One page is 2^LG_PAGE bytes. */ #define LG_PAGE 16 +/* Maximum number of regions in a slab. */ +/* #undef CONFIG_LG_SLAB_MAXREGS */ + /* * One huge page is 2^LG_HUGEPAGE bytes. Note that this is defined even if the * system does not explicitly support huge pages; system calls that require @@ -291,17 +310,46 @@ */ #define JEMALLOC_MADVISE_DONTDUMP +/* + * Defined if MADV_[NO]CORE is supported as an argument to madvise. + */ +/* #undef JEMALLOC_MADVISE_NOCORE */ + +/* Defined if mprotect(2) is available. */ +#define JEMALLOC_HAVE_MPROTECT + /* * Defined if transparent huge pages (THPs) are supported via the * MADV_[NO]HUGEPAGE arguments to madvise(2), and THP support is enabled. */ /* #undef JEMALLOC_THP */ +/* Defined if posix_madvise is available. */ +/* #undef JEMALLOC_HAVE_POSIX_MADVISE */ + +/* + * Method for purging unused pages using posix_madvise. + * + * posix_madvise(..., POSIX_MADV_DONTNEED) + */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS */ + +/* + * Defined if memcntl page admin call is supported + */ +/* #undef JEMALLOC_HAVE_MEMCNTL */ + +/* + * Defined if malloc_size is supported + */ +/* #undef JEMALLOC_HAVE_MALLOC_SIZE */ + /* Define if operating system has alloca.h header. */ -#define JEMALLOC_HAS_ALLOCA_H 1 +#define JEMALLOC_HAS_ALLOCA_H /* C99 restrict keyword supported. */ -#define JEMALLOC_HAS_RESTRICT 1 +#define JEMALLOC_HAS_RESTRICT /* For use by hash code. */ /* #undef JEMALLOC_BIG_ENDIAN */ @@ -342,7 +390,7 @@ /* * If defined, all the features necessary for background threads are present. */ -#define JEMALLOC_BACKGROUND_THREAD 1 +#define JEMALLOC_BACKGROUND_THREAD /* * If defined, jemalloc symbols are not exported (doesn't work when @@ -354,7 +402,7 @@ #define JEMALLOC_CONFIG_MALLOC_CONF "@JEMALLOC_CONFIG_MALLOC_CONF@" /* If defined, jemalloc takes the malloc/free/etc. symbol names. */ -#define JEMALLOC_IS_MALLOC 1 +#define JEMALLOC_IS_MALLOC /* * Defined if strerror_r returns char * if _GNU_SOURCE is defined. @@ -364,4 +412,16 @@ /* Performs additional safety checks when defined. */ /* #undef JEMALLOC_OPT_SAFETY_CHECKS */ +/* Is C++ support being built? */ +/* #undef JEMALLOC_ENABLE_CXX */ + +/* Performs additional size checks when defined. */ +/* #undef JEMALLOC_OPT_SIZE_CHECKS */ + +/* Allows sampled junk and stash for checking use-after-free when defined. */ +/* #undef JEMALLOC_UAF_DETECTION */ + +/* Darwin VM_MAKE_TAG support */ +/* #undef JEMALLOC_HAVE_VM_MAKE_TAG */ + #endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/contrib/jemalloc-cmake/include_linux_riscv64/README b/contrib/jemalloc-cmake/include_linux_riscv64/README deleted file mode 100644 index 01b65655c55..00000000000 --- a/contrib/jemalloc-cmake/include_linux_riscv64/README +++ /dev/null @@ -1,8 +0,0 @@ -Here are pre-generated files from jemalloc on Linux risc-v. -You can obtain these files by running ./autogen.sh inside jemalloc source directory. - -Added #define GNU_SOURCE -Added JEMALLOC_OVERRIDE___POSIX_MEMALIGN because why not. -Removed JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF because it's non standard. -Removed JEMALLOC_PURGE_MADVISE_FREE because it's available only from Linux 4.5. -Added JEMALLOC_CONFIG_MALLOC_CONF substitution diff --git a/contrib/jemalloc-cmake/include_linux_riscv64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_riscv64/jemalloc/internal/jemalloc_internal_defs.h.in index 5e0135cc0d0..ad535e6d773 100644 --- a/contrib/jemalloc-cmake/include_linux_riscv64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_riscv64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -13,12 +13,14 @@ * Define overrides for non-standard allocator-related functions if they are * present on the system. */ -#define JEMALLOC_OVERRIDE___LIBC_CALLOC -#define JEMALLOC_OVERRIDE___LIBC_FREE -#define JEMALLOC_OVERRIDE___LIBC_MALLOC -#define JEMALLOC_OVERRIDE___LIBC_MEMALIGN -#define JEMALLOC_OVERRIDE___LIBC_REALLOC -#define JEMALLOC_OVERRIDE___LIBC_VALLOC +#if !defined(USE_MUSL) + #define JEMALLOC_OVERRIDE___LIBC_CALLOC + #define JEMALLOC_OVERRIDE___LIBC_FREE + #define JEMALLOC_OVERRIDE___LIBC_MALLOC + #define JEMALLOC_OVERRIDE___LIBC_MEMALIGN + #define JEMALLOC_OVERRIDE___LIBC_REALLOC + #define JEMALLOC_OVERRIDE___LIBC_VALLOC +#endif /* #undef JEMALLOC_OVERRIDE___POSIX_MEMALIGN */ /* @@ -45,17 +47,17 @@ #define LG_VADDR 48 /* Defined if C11 atomics are available. */ -#define JEMALLOC_C11_ATOMICS 1 +#define JEMALLOC_C11_ATOMICS /* Defined if GCC __atomic atomics are available. */ -#define JEMALLOC_GCC_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_ATOMIC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS /* Defined if GCC __sync atomics are available. */ -#define JEMALLOC_GCC_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_SYNC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_U8_SYNC_ATOMICS /* * Defined if __builtin_clz() and __builtin_clzl() are available. @@ -73,7 +75,7 @@ /* * Defined if secure_getenv(3) is available. */ -// #define JEMALLOC_HAVE_SECURE_GETENV +/* #undef JEMALLOC_HAVE_SECURE_GETENV */ /* * Defined if issetugid(2) is available. @@ -86,21 +88,32 @@ /* Defined if pthread_setname_np(3) is available. */ #define JEMALLOC_HAVE_PTHREAD_SETNAME_NP +/* Defined if pthread_getname_np(3) is available. */ +#define JEMALLOC_HAVE_PTHREAD_GETNAME_NP + +/* Defined if pthread_get_name_np(3) is available. */ +/* #undef JEMALLOC_HAVE_PTHREAD_GET_NAME_NP */ + /* * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE 1 +#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE /* * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_MONOTONIC 1 +#define JEMALLOC_HAVE_CLOCK_MONOTONIC /* * Defined if mach_absolute_time() is available. */ /* #undef JEMALLOC_HAVE_MACH_ABSOLUTE_TIME */ +/* + * Defined if clock_gettime(CLOCK_REALTIME, ...) is available. + */ +#define JEMALLOC_HAVE_CLOCK_REALTIME + /* * Defined if _malloc_thread_cleanup() exists. At least in the case of * FreeBSD, pthread_key_create() allocates, which if used during malloc @@ -163,6 +176,9 @@ /* Support utrace(2)-based tracing. */ /* #undef JEMALLOC_UTRACE */ +/* Support utrace(2)-based tracing (label based signature). */ +/* #undef JEMALLOC_UTRACE_LABEL */ + /* Support optional abort() on OOM. */ /* #undef JEMALLOC_XMALLOC */ @@ -178,6 +194,9 @@ /* One page is 2^LG_PAGE bytes. */ #define LG_PAGE 16 +/* Maximum number of regions in a slab. */ +/* #undef CONFIG_LG_SLAB_MAXREGS */ + /* * One huge page is 2^LG_HUGEPAGE bytes. Note that this is defined even if the * system does not explicitly support huge pages; system calls that require @@ -291,17 +310,46 @@ */ #define JEMALLOC_MADVISE_DONTDUMP +/* + * Defined if MADV_[NO]CORE is supported as an argument to madvise. + */ +/* #undef JEMALLOC_MADVISE_NOCORE */ + +/* Defined if mprotect(2) is available. */ +#define JEMALLOC_HAVE_MPROTECT + /* * Defined if transparent huge pages (THPs) are supported via the * MADV_[NO]HUGEPAGE arguments to madvise(2), and THP support is enabled. */ /* #undef JEMALLOC_THP */ +/* Defined if posix_madvise is available. */ +/* #undef JEMALLOC_HAVE_POSIX_MADVISE */ + +/* + * Method for purging unused pages using posix_madvise. + * + * posix_madvise(..., POSIX_MADV_DONTNEED) + */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS */ + +/* + * Defined if memcntl page admin call is supported + */ +/* #undef JEMALLOC_HAVE_MEMCNTL */ + +/* + * Defined if malloc_size is supported + */ +/* #undef JEMALLOC_HAVE_MALLOC_SIZE */ + /* Define if operating system has alloca.h header. */ -#define JEMALLOC_HAS_ALLOCA_H 1 +#define JEMALLOC_HAS_ALLOCA_H /* C99 restrict keyword supported. */ -#define JEMALLOC_HAS_RESTRICT 1 +#define JEMALLOC_HAS_RESTRICT /* For use by hash code. */ /* #undef JEMALLOC_BIG_ENDIAN */ @@ -342,7 +390,7 @@ /* * If defined, all the features necessary for background threads are present. */ -#define JEMALLOC_BACKGROUND_THREAD 1 +#define JEMALLOC_BACKGROUND_THREAD /* * If defined, jemalloc symbols are not exported (doesn't work when @@ -354,7 +402,7 @@ #define JEMALLOC_CONFIG_MALLOC_CONF "@JEMALLOC_CONFIG_MALLOC_CONF@" /* If defined, jemalloc takes the malloc/free/etc. symbol names. */ -#define JEMALLOC_IS_MALLOC 1 +#define JEMALLOC_IS_MALLOC /* * Defined if strerror_r returns char * if _GNU_SOURCE is defined. @@ -364,4 +412,16 @@ /* Performs additional safety checks when defined. */ /* #undef JEMALLOC_OPT_SAFETY_CHECKS */ +/* Is C++ support being built? */ +/* #undef JEMALLOC_ENABLE_CXX */ + +/* Performs additional size checks when defined. */ +/* #undef JEMALLOC_OPT_SIZE_CHECKS */ + +/* Allows sampled junk and stash for checking use-after-free when defined. */ +/* #undef JEMALLOC_UAF_DETECTION */ + +/* Darwin VM_MAKE_TAG support */ +/* #undef JEMALLOC_HAVE_VM_MAKE_TAG */ + #endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/README b/contrib/jemalloc-cmake/include_linux_x86_64/README deleted file mode 100644 index 8b93e0d4dcd..00000000000 --- a/contrib/jemalloc-cmake/include_linux_x86_64/README +++ /dev/null @@ -1,8 +0,0 @@ -Here are pre-generated files from jemalloc on Linux x86_64. -You can obtain these files by running ./autogen.sh inside jemalloc source directory. - -Added #define GNU_SOURCE -Added JEMALLOC_OVERRIDE___POSIX_MEMALIGN because why not. -Removed JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF because it's non standard. -Removed JEMALLOC_PURGE_MADVISE_FREE because it's available only from Linux 4.5. -Added JEMALLOC_CONFIG_MALLOC_CONF substitution diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in index 44ff2d9fad1..99ab2d53ca9 100644 --- a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -47,17 +47,17 @@ #define LG_VADDR 48 /* Defined if C11 atomics are available. */ -#define JEMALLOC_C11_ATOMICS 1 +#define JEMALLOC_C11_ATOMICS /* Defined if GCC __atomic atomics are available. */ -#define JEMALLOC_GCC_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_ATOMIC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS /* Defined if GCC __sync atomics are available. */ -#define JEMALLOC_GCC_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_SYNC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_U8_SYNC_ATOMICS /* * Defined if __builtin_clz() and __builtin_clzl() are available. @@ -75,7 +75,7 @@ /* * Defined if secure_getenv(3) is available. */ -// #define JEMALLOC_HAVE_SECURE_GETENV +/* #undef JEMALLOC_HAVE_SECURE_GETENV */ /* * Defined if issetugid(2) is available. @@ -88,21 +88,32 @@ /* Defined if pthread_setname_np(3) is available. */ #define JEMALLOC_HAVE_PTHREAD_SETNAME_NP +/* Defined if pthread_getname_np(3) is available. */ +#define JEMALLOC_HAVE_PTHREAD_GETNAME_NP + +/* Defined if pthread_get_name_np(3) is available. */ +/* #undef JEMALLOC_HAVE_PTHREAD_GET_NAME_NP */ + /* * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE 1 +#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE /* * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_MONOTONIC 1 +#define JEMALLOC_HAVE_CLOCK_MONOTONIC /* * Defined if mach_absolute_time() is available. */ /* #undef JEMALLOC_HAVE_MACH_ABSOLUTE_TIME */ +/* + * Defined if clock_gettime(CLOCK_REALTIME, ...) is available. + */ +#define JEMALLOC_HAVE_CLOCK_REALTIME + /* * Defined if _malloc_thread_cleanup() exists. At least in the case of * FreeBSD, pthread_key_create() allocates, which if used during malloc @@ -165,6 +176,9 @@ /* Support utrace(2)-based tracing. */ /* #undef JEMALLOC_UTRACE */ +/* Support utrace(2)-based tracing (label based signature). */ +/* #undef JEMALLOC_UTRACE_LABEL */ + /* Support optional abort() on OOM. */ /* #undef JEMALLOC_XMALLOC */ @@ -180,6 +194,9 @@ /* One page is 2^LG_PAGE bytes. */ #define LG_PAGE 12 +/* Maximum number of regions in a slab. */ +/* #undef CONFIG_LG_SLAB_MAXREGS */ + /* * One huge page is 2^LG_HUGEPAGE bytes. Note that this is defined even if the * system does not explicitly support huge pages; system calls that require @@ -293,17 +310,46 @@ */ #define JEMALLOC_MADVISE_DONTDUMP +/* + * Defined if MADV_[NO]CORE is supported as an argument to madvise. + */ +/* #undef JEMALLOC_MADVISE_NOCORE */ + +/* Defined if mprotect(2) is available. */ +#define JEMALLOC_HAVE_MPROTECT + /* * Defined if transparent huge pages (THPs) are supported via the * MADV_[NO]HUGEPAGE arguments to madvise(2), and THP support is enabled. */ /* #undef JEMALLOC_THP */ +/* Defined if posix_madvise is available. */ +/* #undef JEMALLOC_HAVE_POSIX_MADVISE */ + +/* + * Method for purging unused pages using posix_madvise. + * + * posix_madvise(..., POSIX_MADV_DONTNEED) + */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS */ + +/* + * Defined if memcntl page admin call is supported + */ +/* #undef JEMALLOC_HAVE_MEMCNTL */ + +/* + * Defined if malloc_size is supported + */ +/* #undef JEMALLOC_HAVE_MALLOC_SIZE */ + /* Define if operating system has alloca.h header. */ -#define JEMALLOC_HAS_ALLOCA_H 1 +#define JEMALLOC_HAS_ALLOCA_H /* C99 restrict keyword supported. */ -#define JEMALLOC_HAS_RESTRICT 1 +#define JEMALLOC_HAS_RESTRICT /* For use by hash code. */ /* #undef JEMALLOC_BIG_ENDIAN */ @@ -344,7 +390,7 @@ /* * If defined, all the features necessary for background threads are present. */ -#define JEMALLOC_BACKGROUND_THREAD 1 +#define JEMALLOC_BACKGROUND_THREAD /* * If defined, jemalloc symbols are not exported (doesn't work when @@ -356,7 +402,7 @@ #define JEMALLOC_CONFIG_MALLOC_CONF "@JEMALLOC_CONFIG_MALLOC_CONF@" /* If defined, jemalloc takes the malloc/free/etc. symbol names. */ -#define JEMALLOC_IS_MALLOC 1 +#define JEMALLOC_IS_MALLOC /* * Defined if strerror_r returns char * if _GNU_SOURCE is defined. @@ -366,4 +412,16 @@ /* Performs additional safety checks when defined. */ /* #undef JEMALLOC_OPT_SAFETY_CHECKS */ +/* Is C++ support being built? */ +/* #undef JEMALLOC_ENABLE_CXX */ + +/* Performs additional size checks when defined. */ +/* #undef JEMALLOC_OPT_SIZE_CHECKS */ + +/* Allows sampled junk and stash for checking use-after-free when defined. */ +/* #undef JEMALLOC_UAF_DETECTION */ + +/* Darwin VM_MAKE_TAG support */ +/* #undef JEMALLOC_HAVE_VM_MAKE_TAG */ + #endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/contrib/llvm-cmake/CMakeLists.txt b/contrib/llvm-cmake/CMakeLists.txt index d240924cac3..e859df65b6f 100644 --- a/contrib/llvm-cmake/CMakeLists.txt +++ b/contrib/llvm-cmake/CMakeLists.txt @@ -6,7 +6,7 @@ endif() option (ENABLE_EMBEDDED_COMPILER "Enable support for 'compile_expressions' option for query execution" ${ENABLE_EMBEDDED_COMPILER_DEFAULT}) if (NOT ENABLE_EMBEDDED_COMPILER) - set (USE_EMBEDDED_COMPILER 0) + message(STATUS "Not using LLVM") return() endif() diff --git a/contrib/re2-cmake/CMakeLists.txt b/contrib/re2-cmake/CMakeLists.txt index e74f488643d..19939c11ebf 100644 --- a/contrib/re2-cmake/CMakeLists.txt +++ b/contrib/re2-cmake/CMakeLists.txt @@ -71,7 +71,6 @@ foreach (FILENAME mutex.h) add_dependencies (re2_st transform_${FILENAME}) endforeach () -# NOTE: you should not change name of library here, since it is used for PVS -# (see docker/test/pvs/Dockerfile), to generate required header (see above) +# NOTE: you should not change name of library here, since it is used to generate required header (see above) add_library(ch_contrib::re2 ALIAS re2) add_library(ch_contrib::re2_st ALIAS re2_st) diff --git a/debian/.gitignore b/debian/.gitignore index 7d927d3a70b..b4432556de7 100644 --- a/debian/.gitignore +++ b/debian/.gitignore @@ -10,7 +10,6 @@ clickhouse-common-static/ clickhouse-server-base/ clickhouse-server-common/ clickhouse-server/ -clickhouse-test/ debhelper-build-stamp files *.debhelper.log diff --git a/debian/clickhouse-test.install b/debian/clickhouse-test.install deleted file mode 100644 index 042a4e02be1..00000000000 --- a/debian/clickhouse-test.install +++ /dev/null @@ -1,2 +0,0 @@ -usr/bin/clickhouse-test -usr/share/clickhouse-test/* diff --git a/debian/control b/debian/control index f22980fdbc4..c5d98d98f41 100644 --- a/debian/control +++ b/debian/control @@ -56,9 +56,3 @@ Replaces: clickhouse-common-dbg Conflicts: clickhouse-common-dbg Description: debugging symbols for clickhouse-common-static This package contains the debugging symbols for clickhouse-common. - -Package: clickhouse-test -Priority: optional -Architecture: all -Depends: ${shlibs:Depends}, ${misc:Depends}, clickhouse-client, bash, expect, python3, python3-lxml, python3-termcolor, python3-requests, curl, perl, sudo, openssl, netcat-openbsd, telnet, brotli, bsdutils -Description: ClickHouse tests diff --git a/docker/images.json b/docker/images.json index 01284d4de69..06d689e8f7c 100644 --- a/docker/images.json +++ b/docker/images.json @@ -7,7 +7,6 @@ "name": "clickhouse/binary-builder", "dependent": [ "docker/test/split_build_smoke_test", - "docker/test/pvs", "docker/test/codebrowser" ] }, @@ -31,11 +30,6 @@ "name": "clickhouse/performance-comparison", "dependent": [] }, - "docker/test/pvs": { - "only_amd64": true, - "name": "clickhouse/pvs-test", - "dependent": [] - }, "docker/test/util": { "name": "clickhouse/test-util", "dependent": [ diff --git a/docker/packager/README.md b/docker/packager/README.md index a745f6225fa..223d70916d4 100644 --- a/docker/packager/README.md +++ b/docker/packager/README.md @@ -14,7 +14,6 @@ $ ls -l deb/test_output -rw-r--r-- 1 root root 14940 clickhouse-server_18.14.2+debug_all.deb -rw-r--r-- 1 root root 340206010 clickhouse-server-base_18.14.2+debug_amd64.deb -rw-r--r-- 1 root root 7900 clickhouse-server-common_18.14.2+debug_all.deb --rw-r--r-- 1 root root 2880432 clickhouse-test_18.14.2+debug_all.deb ``` diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile deleted file mode 100644 index 66d1afb309b..00000000000 --- a/docker/test/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM ubuntu:18.04 - -ARG repository="deb https://repo.clickhouse.com/deb/stable/ main/" -ARG version=22.1.1.* - -RUN apt-get update && \ - apt-get install -y apt-transport-https dirmngr && \ - mkdir -p /etc/apt/sources.list.d && \ - apt-key adv --keyserver keyserver.ubuntu.com --recv E0C56BD4 && \ - echo $repository | tee /etc/apt/sources.list.d/clickhouse.list && \ - apt-get update && \ - env DEBIAN_FRONTEND=noninteractive apt-get install --allow-unauthenticated -y clickhouse-test && \ - rm -rf /var/lib/apt/lists/* /var/cache/debconf && \ - apt-get clean - -ENTRYPOINT ["/usr/bin/clickhouse-test"] diff --git a/docker/test/pvs/Dockerfile b/docker/test/pvs/Dockerfile deleted file mode 100644 index 01cc7c97548..00000000000 --- a/docker/test/pvs/Dockerfile +++ /dev/null @@ -1,50 +0,0 @@ -# rebuild in #33610 -# docker build -t clickhouse/pvs-test . - -ARG FROM_TAG=latest -FROM clickhouse/binary-builder:$FROM_TAG - -RUN apt-get update --yes \ - && apt-get install \ - bash \ - wget \ - software-properties-common \ - gpg-agent \ - debsig-verify \ - strace \ - protobuf-compiler \ - protobuf-compiler-grpc \ - libprotoc-dev \ - libgrpc++-dev \ - libc-ares-dev \ - --yes --no-install-recommends - -#RUN wget -nv -O - http://files.viva64.com/etc/pubkey.txt | sudo apt-key add - -#RUN sudo wget -nv -O /etc/apt/sources.list.d/viva64.list http://files.viva64.com/etc/viva64.list -# -#RUN apt-get --allow-unauthenticated update -y \ -# && env DEBIAN_FRONTEND=noninteractive \ -# apt-get --allow-unauthenticated install --yes --no-install-recommends \ -# pvs-studio - -ENV PKG_VERSION="pvs-studio-latest" - -RUN set -x \ - && export PUBKEY_HASHSUM="ad369a2e9d8b8c30f5a9f2eb131121739b79c78e03fef0f016ea51871a5f78cd4e6257b270dca0ac3be3d1f19d885516" \ - && wget -nv https://files.viva64.com/etc/pubkey.txt -O /tmp/pubkey.txt \ - && echo "${PUBKEY_HASHSUM} /tmp/pubkey.txt" | sha384sum -c \ - && apt-key add /tmp/pubkey.txt \ - && wget -nv "https://files.viva64.com/${PKG_VERSION}.deb" \ - && { debsig-verify ${PKG_VERSION}.deb \ - || echo "WARNING: Some file was just downloaded from the internet without any validation and we are installing it into the system"; } \ - && dpkg -i "${PKG_VERSION}.deb" - -ENV CCACHE_DIR=/test_output/ccache - -CMD echo "Running PVS version $PKG_VERSION" && mkdir -p $CCACHE_DIR && cd /repo_folder && pvs-studio-analyzer credentials $LICENCE_NAME $LICENCE_KEY -o ./licence.lic \ - && cmake . -D"ENABLE_EMBEDDED_COMPILER"=OFF -D"DISABLE_HERMETIC_BUILD"=ON -DCMAKE_C_COMPILER=clang-13 -DCMAKE_CXX_COMPILER=clang\+\+-13 \ - && ninja re2_st clickhouse_grpc_protos \ - && pvs-studio-analyzer analyze -o pvs-studio.log -e contrib -j "$(nproc)" -l ./licence.lic; \ - cp /repo_folder/pvs-studio.log /test_output; \ - plog-converter -a GA:1,2 -t fullhtml -o /test_output/pvs-studio-html-report pvs-studio.log; \ - plog-converter -a GA:1,2 -t tasklist -o /test_output/pvs-studio-task-report.txt pvs-studio.log diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh index c3966fea460..f5e5fe21f30 100755 --- a/docker/test/stateful/run.sh +++ b/docker/test/stateful/run.sh @@ -11,7 +11,8 @@ dpkg -i package_folder/clickhouse-common-static_*.deb; dpkg -i package_folder/clickhouse-common-static-dbg_*.deb dpkg -i package_folder/clickhouse-server_*.deb dpkg -i package_folder/clickhouse-client_*.deb -dpkg -i package_folder/clickhouse-test_*.deb + +ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test # install test configs /usr/share/clickhouse-test/config/install.sh diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile index 9b7fde7d542..bfc6763e8c5 100644 --- a/docker/test/stateless/Dockerfile +++ b/docker/test/stateless/Dockerfile @@ -31,7 +31,8 @@ RUN apt-get update -y \ wget \ mysql-client=8.0* \ postgresql-client \ - sqlite3 + sqlite3 \ + awscli RUN pip3 install numpy scipy pandas Jinja2 diff --git a/docker/test/stateless/clickhouse-statelest-test-runner.Dockerfile b/docker/test/stateless/clickhouse-statelest-test-runner.Dockerfile index 562141ba147..dc83e8b8d2e 100644 --- a/docker/test/stateless/clickhouse-statelest-test-runner.Dockerfile +++ b/docker/test/stateless/clickhouse-statelest-test-runner.Dockerfile @@ -2,7 +2,7 @@ # 1. build base container # 2. run base conatiner with mounted volumes # 3. commit container as image -FROM ubuntu:18.10 as clickhouse-test-runner-base +FROM ubuntu:20.04 as clickhouse-test-runner-base # A volume where directory with clickhouse packages to be mounted, # for later installing. @@ -11,5 +11,4 @@ VOLUME /packages CMD apt-get update ;\ DEBIAN_FRONTEND=noninteractive \ apt install -y /packages/clickhouse-common-static_*.deb \ - /packages/clickhouse-client_*.deb \ - /packages/clickhouse-test_*.deb + /packages/clickhouse-client_*.deb diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index 7c2563f2d86..5fd78502337 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -12,11 +12,8 @@ dpkg -i package_folder/clickhouse-common-static_*.deb dpkg -i package_folder/clickhouse-common-static-dbg_*.deb dpkg -i package_folder/clickhouse-server_*.deb dpkg -i package_folder/clickhouse-client_*.deb -if [[ -n "$TEST_CASES_FROM_DEB" ]] && [[ "$TEST_CASES_FROM_DEB" -eq 1 ]]; then - dpkg -i package_folder/clickhouse-test_*.deb -else - ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test -fi + +ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test # install test configs /usr/share/clickhouse-test/config/install.sh diff --git a/docker/test/stateless/setup_minio.sh b/docker/test/stateless/setup_minio.sh index 7f8b90ee741..df27b21b05b 100755 --- a/docker/test/stateless/setup_minio.sh +++ b/docker/test/stateless/setup_minio.sh @@ -55,3 +55,10 @@ for FILE in $(ls "${MINIO_DATA_PATH}"); do echo "$FILE"; ./mc cp "${MINIO_DATA_PATH}"/"$FILE" clickminio/test/"$FILE"; done + +mkdir -p ~/.aws +cat <> ~/.aws/credentials +[default] +aws_access_key_id=clickhouse +aws_secret_access_key=clickhouse +EOT diff --git a/docker/test/stateless_pytest/Dockerfile b/docker/test/stateless_pytest/Dockerfile index f692f8f39f0..789ee0e9b30 100644 --- a/docker/test/stateless_pytest/Dockerfile +++ b/docker/test/stateless_pytest/Dockerfile @@ -30,5 +30,4 @@ CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \ dpkg -i package_folder/clickhouse-common-static-dbg_*.deb; \ dpkg -i package_folder/clickhouse-server_*.deb; \ dpkg -i package_folder/clickhouse-client_*.deb; \ - dpkg -i package_folder/clickhouse-test_*.deb; \ python3 -m pytest /usr/share/clickhouse-test/queries -n $(nproc) --reruns=1 --timeout=600 --json=test_output/report.json --html=test_output/report.html --self-contained-html diff --git a/docker/test/stress/README.md b/docker/test/stress/README.md index b1519e7968d..96807b9f9a6 100644 --- a/docker/test/stress/README.md +++ b/docker/test/stress/README.md @@ -5,7 +5,7 @@ This allows to find problems like segmentation fault which cause shutdown of ser Usage: ``` $ ls $HOME/someclickhouse -clickhouse-client_18.14.9_all.deb clickhouse-common-static_18.14.9_amd64.deb clickhouse-server_18.14.9_all.deb clickhouse-test_18.14.9_all.deb +clickhouse-client_18.14.9_all.deb clickhouse-common-static_18.14.9_amd64.deb clickhouse-server_18.14.9_all.deb $ docker run --volume=$HOME/someclickhouse:/package_folder --volume=$HOME/test_output:/test_output clickhouse/stress-test Selecting previously unselected package clickhouse-common-static. (Reading database ... 14442 files and directories currently installed.) diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index e57dbc38ded..de77dec03b9 100755 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -31,13 +31,15 @@ dpkg -i package_folder/clickhouse-common-static_*.deb dpkg -i package_folder/clickhouse-common-static-dbg_*.deb dpkg -i package_folder/clickhouse-server_*.deb dpkg -i package_folder/clickhouse-client_*.deb -dpkg -i package_folder/clickhouse-test_*.deb function configure() { # install test configs /usr/share/clickhouse-test/config/install.sh + # we mount tests folder from repo to /usr/share + ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test + # avoid too slow startup sudo cat /etc/clickhouse-server/config.d/keeper_port.xml | sed "s|100000|10000|" > /etc/clickhouse-server/config.d/keeper_port.xml.tmp sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-server/config.d/keeper_port.xml diff --git a/docs/en/development/build.md b/docs/en/development/build.md index 982ba0556a9..aaa3bdfd043 100644 --- a/docs/en/development/build.md +++ b/docs/en/development/build.md @@ -23,7 +23,7 @@ $ sudo apt-get install git cmake python ninja-build Or cmake3 instead of cmake on older systems. -### Install clang-13 (recommended) {#install-clang-13} +### Install the latest clang (recommended) On Ubuntu/Debian you can use the automatic installation script (check [official webpage](https://apt.llvm.org/)) @@ -33,13 +33,15 @@ sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" For other Linux distribution - check the availability of the [prebuild packages](https://releases.llvm.org/download.html) or build clang [from sources](https://clang.llvm.org/get_started.html). -#### Use clang-13 for Builds +#### Use the latest clang for Builds ``` bash -$ export CC=clang-13 -$ export CXX=clang++-13 +$ export CC=clang-14 +$ export CXX=clang++-14 ``` +In this example we use version 14 that is the latest as of Feb 2022. + Gcc can also be used though it is discouraged. ### Checkout ClickHouse Sources {#checkout-clickhouse-sources} @@ -76,7 +78,6 @@ The build requires the following components: - Ninja - C++ compiler: clang-13 or newer - Linker: lld -- Python (is only used inside LLVM build and it is optional) If all the components are installed, you may build in the same way as the steps above. @@ -109,6 +110,29 @@ cmake ../ClickHouse make -j $(nproc) ``` +Here is an example of how to build `clang` and all the llvm infrastructure from sources: + +``` + git clone git@github.com:llvm/llvm-project.git + mkdir llvm-build && cd llvm-build + cmake -DCMAKE_BUILD_TYPE:STRING=Release -DLLVM_ENABLE_PROJECTS=all ../llvm-project/llvm/ + make -j16 + sudo make install + hash clang + clang --version +``` + +You can install the older clang like clang-11 from packages and then use it to build the new clang from sources. + +Here is an example of how to install the new `cmake` from the official website: + +``` +wget https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-linux-x86_64.sh +chmod +x cmake-3.22.2-linux-x86_64.sh +./cmake-3.22.2-linux-x86_64.sh +export PATH=/home/milovidov/work/cmake-3.22.2-linux-x86_64/bin/:${PATH} +hash cmake +``` ## How to Build ClickHouse Debian Package {#how-to-build-clickhouse-debian-package} diff --git a/docs/en/development/continuous-integration.md b/docs/en/development/continuous-integration.md index d2745c0080b..b5b558464ba 100644 --- a/docs/en/development/continuous-integration.md +++ b/docs/en/development/continuous-integration.md @@ -76,15 +76,6 @@ If it fails, fix the style errors following the [code style guide](style.md). - `output.txt` contains the check resulting errors (invalid tabulation etc), blank page means no errors. [Successful result example](https://clickhouse-test-reports.s3.yandex.net/12550/659c78c7abb56141723af6a81bfae39335aa8cb2/style_check/output.txt). -## PVS Check -Check the code with [PVS-studio](https://www.viva64.com/en/pvs-studio/), a static analysis tool. Look at the report to see the exact errors. Fix them if you can, if not -- ask a ClickHouse maintainer for help. - -### Report Details -- [Status page example](https://clickhouse-test-reports.s3.yandex.net/12550/67d716b5cc3987801996c31a67b31bf141bc3486/pvs_check.html) -- `test_run.txt.out.log` contains the building and analyzing log file. It includes only parsing or not-found errors. -- `HTML report` contains the analysis results. For its description visit PVS's [official site](https://www.viva64.com/en/m/0036/#ID14E9A2B2CD). - - ## Fast Test Normally this is the first check that is ran for a PR. It builds ClickHouse and runs most of [stateless functional tests](tests.md#functional-tests), omitting @@ -135,7 +126,6 @@ Builds ClickHouse in various configurations for use in further steps. You have t - `clickhouse-common-static-dbg_XXX[+asan, +msan, +ubsan, +tsan]_amd64.deb` - `clickhouse-common-staticXXX_amd64.deb` - `clickhouse-server_XXX_all.deb` - - `clickhouse-test_XXX_all.deb` - `clickhouse_XXX_amd64.buildinfo` - `clickhouse_XXX_amd64.changes` - `clickhouse`: Main built binary. diff --git a/docs/en/development/tests.md b/docs/en/development/tests.md index 5f3245c4d60..44a552d2a61 100644 --- a/docs/en/development/tests.md +++ b/docs/en/development/tests.md @@ -263,7 +263,7 @@ People from Yandex Security Team did some basic overview of ClickHouse capabilit ## Static Analyzers {#static-analyzers} -We run `clang-tidy` and `PVS-Studio` on per-commit basis. `clang-static-analyzer` checks are also enabled. `clang-tidy` is also used for some style checks. +We run `clang-tidy` on per-commit basis. `clang-static-analyzer` checks are also enabled. `clang-tidy` is also used for some style checks. We have evaluated `clang-tidy`, `Coverity`, `cppcheck`, `PVS-Studio`, `tscancode`, `CodeQL`. You will find instructions for usage in `tests/instructions/` directory. Also you can read [the article in russian](https://habr.com/company/yandex/blog/342018/). diff --git a/docs/en/engines/table-engines/integrations/kafka.md b/docs/en/engines/table-engines/integrations/kafka.md index 9fdc43f3f18..1d80f143098 100644 --- a/docs/en/engines/table-engines/integrations/kafka.md +++ b/docs/en/engines/table-engines/integrations/kafka.md @@ -186,6 +186,7 @@ Example: - `_key` — Key of the message. - `_offset` — Offset of the message. - `_timestamp` — Timestamp of the message. +- `_timestamp_ms` — Timestamp in milliseconds of the message. - `_partition` — Partition of Kafka topic. **See Also** diff --git a/docs/en/operations/external-authenticators/index.md b/docs/en/operations/external-authenticators/index.md index 13b07b8e51e..850b6594b71 100644 --- a/docs/en/operations/external-authenticators/index.md +++ b/docs/en/operations/external-authenticators/index.md @@ -12,5 +12,6 @@ The following external authenticators and directories are supported: - [LDAP](./ldap.md#external-authenticators-ldap) [Authenticator](./ldap.md#ldap-external-authenticator) and [Directory](./ldap.md#ldap-external-user-directory) - Kerberos [Authenticator](./kerberos.md#external-authenticators-kerberos) +- [SSL X.509 authentication](./ssl-x509.md#ssl-external-authentication) [Original article](https://clickhouse.com/docs/en/operations/external-authenticators/index/) diff --git a/docs/en/operations/external-authenticators/ssl-x509.md b/docs/en/operations/external-authenticators/ssl-x509.md new file mode 100644 index 00000000000..dd4f35257bb --- /dev/null +++ b/docs/en/operations/external-authenticators/ssl-x509.md @@ -0,0 +1,24 @@ +# SSL X.509 certificate authentication {#ssl-external-authentication} + +[SSL 'strict' option](../server-configuration-parameters/settings.md#server_configuration_parameters-openssl) enables mandatory certificate validation for the incoming connections. In this case, only connections with trusted certificates can be established. Connections with untrusted certificates will be rejected. Thus, certificate validation allows to uniquely authenticate an incoming connection. `Common Name` field of the certificate is used to identify connected user. This allows to associate multiple certificates with the same user. Additionally, reissuing and revoking of the certificates does not affect the ClickHouse configuration. + +To enable SSL certificate authentication, a list of `Common Name`'s for each ClickHouse user must be sspecified in the settings file `config.xml `: + +**Example** +```xml + + + + + + host.domain.com:example_user + host.domain.com:example_user_dev + + + + + + +``` + +For the SSL [`chain of trust`](https://en.wikipedia.org/wiki/Chain_of_trust) to work correctly, it is also important to make sure that the [`caConfig`](../server-configuration-parameters/settings.md#server_configuration_parameters-openssl) parameter is configured properly. \ No newline at end of file diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantile.md b/docs/en/sql-reference/aggregate-functions/reference/quantile.md index d625ef4cfd9..b6f38e57342 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/quantile.md +++ b/docs/en/sql-reference/aggregate-functions/reference/quantile.md @@ -10,6 +10,8 @@ This function applies [reservoir sampling](https://en.wikipedia.org/wiki/Reservo When using multiple `quantile*` functions with different levels in a query, the internal states are not combined (that is, the query works less efficiently than it could). In this case, use the [quantiles](../../../sql-reference/aggregate-functions/reference/quantiles.md#quantiles) function. +Note that for an empty numeric sequence, `quantile` will return NaN, but its `quantile*` variants will return either NaN or a default value for the sequence type, depending on the variant. + **Syntax** ``` sql diff --git a/docs/en/sql-reference/functions/index.md b/docs/en/sql-reference/functions/index.md index ddc113d31f9..7cceec889bd 100644 --- a/docs/en/sql-reference/functions/index.md +++ b/docs/en/sql-reference/functions/index.md @@ -74,9 +74,10 @@ A function configuration contains the following settings: - `name` - a function name. - `command` - script name to execute or command if `execute_direct` is false. -- `argument` - argument description with the `type` of an argument. Each argument is described in a separate setting. +- `argument` - argument description with the `type`, and optional `name` of an argument. Each argument is described in a separate setting. Specifying name is necessary if argument names are part of serialization for user defined function format like [Native](../../interfaces/formats.md#native) or [JSONEachRow](../../interfaces/formats.md#jsoneachrow). Default argument name value is `c` + argument_number. - `format` - a [format](../../interfaces/formats.md) in which arguments are passed to the command. - `return_type` - the type of a returned value. +- `return_name` - name of retuned value. Specifying return name is necessary if return name is part of serialization for user defined function format like [Native](../../interfaces/formats.md#native) or [JSONEachRow](../../interfaces/formats.md#jsoneachrow). Optional. Default value is `result`. - `type` - an executable type. If `type` is set to `executable` then single command is started. If it is set to `executable_pool` then a pool of commands is created. - `max_command_execution_time` - maximum execution time in seconds for processing block of data. This setting is valid for `executable_pool` commands only. Optional. Default value is `10`. - `command_termination_timeout` - time in seconds during which a command should finish after its pipe is closed. After that time `SIGTERM` is sent to the process executing the command. Optional. Default value is `10`. @@ -100,6 +101,7 @@ File test_function.xml. String UInt64 + value TabSeparated test_function.py @@ -144,9 +146,11 @@ File test_function.xml. UInt64 UInt64 + lhs UInt64 + rhs TabSeparated cd /; clickhouse-local --input-format TabSeparated --output-format TabSeparated --structure 'x UInt64, y UInt64' --query "SELECT x + y FROM table" @@ -169,6 +173,58 @@ Result: └─────────────────────────┘ ``` +Creating `test_function_sum_json` with named arguments and format [JSONEachRow](../../interfaces/formats.md#jsoneachrow) using XML configuration. +File test_function.xml. +```xml + + executable + test_function_sum_json + UInt64 + result_name + + UInt64 + argument_1 + + + UInt64 + argument_2 + + JSONEachRow + test_function_sum_json.py + +``` + +Script file inside `user_scripts` folder `test_function_sum_json.py`. + +```python +#!/usr/bin/python3 + +import sys +import json + +if __name__ == '__main__': + for line in sys.stdin: + value = json.loads(line) + first_arg = int(value['argument_1']) + second_arg = int(value['argument_2']) + result = {'result_name': first_arg + second_arg} + print(json.dumps(result), end='\n') + sys.stdout.flush() +``` + +Query: + +``` sql +SELECT test_function_sum_json(2, 2); +``` + +Result: + +``` text +┌─test_function_sum_json(2, 2)─┐ +│ 4 │ +└──────────────────────────────┘ +``` ## Error Handling {#error-handling} diff --git a/docs/en/sql-reference/functions/url-functions.md b/docs/en/sql-reference/functions/url-functions.md index 98c3135f2b4..c337af9b5b3 100644 --- a/docs/en/sql-reference/functions/url-functions.md +++ b/docs/en/sql-reference/functions/url-functions.md @@ -345,6 +345,21 @@ URLPathHierarchy('https://example.com/browse/CONV-6788') = ] ``` +### encodeURLComponent(URL) {#encodeurlcomponenturl} + +Returns the encoded URL. +Example: + +``` sql +SELECT encodeURLComponent('http://127.0.0.1:8123/?query=SELECT 1;') AS EncodedURL; +``` + +``` text +┌─EncodedURL───────────────────────────────────────────────┐ +│ http%3A%2F%2F127.0.0.1%3A8123%2F%3Fquery%3DSELECT%201%3B │ +└──────────────────────────────────────────────────────────┘ +``` + ### decodeURLComponent(URL) {#decodeurlcomponenturl} Returns the decoded URL. @@ -360,6 +375,21 @@ SELECT decodeURLComponent('http://127.0.0.1:8123/?query=SELECT%201%3B') AS Decod └────────────────────────────────────────┘ ``` +### encodeURLFormComponent(URL) {#encodeurlformcomponenturl} + +Returns the encoded URL. Follows rfc-1866, space(` `) is encoded as plus(`+`). +Example: + +``` sql +SELECT encodeURLFormComponent('http://127.0.0.1:8123/?query=SELECT 1 2+3') AS EncodedURL; +``` + +``` text +┌─EncodedURL────────────────────────────────────────────────┐ +│ http%3A%2F%2F127.0.0.1%3A8123%2F%3Fquery%3DSELECT+1+2%2B3 │ +└───────────────────────────────────────────────────────────┘ +``` + ### decodeURLFormComponent(URL) {#decodeurlformcomponenturl} Returns the decoded URL. Follows rfc-1866, plain plus(`+`) is decoded as space(` `). diff --git a/docs/ja/operations/external-authenticators/ssl-x509.md b/docs/ja/operations/external-authenticators/ssl-x509.md new file mode 120000 index 00000000000..80b18e1b352 --- /dev/null +++ b/docs/ja/operations/external-authenticators/ssl-x509.md @@ -0,0 +1 @@ +../../../en/operations/external-authenticators/ssl-x509.md \ No newline at end of file diff --git a/docs/ru/operations/external-authenticators/index.md b/docs/ru/operations/external-authenticators/index.md index 8465e57d792..9fb07450d7c 100644 --- a/docs/ru/operations/external-authenticators/index.md +++ b/docs/ru/operations/external-authenticators/index.md @@ -12,5 +12,6 @@ ClickHouse поддерживает аутентификацию и управл - [LDAP](./ldap.md#external-authenticators-ldap) [аутентификатор](./ldap.md#ldap-external-authenticator) и [каталог](./ldap.md#ldap-external-user-directory) - Kerberos [аутентификатор](./kerberos.md#external-authenticators-kerberos) +- [SSL X.509 аутентификация](./ssl-x509.md#ssl-external-authentication) [Оригинальная статья](https://clickhouse.com/docs/ru/operations/external-authenticators/index/) diff --git a/docs/ru/operations/external-authenticators/ssl-x509.md b/docs/ru/operations/external-authenticators/ssl-x509.md new file mode 100644 index 00000000000..12ae7e4eec3 --- /dev/null +++ b/docs/ru/operations/external-authenticators/ssl-x509.md @@ -0,0 +1,24 @@ +# Аутентификация по сертификату SSL X.509 {#ssl-external-authentication} + +[Опция 'strict'](../server-configuration-parameters/settings.md#server_configuration_parameters-openssl) включает обязательную проверку сертификатов входящих соединений в библиотеке `SSL`. В этом случае могут быть установлены только соединения, представившие действительный сертификат. Соединения с недоверенными сертификатами будут отвергнуты. Таким образом, проверка сертификата позволяет однозначно аутентифицировать входящее соединение. Идентификация пользователя осуществляется по полю `Common Name` сертификата. Это позволяет ассоциировать несколько сертификатов с одним и тем же пользователем. Дополнительно, перевыпуск и отзыв сертификата не требуют изменения конфигурации ClickHouse. + +Для включения аутентификации по SSL сертификату, необходимо указать список `Common Name` для каждого пользователя ClickHouse в файле настройки `config.xml`: + +**Example** +```xml + + + + + + host.domain.com:example_user + host.domain.com:example_user_dev + + + + + + +``` + +Для правильной работы SSL [`chain of trust`](https://en.wikipedia.org/wiki/Chain_of_trust) важно также убедиться в правильной настройке параметра [`caConfig`](../server-configuration-parameters/settings.md#server_configuration_parameters-openssl) diff --git a/docs/ru/sql-reference/functions/index.md b/docs/ru/sql-reference/functions/index.md index a63c76d8833..30b9f18ec58 100644 --- a/docs/ru/sql-reference/functions/index.md +++ b/docs/ru/sql-reference/functions/index.md @@ -72,37 +72,89 @@ ClickHouse может вызывать внешнюю программу или Конфигурация функции содержит следующие настройки: - `name` - имя функции. -- `command` - исполняемая команда или скрипт. -- `argument` - описание аргумента, содержащее его тип во вложенной настройке `type`. Каждый аргумент описывается отдельно. +- `command` - имя скрипта для выполнения или команды, если `execute_direct` равно false. +- `argument` - описание аргумента, содержащее его тип во вложенной настройке `type`, и опционально его имя во вложенной настройке `name`. Каждый аргумент описывается отдельно. Указание имени для аргумента необходимо, если имена аргументов являются частью сериализации для пользовательского формата функции, например [Native](../../interfaces/formats.md#native) или [JSONEachRow](../../interfaces/formats.md#jsoneachrow). Значение имени аргумента по умолчанию `c` + номер аргумента. - `format` - [формат](../../interfaces/formats.md) передачи аргументов. - `return_type` - тип возвращаемого значения. +- `return_name` - имя возвращаемого значения. Указание имени возвращаемого значения необходимо, если имя возвращаемого значения является частью сериализации для пользовательского формата функции, например [Native](../../interfaces/formats.md#native) или [JSONEachRow](../../interfaces/formats.md#jsoneachrow). Необязательный. Значение по умолчанию — `result`. - `type` - вариант запуска команды. Если задан вариант `executable`, то запускается одна команда. При указании `executable_pool` создается пул команд. - `max_command_execution_time` - максимальное время в секундах, которое отводится на обработку блока данных. Эта настройка применима только для команд с вариантом запуска `executable_pool`. Необязательная настройка. Значение по умолчанию `10`. - `command_termination_timeout` - максимальное время завершения команды в секундах после закрытия конвейера. Если команда не завершается, то процессу отправляется сигнал `SIGTERM`. Эта настройка применима только для команд с вариантом запуска `executable_pool`. Необязательная настройка. Значение по умолчанию `10`. +- `command_read_timeout` - время ожидания чтения данных из команды stdout в миллисекундах. Значение по умолчанию 10000. Необязательная настройка. +- `command_write_timeout` - время ожидания записи данных в команду stdin в миллисекундах. Значение по умолчанию 10000. Необязательная настройка. - `pool_size` - размер пула команд. Необязательная настройка. Значение по умолчанию `16`. -- `lifetime` - интервал перезагрузки функций в секундах. Если задан `0`, то функция не перезагружается. - `send_chunk_header` - управляет отправкой количества строк перед отправкой блока данных для обработки. Необязательная настройка. Значение по умолчанию `false`. +- `execute_direct` - Если `execute_direct` = `1`, то будет произведен поиск `command` в папке user_scripts. Дополнительные аргументы скрипта можно указать с помощью разделителя пробелов. Пример: `script_name arg1 arg2`. Если `execute_direct` = `0`, `command` передается как аргумент для `bin/sh -c`. Значение по умолчанию `1`. Необязательный параметр. +- `lifetime` - интервал перезагрузки функций в секундах. Если задан `0`, то функция не перезагружается. Команда должна читать аргументы из `STDIN` и выводить результат в `STDOUT`. Обработка должна выполняться в цикле. То есть после обработки группы аргументов команда должна ожидать следующую группу. **Пример** -XML конфигурация, описывающая функцию `test_function`: -``` +Создание `test_function` с использованием конфигурации XML. +Файл test_function.xml. +```xml executable - test_function + test_function_python + String + + UInt64 + value + + TabSeparated + test_function.py + + +``` + +Файл скрипта внутри папки `user_scripts` `test_function.py`. + +```python +#!/usr/bin/python3 + +import sys + +if __name__ == '__main__': + for line in sys.stdin: + print("Value " + line, end='') + sys.stdout.flush() +``` + +Запрос: + +``` sql +SELECT test_function_python(toUInt64(2)); +``` + +Результат: + +``` text +┌─test_function_python(2)─┐ +│ Value 2 │ +└─────────────────────────┘ +``` + +Создание `test_function_sum`, указав для `execute_direct` значение `0`, используя конфигурацию XML. +File test_function.xml. +```xml + + + executable + test_function_sum UInt64 UInt64 + lhs UInt64 + rhs TabSeparated cd /; clickhouse-local --input-format TabSeparated --output-format TabSeparated --structure 'x UInt64, y UInt64' --query "SELECT x + y FROM table" - 0 + 0 ``` @@ -110,15 +162,68 @@ XML конфигурация, описывающая функцию `test_functi Запрос: ``` sql -SELECT test_function(toUInt64(2), toUInt64(2)); +SELECT test_function_sum(2, 2); ``` Результат: ``` text -┌─test_function(toUInt64(2), toUInt64(2))─┐ -│ 4 │ -└─────────────────────────────────────────┘ +┌─test_function_sum(2, 2)─┐ +│ 4 │ +└─────────────────────────┘ +``` + +Создание `test_function_sum_json` с именноваными аргументами и форматом [JSONEachRow](../../interfaces/formats.md#jsoneachrow) с использованием конфигурации XML. +Файл test_function.xml. +```xml + + executable + test_function_sum_json + UInt64 + result_name + + UInt64 + argument_1 + + + UInt64 + argument_2 + + JSONEachRow + test_function_sum_json.py + +``` + +Файл скрипта внутри папки `user_scripts` `test_function_sum_json.py`. + +```python +#!/usr/bin/python3 + +import sys +import json + +if __name__ == '__main__': + for line in sys.stdin: + value = json.loads(line) + first_arg = int(value['argument_1']) + second_arg = int(value['argument_2']) + result = {'result_name': first_arg + second_arg} + print(json.dumps(result), end='\n') + sys.stdout.flush() +``` + +Запрос: + +``` sql +SELECT test_function_sum_json(2, 2); +``` + +Результат: + +``` text +┌─test_function_sum_json(2, 2)─┐ +│ 4 │ +└──────────────────────────────┘ ``` ## Обработка ошибок {#obrabotka-oshibok} diff --git a/docs/zh/development/continuous-integration.md b/docs/zh/development/continuous-integration.md index 6cff83067de..ae0367f276e 100644 --- a/docs/zh/development/continuous-integration.md +++ b/docs/zh/development/continuous-integration.md @@ -46,14 +46,6 @@ git push - [状态页示例](https://clickhouse-test-reports.s3.yandex.net/12550/659c78c7abb56141723af6a81bfae39335aa8cb2/style_check.html) - `docs_output.txt`记录了查结果错误(无效表格等), 空白页表示没有错误. [成功结果案例](https://clickhouse-test-reports.s3.yandex.net/12550/659c78c7abb56141723af6a81bfae39335aa8cb2/style_check/output.txt) -### PVS 检查 {#pvs-check} -使用静态分析工具[PVS-studio](https://www.viva64.com/en/pvs-studio/)检查代码. 查看报告以查看确切的错误.如果可以则修复它们, 如果不行, 可以向ClickHouse的维护人员寻求帮忙. - -### 报告详情 {#report-details} -- [状态页示例](https://clickhouse-test-reports.s3.yandex.net/12550/67d716b5cc3987801996c31a67b31bf141bc3486/pvs_check.html) -- `test_run.txt.out.log`包含构建和分析日志文件.它只包含解析或未找到的错误. -- `HTML report`包含分析结果.有关说明请访问PVS的[官方网站](https://www.viva64.com/en/m/0036/#ID14E9A2B2CD) - ## 快速测试 {#fast-test} 通常情况下这是PR运行的第一个检查.它构建ClickHouse以及大多数无状态运行测试, 其中省略了一些.如果失败，在修复之前不会开始进一步的检查. 查看报告以了解哪些测试失败, 然后按照[此处](./tests.md#functional-test-locally)描述的在本地重现失败. diff --git a/docs/zh/development/tests.md b/docs/zh/development/tests.md index 9389e8871c4..43b37fffa9f 100644 --- a/docs/zh/development/tests.md +++ b/docs/zh/development/tests.md @@ -254,7 +254,7 @@ Yandex安全团队的人员从安全的角度对ClickHouse的功能做了一些 ## 静态分析仪 {#static-analyzers} -我们在每次提交的基础上运行 `clang-tidy` 和 `PVS-Studio`. `clang-static-analyzer` 检查也被启用. `clang-tidy` 也用于一些样式检查. +我们在每次提交的基础上运行 `clang-tidy`. `clang-static-analyzer` 检查也被启用. `clang-tidy` 也用于一些样式检查. 我们已经评估了 `clang-tidy`、`Coverity`、`cppcheck`、`PVS-Studio`、`tscancode`、`CodeQL`. 您将在 `tests/instructions/` 目录中找到使用说明. 你也可以阅读[俄文文章](https://habr.com/company/yandex/blog/342018/). diff --git a/docs/zh/engines/table-engines/integrations/kafka.md b/docs/zh/engines/table-engines/integrations/kafka.md index ee6bbbe67fc..0a8fbdbf5ae 100644 --- a/docs/zh/engines/table-engines/integrations/kafka.md +++ b/docs/zh/engines/table-engines/integrations/kafka.md @@ -153,8 +153,9 @@ clickhouse也支持自己使用keyfile的方式来维护kerbros的凭证。配 - `_topic` – Kafka 主题。 - `_key` – 信息的键。 - `_offset` – 消息的偏移量。 -- `_timestamp ` – 消息的时间戳。 -- `_partition ` – Kafka 主题的分区。 +- `_timestamp` – 消息的时间戳。 +- `_timestamp_ms` – 消息的时间戳(毫秒)。 +- `_partition` – Kafka 主题的分区。 **另请参阅** diff --git a/docs/zh/engines/table-engines/integrations/sqlite.md b/docs/zh/engines/table-engines/integrations/sqlite.md deleted file mode 120000 index 56d0a490f52..00000000000 --- a/docs/zh/engines/table-engines/integrations/sqlite.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/engines/table-engines/integrations/sqlite.md \ No newline at end of file diff --git a/docs/zh/engines/table-engines/integrations/sqlite.md b/docs/zh/engines/table-engines/integrations/sqlite.md new file mode 100644 index 00000000000..f492148ed10 --- /dev/null +++ b/docs/zh/engines/table-engines/integrations/sqlite.md @@ -0,0 +1,59 @@ +--- +toc_priority: 7 +toc_title: SQLite +--- + +# SQLite {#sqlite} + +该引擎允许导入和导出数据到SQLite，并支持查询SQLite表直接从ClickHouse。 + +## 创建数据表 {#creating-a-table} + +``` sql + CREATE TABLE [IF NOT EXISTS] [db.]table_name + ( + name1 [type1], + name2 [type2], ... + ) ENGINE = SQLite('db_path', 'table') +``` + +**引擎参数** + +- `db_path` — SQLite数据库文件的具体路径地址。 +- `table` — SQLite数据库中的表名。 + +## 使用示例 {#usage-example} + +显示创建表的查询语句： + +```sql +SHOW CREATE TABLE sqlite_db.table2; +``` + +``` text +CREATE TABLE SQLite.table2 +( + `col1` Nullable(Int32), + `col2` Nullable(String) +) +ENGINE = SQLite('sqlite.db','table2'); +``` + +从数据表查询数据： + +``` sql +SELECT * FROM sqlite_db.table2 ORDER BY col1; +``` + +```text +┌─col1─┬─col2──┐ +│ 1 │ text1 │ +│ 2 │ text2 │ +│ 3 │ text3 │ +└──────┴───────┘ +``` + +**详见** + +- [SQLite](../../../engines/database-engines/sqlite.md) 引擎 +- [sqlite](../../../sql-reference/table-functions/sqlite.md) 表方法函数 \ No newline at end of file diff --git a/docs/zh/engines/table-engines/special/memory.md b/docs/zh/engines/table-engines/special/memory.md index 2e880934652..f906d64d4b3 100644 --- a/docs/zh/engines/table-engines/special/memory.md +++ b/docs/zh/engines/table-engines/special/memory.md @@ -1,7 +1,12 @@ +--- +toc_priority: 44 +toc_title: Memory +--- + # 内存表 {#memory} Memory 引擎以未压缩的形式将数据存储在 RAM 中。数据完全以读取时获得的形式存储。换句话说，从这张表中读取是很轻松的。并发数据访问是同步的。锁范围小：读写操作不会相互阻塞。不支持索引。查询是并行化的。在简单查询上达到最大速率（超过10 GB /秒），因为没有磁盘读取，不需要解压缩或反序列化数据。（值得注意的是，在许多情况下，与 MergeTree 引擎的性能几乎一样高）。重新启动服务器时，表中的数据消失，表将变为空。通常，使用此表引擎是不合理的。但是，它可用于测试，以及在相对较少的行（最多约100,000,000）上需要最高性能的查询。 Memory 引擎是由系统用于临时表进行外部数据的查询（请参阅 «外部数据用于请求处理» 部分），以及用于实现 `GLOBAL IN`（请参见 «IN 运算符» 部分）。 -[原始文章](https://clickhouse.com/docs/zh/operations/table_engines/memory/) +[原始文章](https://clickhouse.com/docs/en/operations/table_engines/memory/) diff --git a/docs/zh/getting-started/example-datasets/github-events.md b/docs/zh/getting-started/example-datasets/github-events.md deleted file mode 120000 index c9649c0a61f..00000000000 --- a/docs/zh/getting-started/example-datasets/github-events.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/getting-started/example-datasets/github-events.md \ No newline at end of file diff --git a/docs/zh/getting-started/example-datasets/github-events.md b/docs/zh/getting-started/example-datasets/github-events.md new file mode 100644 index 00000000000..eabc3c1bf1f --- /dev/null +++ b/docs/zh/getting-started/example-datasets/github-events.md @@ -0,0 +1,10 @@ +--- +toc_priority: 11 +toc_title: GitHub 事件数据集 +--- + +# GitHub 事件数据集 + +数据集包含了GitHub上从2011年到2020年12月6日的所有事件，大小为31亿条记录。下载大小为75 GB，如果存储在使用lz4压缩的表中，则需要多达200 GB的磁盘空间。 + +完整的数据集描述，见解，下载说明和交互式查询请参考[这里](https://ghe.clickhouse.tech/)。 \ No newline at end of file diff --git a/docs/zh/getting-started/install.md b/docs/zh/getting-started/install.md index 1bf8ecd9826..eec3aabe2a1 100644 --- a/docs/zh/getting-started/install.md +++ b/docs/zh/getting-started/install.md @@ -66,14 +66,14 @@ sudo yum install clickhouse-server clickhouse-client 所需的版本可以通过`curl`或`wget`从存储库`https://repo.clickhouse.com/tgz/`下载。 -下载后解压缩下载资源文件并使用安装脚本进行安装。以下是一个最新版本的安装示例: +下载后解压缩下载资源文件并使用安装脚本进行安装。以下是一个最新稳定版本的安装示例: ``` bash -export LATEST_VERSION=`curl https://api.github.com/repos/ClickHouse/ClickHouse/tags 2>/dev/null | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -n 1` -curl -O https://repo.clickhouse.com/tgz/clickhouse-common-static-$LATEST_VERSION.tgz -curl -O https://repo.clickhouse.com/tgz/clickhouse-common-static-dbg-$LATEST_VERSION.tgz -curl -O https://repo.clickhouse.com/tgz/clickhouse-server-$LATEST_VERSION.tgz -curl -O https://repo.clickhouse.com/tgz/clickhouse-client-$LATEST_VERSION.tgz +export LATEST_VERSION=`curl https://api.github.com/repos/ClickHouse/ClickHouse/tags 2>/dev/null | grep stable | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -n 1` +curl -O https://repo.clickhouse.com/tgz/stable/clickhouse-common-static-$LATEST_VERSION.tgz +curl -O https://repo.clickhouse.com/tgz/stable/clickhouse-common-static-dbg-$LATEST_VERSION.tgz +curl -O https://repo.clickhouse.com/tgz/stable/clickhouse-server-$LATEST_VERSION.tgz +curl -O https://repo.clickhouse.com/tgz/stable/clickhouse-client-$LATEST_VERSION.tgz tar -xzvf clickhouse-common-static-$LATEST_VERSION.tgz sudo clickhouse-common-static-$LATEST_VERSION/install/doinst.sh diff --git a/docs/zh/operations/external-authenticators/ssl-x509.md b/docs/zh/operations/external-authenticators/ssl-x509.md new file mode 120000 index 00000000000..80b18e1b352 --- /dev/null +++ b/docs/zh/operations/external-authenticators/ssl-x509.md @@ -0,0 +1 @@ +../../../en/operations/external-authenticators/ssl-x509.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/alter/delete.md b/docs/zh/sql-reference/statements/alter/delete.md deleted file mode 120000 index b9d6a233197..00000000000 --- a/docs/zh/sql-reference/statements/alter/delete.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/statements/alter/delete.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/alter/delete.md b/docs/zh/sql-reference/statements/alter/delete.md new file mode 100644 index 00000000000..390c7becb14 --- /dev/null +++ b/docs/zh/sql-reference/statements/alter/delete.md @@ -0,0 +1,27 @@ +--- +toc_priority: 39 +toc_title: DELETE +--- + +# ALTER TABLE … DELETE 语句 {#alter-mutations} + +``` sql +ALTER TABLE [db.]table [ON CLUSTER cluster] DELETE WHERE filter_expr +``` + +删除匹配指定过滤表达式的数据。实现为[突变](../../../sql-reference/statements/alter/index.md#mutations). + +!!! note "备注" + `ALTER TABLE`前缀使得这个语法不同于大多数其他支持SQL的系统。它的目的是表示，与OLTP数据库中的类似查询不同，这是一个不为经常使用而设计的繁重操作。 + +`filter_expr` 的类型必须是`UInt8`。该查询删除表中该表达式接受非零值的行。 + +一个查询可以包含多个用逗号分隔的命令。 + +查询处理的同步性由[mutations_sync](../../../operations/settings/settings.md#mutations_sync)设置定义。缺省情况下，是异步的。 + +**详见** + +- [突变](../../../sql-reference/statements/alter/index.md#mutations) +- [ALTER查询的同步性](../../../sql-reference/statements/alter/index.md#synchronicity-of-alter-queries) +- [mutations_sync](../../../operations/settings/settings.md#mutations_sync) setting diff --git a/docs/zh/sql-reference/statements/alter/order-by.md b/docs/zh/sql-reference/statements/alter/order-by.md deleted file mode 120000 index ef5a22dc165..00000000000 --- a/docs/zh/sql-reference/statements/alter/order-by.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/statements/alter/order-by.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/alter/order-by.md b/docs/zh/sql-reference/statements/alter/order-by.md new file mode 100644 index 00000000000..112571b55f6 --- /dev/null +++ b/docs/zh/sql-reference/statements/alter/order-by.md @@ -0,0 +1,17 @@ +--- +toc_priority: 41 +toc_title: ORDER BY +--- + +# 操作排序键表达式 {#manipulations-with-key-expressions} + +```sql +ALTER TABLE [db].name [ON CLUSTER cluster] MODIFY ORDER BY new_expression +``` +该命令将表的[排序键](../../../engines/table-engines/mergetree-family/mergetree.md)更改为 `new_expression`(表达式或表达式元组)。主键保持不变。 + +从某种意义上说，该命令是轻量级的，它只更改元数据。要保持数据部分行按排序键表达式排序的属性，您不能向排序键添加包含现有列的表达式(仅在相同的`ALTER`查询中由`ADD COLUMN`命令添加的列，没有默认的列值)。 + + +!!! note "备注" + 它只适用于[`MergeTree`](../../../engines/table-engines/mergetree-family/mergetree.md)表族(包括[replicated](../../../engines/table-engines/mergetree-family/replication.md)表)。 diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 2cbf39a61da..c40f41cd8d1 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -486,12 +486,19 @@ void Client::connect() UInt64 server_version_minor = 0; UInt64 server_version_patch = 0; + if (hosts_and_ports.empty()) + { + String host = config().getString("host", "localhost"); + UInt16 port = static_cast(ConnectionParameters::getPortFromConfig(config())); + hosts_and_ports.emplace_back(HostAndPort{host, port}); + } + for (size_t attempted_address_index = 0; attempted_address_index < hosts_and_ports.size(); ++attempted_address_index) { try { - connection_parameters - = ConnectionParameters(config(), hosts_and_ports[attempted_address_index].host, hosts_and_ports[attempted_address_index].port); + connection_parameters = ConnectionParameters( + config(), hosts_and_ports[attempted_address_index].host, hosts_and_ports[attempted_address_index].port); if (is_interactive) std::cout << "Connecting to " @@ -1085,22 +1092,15 @@ void Client::processOptions(const OptionsDescription & options_description, } } - if (hosts_and_ports_arguments.empty()) + for (const auto & hosts_and_ports_argument : hosts_and_ports_arguments) { - hosts_and_ports.emplace_back(HostAndPort{"localhost", DBMS_DEFAULT_PORT}); - } - else - { - for (const auto & hosts_and_ports_argument : hosts_and_ports_arguments) - { - /// Parse commandline options related to external tables. - po::parsed_options parsed_hosts_and_ports - = po::command_line_parser(hosts_and_ports_argument).options(options_description.hosts_and_ports_description.value()).run(); - po::variables_map host_and_port_options; - po::store(parsed_hosts_and_ports, host_and_port_options); - hosts_and_ports.emplace_back( - HostAndPort{host_and_port_options["host"].as(), host_and_port_options["port"].as()}); - } + /// Parse commandline options related to external tables. + po::parsed_options parsed_hosts_and_ports + = po::command_line_parser(hosts_and_ports_argument).options(options_description.hosts_and_ports_description.value()).run(); + po::variables_map host_and_port_options; + po::store(parsed_hosts_and_ports, host_and_port_options); + hosts_and_ports.emplace_back( + HostAndPort{host_and_port_options["host"].as(), host_and_port_options["port"].as()}); } send_external_tables = true; diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index 6bc9aeec4c2..f82cb564db5 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -87,6 +87,9 @@ bool Authentication::areCredentialsValid(const Credentials & credentials, const case AuthenticationType::KERBEROS: return external_authenticators.checkKerberosCredentials(auth_data.getKerberosRealm(), *gss_acceptor_context); + case AuthenticationType::SSL_CERTIFICATE: + throw Authentication::Require("ClickHouse X.509 Authentication"); + case AuthenticationType::MAX: break; } @@ -110,6 +113,9 @@ bool Authentication::areCredentialsValid(const Credentials & credentials, const case AuthenticationType::KERBEROS: throw Authentication::Require("ClickHouse Basic Authentication"); + case AuthenticationType::SSL_CERTIFICATE: + throw Authentication::Require("ClickHouse X.509 Authentication"); + case AuthenticationType::MAX: break; } @@ -137,6 +143,31 @@ bool Authentication::areCredentialsValid(const Credentials & credentials, const case AuthenticationType::KERBEROS: throw Authentication::Require(auth_data.getKerberosRealm()); + case AuthenticationType::SSL_CERTIFICATE: + throw Authentication::Require("ClickHouse X.509 Authentication"); + + case AuthenticationType::MAX: + break; + } + } + + if (const auto * ssl_certificate_credentials = typeid_cast(&credentials)) + { + switch (auth_data.getType()) + { + case AuthenticationType::NO_PASSWORD: + case AuthenticationType::PLAINTEXT_PASSWORD: + case AuthenticationType::SHA256_PASSWORD: + case AuthenticationType::DOUBLE_SHA1_PASSWORD: + case AuthenticationType::LDAP: + throw Authentication::Require("ClickHouse Basic Authentication"); + + case AuthenticationType::KERBEROS: + throw Authentication::Require(auth_data.getKerberosRealm()); + + case AuthenticationType::SSL_CERTIFICATE: + return auth_data.getSSLCertificateCommonNames().contains(ssl_certificate_credentials->getCommonName()); + case AuthenticationType::MAX: break; } diff --git a/src/Access/Authentication.h b/src/Access/Authentication.h index 000ba8ca324..ab787851cb2 100644 --- a/src/Access/Authentication.h +++ b/src/Access/Authentication.h @@ -15,7 +15,6 @@ namespace ErrorCodes class Credentials; class ExternalAuthenticators; - /// TODO: Try to move this checking to Credentials. struct Authentication { diff --git a/src/Access/Common/AuthenticationData.cpp b/src/Access/Common/AuthenticationData.cpp index 7412d7336e3..e945b5e435b 100644 --- a/src/Access/Common/AuthenticationData.cpp +++ b/src/Access/Common/AuthenticationData.cpp @@ -59,6 +59,11 @@ const AuthenticationTypeInfo & AuthenticationTypeInfo::get(AuthenticationType ty static const auto info = make_info("KERBEROS"); return info; } + case AuthenticationType::SSL_CERTIFICATE: + { + static const auto info = make_info("SSL_CERTIFICATE"); + return info; + } case AuthenticationType::MAX: break; } @@ -92,7 +97,8 @@ AuthenticationData::Digest AuthenticationData::Util::encodeSHA1(const std::strin bool operator ==(const AuthenticationData & lhs, const AuthenticationData & rhs) { return (lhs.type == rhs.type) && (lhs.password_hash == rhs.password_hash) - && (lhs.ldap_server_name == rhs.ldap_server_name) && (lhs.kerberos_realm == rhs.kerberos_realm); + && (lhs.ldap_server_name == rhs.ldap_server_name) && (lhs.kerberos_realm == rhs.kerberos_realm) + && (lhs.ssl_certificate_common_names == rhs.ssl_certificate_common_names); } @@ -112,6 +118,7 @@ void AuthenticationData::setPassword(const String & password_) case AuthenticationType::NO_PASSWORD: case AuthenticationType::LDAP: case AuthenticationType::KERBEROS: + case AuthenticationType::SSL_CERTIFICATE: throw Exception("Cannot specify password for authentication type " + toString(type), ErrorCodes::LOGICAL_ERROR); case AuthenticationType::MAX: @@ -149,7 +156,7 @@ void AuthenticationData::setPasswordHashHex(const String & hash) String AuthenticationData::getPasswordHashHex() const { - if (type == AuthenticationType::LDAP || type == AuthenticationType::KERBEROS) + if (type == AuthenticationType::LDAP || type == AuthenticationType::KERBEROS || type == AuthenticationType::SSL_CERTIFICATE) throw Exception("Cannot get password hex hash for authentication type " + toString(type), ErrorCodes::LOGICAL_ERROR); String hex; @@ -194,6 +201,7 @@ void AuthenticationData::setPasswordHashBinary(const Digest & hash) case AuthenticationType::NO_PASSWORD: case AuthenticationType::LDAP: case AuthenticationType::KERBEROS: + case AuthenticationType::SSL_CERTIFICATE: throw Exception("Cannot specify password binary hash for authentication type " + toString(type), ErrorCodes::LOGICAL_ERROR); case AuthenticationType::MAX: @@ -202,4 +210,12 @@ void AuthenticationData::setPasswordHashBinary(const Digest & hash) throw Exception("setPasswordHashBinary(): authentication type " + toString(type) + " not supported", ErrorCodes::NOT_IMPLEMENTED); } + +void AuthenticationData::setSSLCertificateCommonNames(boost::container::flat_set common_names_) +{ + if (common_names_.empty()) + throw Exception("The 'SSL CERTIFICATE' authentication type requires a non-empty list of common names.", ErrorCodes::BAD_ARGUMENTS); + ssl_certificate_common_names = std::move(common_names_); +} + } diff --git a/src/Access/Common/AuthenticationData.h b/src/Access/Common/AuthenticationData.h index 8b390fd0900..3573d28d45e 100644 --- a/src/Access/Common/AuthenticationData.h +++ b/src/Access/Common/AuthenticationData.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include namespace DB @@ -27,6 +28,10 @@ enum class AuthenticationType /// Kerberos authentication performed through GSS-API negotiation loop. KERBEROS, + /// Authentication is done in SSL by checking user certificate. + /// Certificates may only be trusted if 'strict' SSL mode is enabled. + SSL_CERTIFICATE, + MAX, }; @@ -79,6 +84,9 @@ public: const String & getKerberosRealm() const { return kerberos_realm; } void setKerberosRealm(const String & realm) { kerberos_realm = realm; } + const boost::container::flat_set & getSSLCertificateCommonNames() const { return ssl_certificate_common_names; } + void setSSLCertificateCommonNames(boost::container::flat_set common_names_); + friend bool operator ==(const AuthenticationData & lhs, const AuthenticationData & rhs); friend bool operator !=(const AuthenticationData & lhs, const AuthenticationData & rhs) { return !(lhs == rhs); } @@ -97,6 +105,7 @@ private: Digest password_hash; String ldap_server_name; String kerberos_realm; + boost::container::flat_set ssl_certificate_common_names; }; } diff --git a/src/Access/Credentials.cpp b/src/Access/Credentials.cpp index c2850ad4d4f..9878cdb9df3 100644 --- a/src/Access/Credentials.cpp +++ b/src/Access/Credentials.cpp @@ -48,6 +48,20 @@ void AlwaysAllowCredentials::setUserName(const String & user_name_) user_name = user_name_; } +SSLCertificateCredentials::SSLCertificateCredentials(const String & user_name_, const String & common_name_) + : Credentials(user_name_) + , common_name(common_name_) +{ + is_ready = true; +} + +const String & SSLCertificateCredentials::getCommonName() const +{ + if (!isReady()) + throwNotReady(); + return common_name; +} + BasicCredentials::BasicCredentials() { is_ready = true; diff --git a/src/Access/Credentials.h b/src/Access/Credentials.h index 4992199f64c..d0c9efcfd8c 100644 --- a/src/Access/Credentials.h +++ b/src/Access/Credentials.h @@ -38,6 +38,17 @@ public: void setUserName(const String & user_name_); }; +class SSLCertificateCredentials + : public Credentials +{ +public: + explicit SSLCertificateCredentials(const String & user_name_, const String & common_name_); + const String & getCommonName() const; + +private: + String common_name; +}; + class BasicCredentials : public Credentials { diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index 5bd2da97445..a06ceabd808 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -62,13 +62,16 @@ namespace bool has_ldap = config.has(user_config + ".ldap"); bool has_kerberos = config.has(user_config + ".kerberos"); - size_t num_password_fields = has_no_password + has_password_plaintext + has_password_sha256_hex + has_password_double_sha1_hex + has_ldap + has_kerberos; + const auto certificates_config = user_config + ".ssl_certificates"; + bool has_certificates = config.has(certificates_config); + + size_t num_password_fields = has_no_password + has_password_plaintext + has_password_sha256_hex + has_password_double_sha1_hex + has_ldap + has_kerberos + has_certificates; if (num_password_fields > 1) - throw Exception("More than one field of 'password', 'password_sha256_hex', 'password_double_sha1_hex', 'no_password', 'ldap', 'kerberos' are used to specify password for user " + user_name + ". Must be only one of them.", + throw Exception("More than one field of 'password', 'password_sha256_hex', 'password_double_sha1_hex', 'no_password', 'ldap', 'kerberos', 'certificates' are used to specify authentication info for user " + user_name + ". Must be only one of them.", ErrorCodes::BAD_ARGUMENTS); if (num_password_fields < 1) - throw Exception("Either 'password' or 'password_sha256_hex' or 'password_double_sha1_hex' or 'no_password' or 'ldap' or 'kerberos' must be specified for user " + user_name + ".", ErrorCodes::BAD_ARGUMENTS); + throw Exception("Either 'password' or 'password_sha256_hex' or 'password_double_sha1_hex' or 'no_password' or 'ldap' or 'kerberos' or 'certificates' must be specified for user " + user_name + ".", ErrorCodes::BAD_ARGUMENTS); if (has_password_plaintext) { @@ -105,6 +108,26 @@ namespace user->auth_data = AuthenticationData{AuthenticationType::KERBEROS}; user->auth_data.setKerberosRealm(realm); } + else if (has_certificates) + { + user->auth_data = AuthenticationData{AuthenticationType::SSL_CERTIFICATE}; + + /// Fill list of allowed certificates. + Poco::Util::AbstractConfiguration::Keys keys; + config.keys(certificates_config, keys); + boost::container::flat_set common_names; + for (const String & key : keys) + { + if (key.starts_with("common_name")) + { + String value = config.getString(certificates_config + "." + key); + common_names.insert(std::move(value)); + } + else + throw Exception("Unknown certificate pattern type: " + key, ErrorCodes::BAD_ARGUMENTS); + } + user->auth_data.setSSLCertificateCommonNames(std::move(common_names)); + } const auto profile_name_config = user_config + ".profile"; if (config.has(profile_name_config)) diff --git a/src/AggregateFunctions/AggregateFunctionMinMaxAny.h b/src/AggregateFunctions/AggregateFunctionMinMaxAny.h index ce54cce9752..88400b2054d 100644 --- a/src/AggregateFunctions/AggregateFunctionMinMaxAny.h +++ b/src/AggregateFunctions/AggregateFunctionMinMaxAny.h @@ -49,6 +49,7 @@ private: public: static constexpr bool is_nullable = false; + static constexpr bool is_any = false; bool has() const { @@ -471,6 +472,7 @@ private: public: static constexpr bool is_nullable = false; + static constexpr bool is_any = false; bool has() const { @@ -696,6 +698,7 @@ private: public: static constexpr bool is_nullable = false; + static constexpr bool is_any = false; bool has() const { @@ -930,6 +933,7 @@ template struct AggregateFunctionAnyData : Data { using Self = AggregateFunctionAnyData; + static constexpr bool is_any = true; bool changeIfBetter(const IColumn & column, size_t row_num, Arena * arena) { return this->changeFirstTime(column, row_num, arena); } bool changeIfBetter(const Self & to, Arena * arena) { return this->changeFirstTime(to, arena); } @@ -1120,6 +1124,8 @@ struct AggregateFunctionAnyHeavyData : Data template class AggregateFunctionsSingleValue final : public IAggregateFunctionDataHelper> { + static constexpr bool is_any = Data::is_any; + private: SerializationPtr serialization; @@ -1152,6 +1158,75 @@ public: this->data(place).changeIfBetter(*columns[0], row_num, arena); } + void addBatchSinglePlace( + size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, ssize_t if_argument_pos) const override + { + if constexpr (is_any) + if (this->data(place).has()) + return; + if (if_argument_pos >= 0) + { + const auto & flags = assert_cast(*columns[if_argument_pos]).getData(); + for (size_t i = 0; i < batch_size; ++i) + { + if (flags[i]) + { + this->data(place).changeIfBetter(*columns[0], i, arena); + if constexpr (is_any) + break; + } + } + } + else + { + for (size_t i = 0; i < batch_size; ++i) + { + this->data(place).changeIfBetter(*columns[0], i, arena); + if constexpr (is_any) + break; + } + } + } + + void addBatchSinglePlaceNotNull( + size_t batch_size, + AggregateDataPtr place, + const IColumn ** columns, + const UInt8 * null_map, + Arena * arena, + ssize_t if_argument_pos = -1) const override + { + if constexpr (is_any) + if (this->data(place).has()) + return; + + if (if_argument_pos >= 0) + { + const auto & flags = assert_cast(*columns[if_argument_pos]).getData(); + for (size_t i = 0; i < batch_size; ++i) + { + if (!null_map[i] && flags[i]) + { + this->data(place).changeIfBetter(*columns[0], i, arena); + if constexpr (is_any) + break; + } + } + } + else + { + for (size_t i = 0; i < batch_size; ++i) + { + if (!null_map[i]) + { + this->data(place).changeIfBetter(*columns[0], i, arena); + if constexpr (is_any) + break; + } + } + } + } + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena * arena) const override { this->data(place).changeIfBetter(this->data(rhs), arena); diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 6fd8c0cb9bc..2829f9e9f46 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -120,8 +120,7 @@ static void incrementProfileEventsBlock(Block & dst, const Block & src) { if (!dst) { - dst = src; - return; + dst = src.cloneEmpty(); } assertBlocksHaveEqualStructure(src, dst, "ProfileEvents"); @@ -142,7 +141,7 @@ static void incrementProfileEventsBlock(Block & dst, const Block & src) const auto & src_column_host_name = typeid_cast(*src.getByName("host_name").column); const auto & src_array_current_time = typeid_cast(*src.getByName("current_time").column).getData(); - const auto & src_array_thread_id = typeid_cast(*src.getByName("thread_id").column).getData(); + // const auto & src_array_thread_id = typeid_cast(*src.getByName("thread_id").column).getData(); const auto & src_column_name = typeid_cast(*src.getByName("name").column); const auto & src_array_value = typeid_cast(*src.getByName("value").column).getData(); @@ -150,12 +149,11 @@ static void incrementProfileEventsBlock(Block & dst, const Block & src) { StringRef name; StringRef host_name; - UInt64 thread_id; bool operator<(const Id & rhs) const { - return std::tie(name, host_name, thread_id) - < std::tie(rhs.name, rhs.host_name, rhs.thread_id); + return std::tie(name, host_name) + < std::tie(rhs.name, rhs.host_name); } }; std::map rows_by_name; @@ -164,7 +162,6 @@ static void incrementProfileEventsBlock(Block & dst, const Block & src) Id id{ src_column_name.getDataAt(src_row), src_column_host_name.getDataAt(src_row), - src_array_thread_id[src_row], }; rows_by_name[id] = src_row; } @@ -175,7 +172,6 @@ static void incrementProfileEventsBlock(Block & dst, const Block & src) Id id{ dst_column_name.getDataAt(dst_row), dst_column_host_name.getDataAt(dst_row), - dst_array_thread_id[dst_row], }; if (auto it = rows_by_name.find(id); it != rows_by_name.end()) @@ -206,7 +202,18 @@ static void incrementProfileEventsBlock(Block & dst, const Block & src) } } + /// Filter out snapshots + std::set thread_id_filter_mask; + for (size_t i = 0; i < dst_array_thread_id.size(); ++i) + { + if (dst_array_thread_id[i] != 0) + { + thread_id_filter_mask.emplace(i); + } + } + dst.setColumns(std::move(mutable_columns)); + dst.erase(thread_id_filter_mask); } diff --git a/src/Common/MemoryTracker.h b/src/Common/MemoryTracker.h index 780a869acad..a606086ab1c 100644 --- a/src/Common/MemoryTracker.h +++ b/src/Common/MemoryTracker.h @@ -117,6 +117,10 @@ public: void setSoftLimit(Int64 value); void setHardLimit(Int64 value); + Int64 getHardLimit() const + { + return hard_limit.load(std::memory_order_relaxed); + } Int64 getSoftLimit() const { return soft_limit.load(std::memory_order_relaxed); diff --git a/src/Common/OvercommitTracker.cpp b/src/Common/OvercommitTracker.cpp index 4be7096aa60..0e70619f628 100644 --- a/src/Common/OvercommitTracker.cpp +++ b/src/Common/OvercommitTracker.cpp @@ -6,10 +6,13 @@ using namespace std::chrono_literals; -OvercommitTracker::OvercommitTracker() - : max_wait_time(0us) +constexpr std::chrono::microseconds ZERO_MICROSEC = 0us; + +OvercommitTracker::OvercommitTracker(std::mutex & global_mutex_) + : max_wait_time(ZERO_MICROSEC) , picked_tracker(nullptr) , cancelation_state(QueryCancelationState::NONE) + , global_mutex(global_mutex_) {} void OvercommitTracker::setMaxWaitTime(UInt64 wait_time) @@ -20,10 +23,15 @@ void OvercommitTracker::setMaxWaitTime(UInt64 wait_time) bool OvercommitTracker::needToStopQuery(MemoryTracker * tracker) { + std::unique_lock global_lock(global_mutex); std::unique_lock lk(overcommit_m); + if (max_wait_time == ZERO_MICROSEC) + return true; + pickQueryToExclude(); assert(cancelation_state == QueryCancelationState::RUNNING); + global_lock.unlock(); // If no query was chosen we need to stop current query. // This may happen if no soft limit is set. @@ -34,10 +42,15 @@ bool OvercommitTracker::needToStopQuery(MemoryTracker * tracker) } if (picked_tracker == tracker) return true; - return !cv.wait_for(lk, max_wait_time, [this]() + bool timeout = !cv.wait_for(lk, max_wait_time, [this]() { return cancelation_state == QueryCancelationState::NONE; }); + if (timeout) + LOG_DEBUG(getLogger(), "Need to stop query because reached waiting timeout"); + else + LOG_DEBUG(getLogger(), "Memory freed within timeout"); + return timeout; } void OvercommitTracker::unsubscribe(MemoryTracker * tracker) @@ -53,8 +66,9 @@ void OvercommitTracker::unsubscribe(MemoryTracker * tracker) } } -UserOvercommitTracker::UserOvercommitTracker(DB::ProcessListForUser * user_process_list_) - : user_process_list(user_process_list_) +UserOvercommitTracker::UserOvercommitTracker(DB::ProcessList * process_list, DB::ProcessListForUser * user_process_list_) + : OvercommitTracker(process_list->mutex) + , user_process_list(user_process_list_) {} void UserOvercommitTracker::pickQueryToExcludeImpl() @@ -87,10 +101,18 @@ void UserOvercommitTracker::pickQueryToExcludeImpl() picked_tracker = query_tracker; } +GlobalOvercommitTracker::GlobalOvercommitTracker(DB::ProcessList * process_list_) + : OvercommitTracker(process_list_->mutex) + , process_list(process_list_) +{} + void GlobalOvercommitTracker::pickQueryToExcludeImpl() { MemoryTracker * query_tracker = nullptr; OvercommitRatio current_ratio{0, 0}; + // At this moment query list must be read only. + // BlockQueryIfMemoryLimit is used in ProcessList to guarantee this. + LOG_DEBUG(logger, "Trying to choose query to stop"); process_list->processEachQueryStatus([&](DB::QueryStatus const & query) { if (query.isKilled()) diff --git a/src/Common/OvercommitTracker.h b/src/Common/OvercommitTracker.h index 2286ad4bde2..7c7974f0a24 100644 --- a/src/Common/OvercommitTracker.h +++ b/src/Common/OvercommitTracker.h @@ -43,7 +43,7 @@ class MemoryTracker; // is killed to free memory. struct OvercommitTracker : boost::noncopyable { - OvercommitTracker(); + explicit OvercommitTracker(std::mutex & global_mutex_); void setMaxWaitTime(UInt64 wait_time); @@ -87,7 +87,7 @@ private: } } - friend struct BlockQueryIfMemoryLimit; + std::mutex & global_mutex; }; namespace DB @@ -98,7 +98,7 @@ namespace DB struct UserOvercommitTracker : OvercommitTracker { - explicit UserOvercommitTracker(DB::ProcessListForUser * user_process_list_); + explicit UserOvercommitTracker(DB::ProcessList * process_list, DB::ProcessListForUser * user_process_list_); ~UserOvercommitTracker() override = default; @@ -113,9 +113,7 @@ private: struct GlobalOvercommitTracker : OvercommitTracker { - explicit GlobalOvercommitTracker(DB::ProcessList * process_list_) - : process_list(process_list_) - {} + explicit GlobalOvercommitTracker(DB::ProcessList * process_list_); ~GlobalOvercommitTracker() override = default; @@ -127,29 +125,3 @@ private: DB::ProcessList * process_list; Poco::Logger * logger = &Poco::Logger::get("GlobalOvercommitTracker"); }; - -// UserOvercommitTracker requires to check the whole list of user's queries -// to pick one to stop. BlockQueryIfMemoryLimit struct allows to wait until -// query selection is finished. It's used in ProcessList to make user query -// list immutable when UserOvercommitTracker reads it. -struct BlockQueryIfMemoryLimit -{ - BlockQueryIfMemoryLimit(OvercommitTracker const & overcommit_tracker) - : mutex(overcommit_tracker.overcommit_m) - , lk(mutex) - { - if (overcommit_tracker.cancelation_state == OvercommitTracker::QueryCancelationState::RUNNING) - { - overcommit_tracker.cv.wait_for(lk, overcommit_tracker.max_wait_time, [&overcommit_tracker]() - { - return overcommit_tracker.cancelation_state == OvercommitTracker::QueryCancelationState::NONE; - }); - } - } - - ~BlockQueryIfMemoryLimit() = default; - -private: - std::mutex & mutex; - std::unique_lock lk; -}; diff --git a/src/Common/StringSearcher.h b/src/Common/StringSearcher.h index 40629838284..c626231bc55 100644 --- a/src/Common/StringSearcher.h +++ b/src/Common/StringSearcher.h @@ -158,8 +158,8 @@ public: int c_l_u32 = Poco::Unicode::toLower(*c_u32); int c_u_u32 = Poco::Unicode::toUpper(*c_u32); - uint8_t dst_l_len = static_cast(UTF8::convertCodePointToUTF8(c_l_u32, l_seq, sizeof(l_seq))); - uint8_t dst_u_len = static_cast(UTF8::convertCodePointToUTF8(c_u_u32, u_seq, sizeof(u_seq))); + size_t dst_l_len = UTF8::convertCodePointToUTF8(c_l_u32, l_seq, sizeof(l_seq)); + size_t dst_u_len = UTF8::convertCodePointToUTF8(c_u_u32, u_seq, sizeof(u_seq)); /// @note Unicode standard states it is a rare but possible occasion if (!(dst_l_len == dst_u_len && dst_u_len == src_len)) diff --git a/src/Common/UTF8Helpers.h b/src/Common/UTF8Helpers.h index e33a3f1427c..9641518de37 100644 --- a/src/Common/UTF8Helpers.h +++ b/src/Common/UTF8Helpers.h @@ -76,7 +76,7 @@ inline size_t countCodePoints(const UInt8 * data, size_t size) template > -size_t convertCodePointToUTF8(uint32_t code_point, CharT * out_bytes, size_t out_length) +size_t convertCodePointToUTF8(int code_point, CharT * out_bytes, size_t out_length) { static const Poco::UTF8Encoding utf8; int res = utf8.convert(code_point, reinterpret_cast(out_bytes), out_length); diff --git a/src/Common/Volnitsky.h b/src/Common/Volnitsky.h index 881817f33be..cbdaf7048eb 100644 --- a/src/Common/Volnitsky.h +++ b/src/Common/Volnitsky.h @@ -60,7 +60,7 @@ namespace VolnitskyTraits static inline Ngram toNGram(const UInt8 * const pos) { return unalignedLoad(pos); } template - static inline void putNGramASCIICaseInsensitive(const UInt8 * pos, int offset, Callback && putNGramBase) + static inline bool putNGramASCIICaseInsensitive(const UInt8 * pos, int offset, Callback && putNGramBase) { struct Chars { @@ -107,10 +107,12 @@ namespace VolnitskyTraits else /// 1 combination: 01 putNGramBase(n, offset); + + return true; } template - static inline void putNGramUTF8CaseInsensitive( + static inline bool putNGramUTF8CaseInsensitive( const UInt8 * pos, int offset, const UInt8 * begin, size_t size, Callback && putNGramBase) { const UInt8 * end = begin + size; @@ -131,7 +133,7 @@ namespace VolnitskyTraits if (isascii(chars.c0) && isascii(chars.c1)) { - putNGramASCIICaseInsensitive(pos, offset, putNGramBase); + return putNGramASCIICaseInsensitive(pos, offset, putNGramBase); } else { @@ -177,21 +179,25 @@ namespace VolnitskyTraits /// where is the given ngram in respect to the start of UTF-8 sequence? size_t seq_ngram_offset = pos - seq_pos; - Seq seq; + Seq seq_l; + size_t length_l = UTF8::convertCodePointToUTF8(l_u32, seq_l, sizeof(seq_l)); - /// put ngram for lowercase - size_t length_l [[maybe_unused]] = UTF8::convertCodePointToUTF8(l_u32, seq, sizeof(seq)); - assert(length_l >= 2); - chars.c0 = seq[seq_ngram_offset]; - chars.c1 = seq[seq_ngram_offset + 1]; + Seq seq_r; + size_t length_r = UTF8::convertCodePointToUTF8(u_u32, seq_r, sizeof(seq_r)); + + if (length_l != length_r) + return false; + + assert(length_l >= 2 && length_r >= 2); + + chars.c0 = seq_l[seq_ngram_offset]; + chars.c1 = seq_l[seq_ngram_offset + 1]; putNGramBase(n, offset); - /// put ngram for uppercase - size_t length_r [[maybe_unused]] = UTF8::convertCodePointToUTF8(u_u32, seq, sizeof(seq)); - assert(length_r >= 2); - chars.c0 = seq[seq_ngram_offset]; //-V519 - chars.c1 = seq[seq_ngram_offset + 1]; //-V519 + chars.c0 = seq_r[seq_ngram_offset]; //-V519 + chars.c1 = seq_r[seq_ngram_offset + 1]; //-V519 putNGramBase(n, offset); + } } } @@ -235,40 +241,47 @@ namespace VolnitskyTraits else if (first_l_u32 == first_u_u32) { /// first symbol is case-independent - Seq seq; + Seq seq_l; + size_t size_l = UTF8::convertCodePointToUTF8(second_l_u32, seq_l, sizeof(seq_l)); - /// put ngram for lowercase - size_t size_l [[maybe_unused]] = UTF8::convertCodePointToUTF8(second_l_u32, seq, sizeof(seq)); - assert(size_l >= 1); - chars.c1 = seq[0]; + Seq seq_u; + size_t size_u = UTF8::convertCodePointToUTF8(second_u_u32, seq_u, sizeof(seq_u)); + + if (size_l != size_u) + return false; + + assert(size_l >= 1 && size_u >= 1); + chars.c1 = seq_l[0]; putNGramBase(n, offset); /// put ngram from uppercase, if it is different - size_t size_u [[maybe_unused]] = UTF8::convertCodePointToUTF8(second_u_u32, seq, sizeof(seq)); - assert(size_u >= 1); - if (chars.c1 != seq[0]) + if (chars.c1 != seq_u[0]) { - chars.c1 = seq[0]; + chars.c1 = seq_u[0]; putNGramBase(n, offset); } } else if (second_l_u32 == second_u_u32) { /// second symbol is case-independent - Seq seq; - /// put ngram for lowercase - size_t size_l [[maybe_unused]] = UTF8::convertCodePointToUTF8(first_l_u32, seq, sizeof(seq)); - assert(size_l > seq_ngram_offset); - chars.c0 = seq[seq_ngram_offset]; + Seq seq_l; + size_t size_l = UTF8::convertCodePointToUTF8(first_l_u32, seq_l, sizeof(seq_l)); + Seq seq_u; + size_t size_u = UTF8::convertCodePointToUTF8(first_u_u32, seq_u, sizeof(seq_u)); + + if (size_l != size_u) + return false; + + assert(size_l > seq_ngram_offset && size_u > seq_ngram_offset); + + chars.c0 = seq_l[seq_ngram_offset]; putNGramBase(n, offset); /// put ngram for uppercase, if it is different - size_t size_u [[maybe_unused]] = UTF8::convertCodePointToUTF8(first_u_u32, seq, sizeof(seq)); - assert(size_u > seq_ngram_offset); - if (chars.c0 != seq[seq_ngram_offset]) + if (chars.c0 != seq_u[seq_ngram_offset]) { - chars.c0 = seq[seq_ngram_offset]; + chars.c0 = seq_u[seq_ngram_offset]; putNGramBase(n, offset); } } @@ -279,10 +292,12 @@ namespace VolnitskyTraits Seq second_l_seq; Seq second_u_seq; - size_t size_first_l [[maybe_unused]] = UTF8::convertCodePointToUTF8(first_l_u32, first_l_seq, sizeof(first_l_seq)); - size_t size_first_u [[maybe_unused]] = UTF8::convertCodePointToUTF8(first_u_u32, first_u_seq, sizeof(first_u_seq)); - size_t size_second_l [[maybe_unused]] = UTF8::convertCodePointToUTF8(second_l_u32, second_l_seq, sizeof(second_l_seq)); - size_t size_second_u [[maybe_unused]] = UTF8::convertCodePointToUTF8(second_u_u32, second_u_seq, sizeof(second_u_seq)); + size_t size_first_l = UTF8::convertCodePointToUTF8(first_l_u32, first_l_seq, sizeof(first_l_seq)); + size_t size_first_u = UTF8::convertCodePointToUTF8(first_u_u32, first_u_seq, sizeof(first_u_seq)); + size_t size_second_l = UTF8::convertCodePointToUTF8(second_l_u32, second_l_seq, sizeof(second_l_seq)); + size_t size_second_u = UTF8::convertCodePointToUTF8(second_u_u32, second_u_seq, sizeof(second_u_seq)); + if (size_first_l != size_first_u || size_second_l != size_second_u) + return false; assert(size_first_l > seq_ngram_offset); assert(size_first_u > seq_ngram_offset); @@ -325,17 +340,25 @@ namespace VolnitskyTraits } } } + return true; } template - static inline void putNGram(const UInt8 * pos, int offset, [[maybe_unused]] const UInt8 * begin, size_t size, Callback && putNGramBase) + static inline bool putNGram(const UInt8 * pos, int offset, [[maybe_unused]] const UInt8 * begin, size_t size, Callback && putNGramBase) { if constexpr (CaseSensitive) + { putNGramBase(toNGram(pos), offset); + return true; + } else if constexpr (ASCII) - putNGramASCIICaseInsensitive(pos, offset, std::forward(putNGramBase)); + { + return putNGramASCIICaseInsensitive(pos, offset, std::forward(putNGramBase)); + } else - putNGramUTF8CaseInsensitive(pos, offset, begin, size, std::forward(putNGramBase)); + { + return putNGramUTF8CaseInsensitive(pos, offset, begin, size, std::forward(putNGramBase)); + } } } @@ -381,7 +404,20 @@ public: /// ssize_t is used here because unsigned can't be used with condition like `i >= 0`, unsigned always >= 0 /// And also adding from the end guarantees that we will find first occurrence because we will lookup bigger offsets first. for (auto i = static_cast(needle_size - sizeof(VolnitskyTraits::Ngram)); i >= 0; --i) - VolnitskyTraits::putNGram(needle + i, i + 1, needle, needle_size, callback); + { + bool ok = VolnitskyTraits::putNGram(needle + i, i + 1, needle, needle_size, callback); + + /** `putNGramUTF8CaseInsensitive` does not work if characters with lower and upper cases + * are represented by different number of bytes or code points. + * So, use fallback if error occurred. + */ + if (!ok) + { + fallback_searcher.force_fallback = true; + hash = nullptr; + return; + } + } } diff --git a/src/Compression/CachedCompressedReadBuffer.h b/src/Compression/CachedCompressedReadBuffer.h index 8394c478319..4493a40156c 100644 --- a/src/Compression/CachedCompressedReadBuffer.h +++ b/src/Compression/CachedCompressedReadBuffer.h @@ -51,7 +51,7 @@ public: /// Seek is lazy. It doesn't move the position anywhere, just remember them and perform actual /// seek inside nextImpl. - void seek(size_t offset_in_compressed_file, size_t offset_in_decompressed_block); + void seek(size_t offset_in_compressed_file, size_t offset_in_decompressed_block) override; void setProfileCallback(const ReadBufferFromFileBase::ProfileCallback & profile_callback_, clockid_t clock_type_ = CLOCK_MONOTONIC_COARSE) { diff --git a/src/Compression/CompressedReadBufferBase.h b/src/Compression/CompressedReadBufferBase.h index c32a169aecc..152447c0b64 100644 --- a/src/Compression/CompressedReadBufferBase.h +++ b/src/Compression/CompressedReadBufferBase.h @@ -48,8 +48,8 @@ protected: public: /// 'compressed_in' could be initialized lazily, but before first call of 'readCompressedData'. - CompressedReadBufferBase(ReadBuffer * in = nullptr, bool allow_different_codecs_ = false); - ~CompressedReadBufferBase(); + explicit CompressedReadBufferBase(ReadBuffer * in = nullptr, bool allow_different_codecs_ = false); + virtual ~CompressedReadBufferBase(); /** Disable checksums. * For example, may be used when @@ -60,7 +60,9 @@ public: disable_checksum = true; } -public: + /// Some compressed read buffer can do useful seek operation + virtual void seek(size_t /* offset_in_compressed_file */, size_t /* offset_in_decompressed_block */) {} + CompressionCodecPtr codec; }; diff --git a/src/Compression/CompressedReadBufferFromFile.h b/src/Compression/CompressedReadBufferFromFile.h index 3119873db4f..719959b96f4 100644 --- a/src/Compression/CompressedReadBufferFromFile.h +++ b/src/Compression/CompressedReadBufferFromFile.h @@ -51,7 +51,7 @@ public: /// Seek is lazy in some sense. We move position in compressed file_in to offset_in_compressed_file, but don't /// read data into working_buffer and don't shift our position to offset_in_decompressed_block. Instead /// we store this offset inside nextimpl_working_buffer_offset. - void seek(size_t offset_in_compressed_file, size_t offset_in_decompressed_block); + void seek(size_t offset_in_compressed_file, size_t offset_in_decompressed_block) override; size_t readBig(char * to, size_t n) override; diff --git a/src/Coordination/KeeperSnapshotManager.cpp b/src/Coordination/KeeperSnapshotManager.cpp index 8d5df7c35e9..5c009116010 100644 --- a/src/Coordination/KeeperSnapshotManager.cpp +++ b/src/Coordination/KeeperSnapshotManager.cpp @@ -526,5 +526,33 @@ void KeeperSnapshotManager::removeSnapshot(uint64_t log_idx) existing_snapshots.erase(itr); } +std::pair KeeperSnapshotManager::serializeSnapshotToDisk(const KeeperStorageSnapshot & snapshot) +{ + auto up_to_log_idx = snapshot.snapshot_meta->get_last_log_idx(); + auto snapshot_file_name = getSnapshotFileName(up_to_log_idx, compress_snapshots_zstd); + auto tmp_snapshot_file_name = "tmp_" + snapshot_file_name; + std::string tmp_snapshot_path = std::filesystem::path{snapshots_path} / tmp_snapshot_file_name; + std::string new_snapshot_path = std::filesystem::path{snapshots_path} / snapshot_file_name; + + auto writer = std::make_unique(tmp_snapshot_path, O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC| O_APPEND); + std::unique_ptr compressed_writer; + if (compress_snapshots_zstd) + compressed_writer = wrapWriteBufferWithCompressionMethod(std::move(writer), CompressionMethod::Zstd, 3); + else + compressed_writer = std::make_unique(*writer); + + KeeperStorageSnapshot::serialize(snapshot, *compressed_writer); + compressed_writer->finalize(); + compressed_writer->sync(); + + std::error_code ec; + std::filesystem::rename(tmp_snapshot_path, new_snapshot_path, ec); + if (!ec) + { + existing_snapshots.emplace(up_to_log_idx, new_snapshot_path); + removeOutdatedSnapshotsIfNeeded(); + } + return {new_snapshot_path, ec}; +} } diff --git a/src/Coordination/KeeperSnapshotManager.h b/src/Coordination/KeeperSnapshotManager.h index 174864a0ceb..37341ef546a 100644 --- a/src/Coordination/KeeperSnapshotManager.h +++ b/src/Coordination/KeeperSnapshotManager.h @@ -1,4 +1,6 @@ #pragma once +#include +#include #include #include #include @@ -101,6 +103,9 @@ public: /// Serialize already compressed snapshot to disk (return path) std::string serializeSnapshotBufferToDisk(nuraft::buffer & buffer, uint64_t up_to_log_idx); + /// Serialize snapshot directly to disk + std::pair serializeSnapshotToDisk(const KeeperStorageSnapshot & snapshot); + SnapshotDeserializationResult deserializeSnapshotFromBuffer(nuraft::ptr buffer) const; /// Deserialize snapshot with log index up_to_log_idx from disk into compressed nuraft buffer. @@ -126,6 +131,18 @@ public: return 0; } + std::string getLatestSnapshotPath() const + { + if (!existing_snapshots.empty()) + { + const auto & path = existing_snapshots.at(getLatestSnapshotIndex()); + std::error_code ec; + if (std::filesystem::exists(path, ec)) + return path; + } + return ""; + } + private: void removeOutdatedSnapshotsIfNeeded(); diff --git a/src/Coordination/KeeperStateMachine.cpp b/src/Coordination/KeeperStateMachine.cpp index 20d3bcbfd30..4c8be4abe24 100644 --- a/src/Coordination/KeeperStateMachine.cpp +++ b/src/Coordination/KeeperStateMachine.cpp @@ -1,3 +1,5 @@ +#include +#include #include #include #include @@ -73,8 +75,8 @@ void KeeperStateMachine::init() try { - latest_snapshot_buf = snapshot_manager.deserializeSnapshotBufferFromDisk(latest_log_index); - auto snapshot_deserialization_result = snapshot_manager.deserializeSnapshotFromBuffer(latest_snapshot_buf); + auto snapshot_deserialization_result = snapshot_manager.deserializeSnapshotFromBuffer(snapshot_manager.deserializeSnapshotBufferFromDisk(latest_log_index)); + latest_snapshot_path = snapshot_manager.getLatestSnapshotPath(); storage = std::move(snapshot_deserialization_result.storage); latest_snapshot_meta = snapshot_deserialization_result.snapshot_meta; cluster_config = snapshot_deserialization_result.cluster_config; @@ -155,7 +157,7 @@ bool KeeperStateMachine::apply_snapshot(nuraft::snapshot & s) { /// deserialize and apply snapshot to storage std::lock_guard lock(storage_and_responses_lock); - auto snapshot_deserialization_result = snapshot_manager.deserializeSnapshotFromBuffer(latest_snapshot_ptr); + auto snapshot_deserialization_result = snapshot_manager.deserializeSnapshotFromBuffer(snapshot_manager.deserializeSnapshotBufferFromDisk(s.get_last_log_idx())); storage = std::move(snapshot_deserialization_result.storage); latest_snapshot_meta = snapshot_deserialization_result.snapshot_meta; cluster_config = snapshot_deserialization_result.cluster_config; @@ -203,12 +205,15 @@ void KeeperStateMachine::create_snapshot( { { /// Read storage data without locks and create snapshot std::lock_guard lock(snapshots_lock); - auto snapshot_buf = snapshot_manager.serializeSnapshotToBuffer(*snapshot); - auto result_path = snapshot_manager.serializeSnapshotBufferToDisk(*snapshot_buf, snapshot->snapshot_meta->get_last_log_idx()); - latest_snapshot_buf = snapshot_buf; + auto [path, error_code]= snapshot_manager.serializeSnapshotToDisk(*snapshot); + if (error_code) + { + throw Exception(ErrorCodes::SYSTEM_ERROR, "Snapshot {} was created failed, error: {}", + snapshot->snapshot_meta->get_last_log_idx(), error_code.message()); + } + latest_snapshot_path = path; latest_snapshot_meta = snapshot->snapshot_meta; - - LOG_DEBUG(log, "Created persistent snapshot {} with path {}", latest_snapshot_meta->get_last_log_idx(), result_path); + LOG_DEBUG(log, "Created persistent snapshot {} with path {}", latest_snapshot_meta->get_last_log_idx(), path); } { @@ -258,7 +263,6 @@ void KeeperStateMachine::save_logical_snp_obj( else { /// copy snapshot into memory - cloned_buffer = nuraft::buffer::clone(data); } /// copy snapshot meta into memory @@ -268,9 +272,9 @@ void KeeperStateMachine::save_logical_snp_obj( try { std::lock_guard lock(snapshots_lock); - /// Serialize snapshot to disk and switch in memory pointers. - auto result_path = snapshot_manager.serializeSnapshotBufferToDisk(*cloned_buffer, s.get_last_log_idx()); - latest_snapshot_buf = cloned_buffer; + /// Serialize snapshot to disk + auto result_path = snapshot_manager.serializeSnapshotBufferToDisk(data, s.get_last_log_idx()); + latest_snapshot_path = result_path; latest_snapshot_meta = cloned_meta; LOG_DEBUG(log, "Saved snapshot {} to path {}", s.get_last_log_idx(), result_path); obj_id++; @@ -281,6 +285,37 @@ void KeeperStateMachine::save_logical_snp_obj( } } +static int bufferFromFile(Poco::Logger * log, const std::string & path, nuraft::ptr & data_out) +{ + if (path.empty() || !std::filesystem::exists(path)) + { + LOG_WARNING(log, "Snapshot file {} does not exist", path); + return -1; + } + + int fd = ::open(path.c_str(), O_RDONLY); + LOG_INFO(log, "Opening file {} for read_logical_snp_obj", path); + if (fd < 0) + { + LOG_WARNING(log, "Error opening {}, error: {}, errno: {}", path, std::strerror(errno), errno); + return errno; + } + auto file_size = ::lseek(fd, 0, SEEK_END); + ::lseek(fd, 0, SEEK_SET); + auto * chunk = reinterpret_cast(::mmap(nullptr, file_size, PROT_READ, MAP_FILE | MAP_SHARED, fd, 0)); + if (chunk == MAP_FAILED) + { + LOG_WARNING(log, "Error mmapping {}, error: {}, errno: {}", path, std::strerror(errno), errno); + ::close(fd); + return errno; + } + data_out = nuraft::buffer::alloc(file_size); + data_out->put_raw(chunk, file_size); + ::munmap(chunk, file_size); + ::close(fd); + return 0; +} + int KeeperStateMachine::read_logical_snp_obj( nuraft::snapshot & s, void* & /*user_snp_ctx*/, @@ -308,7 +343,11 @@ int KeeperStateMachine::read_logical_snp_obj( s.get_last_log_idx(), latest_snapshot_meta->get_last_log_idx()); return -1; } - data_out = nuraft::buffer::clone(*latest_snapshot_buf); + if (bufferFromFile(log, latest_snapshot_path, data_out)) + { + LOG_WARNING(log, "Error reading snapshot {} from {}", s.get_last_log_idx(), latest_snapshot_path); + return -1; + } is_last_obj = true; } diff --git a/src/Coordination/KeeperStateMachine.h b/src/Coordination/KeeperStateMachine.h index 291b58e2498..a6d116ca745 100644 --- a/src/Coordination/KeeperStateMachine.h +++ b/src/Coordination/KeeperStateMachine.h @@ -105,6 +105,7 @@ private: /// In our state machine we always have a single snapshot which is stored /// in memory in compressed (serialized) format. SnapshotMetadataPtr latest_snapshot_meta = nullptr; + std::string latest_snapshot_path; nuraft::ptr latest_snapshot_buf = nullptr; CoordinationSettingsPtr coordination_settings; diff --git a/src/Core/AccurateComparison.h b/src/Core/AccurateComparison.h index ab82a966a76..09014b06710 100644 --- a/src/Core/AccurateComparison.h +++ b/src/Core/AccurateComparison.h @@ -151,9 +151,9 @@ bool notEqualsOp(A a, B b) return !equalsOp(a, b); } - /// Converts numeric to an equal numeric of other type. -template +/// When `strict` is `true` check that result exactly same as input, otherwise just check overflow +template inline bool NO_SANITIZE_UNDEFINED convertNumeric(From value, To & result) { /// If the type is actually the same it's not necessary to do any checks. @@ -192,7 +192,9 @@ inline bool NO_SANITIZE_UNDEFINED convertNumeric(From value, To & result) } result = static_cast(value); - return equalsOp(value, result); + if constexpr (strict) + return equalsOp(value, result); + return true; } } diff --git a/src/DataTypes/EnumValues.h b/src/DataTypes/EnumValues.h index 17c292c5551..0747cd4aed8 100644 --- a/src/DataTypes/EnumValues.h +++ b/src/DataTypes/EnumValues.h @@ -29,7 +29,7 @@ private: void fillMaps(); public: - EnumValues(const Values & values_); + explicit EnumValues(const Values & values_); const Values & getValues() const { return values; } diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index bd9a1f7776e..deecc3c983e 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -277,7 +277,7 @@ void registerDictionarySourceClickHouse(DictionarySourceFactory & factory) { /// We should set user info even for the case when the dictionary is loaded in-process (without TCP communication). Session session(global_context, ClientInfo::Interface::LOCAL); - session.authenticate(configuration.user, configuration.password, {}); + session.authenticate(configuration.user, configuration.password, Poco::Net::SocketAddress{}); context = session.makeQueryContext(); } else diff --git a/src/Disks/AzureBlobStorage/DiskAzureBlobStorage.cpp b/src/Disks/AzureBlobStorage/DiskAzureBlobStorage.cpp index 7ff4b21965f..fb07d8c356b 100644 --- a/src/Disks/AzureBlobStorage/DiskAzureBlobStorage.cpp +++ b/src/Disks/AzureBlobStorage/DiskAzureBlobStorage.cpp @@ -108,7 +108,7 @@ std::unique_ptr DiskAzureBlobStorage::writeFile( readOrCreateUpdateAndStoreMetadata(path, mode, false, [blob_path, count] (Metadata & metadata) { metadata.addObject(blob_path, count); return true; }); }; - return std::make_unique>(std::move(buffer), std::move(create_metadata_callback), path); + return std::make_unique(std::move(buffer), std::move(create_metadata_callback), path); } diff --git a/src/Disks/DiskRestartProxy.cpp b/src/Disks/DiskRestartProxy.cpp index e993f5e4f27..43011a4cf72 100644 --- a/src/Disks/DiskRestartProxy.cpp +++ b/src/Disks/DiskRestartProxy.cpp @@ -20,11 +20,26 @@ public: RestartAwareReadBuffer(const DiskRestartProxy & disk, std::unique_ptr impl_) : ReadBufferFromFileDecorator(std::move(impl_)), lock(disk.mutex) { } - void prefetch() override { impl->prefetch(); } + void prefetch() override + { + swap(*impl); + impl->prefetch(); + swap(*impl); + } - void setReadUntilPosition(size_t position) override { impl->setReadUntilPosition(position); } + void setReadUntilPosition(size_t position) override + { + swap(*impl); + impl->setReadUntilPosition(position); + swap(*impl); + } - void setReadUntilEnd() override { impl->setReadUntilEnd(); } + void setReadUntilEnd() override + { + swap(*impl); + impl->setReadUntilEnd(); + swap(*impl); + } String getInfoForLog() override { return impl->getInfoForLog(); } diff --git a/src/Disks/HDFS/DiskHDFS.cpp b/src/Disks/HDFS/DiskHDFS.cpp index fb3f2149565..7f60b219a4b 100644 --- a/src/Disks/HDFS/DiskHDFS.cpp +++ b/src/Disks/HDFS/DiskHDFS.cpp @@ -106,8 +106,7 @@ std::unique_ptr DiskHDFS::writeFile(const String & path readOrCreateUpdateAndStoreMetadata(path, mode, false, [file_name, count] (Metadata & metadata) { metadata.addObject(file_name, count); return true; }); }; - return std::make_unique>( - std::move(hdfs_buffer), std::move(create_metadata_callback), path); + return std::make_unique(std::move(hdfs_buffer), std::move(create_metadata_callback), path); } diff --git a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp index a5c2f6b9816..87891f7b7d2 100644 --- a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp +++ b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp @@ -32,7 +32,7 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; - extern const int CANNOT_SEEK_THROUGH_FILE; + extern const int ARGUMENT_OUT_OF_BOUND; } @@ -157,127 +157,133 @@ bool AsynchronousReadIndirectBufferFromRemoteFS::nextImpl() return false; size_t size = 0; - if (prefetch_future.valid()) { ProfileEvents::increment(ProfileEvents::RemoteFSPrefetchedReads); - CurrentMetrics::Increment metric_increment{CurrentMetrics::AsynchronousReadWait}; - Stopwatch watch; + size_t offset = 0; { + Stopwatch watch; + CurrentMetrics::Increment metric_increment{CurrentMetrics::AsynchronousReadWait}; auto result = prefetch_future.get(); size = result.size; - auto offset = result.offset; + offset = result.offset; LOG_TEST(log, "Current size: {}, offset: {}", size, offset); - assert(offset < size); - if (size) - { - memory.swap(prefetch_buffer); - size -= offset; - set(memory.data() + offset, size); - working_buffer.resize(size); - file_offset_of_buffer_end += size; - } + /// If prefetch_future is valid, size should always be greater than zero. + assert(offset < size && size > 0); + ProfileEvents::increment(ProfileEvents::AsynchronousReadWaitMicroseconds, watch.elapsedMicroseconds()); } - watch.stop(); - ProfileEvents::increment(ProfileEvents::AsynchronousReadWaitMicroseconds, watch.elapsedMicroseconds()); + prefetch_buffer.swap(memory); + /// Adjust the working buffer so that it ignores `offset` bytes. + setWithBytesToIgnore(memory.data(), size, offset); } else { ProfileEvents::increment(ProfileEvents::RemoteFSUnprefetchedReads); + auto result = readInto(memory.data(), memory.size()).get(); size = result.size; auto offset = result.offset; + LOG_TEST(log, "Current size: {}, offset: {}", size, offset); - assert(offset < size); + assert(offset < size || size == 0); if (size) { - size -= offset; - set(memory.data() + offset, size); - working_buffer.resize(size); - file_offset_of_buffer_end += size; + /// Adjust the working buffer so that it ignores `offset` bytes. + setWithBytesToIgnore(memory.data(), size, offset); } } - if (file_offset_of_buffer_end != impl->getFileOffsetOfBufferEnd()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected equality {} == {}. It's a bug", file_offset_of_buffer_end, impl->getFileOffsetOfBufferEnd()); + file_offset_of_buffer_end = impl->offset(); prefetch_future = {}; return size; } -off_t AsynchronousReadIndirectBufferFromRemoteFS::seek(off_t offset_, int whence) +off_t AsynchronousReadIndirectBufferFromRemoteFS::seek(off_t offset, int whence) { ProfileEvents::increment(ProfileEvents::RemoteFSSeeks); - if (whence == SEEK_CUR) + size_t new_pos; + if (whence == SEEK_SET) { - /// If position within current working buffer - shift pos. - if (!working_buffer.empty() && static_cast(getPosition() + offset_) < file_offset_of_buffer_end) - { - pos += offset_; - return getPosition(); - } - else - { - file_offset_of_buffer_end += offset_; - } + assert(offset >= 0); + new_pos = offset; } - else if (whence == SEEK_SET) + else if (whence == SEEK_CUR) { - /// If position is within current working buffer - shift pos. - if (!working_buffer.empty() - && static_cast(offset_) >= file_offset_of_buffer_end - working_buffer.size() - && size_t(offset_) < file_offset_of_buffer_end) - { - pos = working_buffer.end() - (file_offset_of_buffer_end - offset_); + new_pos = file_offset_of_buffer_end - (working_buffer.end() - pos) + offset; + } + else + { + throw Exception("ReadBufferFromFileDescriptor::seek expects SEEK_SET or SEEK_CUR as whence", ErrorCodes::ARGUMENT_OUT_OF_BOUND); + } + /// Position is unchanged. + if (new_pos + (working_buffer.end() - pos) == file_offset_of_buffer_end) + return new_pos; + + bool read_from_prefetch = false; + while (true) + { + if (file_offset_of_buffer_end - working_buffer.size() <= new_pos && new_pos <= file_offset_of_buffer_end) + { + /// Position is still inside the buffer. + /// Probably it is at the end of the buffer - then we will load data on the following 'next' call. + + pos = working_buffer.end() - file_offset_of_buffer_end + new_pos; assert(pos >= working_buffer.begin()); assert(pos <= working_buffer.end()); - return getPosition(); + return new_pos; } - else + else if (prefetch_future.valid()) { - file_offset_of_buffer_end = offset_; + /// Read from prefetch buffer and recheck if the new position is valid inside. + + if (nextImpl()) + { + read_from_prefetch = true; + continue; + } } - } - else - throw Exception("Only SEEK_SET or SEEK_CUR modes are allowed.", ErrorCodes::CANNOT_SEEK_THROUGH_FILE); - if (prefetch_future.valid()) - { - ProfileEvents::increment(ProfileEvents::RemoteFSCancelledPrefetches); - prefetch_future.wait(); - prefetch_future = {}; + /// Prefetch is cancelled because of seek. + if (read_from_prefetch) + ProfileEvents::increment(ProfileEvents::RemoteFSCancelledPrefetches); + + break; } + assert(!prefetch_future.valid()); + + /// First reset the buffer so the next read will fetch new data to the buffer. resetWorkingBuffer(); /** * Lazy ignore. Save number of bytes to ignore and ignore it either for prefetch buffer or current buffer. * Note: we read in range [file_offset_of_buffer_end, read_until_position). */ - off_t file_offset_before_seek = impl->getFileOffsetOfBufferEnd(); if (impl->initialized() - && read_until_position && file_offset_of_buffer_end < *read_until_position - && static_cast(file_offset_of_buffer_end) > file_offset_before_seek - && static_cast(file_offset_of_buffer_end) < file_offset_before_seek + static_cast(min_bytes_for_seek)) + && read_until_position && new_pos < *read_until_position + && new_pos > file_offset_of_buffer_end + && new_pos < file_offset_of_buffer_end + min_bytes_for_seek) { ProfileEvents::increment(ProfileEvents::RemoteFSLazySeeks); - bytes_to_ignore = file_offset_of_buffer_end - file_offset_before_seek; + bytes_to_ignore = new_pos - file_offset_of_buffer_end; } else { ProfileEvents::increment(ProfileEvents::RemoteFSSeeksWithReset); impl->reset(); + file_offset_of_buffer_end = new_pos; } - return file_offset_of_buffer_end; + return new_pos; } diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp index 9cc393b5fd0..246d02ce6ea 100644 --- a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp @@ -194,6 +194,7 @@ bool ReadBufferFromRemoteFSGather::readImpl() { current_buf->ignore(bytes_to_ignore); result = current_buf->hasPendingData(); + file_offset_of_buffer_end += bytes_to_ignore; bytes_to_ignore = 0; } diff --git a/src/Disks/IO/WriteIndirectBufferFromRemoteFS.cpp b/src/Disks/IO/WriteIndirectBufferFromRemoteFS.cpp index f6a80ece3b3..9b604341da9 100644 --- a/src/Disks/IO/WriteIndirectBufferFromRemoteFS.cpp +++ b/src/Disks/IO/WriteIndirectBufferFromRemoteFS.cpp @@ -9,9 +9,8 @@ namespace DB { -template -WriteIndirectBufferFromRemoteFS::WriteIndirectBufferFromRemoteFS( - std::unique_ptr impl_, +WriteIndirectBufferFromRemoteFS::WriteIndirectBufferFromRemoteFS( + std::unique_ptr impl_, CreateMetadataCallback && create_callback_, const String & metadata_file_path_) : WriteBufferFromFileDecorator(std::move(impl_)) @@ -20,8 +19,8 @@ WriteIndirectBufferFromRemoteFS::WriteIndirectBufferFromRemoteFS( { } -template -WriteIndirectBufferFromRemoteFS::~WriteIndirectBufferFromRemoteFS() + +WriteIndirectBufferFromRemoteFS::~WriteIndirectBufferFromRemoteFS() { try { @@ -33,29 +32,12 @@ WriteIndirectBufferFromRemoteFS::~WriteIndirectBufferFromRemoteFS() } } -template -void WriteIndirectBufferFromRemoteFS::finalizeImpl() + +void WriteIndirectBufferFromRemoteFS::finalizeImpl() { WriteBufferFromFileDecorator::finalizeImpl(); create_metadata_callback(count()); } -#if USE_AWS_S3 -template -class WriteIndirectBufferFromRemoteFS; -#endif - -#if USE_AZURE_BLOB_STORAGE -template -class WriteIndirectBufferFromRemoteFS; -#endif - -#if USE_HDFS -template -class WriteIndirectBufferFromRemoteFS; -#endif - -template -class WriteIndirectBufferFromRemoteFS; } diff --git a/src/Disks/IO/WriteIndirectBufferFromRemoteFS.h b/src/Disks/IO/WriteIndirectBufferFromRemoteFS.h index 54b08ced191..25a93e2fe07 100644 --- a/src/Disks/IO/WriteIndirectBufferFromRemoteFS.h +++ b/src/Disks/IO/WriteIndirectBufferFromRemoteFS.h @@ -12,12 +12,11 @@ namespace DB using CreateMetadataCallback = std::function; /// Stores data in S3/HDFS and adds the object path and object size to metadata file on local FS. -template class WriteIndirectBufferFromRemoteFS final : public WriteBufferFromFileDecorator { public: WriteIndirectBufferFromRemoteFS( - std::unique_ptr impl_, + std::unique_ptr impl_, CreateMetadataCallback && create_callback_, const String & metadata_file_path_); diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index c6edd908c45..43917e0da1e 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -264,21 +264,20 @@ std::unique_ptr DiskS3::writeFile(const String & path, LOG_TRACE(log, "{} to file by path: {}. S3 path: {}", mode == WriteMode::Rewrite ? "Write" : "Append", backQuote(metadata_disk->getPath() + path), remote_fs_root_path + blob_name); - /// FIXME -- thread pool lead to obscure segfaults - /// ScheduleFunc schedule = [pool = &getThreadPoolWriter(), thread_group = CurrentThread::getGroup()](auto callback) - /// { - /// pool->scheduleOrThrow([callback = std::move(callback), thread_group]() - /// { - /// if (thread_group) - /// CurrentThread::attachTo(thread_group); + ScheduleFunc schedule = [pool = &getThreadPoolWriter(), thread_group = CurrentThread::getGroup()](auto callback) + { + pool->scheduleOrThrow([callback = std::move(callback), thread_group]() + { + if (thread_group) + CurrentThread::attachTo(thread_group); - /// SCOPE_EXIT_SAFE( - /// if (thread_group) - /// CurrentThread::detachQueryIfNotDetached(); - /// ); - /// callback(); - /// }); - /// }; + SCOPE_EXIT_SAFE( + if (thread_group) + CurrentThread::detachQueryIfNotDetached(); + ); + callback(); + }); + }; auto s3_buffer = std::make_unique( settings->client, @@ -296,7 +295,7 @@ std::unique_ptr DiskS3::writeFile(const String & path, readOrCreateUpdateAndStoreMetadata(path, mode, false, [blob_name, count] (Metadata & metadata) { metadata.addObject(blob_name, count); return true; }); }; - return std::make_unique>(std::move(s3_buffer), std::move(create_metadata_callback), path); + return std::make_unique(std::move(s3_buffer), std::move(create_metadata_callback), path); } void DiskS3::createHardLink(const String & src_path, const String & dst_path) diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index b7020ea128e..2596b10503f 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -48,7 +48,7 @@ endif() option(STRIP_DEBUG_SYMBOLS_FUNCTIONS "Do not generate debugger info for ClickHouse functions" ${STRIP_DSF_DEFAULT}) if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) - message(WARNING "Not generating debugger info for ClickHouse functions") + message(INFO "Not generating debugger info for ClickHouse functions") target_compile_options(clickhouse_functions PRIVATE "-g0") else() message(STATUS "Generating debugger info for ClickHouse functions") diff --git a/src/Functions/FunctionDateOrDateTimeAddInterval.h b/src/Functions/FunctionDateOrDateTimeAddInterval.h index 8f6b1370935..cb48b819481 100644 --- a/src/Functions/FunctionDateOrDateTimeAddInterval.h +++ b/src/Functions/FunctionDateOrDateTimeAddInterval.h @@ -1,4 +1,6 @@ #pragma once +#include +#include #include #include @@ -22,9 +24,10 @@ namespace DB namespace ErrorCodes { - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int DECIMAL_OVERFLOW; extern const int ILLEGAL_COLUMN; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } /// Type of first argument of 'execute' function overload defines what INPUT DataType it is used for. @@ -280,7 +283,7 @@ struct Adder vec_to.resize(size); for (size_t i = 0; i < size; ++i) - vec_to[i] = transform.execute(vec_from[i], delta, time_zone); + vec_to[i] = transform.execute(vec_from[i], checkOverflow(delta), time_zone); } template @@ -310,12 +313,22 @@ struct Adder } private: + + template + static Int64 checkOverflow(Value val) + { + Int64 result; + if (accurate::convertNumeric(val, result)) + return result; + throw DB::Exception("Numeric overflow", ErrorCodes::DECIMAL_OVERFLOW); + } + template NO_INLINE NO_SANITIZE_UNDEFINED void vectorVector( const FromVectorType & vec_from, ToVectorType & vec_to, const DeltaColumnType & delta, const DateLUTImpl & time_zone, size_t size) const { for (size_t i = 0; i < size; ++i) - vec_to[i] = transform.execute(vec_from[i], delta.getData()[i], time_zone); + vec_to[i] = transform.execute(vec_from[i], checkOverflow(delta.getData()[i]), time_zone); } template @@ -323,7 +336,7 @@ private: const FromType & from, ToVectorType & vec_to, const DeltaColumnType & delta, const DateLUTImpl & time_zone, size_t size) const { for (size_t i = 0; i < size; ++i) - vec_to[i] = transform.execute(from, delta.getData()[i], time_zone); + vec_to[i] = transform.execute(from, checkOverflow(delta.getData()[i]), time_zone); } }; @@ -346,10 +359,9 @@ struct DateTimeAddIntervalImpl auto result_col = result_type->createColumn(); auto col_to = assert_cast(result_col.get()); + const IColumn & delta_column = *arguments[1].column; if (const auto * sources = checkAndGetColumn(source_col.get())) { - const IColumn & delta_column = *arguments[1].column; - if (const auto * delta_const_column = typeid_cast(&delta_column)) op.vectorConstant(sources->getData(), col_to->getData(), delta_const_column->getInt(0), time_zone); else @@ -360,13 +372,12 @@ struct DateTimeAddIntervalImpl op.constantVector( sources_const->template getValue(), col_to->getData(), - *arguments[1].column, time_zone); + delta_column, time_zone); } else { - throw Exception("Illegal column " + arguments[0].column->getName() - + " of first argument of function " + Transform::name, - ErrorCodes::ILLEGAL_COLUMN); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of first argument of function {}", + arguments[0].column->getName(), Transform::name); } return result_col; diff --git a/src/Functions/FunctionJoinGet.cpp b/src/Functions/FunctionJoinGet.cpp index df131538275..da1061e4b3e 100644 --- a/src/Functions/FunctionJoinGet.cpp +++ b/src/Functions/FunctionJoinGet.cpp @@ -74,9 +74,9 @@ FunctionBasePtr JoinGetOverloadResolver::buildImpl(const ColumnsWithTyp { if (arguments.size() < 3) throw Exception( - "Number of arguments for function " + getName() + " doesn't match: passed " + toString(arguments.size()) - + ", should be greater or equal to 3", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Number of arguments for function '{}' doesn't match: passed {}, should be greater or equal to 3", + getName() , arguments.size()); auto [storage_join, attr_name] = getJoin(arguments, getContext()); DataTypes data_types(arguments.size() - 2); DataTypes argument_types(arguments.size()); @@ -86,9 +86,13 @@ FunctionBasePtr JoinGetOverloadResolver::buildImpl(const ColumnsWithTyp data_types[i - 2] = arguments[i].type; argument_types[i] = arguments[i].type; } - auto return_type = storage_join->joinGetCheckAndGetReturnType(data_types, attr_name, or_null); + + auto return_type = storage_join->joinGetCheckAndGetReturnType(data_types, attr_name, or_null || storage_join->useNulls()); auto table_lock = storage_join->lockForShare(getContext()->getInitialQueryId(), getContext()->getSettingsRef().lock_acquire_timeout); + if (storage_join->useNulls()) + return std::make_unique>(getContext(), table_lock, storage_join, attr_name, argument_types, return_type); + return std::make_unique>(getContext(), table_lock, storage_join, attr_name, argument_types, return_type); } diff --git a/src/Functions/FunctionStartsEndsWith.h b/src/Functions/FunctionStartsEndsWith.h index 5a3aba62f26..bbe1631fdf9 100644 --- a/src/Functions/FunctionStartsEndsWith.h +++ b/src/Functions/FunctionStartsEndsWith.h @@ -1,4 +1,6 @@ #pragma once +#include + #include #include #include @@ -7,7 +9,9 @@ #include #include #include +#include #include +#include namespace DB { @@ -17,6 +21,7 @@ using namespace GatherUtils; namespace ErrorCodes { extern const int ILLEGAL_COLUMN; + extern const int LOGICAL_ERROR; extern const int ILLEGAL_TYPE_OF_ARGUMENT; } @@ -59,16 +64,65 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { - if (!isStringOrFixedString(arguments[0])) - throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (isStringOrFixedString(arguments[0]) && isStringOrFixedString(arguments[1])) + return std::make_shared(); - if (!isStringOrFixedString(arguments[1])) - throw Exception("Illegal type " + arguments[1]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (isArray(arguments[0]) && isArray(arguments[1])) + return std::make_shared(); - return std::make_shared(); + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal types {} {} of arguments of function {}. Both must be String or Array", + arguments[0]->getName(), arguments[1]->getName(), getName()); } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + auto data_type = arguments[0].type; + if (isStringOrFixedString(*data_type)) + return executeImplString(arguments, {}, input_rows_count); + if (isArray(data_type)) + return executeImplArray(arguments, {}, input_rows_count); + return {}; + } + +private: + ColumnPtr executeImplArray(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const + { + DataTypePtr common_type = getLeastSupertype(collections::map(arguments, [](auto & arg) { return arg.type; })); + + Columns preprocessed_columns(2); + for (size_t i = 0; i < 2; ++i) + preprocessed_columns[i] = castColumn(arguments[i], common_type); + + std::vector> sources; + for (auto & argument_column : preprocessed_columns) + { + bool is_const = false; + + if (const auto * argument_column_const = typeid_cast(argument_column.get())) + { + is_const = true; + argument_column = argument_column_const->getDataColumnPtr(); + } + + if (const auto * argument_column_array = typeid_cast(argument_column.get())) + sources.emplace_back(GatherUtils::createArraySource(*argument_column_array, is_const, input_rows_count)); + else + throw Exception{"Arguments for function " + getName() + " must be arrays.", ErrorCodes::LOGICAL_ERROR}; + } + + auto result_column = ColumnUInt8::create(input_rows_count); + auto * result_column_ptr = typeid_cast(result_column.get()); + + if constexpr (std::is_same_v) + GatherUtils::sliceHas(*sources[0], *sources[1], GatherUtils::ArraySearchType::StartsWith, *result_column_ptr); + else + GatherUtils::sliceHas(*sources[0], *sources[1], GatherUtils::ArraySearchType::EndsWith, *result_column_ptr); + + return result_column; + } + + ColumnPtr executeImplString(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const { const IColumn * haystack_column = arguments[0].column.get(); const IColumn * needle_column = arguments[1].column.get(); @@ -92,7 +146,6 @@ public: return col_res; } -private: template void dispatch(HaystackSource haystack_source, const IColumn * needle_column, PaddedPODArray & res_data) const { diff --git a/src/Functions/GatherUtils/Algorithms.h b/src/Functions/GatherUtils/Algorithms.h index 046e2dcf70f..2d4544b2167 100644 --- a/src/Functions/GatherUtils/Algorithms.h +++ b/src/Functions/GatherUtils/Algorithms.h @@ -496,6 +496,31 @@ bool sliceHasImplAnyAll(const FirstSliceType & first, const SecondSliceType & se return search_type == ArraySearchType::All; } +template < + ArraySearchType search_type, + typename FirstSliceType, + typename SecondSliceType, + bool (*isEqual)(const FirstSliceType &, const SecondSliceType &, size_t, size_t)> +bool sliceHasImplStartsEndsWith(const FirstSliceType & first, const SecondSliceType & second, const UInt8 * first_null_map, const UInt8 * second_null_map) +{ + const bool has_first_null_map = first_null_map != nullptr; + const bool has_second_null_map = second_null_map != nullptr; + + if (first.size < second.size) + return false; + + size_t first_index = (search_type == ArraySearchType::StartsWith) ? 0 : first.size - second.size; + for (size_t second_index = 0; second_index < second.size; ++second_index, ++first_index) + { + const bool is_first_null = has_first_null_map && first_null_map[first_index]; + const bool is_second_null = has_second_null_map && second_null_map[second_index]; + if (is_first_null != is_second_null) + return false; + if (!is_first_null && !is_second_null && !isEqual(first, second, first_index, second_index)) + return false; + } + return true; +} /// For details of Knuth-Morris-Pratt string matching algorithm see /// https://en.wikipedia.org/wiki/Knuth%E2%80%93Morris%E2%80%93Pratt_algorithm. @@ -589,6 +614,8 @@ bool sliceHasImpl(const FirstSliceType & first, const SecondSliceType & second, { if constexpr (search_type == ArraySearchType::Substr) return sliceHasImplSubstr(first, second, first_null_map, second_null_map); + else if constexpr (search_type == ArraySearchType::StartsWith || search_type == ArraySearchType::EndsWith) + return sliceHasImplStartsEndsWith(first, second, first_null_map, second_null_map); else return sliceHasImplAnyAll(first, second, first_null_map, second_null_map); } diff --git a/src/Functions/GatherUtils/GatherUtils.h b/src/Functions/GatherUtils/GatherUtils.h index 8a623caa297..52a01b6ff62 100644 --- a/src/Functions/GatherUtils/GatherUtils.h +++ b/src/Functions/GatherUtils/GatherUtils.h @@ -34,7 +34,9 @@ enum class ArraySearchType { Any, // Corresponds to the hasAny array function All, // Corresponds to the hasAll array function - Substr // Corresponds to the hasSubstr array function + Substr, // Corresponds to the hasSubstr array function + StartsWith, + EndsWith }; std::unique_ptr createArraySource(const ColumnArray & col, bool is_const, size_t total_rows); @@ -58,6 +60,8 @@ ColumnArray::MutablePtr sliceFromRightDynamicLength(IArraySource & src, const IC void sliceHasAny(IArraySource & first, IArraySource & second, ColumnUInt8 & result); void sliceHasAll(IArraySource & first, IArraySource & second, ColumnUInt8 & result); void sliceHasSubstr(IArraySource & first, IArraySource & second, ColumnUInt8 & result); +void sliceHasStartsWith(IArraySource & first, IArraySource & second, ColumnUInt8 & result); +void sliceHasEndsWith(IArraySource & first, IArraySource & second, ColumnUInt8 & result); inline void sliceHas(IArraySource & first, IArraySource & second, ArraySearchType search_type, ColumnUInt8 & result) { @@ -72,7 +76,12 @@ inline void sliceHas(IArraySource & first, IArraySource & second, ArraySearchTyp case ArraySearchType::Substr: sliceHasSubstr(first, second, result); break; - + case ArraySearchType::StartsWith: + sliceHasStartsWith(first, second, result); + break; + case ArraySearchType::EndsWith: + sliceHasEndsWith(first, second, result); + break; } } diff --git a/src/Functions/GatherUtils/ends_with.cpp b/src/Functions/GatherUtils/ends_with.cpp new file mode 100644 index 00000000000..579d903005a --- /dev/null +++ b/src/Functions/GatherUtils/ends_with.cpp @@ -0,0 +1,71 @@ +#include "GatherUtils.h" +#include "Selectors.h" +#include "Algorithms.h" + +namespace DB::GatherUtils +{ + +namespace +{ + +struct ArrayEndsWithSelectArraySourcePair : public ArraySourcePairSelector +{ + template + static void callFunction(FirstSource && first, + bool is_second_const, bool is_second_nullable, SecondSource && second, + ColumnUInt8 & result) + { + using SourceType = typename std::decay::type; + + if (is_second_nullable) + { + using NullableSource = NullableArraySource; + + if (is_second_const) + arrayAllAny(first, static_cast &>(second), result); + else + arrayAllAny(first, static_cast(second), result); + } + else + { + if (is_second_const) + arrayAllAny(first, static_cast &>(second), result); + else + arrayAllAny(first, second, result); + } + } + + template + static void selectSourcePair(bool is_first_const, bool is_first_nullable, FirstSource && first, + bool is_second_const, bool is_second_nullable, SecondSource && second, + ColumnUInt8 & result) + { + using SourceType = typename std::decay::type; + + if (is_first_nullable) + { + using NullableSource = NullableArraySource; + + if (is_first_const) + callFunction(static_cast &>(first), is_second_const, is_second_nullable, second, result); + else + callFunction(static_cast(first), is_second_const, is_second_nullable, second, result); + } + else + { + if (is_first_const) + callFunction(static_cast &>(first), is_second_const, is_second_nullable, second, result); + else + callFunction(first, is_second_const, is_second_nullable, second, result); + } + } +}; + +} + +void sliceHasEndsWith(IArraySource & first, IArraySource & second, ColumnUInt8 & result) +{ + ArrayEndsWithSelectArraySourcePair::select(first, second, result); +} + +} diff --git a/src/Functions/GatherUtils/starts_with.cpp b/src/Functions/GatherUtils/starts_with.cpp new file mode 100644 index 00000000000..813294bc092 --- /dev/null +++ b/src/Functions/GatherUtils/starts_with.cpp @@ -0,0 +1,71 @@ +#include "GatherUtils.h" +#include "Selectors.h" +#include "Algorithms.h" + +namespace DB::GatherUtils +{ + +namespace +{ + +struct ArrayStartsWithSelectArraySourcePair : public ArraySourcePairSelector +{ + template + static void callFunction(FirstSource && first, + bool is_second_const, bool is_second_nullable, SecondSource && second, + ColumnUInt8 & result) + { + using SourceType = typename std::decay::type; + + if (is_second_nullable) + { + using NullableSource = NullableArraySource; + + if (is_second_const) + arrayAllAny(first, static_cast &>(second), result); + else + arrayAllAny(first, static_cast(second), result); + } + else + { + if (is_second_const) + arrayAllAny(first, static_cast &>(second), result); + else + arrayAllAny(first, second, result); + } + } + + template + static void selectSourcePair(bool is_first_const, bool is_first_nullable, FirstSource && first, + bool is_second_const, bool is_second_nullable, SecondSource && second, + ColumnUInt8 & result) + { + using SourceType = typename std::decay::type; + + if (is_first_nullable) + { + using NullableSource = NullableArraySource; + + if (is_first_const) + callFunction(static_cast &>(first), is_second_const, is_second_nullable, second, result); + else + callFunction(static_cast(first), is_second_const, is_second_nullable, second, result); + } + else + { + if (is_first_const) + callFunction(static_cast &>(first), is_second_const, is_second_nullable, second, result); + else + callFunction(first, is_second_const, is_second_nullable, second, result); + } + } +}; + +} + +void sliceHasStartsWith(IArraySource & first, IArraySource & second, ColumnUInt8 & result) +{ + ArrayStartsWithSelectArraySourcePair::select(first, second, result); +} + +} diff --git a/src/Functions/URL/decodeURLComponent.cpp b/src/Functions/URL/decodeURLComponent.cpp index 9ed290b1832..a4c9e1f4eec 100644 --- a/src/Functions/URL/decodeURLComponent.cpp +++ b/src/Functions/URL/decodeURLComponent.cpp @@ -11,8 +11,33 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; } +static size_t encodeURL(const char * __restrict src, size_t src_size, char * __restrict dst, bool space_as_plus) +{ + char * dst_pos = dst; + for (size_t i = 0; i < src_size; i++) + { + if ((src[i] >= '0' && src[i] <= '9') || (src[i] >= 'a' && src[i] <= 'z') || (src[i] >= 'A' && src[i] <= 'Z') + || src[i] == '-' || src[i] == '_' || src[i] == '.' || src[i] == '~') + { + *dst_pos++ = src[i]; + } + else if (src[i] == ' ' && space_as_plus) + { + *dst_pos++ = '+'; + } + else + { + *dst_pos++ = '%'; + *dst_pos++ = hexDigitUppercase(src[i] >> 4); + *dst_pos++ = hexDigitUppercase(src[i] & 0xf); + } + } + *dst_pos++ = src[src_size]; + return dst_pos - dst; +} + /// We assume that size of the dst buf isn't less than src_size. -static size_t decodeURL(const char * src, size_t src_size, char * dst, bool plus_as_space) +static size_t decodeURL(const char * __restrict src, size_t src_size, char * __restrict dst, bool plus_as_space) { const char * src_prev_pos = src; const char * src_curr_pos = src; @@ -81,15 +106,25 @@ static size_t decodeURL(const char * src, size_t src_size, char * dst, bool plus return dst_pos - dst; } +enum URLCodeStrategy +{ + encode, + decode +}; /// Percent decode of URL data. -template -struct DecodeURLComponentImpl +template +struct CodeURLComponentImpl { static void vector(const ColumnString::Chars & data, const ColumnString::Offsets & offsets, ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets) { - res_data.resize(data.size()); + if (code_strategy == encode) + //the destination(res_data) string is at most three times the length of the source string + res_data.resize(data.size() * 3); + else + res_data.resize(data.size()); + size_t size = offsets.size(); res_offsets.resize(size); @@ -100,7 +135,18 @@ struct DecodeURLComponentImpl { const char * src_data = reinterpret_cast(&data[prev_offset]); size_t src_size = offsets[i] - prev_offset; - size_t dst_size = decodeURL(src_data, src_size, reinterpret_cast(res_data.data() + res_offset), plus_as_space); + size_t dst_size; + + if constexpr (code_strategy == encode) + { + /// Skip encoding of zero terminated character + size_t src_encode_size = src_size - 1; + dst_size = encodeURL(src_data, src_encode_size, reinterpret_cast(res_data.data() + res_offset), space_as_plus); + } + else + { + dst_size = decodeURL(src_data, src_size, reinterpret_cast(res_data.data() + res_offset), space_as_plus); + } res_offset += dst_size; res_offsets[i] = res_offset; @@ -118,14 +164,20 @@ struct DecodeURLComponentImpl struct NameDecodeURLComponent { static constexpr auto name = "decodeURLComponent"; }; +struct NameEncodeURLComponent { static constexpr auto name = "encodeURLComponent"; }; struct NameDecodeURLFormComponent { static constexpr auto name = "decodeURLFormComponent"; }; -using FunctionDecodeURLComponent = FunctionStringToString, NameDecodeURLComponent>; -using FunctionDecodeURLFormComponent = FunctionStringToString, NameDecodeURLFormComponent>; +struct NameEncodeURLFormComponent { static constexpr auto name = "encodeURLFormComponent"; }; +using FunctionDecodeURLComponent = FunctionStringToString, NameDecodeURLComponent>; +using FunctionEncodeURLComponent = FunctionStringToString, NameEncodeURLComponent>; +using FunctionDecodeURLFormComponent = FunctionStringToString, NameDecodeURLFormComponent>; +using FunctionEncodeURLFormComponent = FunctionStringToString, NameEncodeURLFormComponent>; -void registerFunctionDecodeURLComponent(FunctionFactory & factory) +void registerFunctionEncodeAndDecodeURLComponent(FunctionFactory & factory) { factory.registerFunction(); + factory.registerFunction(); factory.registerFunction(); + factory.registerFunction(); } } diff --git a/src/Functions/URL/registerFunctionsURL.cpp b/src/Functions/URL/registerFunctionsURL.cpp index 91118074b7a..91142a593f2 100644 --- a/src/Functions/URL/registerFunctionsURL.cpp +++ b/src/Functions/URL/registerFunctionsURL.cpp @@ -27,7 +27,7 @@ void registerFunctionCutQueryString(FunctionFactory & factory); void registerFunctionCutFragment(FunctionFactory & factory); void registerFunctionCutQueryStringAndFragment(FunctionFactory & factory); void registerFunctionCutURLParameter(FunctionFactory & factory); -void registerFunctionDecodeURLComponent(FunctionFactory & factory); +void registerFunctionEncodeAndDecodeURLComponent(FunctionFactory & factory); void registerFunctionNetloc(FunctionFactory & factory); void registerFunctionsURL(FunctionFactory & factory) @@ -56,7 +56,7 @@ void registerFunctionsURL(FunctionFactory & factory) registerFunctionCutFragment(factory); registerFunctionCutQueryStringAndFragment(factory); registerFunctionCutURLParameter(factory); - registerFunctionDecodeURLComponent(factory); + registerFunctionEncodeAndDecodeURLComponent(factory); registerFunctionNetloc(factory); } diff --git a/src/Functions/geoToH3.cpp b/src/Functions/geoToH3.cpp index fb7301de776..ce0bdee74a3 100644 --- a/src/Functions/geoToH3.cpp +++ b/src/Functions/geoToH3.cpp @@ -71,7 +71,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_lon = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_lon = checkAndGetColumn(non_const_arguments[0].column.get()); if (!col_lon) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -81,7 +85,7 @@ public: getName()); const auto & data_lon = col_lon->getData(); - const auto * col_lat = checkAndGetColumn(arguments[1].column.get()); + const auto * col_lat = checkAndGetColumn(non_const_arguments[1].column.get()); if (!col_lat) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -91,7 +95,7 @@ public: getName()); const auto & data_lat = col_lat->getData(); - const auto * col_res = checkAndGetColumn(arguments[2].column.get()); + const auto * col_res = checkAndGetColumn(non_const_arguments[2].column.get()); if (!col_res) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/geoToS2.cpp b/src/Functions/geoToS2.cpp index d69c15bdbe0..8d90552652a 100644 --- a/src/Functions/geoToS2.cpp +++ b/src/Functions/geoToS2.cpp @@ -67,7 +67,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_lon = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_lon = checkAndGetColumn(non_const_arguments[0].column.get()); if (!col_lon) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -77,7 +81,7 @@ public: getName()); const auto & data_col_lon = col_lon->getData(); - const auto * col_lat = checkAndGetColumn(arguments[1].column.get()); + const auto * col_lat = checkAndGetColumn(non_const_arguments[1].column.get()); if (!col_lat) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3CellAreaM2.cpp b/src/Functions/h3CellAreaM2.cpp index d110d0d92f9..e5fb5aa2ed7 100644 --- a/src/Functions/h3CellAreaM2.cpp +++ b/src/Functions/h3CellAreaM2.cpp @@ -20,6 +20,7 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int ILLEGAL_COLUMN; +extern const int INCORRECT_DATA; } namespace @@ -52,7 +53,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -70,6 +75,12 @@ public: for (size_t row = 0; row < input_rows_count; ++row) { const UInt64 index = data[row]; + + CellBoundary boundary{}; + auto err = cellToBoundary(index, &boundary); + if (err) + throw Exception(ErrorCodes::INCORRECT_DATA, "Incorrect H3 index: {}, error: {}", index, err); + Float64 res = cellAreaM2(index); dst_data[row] = res; } diff --git a/src/Functions/h3CellAreaRads2.cpp b/src/Functions/h3CellAreaRads2.cpp index 1a257b0d9d3..15d18613b72 100644 --- a/src/Functions/h3CellAreaRads2.cpp +++ b/src/Functions/h3CellAreaRads2.cpp @@ -20,6 +20,7 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int ILLEGAL_COLUMN; +extern const int INCORRECT_DATA; } namespace @@ -52,7 +53,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -60,7 +65,6 @@ public: arguments[0].type->getName(), 1, getName()); - const auto & data = column->getData(); auto dst = ColumnVector::create(); @@ -70,6 +74,12 @@ public: for (size_t row = 0; row < input_rows_count; ++row) { const UInt64 index = data[row]; + + CellBoundary boundary{}; + auto err = cellToBoundary(index, &boundary); + if (err) + throw Exception(ErrorCodes::INCORRECT_DATA, "Incorrect H3 index: {}, error: {}", index, err); + Float64 res = cellAreaRads2(index); dst_data[row] = res; } diff --git a/src/Functions/h3EdgeAngle.cpp b/src/Functions/h3EdgeAngle.cpp index a65fde285f1..8240f092ee8 100644 --- a/src/Functions/h3EdgeAngle.cpp +++ b/src/Functions/h3EdgeAngle.cpp @@ -52,7 +52,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3EdgeLengthKm.cpp b/src/Functions/h3EdgeLengthKm.cpp index 0cc485e93b1..b7072a2f309 100644 --- a/src/Functions/h3EdgeLengthKm.cpp +++ b/src/Functions/h3EdgeLengthKm.cpp @@ -53,7 +53,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3EdgeLengthM.cpp b/src/Functions/h3EdgeLengthM.cpp index 08b78517be9..2c934bc6c05 100644 --- a/src/Functions/h3EdgeLengthM.cpp +++ b/src/Functions/h3EdgeLengthM.cpp @@ -58,7 +58,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3ExactEdgeLengthKm.cpp b/src/Functions/h3ExactEdgeLengthKm.cpp index 7aa9e573bed..f37d93fd715 100644 --- a/src/Functions/h3ExactEdgeLengthKm.cpp +++ b/src/Functions/h3ExactEdgeLengthKm.cpp @@ -52,7 +52,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3ExactEdgeLengthM.cpp b/src/Functions/h3ExactEdgeLengthM.cpp index 5b7cb91e427..99acbb757c3 100644 --- a/src/Functions/h3ExactEdgeLengthM.cpp +++ b/src/Functions/h3ExactEdgeLengthM.cpp @@ -52,7 +52,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3ExactEdgeLengthRads.cpp b/src/Functions/h3ExactEdgeLengthRads.cpp index d2b9345c989..a2937e85c65 100644 --- a/src/Functions/h3ExactEdgeLengthRads.cpp +++ b/src/Functions/h3ExactEdgeLengthRads.cpp @@ -52,7 +52,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3GetBaseCell.cpp b/src/Functions/h3GetBaseCell.cpp index 7865f454815..a3023561824 100644 --- a/src/Functions/h3GetBaseCell.cpp +++ b/src/Functions/h3GetBaseCell.cpp @@ -50,7 +50,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3GetFaces.cpp b/src/Functions/h3GetFaces.cpp index b85ad37c04f..0344ccc7944 100644 --- a/src/Functions/h3GetFaces.cpp +++ b/src/Functions/h3GetFaces.cpp @@ -52,7 +52,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3GetResolution.cpp b/src/Functions/h3GetResolution.cpp index 7cc7dab8916..4d0a92a1414 100644 --- a/src/Functions/h3GetResolution.cpp +++ b/src/Functions/h3GetResolution.cpp @@ -50,7 +50,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3HexAreaKm2.cpp b/src/Functions/h3HexAreaKm2.cpp index 74b74a351a1..c4a3c1258ab 100644 --- a/src/Functions/h3HexAreaKm2.cpp +++ b/src/Functions/h3HexAreaKm2.cpp @@ -53,7 +53,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3HexAreaM2.cpp b/src/Functions/h3HexAreaM2.cpp index ff68d01bf68..f8599651b0f 100644 --- a/src/Functions/h3HexAreaM2.cpp +++ b/src/Functions/h3HexAreaM2.cpp @@ -53,7 +53,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3IndexesAreNeighbors.cpp b/src/Functions/h3IndexesAreNeighbors.cpp index 97b8461fc5a..ee603f7de49 100644 --- a/src/Functions/h3IndexesAreNeighbors.cpp +++ b/src/Functions/h3IndexesAreNeighbors.cpp @@ -57,7 +57,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_hindex_origin = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_hindex_origin = checkAndGetColumn(non_const_arguments[0].column.get()); if (!col_hindex_origin) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -68,7 +72,7 @@ public: const auto & data_hindex_origin = col_hindex_origin->getData(); - const auto * col_hindex_dest = checkAndGetColumn(arguments[1].column.get()); + const auto * col_hindex_dest = checkAndGetColumn(non_const_arguments[1].column.get()); if (!col_hindex_dest) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3IsPentagon.cpp b/src/Functions/h3IsPentagon.cpp index 78c93be75a6..b7374bc1d8d 100644 --- a/src/Functions/h3IsPentagon.cpp +++ b/src/Functions/h3IsPentagon.cpp @@ -50,7 +50,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3IsResClassIII.cpp b/src/Functions/h3IsResClassIII.cpp index 3fd9e1e62b0..08025c966cd 100644 --- a/src/Functions/h3IsResClassIII.cpp +++ b/src/Functions/h3IsResClassIII.cpp @@ -50,7 +50,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3IsValid.cpp b/src/Functions/h3IsValid.cpp index 7c97e77250c..b6701d89de6 100644 --- a/src/Functions/h3IsValid.cpp +++ b/src/Functions/h3IsValid.cpp @@ -49,7 +49,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3NumHexagons.cpp b/src/Functions/h3NumHexagons.cpp index 4336e441cfb..009ff182940 100644 --- a/src/Functions/h3NumHexagons.cpp +++ b/src/Functions/h3NumHexagons.cpp @@ -52,7 +52,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3ToCenterChild.cpp b/src/Functions/h3ToCenterChild.cpp index d9bc31b8f19..6b9724c297c 100644 --- a/src/Functions/h3ToCenterChild.cpp +++ b/src/Functions/h3ToCenterChild.cpp @@ -61,7 +61,11 @@ namespace ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_hindex = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_hindex = checkAndGetColumn(non_const_arguments[0].column.get()); if (!col_hindex) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -71,7 +75,7 @@ namespace getName()); const auto & data_hindex = col_hindex->getData(); - const auto * col_resolution = checkAndGetColumn(arguments[1].column.get()); + const auto * col_resolution = checkAndGetColumn(non_const_arguments[1].column.get()); if (!col_resolution) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3ToChildren.cpp b/src/Functions/h3ToChildren.cpp index a825255495d..26d3e3dc204 100644 --- a/src/Functions/h3ToChildren.cpp +++ b/src/Functions/h3ToChildren.cpp @@ -66,7 +66,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_hindex = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_hindex = checkAndGetColumn(non_const_arguments[0].column.get()); if (!col_hindex) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -77,7 +81,7 @@ public: const auto & data_hindex = col_hindex->getData(); - const auto * col_resolution = checkAndGetColumn(arguments[1].column.get()); + const auto * col_resolution = checkAndGetColumn(non_const_arguments[1].column.get()); if (!col_resolution) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3ToGeoBoundary.cpp b/src/Functions/h3ToGeoBoundary.cpp index 2c892fb59ae..acf1ec64020 100644 --- a/src/Functions/h3ToGeoBoundary.cpp +++ b/src/Functions/h3ToGeoBoundary.cpp @@ -52,7 +52,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3ToParent.cpp b/src/Functions/h3ToParent.cpp index b2262d3ac22..8fd9931b81b 100644 --- a/src/Functions/h3ToParent.cpp +++ b/src/Functions/h3ToParent.cpp @@ -60,7 +60,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_hindex = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_hindex = checkAndGetColumn(non_const_arguments[0].column.get()); if (!col_hindex) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -71,7 +75,7 @@ public: const auto & data_hindex = col_hindex->getData(); - const auto * col_resolution = checkAndGetColumn(arguments[1].column.get()); + const auto * col_resolution = checkAndGetColumn(non_const_arguments[1].column.get()); if (!col_resolution) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3ToString.cpp b/src/Functions/h3ToString.cpp index 0152dd69b6a..7ca46c3b064 100644 --- a/src/Functions/h3ToString.cpp +++ b/src/Functions/h3ToString.cpp @@ -53,7 +53,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3kRing.cpp b/src/Functions/h3kRing.cpp index baa74b6698b..a801fd299d6 100644 --- a/src/Functions/h3kRing.cpp +++ b/src/Functions/h3kRing.cpp @@ -64,7 +64,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_hindex = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_hindex = checkAndGetColumn(non_const_arguments[0].column.get()); if (!col_hindex) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -76,7 +80,7 @@ public: const auto & data_hindex = col_hindex->getData(); /// ColumnUInt16 is sufficient as the max value of 2nd arg is checked (arg > 0 < 10000) in implementation below - const auto * col_k = checkAndGetColumn(arguments[1].column.get()); + const auto * col_k = checkAndGetColumn(non_const_arguments[1].column.get()); if (!col_k) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3toGeo.cpp b/src/Functions/h3toGeo.cpp index 403bcc3c3d6..9c5eb1b11bf 100644 --- a/src/Functions/h3toGeo.cpp +++ b/src/Functions/h3toGeo.cpp @@ -59,7 +59,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/s2CapContains.cpp b/src/Functions/s2CapContains.cpp index 6604ff9707e..482c4a22c63 100644 --- a/src/Functions/s2CapContains.cpp +++ b/src/Functions/s2CapContains.cpp @@ -84,7 +84,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_center = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_center = checkAndGetColumn(non_const_arguments[0].column.get()); if (!col_center) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -94,7 +98,7 @@ public: getName()); const auto & data_center = col_center->getData(); - const auto * col_degrees = checkAndGetColumn(arguments[1].column.get()); + const auto * col_degrees = checkAndGetColumn(non_const_arguments[1].column.get()); if (!col_degrees) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -105,7 +109,7 @@ public: const auto & data_degrees = col_degrees->getData(); - const auto * col_point = checkAndGetColumn(arguments[2].column.get()); + const auto * col_point = checkAndGetColumn(non_const_arguments[2].column.get()); if (!col_point) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/s2CapUnion.cpp b/src/Functions/s2CapUnion.cpp index 7af6324a7d5..ea1f2f7534d 100644 --- a/src/Functions/s2CapUnion.cpp +++ b/src/Functions/s2CapUnion.cpp @@ -82,7 +82,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_center1 = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_center1 = checkAndGetColumn(non_const_arguments[0].column.get()); if (!col_center1) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -92,7 +96,7 @@ public: getName()); const auto & data_center1 = col_center1->getData(); - const auto * col_radius1 = checkAndGetColumn(arguments[1].column.get()); + const auto * col_radius1 = checkAndGetColumn(non_const_arguments[1].column.get()); if (!col_radius1) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -102,7 +106,7 @@ public: getName()); const auto & data_radius1 = col_radius1->getData(); - const auto * col_center2 = checkAndGetColumn(arguments[2].column.get()); + const auto * col_center2 = checkAndGetColumn(non_const_arguments[2].column.get()); if (!col_center2) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -112,7 +116,7 @@ public: getName()); const auto & data_center2 = col_center2->getData(); - const auto * col_radius2 = checkAndGetColumn(arguments[3].column.get()); + const auto * col_radius2 = checkAndGetColumn(non_const_arguments[3].column.get()); if (!col_radius2) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/s2CellsIntersect.cpp b/src/Functions/s2CellsIntersect.cpp index 5d9796ea26c..617910e1a59 100644 --- a/src/Functions/s2CellsIntersect.cpp +++ b/src/Functions/s2CellsIntersect.cpp @@ -66,7 +66,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_id_first = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_id_first = checkAndGetColumn(non_const_arguments[0].column.get()); if (!col_id_first) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -76,7 +80,7 @@ public: getName()); const auto & data_id_first = col_id_first->getData(); - const auto * col_id_second = checkAndGetColumn(arguments[1].column.get()); + const auto * col_id_second = checkAndGetColumn(non_const_arguments[1].column.get()); if (!col_id_second) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/s2GetNeighbors.cpp b/src/Functions/s2GetNeighbors.cpp index 32eaff90740..f16e531a645 100644 --- a/src/Functions/s2GetNeighbors.cpp +++ b/src/Functions/s2GetNeighbors.cpp @@ -67,7 +67,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_id = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_id = checkAndGetColumn(non_const_arguments[0].column.get()); if (!col_id) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/s2RectAdd.cpp b/src/Functions/s2RectAdd.cpp index 75da7de8f7e..44f240588be 100644 --- a/src/Functions/s2RectAdd.cpp +++ b/src/Functions/s2RectAdd.cpp @@ -65,7 +65,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_lo = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_lo = checkAndGetColumn(non_const_arguments[0].column.get()); if (!col_lo) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -75,7 +79,7 @@ public: getName()); const auto & data_low = col_lo->getData(); - const auto * col_hi = checkAndGetColumn(arguments[1].column.get()); + const auto * col_hi = checkAndGetColumn(non_const_arguments[1].column.get()); if (!col_hi) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -85,7 +89,7 @@ public: getName()); const auto & data_hi = col_hi->getData(); - const auto * col_point = checkAndGetColumn(arguments[2].column.get()); + const auto * col_point = checkAndGetColumn(non_const_arguments[2].column.get()); if (!col_point) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/s2RectContains.cpp b/src/Functions/s2RectContains.cpp index be46253f70e..84218704ef1 100644 --- a/src/Functions/s2RectContains.cpp +++ b/src/Functions/s2RectContains.cpp @@ -63,7 +63,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_lo = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_lo = checkAndGetColumn(non_const_arguments[0].column.get()); if (!col_lo) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -73,7 +77,7 @@ public: getName()); const auto & data_low = col_lo->getData(); - const auto * col_hi = checkAndGetColumn(arguments[1].column.get()); + const auto * col_hi = checkAndGetColumn(non_const_arguments[1].column.get()); if (!col_hi) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -83,7 +87,7 @@ public: getName()); const auto & data_hi = col_hi->getData(); - const auto * col_point = checkAndGetColumn(arguments[2].column.get()); + const auto * col_point = checkAndGetColumn(non_const_arguments[2].column.get()); if (!col_point) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/s2RectIntersection.cpp b/src/Functions/s2RectIntersection.cpp index d4339b4d601..064dad4166d 100644 --- a/src/Functions/s2RectIntersection.cpp +++ b/src/Functions/s2RectIntersection.cpp @@ -68,7 +68,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_lo1 = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_lo1 = checkAndGetColumn(non_const_arguments[0].column.get()); if (!col_lo1) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -78,7 +82,7 @@ public: getName()); const auto & data_lo1 = col_lo1->getData(); - const auto * col_hi1 = checkAndGetColumn(arguments[1].column.get()); + const auto * col_hi1 = checkAndGetColumn(non_const_arguments[1].column.get()); if (!col_hi1) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -88,7 +92,7 @@ public: getName()); const auto & data_hi1 = col_hi1->getData(); - const auto * col_lo2 = checkAndGetColumn(arguments[2].column.get()); + const auto * col_lo2 = checkAndGetColumn(non_const_arguments[2].column.get()); if (!col_lo2) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -98,7 +102,7 @@ public: getName()); const auto & data_lo2 = col_lo2->getData(); - const auto * col_hi2 = checkAndGetColumn(arguments[3].column.get()); + const auto * col_hi2 = checkAndGetColumn(non_const_arguments[3].column.get()); if (!col_hi2) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/s2RectUnion.cpp b/src/Functions/s2RectUnion.cpp index 047d331e711..91664d3b65f 100644 --- a/src/Functions/s2RectUnion.cpp +++ b/src/Functions/s2RectUnion.cpp @@ -66,7 +66,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_lo1 = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_lo1 = checkAndGetColumn(non_const_arguments[0].column.get()); if (!col_lo1) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -76,7 +80,7 @@ public: getName()); const auto & data_lo1 = col_lo1->getData(); - const auto * col_hi1 = checkAndGetColumn(arguments[1].column.get()); + const auto * col_hi1 = checkAndGetColumn(non_const_arguments[1].column.get()); if (!col_hi1) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -86,7 +90,7 @@ public: getName()); const auto & data_hi1 = col_hi1->getData(); - const auto * col_lo2 = checkAndGetColumn(arguments[2].column.get()); + const auto * col_lo2 = checkAndGetColumn(non_const_arguments[2].column.get()); if (!col_lo2) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -96,7 +100,7 @@ public: getName()); const auto & data_lo2 = col_lo2->getData(); - const auto * col_hi2 = checkAndGetColumn(arguments[3].column.get()); + const auto * col_hi2 = checkAndGetColumn(non_const_arguments[3].column.get()); if (!col_hi2) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/s2ToGeo.cpp b/src/Functions/s2ToGeo.cpp index 3d11c21a353..082c300536d 100644 --- a/src/Functions/s2ToGeo.cpp +++ b/src/Functions/s2ToGeo.cpp @@ -68,7 +68,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_id = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_id = checkAndGetColumn(non_const_arguments[0].column.get()); if (!col_id) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/IO/AsynchronousReadBufferFromFileDescriptor.cpp b/src/IO/AsynchronousReadBufferFromFileDescriptor.cpp index 877702f9705..a8c3e23815d 100644 --- a/src/IO/AsynchronousReadBufferFromFileDescriptor.cpp +++ b/src/IO/AsynchronousReadBufferFromFileDescriptor.cpp @@ -26,6 +26,7 @@ namespace DB namespace ErrorCodes { extern const int ARGUMENT_OUT_OF_BOUND; + extern const int LOGICAL_ERROR; } @@ -43,6 +44,8 @@ std::future AsynchronousReadBufferFromFileDescripto request.size = size; request.offset = file_offset_of_buffer_end; request.priority = priority; + request.ignore = bytes_to_ignore; + bytes_to_ignore = 0; /// This is a workaround of a read pass EOF bug in linux kernel with pread() if (file_size.has_value() && file_offset_of_buffer_end >= *file_size) @@ -75,11 +78,14 @@ bool AsynchronousReadBufferFromFileDescriptor::nextImpl() /// Read request already in flight. Wait for its completion. size_t size = 0; + size_t offset = 0; { Stopwatch watch; CurrentMetrics::Increment metric_increment{CurrentMetrics::AsynchronousReadWait}; auto result = prefetch_future.get(); size = result.size; + offset = result.offset; + assert(offset < size || size == 0); ProfileEvents::increment(ProfileEvents::AsynchronousReadWaitMicroseconds, watch.elapsedMicroseconds()); } @@ -89,8 +95,8 @@ bool AsynchronousReadBufferFromFileDescriptor::nextImpl() if (size) { prefetch_buffer.swap(memory); - set(memory.data(), memory.size()); - working_buffer.resize(size); + /// Adjust the working buffer so that it ignores `offset` bytes. + setWithBytesToIgnore(memory.data(), size, offset); return true; } @@ -100,13 +106,13 @@ bool AsynchronousReadBufferFromFileDescriptor::nextImpl() { /// No pending request. Do synchronous read. - auto [size, _] = readInto(memory.data(), memory.size()).get(); + auto [size, offset] = readInto(memory.data(), memory.size()).get(); file_offset_of_buffer_end += size; if (size) { - set(memory.data(), memory.size()); - working_buffer.resize(size); + /// Adjust the working buffer so that it ignores `offset` bytes. + setWithBytesToIgnore(memory.data(), size, offset); return true; } @@ -125,6 +131,30 @@ void AsynchronousReadBufferFromFileDescriptor::finalize() } +AsynchronousReadBufferFromFileDescriptor::AsynchronousReadBufferFromFileDescriptor( + AsynchronousReaderPtr reader_, + Int32 priority_, + int fd_, + size_t buf_size, + char * existing_memory, + size_t alignment, + std::optional file_size_) + : ReadBufferFromFileBase(buf_size, existing_memory, alignment, file_size_) + , reader(std::move(reader_)) + , priority(priority_) + , required_alignment(alignment) + , fd(fd_) +{ + if (required_alignment > buf_size) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Too large alignment. Cannot have required_alignment greater than buf_size: {} > {}. It is a bug", + required_alignment, + buf_size); + + prefetch_buffer.alignment = alignment; +} + AsynchronousReadBufferFromFileDescriptor::~AsynchronousReadBufferFromFileDescriptor() { finalize(); @@ -153,46 +183,48 @@ off_t AsynchronousReadBufferFromFileDescriptor::seek(off_t offset, int whence) if (new_pos + (working_buffer.end() - pos) == file_offset_of_buffer_end) return new_pos; - if (file_offset_of_buffer_end - working_buffer.size() <= static_cast(new_pos) - && new_pos <= file_offset_of_buffer_end) + while (true) { - /// Position is still inside the buffer. - /// Probably it is at the end of the buffer - then we will load data on the following 'next' call. - - pos = working_buffer.end() - file_offset_of_buffer_end + new_pos; - assert(pos >= working_buffer.begin()); - assert(pos <= working_buffer.end()); - - return new_pos; - } - else - { - if (prefetch_future.valid()) + if (file_offset_of_buffer_end - working_buffer.size() <= new_pos && new_pos <= file_offset_of_buffer_end) { - //std::cerr << "Ignoring prefetched data" << "\n"; - prefetch_future.wait(); - prefetch_future = {}; + /// Position is still inside the buffer. + /// Probably it is at the end of the buffer - then we will load data on the following 'next' call. + + pos = working_buffer.end() - file_offset_of_buffer_end + new_pos; + assert(pos >= working_buffer.begin()); + assert(pos <= working_buffer.end()); + + return new_pos; + } + else if (prefetch_future.valid()) + { + /// Read from prefetch buffer and recheck if the new position is valid inside. + + if (nextImpl()) + continue; } - /// Position is out of the buffer, we need to do real seek. - off_t seek_pos = required_alignment > 1 - ? new_pos / required_alignment * required_alignment - : new_pos; - - off_t offset_after_seek_pos = new_pos - seek_pos; - - /// First reset the buffer so the next read will fetch new data to the buffer. - resetWorkingBuffer(); - - /// Just update the info about the next position in file. - - file_offset_of_buffer_end = seek_pos; - - if (offset_after_seek_pos > 0) - ignore(offset_after_seek_pos); - - return seek_pos; + break; } + + assert(!prefetch_future.valid()); + + /// Position is out of the buffer, we need to do real seek. + off_t seek_pos = required_alignment > 1 + ? new_pos / required_alignment * required_alignment + : new_pos; + + /// First reset the buffer so the next read will fetch new data to the buffer. + resetWorkingBuffer(); + + /// Just update the info about the next position in file. + + file_offset_of_buffer_end = seek_pos; + bytes_to_ignore = new_pos - seek_pos; + + assert(bytes_to_ignore < internal_buffer.size()); + + return seek_pos; } diff --git a/src/IO/AsynchronousReadBufferFromFileDescriptor.h b/src/IO/AsynchronousReadBufferFromFileDescriptor.h index 2a16148812e..a74f24d62a0 100644 --- a/src/IO/AsynchronousReadBufferFromFileDescriptor.h +++ b/src/IO/AsynchronousReadBufferFromFileDescriptor.h @@ -24,6 +24,7 @@ protected: const size_t required_alignment = 0; /// For O_DIRECT both file offsets and memory addresses have to be aligned. size_t file_offset_of_buffer_end = 0; /// What offset in file corresponds to working_buffer.end(). + size_t bytes_to_ignore = 0; /// How many bytes should we ignore upon a new read request. int fd; bool nextImpl() override; @@ -41,15 +42,7 @@ public: size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, size_t alignment = 0, - std::optional file_size_ = std::nullopt) - : ReadBufferFromFileBase(buf_size, existing_memory, alignment, file_size_) - , reader(std::move(reader_)) - , priority(priority_) - , required_alignment(alignment) - , fd(fd_) - { - prefetch_buffer.alignment = alignment; - } + std::optional file_size_ = std::nullopt); ~AsynchronousReadBufferFromFileDescriptor() override; diff --git a/src/IO/ReadBuffer.h b/src/IO/ReadBuffer.h index bc64a845100..b620f0c49c6 100644 --- a/src/IO/ReadBuffer.h +++ b/src/IO/ReadBuffer.h @@ -50,6 +50,29 @@ public: // FIXME: behavior differs greately from `BufferBase::set()` and it's very confusing. void set(Position ptr, size_t size) { BufferBase::set(ptr, size, 0); working_buffer.resize(0); } + /// Set buffer to given piece of memory but with certain bytes ignored from beginning. + /// + /// internal_buffer: |__________________| + /// working_buffer: |xxxxx|____________| + /// ^ ^ + /// bytes_to_ignore + /// + /// It's used for lazy seek. We also have another lazy seek mechanism that uses + /// `nextimpl_working_buffer_offset` to set offset in `next` method. It's important that we + /// don't do double lazy seek, which means `nextimpl_working_buffer_offset` should be zero. It's + /// useful to keep internal_buffer points to the real span of the underlying memory, because its + /// size might be used to allocate other buffers. It's also important to have pos starts at + /// working_buffer.begin(), because some buffers assume this condition to be true and uses + /// offset() to check read bytes. + void setWithBytesToIgnore(Position ptr, size_t size, size_t bytes_to_ignore) + { + assert(bytes_to_ignore < size); + assert(nextimpl_working_buffer_offset == 0); + internal_buffer = Buffer(ptr, ptr + size); + working_buffer = Buffer(ptr + bytes_to_ignore, ptr + size); + pos = ptr + bytes_to_ignore; + } + /** read next data and fill a buffer with it; set position to the beginning; * return `false` in case of end, `true` otherwise; throw an exception, if something is wrong */ diff --git a/src/IO/ReadBufferFromPocoSocket.cpp b/src/IO/ReadBufferFromPocoSocket.cpp index 527b68e623a..988ad75cdf4 100644 --- a/src/IO/ReadBufferFromPocoSocket.cpp +++ b/src/IO/ReadBufferFromPocoSocket.cpp @@ -1,5 +1,7 @@ #include +#include + #include #include #include @@ -35,6 +37,12 @@ bool ReadBufferFromPocoSocket::nextImpl() ssize_t bytes_read = 0; Stopwatch watch; + SCOPE_EXIT({ + // / NOTE: it is quite inaccurate on high loads since the thread could be replaced by another one + ProfileEvents::increment(ProfileEvents::NetworkReceiveElapsedMicroseconds, watch.elapsedMicroseconds()); + ProfileEvents::increment(ProfileEvents::NetworkReceiveBytes, bytes_read); + }); + /// Add more details to exceptions. try { @@ -66,10 +74,6 @@ bool ReadBufferFromPocoSocket::nextImpl() if (bytes_read < 0) throw NetException("Cannot read from socket (" + peer_address.toString() + ")", ErrorCodes::CANNOT_READ_FROM_SOCKET); - /// NOTE: it is quite inaccurate on high loads since the thread could be replaced by another one - ProfileEvents::increment(ProfileEvents::NetworkReceiveElapsedMicroseconds, watch.elapsedMicroseconds()); - ProfileEvents::increment(ProfileEvents::NetworkReceiveBytes, bytes_read); - if (bytes_read) working_buffer.resize(bytes_read); else diff --git a/src/IO/SynchronousReader.cpp b/src/IO/SynchronousReader.cpp index 4414da28d28..7cef3bd8963 100644 --- a/src/IO/SynchronousReader.cpp +++ b/src/IO/SynchronousReader.cpp @@ -82,8 +82,7 @@ std::future SynchronousReader::submit(Request reque watch.stop(); ProfileEvents::increment(ProfileEvents::DiskReadElapsedMicroseconds, watch.elapsedMicroseconds()); - return Result{ .size = bytes_read, .offset = 0}; - + return Result{ .size = bytes_read, .offset = request.ignore }; }); } diff --git a/src/IO/ThreadPoolReader.cpp b/src/IO/ThreadPoolReader.cpp index 63bc8fe7c49..0c2791c6f68 100644 --- a/src/IO/ThreadPoolReader.cpp +++ b/src/IO/ThreadPoolReader.cpp @@ -176,7 +176,7 @@ std::future ThreadPoolReader::submit(Request reques ProfileEvents::increment(ProfileEvents::ThreadPoolReaderPageCacheHitElapsedMicroseconds, watch.elapsedMicroseconds()); ProfileEvents::increment(ProfileEvents::DiskReadElapsedMicroseconds, watch.elapsedMicroseconds()); - promise.set_value({bytes_read, 0}); + promise.set_value({bytes_read, request.ignore}); return future; } } @@ -219,7 +219,7 @@ std::future ThreadPoolReader::submit(Request reques ProfileEvents::increment(ProfileEvents::ThreadPoolReaderPageCacheMissElapsedMicroseconds, watch.elapsedMicroseconds()); ProfileEvents::increment(ProfileEvents::DiskReadElapsedMicroseconds, watch.elapsedMicroseconds()); - return Result{ .size = bytes_read, .offset = 0 }; + return Result{ .size = bytes_read, .offset = request.ignore }; }); auto future = task->get_future(); diff --git a/src/IO/WriteBufferFromPocoSocket.cpp b/src/IO/WriteBufferFromPocoSocket.cpp index 4dc703c34eb..fb4e5df9b59 100644 --- a/src/IO/WriteBufferFromPocoSocket.cpp +++ b/src/IO/WriteBufferFromPocoSocket.cpp @@ -1,5 +1,7 @@ #include +#include + #include #include @@ -38,8 +40,13 @@ void WriteBufferFromPocoSocket::nextImpl() return; Stopwatch watch; - size_t bytes_written = 0; + + SCOPE_EXIT({ + ProfileEvents::increment(ProfileEvents::NetworkSendElapsedMicroseconds, watch.elapsedMicroseconds()); + ProfileEvents::increment(ProfileEvents::NetworkSendBytes, bytes_written); + }); + while (bytes_written < offset()) { ssize_t res = 0; @@ -70,9 +77,6 @@ void WriteBufferFromPocoSocket::nextImpl() bytes_written += res; } - - ProfileEvents::increment(ProfileEvents::NetworkSendElapsedMicroseconds, watch.elapsedMicroseconds()); - ProfileEvents::increment(ProfileEvents::NetworkSendBytes, bytes_written); } WriteBufferFromPocoSocket::WriteBufferFromPocoSocket(Poco::Net::Socket & socket_, size_t buf_size) diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index d91b1e94297..60c75b3c90c 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -152,6 +152,10 @@ void WriteBufferFromS3::createMultipartUpload() Aws::S3::Model::CreateMultipartUploadRequest req; req.SetBucket(bucket); req.SetKey(key); + + /// If we don't do it, AWS SDK can mistakenly set it to application/xml, see https://github.com/aws/aws-sdk-cpp/issues/1840 + req.SetContentType("binary/octet-stream"); + if (object_metadata.has_value()) req.SetMetadata(object_metadata.value()); @@ -160,7 +164,7 @@ void WriteBufferFromS3::createMultipartUpload() if (outcome.IsSuccess()) { multipart_upload_id = outcome.GetResult().GetUploadId(); - LOG_DEBUG(log, "Multipart upload has created. Bucket: {}, Key: {}, Upload id: {}", bucket, key, multipart_upload_id); + LOG_TRACE(log, "Multipart upload has created. Bucket: {}, Key: {}, Upload id: {}", bucket, key, multipart_upload_id); } else throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); @@ -170,7 +174,7 @@ void WriteBufferFromS3::writePart() { auto size = temporary_buffer->tellp(); - LOG_DEBUG(log, "Writing part. Bucket: {}, Key: {}, Upload_id: {}, Size: {}", bucket, key, multipart_upload_id, size); + LOG_TRACE(log, "Writing part. Bucket: {}, Key: {}, Upload_id: {}, Size: {}", bucket, key, multipart_upload_id, size); if (size < 0) { @@ -180,7 +184,7 @@ void WriteBufferFromS3::writePart() if (size == 0) { - LOG_DEBUG(log, "Skipping writing part. Buffer is empty."); + LOG_TRACE(log, "Skipping writing part. Buffer is empty."); return; } @@ -242,6 +246,9 @@ void WriteBufferFromS3::fillUploadRequest(Aws::S3::Model::UploadPartRequest & re req.SetUploadId(multipart_upload_id); req.SetContentLength(temporary_buffer->tellp()); req.SetBody(temporary_buffer); + + /// If we don't do it, AWS SDK can mistakenly set it to application/xml, see https://github.com/aws/aws-sdk-cpp/issues/1840 + req.SetContentType("binary/octet-stream"); } void WriteBufferFromS3::processUploadRequest(UploadPartTask & task) @@ -251,7 +258,7 @@ void WriteBufferFromS3::processUploadRequest(UploadPartTask & task) if (outcome.IsSuccess()) { task.tag = outcome.GetResult().GetETag(); - LOG_DEBUG(log, "Writing part finished. Bucket: {}, Key: {}, Upload_id: {}, Etag: {}, Parts: {}", bucket, key, multipart_upload_id, task.tag, part_tags.size()); + LOG_TRACE(log, "Writing part finished. Bucket: {}, Key: {}, Upload_id: {}, Etag: {}, Parts: {}", bucket, key, multipart_upload_id, task.tag, part_tags.size()); } else throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); @@ -261,7 +268,7 @@ void WriteBufferFromS3::processUploadRequest(UploadPartTask & task) void WriteBufferFromS3::completeMultipartUpload() { - LOG_DEBUG(log, "Completing multipart upload. Bucket: {}, Key: {}, Upload_id: {}, Parts: {}", bucket, key, multipart_upload_id, part_tags.size()); + LOG_TRACE(log, "Completing multipart upload. Bucket: {}, Key: {}, Upload_id: {}, Parts: {}", bucket, key, multipart_upload_id, part_tags.size()); if (part_tags.empty()) throw Exception("Failed to complete multipart upload. No parts have uploaded", ErrorCodes::S3_ERROR); @@ -283,7 +290,7 @@ void WriteBufferFromS3::completeMultipartUpload() auto outcome = client_ptr->CompleteMultipartUpload(req); if (outcome.IsSuccess()) - LOG_DEBUG(log, "Multipart upload has completed. Bucket: {}, Key: {}, Upload_id: {}, Parts: {}", bucket, key, multipart_upload_id, part_tags.size()); + LOG_TRACE(log, "Multipart upload has completed. Bucket: {}, Key: {}, Upload_id: {}, Parts: {}", bucket, key, multipart_upload_id, part_tags.size()); else { throw Exception(ErrorCodes::S3_ERROR, "{} Tags:{}", @@ -297,7 +304,7 @@ void WriteBufferFromS3::makeSinglepartUpload() auto size = temporary_buffer->tellp(); bool with_pool = bool(schedule); - LOG_DEBUG(log, "Making single part upload. Bucket: {}, Key: {}, Size: {}, WithPool: {}", bucket, key, size, with_pool); + LOG_TRACE(log, "Making single part upload. Bucket: {}, Key: {}, Size: {}, WithPool: {}", bucket, key, size, with_pool); if (size < 0) { @@ -307,7 +314,7 @@ void WriteBufferFromS3::makeSinglepartUpload() if (size == 0) { - LOG_DEBUG(log, "Skipping single part upload. Buffer is empty."); + LOG_TRACE(log, "Skipping single part upload. Buffer is empty."); return; } @@ -354,6 +361,9 @@ void WriteBufferFromS3::fillPutRequest(Aws::S3::Model::PutObjectRequest & req) req.SetBody(temporary_buffer); if (object_metadata.has_value()) req.SetMetadata(object_metadata.value()); + + /// If we don't do it, AWS SDK can mistakenly set it to application/xml, see https://github.com/aws/aws-sdk-cpp/issues/1840 + req.SetContentType("binary/octet-stream"); } void WriteBufferFromS3::processPutRequest(PutObjectTask & task) @@ -362,7 +372,7 @@ void WriteBufferFromS3::processPutRequest(PutObjectTask & task) bool with_pool = bool(schedule); if (outcome.IsSuccess()) - LOG_DEBUG(log, "Single part upload has completed. Bucket: {}, Key: {}, Object size: {}, WithPool: {}", bucket, key, task.req.GetContentLength(), with_pool); + LOG_TRACE(log, "Single part upload has completed. Bucket: {}, Key: {}, Object size: {}, WithPool: {}", bucket, key, task.req.GetContentLength(), with_pool); else throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); } diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 6ed35210251..a5122cd54c7 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -1061,7 +1061,6 @@ ActionsDAGPtr ActionsDAG::makeConvertingActions( throw Exception(ErrorCodes::THERE_IS_NO_COLUMN, "Cannot find column `{}` in source stream, there are only columns: [{}]", res_elem.name, Block(source).dumpNames()); - } else { @@ -1080,14 +1079,16 @@ ActionsDAGPtr ActionsDAG::makeConvertingActions( if (ignore_constant_values) dst_node = &actions_dag->addColumn(res_elem); else if (res_const->getField() != src_const->getField()) - throw Exception("Cannot convert column " + backQuote(res_elem.name) + " because " - "it is constant but values of constants are different in source and result", - ErrorCodes::ILLEGAL_COLUMN); + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Cannot convert column `{}` because it is constant but values of constants are different in source and result", + res_elem.name); } else - throw Exception("Cannot convert column " + backQuote(res_elem.name) + " because " - "it is non constant in source stream but must be constant in result", - ErrorCodes::ILLEGAL_COLUMN); + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Cannot convert column `{}` because it is non constant in source stream but must be constant in result", + res_elem.name); } /// Add CAST function to convert into result type if needed. @@ -1119,10 +1120,8 @@ ActionsDAGPtr ActionsDAG::makeConvertingActions( if (add_casted_columns) { if (inputs.contains(dst_node->result_name)) - throw Exception("Cannot convert column " + backQuote(res_elem.name) + - " to "+ backQuote(dst_node->result_name) + - " because other column have same name", - ErrorCodes::ILLEGAL_COLUMN); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Cannot convert column `{}` to `{}` because other column have same name", + res_elem.name, dst_node->result_name); if (new_names) new_names->emplace(res_elem.name, dst_node->result_name); diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index 05c9133cb35..3457e01f98f 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -184,7 +184,9 @@ struct AggregationMethodOneNumber AggregationMethodOneNumber() = default; template - AggregationMethodOneNumber(const Other & other) : data(other.data) {} + explicit AggregationMethodOneNumber(const Other & other) : data(other.data) + { + } /// To use one `Method` in different threads, use different `State`. using State = ColumnsHashing::HashMethodOneNumber - AggregationMethodString(const Other & other) : data(other.data) {} + explicit AggregationMethodString(const Other & other) : data(other.data) + { + } using State = ColumnsHashing::HashMethodString; @@ -247,7 +251,9 @@ struct AggregationMethodStringNoCache AggregationMethodStringNoCache() = default; template - AggregationMethodStringNoCache(const Other & other) : data(other.data) {} + explicit AggregationMethodStringNoCache(const Other & other) : data(other.data) + { + } using State = ColumnsHashing::HashMethodString; @@ -275,7 +281,9 @@ struct AggregationMethodFixedString AggregationMethodFixedString() = default; template - AggregationMethodFixedString(const Other & other) : data(other.data) {} + explicit AggregationMethodFixedString(const Other & other) : data(other.data) + { + } using State = ColumnsHashing::HashMethodFixedString; @@ -302,7 +310,9 @@ struct AggregationMethodFixedStringNoCache AggregationMethodFixedStringNoCache() = default; template - AggregationMethodFixedStringNoCache(const Other & other) : data(other.data) {} + explicit AggregationMethodFixedStringNoCache(const Other & other) : data(other.data) + { + } using State = ColumnsHashing::HashMethodFixedString; @@ -373,7 +383,9 @@ struct AggregationMethodKeysFixed AggregationMethodKeysFixed() = default; template - AggregationMethodKeysFixed(const Other & other) : data(other.data) {} + explicit AggregationMethodKeysFixed(const Other & other) : data(other.data) + { + } using State = ColumnsHashing::HashMethodKeysFixed< typename Data::value_type, @@ -462,7 +474,9 @@ struct AggregationMethodSerialized AggregationMethodSerialized() = default; template - AggregationMethodSerialized(const Other & other) : data(other.data) {} + explicit AggregationMethodSerialized(const Other & other) : data(other.data) + { + } using State = ColumnsHashing::HashMethodSerialized; diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index 72e49505b54..b8262370a0d 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -373,7 +373,7 @@ static void calculateMaxAndSum(Max & max, Sum & sum, T x) max = x; } -#if USE_JEMALLOC && JEMALLOC_VERSION_MAJOR >= 4 +#if USE_JEMALLOC uint64_t updateJemallocEpoch() { uint64_t value = 0; @@ -1429,7 +1429,7 @@ void AsynchronousMetrics::update(std::chrono::system_clock::time_point update_ti } } -#if USE_JEMALLOC && JEMALLOC_VERSION_MAJOR >= 4 +#if USE_JEMALLOC // 'epoch' is a special mallctl -- it updates the statistics. Without it, all // the following calls will return stale values. It increments and returns // the current epoch number, which might be useful to log as a sanity check. diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 5577d75fb4e..c9ec0606ebd 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -422,7 +422,7 @@ Context::Context(const Context &) = default; Context & Context::operator=(const Context &) = default; SharedContextHolder::SharedContextHolder(SharedContextHolder &&) noexcept = default; -SharedContextHolder & SharedContextHolder::operator=(SharedContextHolder &&) = default; +SharedContextHolder & SharedContextHolder::operator=(SharedContextHolder &&) noexcept = default; SharedContextHolder::SharedContextHolder() = default; SharedContextHolder::~SharedContextHolder() = default; SharedContextHolder::SharedContextHolder(std::unique_ptr shared_context) diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 2a2783603a2..c3615db9068 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -170,7 +170,7 @@ struct SharedContextHolder explicit SharedContextHolder(std::unique_ptr shared_context); SharedContextHolder(SharedContextHolder &&) noexcept; - SharedContextHolder & operator=(SharedContextHolder &&); + SharedContextHolder & operator=(SharedContextHolder &&) noexcept; ContextSharedPart * get() const { return shared.get(); } void reset(); diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 4a5f18a408f..2e464053131 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -301,7 +301,7 @@ NamesAndTypesList ExpressionAnalyzer::analyzeJoin(ActionsDAGPtr & actions, const } NamesAndTypesList result_columns = src_columns; - syntax->analyzed_join->addJoinedColumnsAndCorrectTypes(result_columns,false); + syntax->analyzed_join->addJoinedColumnsAndCorrectTypes(result_columns, false); return result_columns; } @@ -991,7 +991,8 @@ JoinPtr SelectQueryExpressionAnalyzer::makeTableJoin( if (auto storage = analyzed_join->getStorageJoin()) { - std::tie(left_convert_actions, right_convert_actions) = analyzed_join->createConvertingActions(left_columns, {}); + auto right_columns = storage->getRightSampleBlock().getColumnsWithTypeAndName(); + std::tie(left_convert_actions, right_convert_actions) = analyzed_join->createConvertingActions(left_columns, right_columns); return storage->getJoinLocked(analyzed_join, getContext()); } diff --git a/src/Interpreters/ExternalUserDefinedExecutableFunctionsLoader.cpp b/src/Interpreters/ExternalUserDefinedExecutableFunctionsLoader.cpp index b266746642f..31b4d4a621c 100644 --- a/src/Interpreters/ExternalUserDefinedExecutableFunctionsLoader.cpp +++ b/src/Interpreters/ExternalUserDefinedExecutableFunctionsLoader.cpp @@ -83,6 +83,10 @@ ExternalLoader::LoadablePtr ExternalUserDefinedExecutableFunctionsLoader::create String format = config.getString(key_in_config + ".format"); DataTypePtr result_type = DataTypeFactory::instance().get(config.getString(key_in_config + ".return_type")); + String result_name = "result"; + if (config.has(key_in_config + ".return_name")) + result_name = config.getString(key_in_config + ".return_name"); + bool send_chunk_header = config.getBool(key_in_config + ".send_chunk_header", false); size_t command_termination_timeout_seconds = config.getUInt64(key_in_config + ".command_termination_timeout", 10); size_t command_read_timeout_milliseconds = config.getUInt64(key_in_config + ".command_read_timeout", 10000); @@ -106,33 +110,46 @@ ExternalLoader::LoadablePtr ExternalUserDefinedExecutableFunctionsLoader::create if (config.has(key_in_config + ".lifetime")) lifetime = ExternalLoadableLifetime(config, key_in_config + ".lifetime"); - std::vector argument_types; + std::vector arguments; Poco::Util::AbstractConfiguration::Keys config_elems; config.keys(key_in_config, config_elems); + size_t argument_number = 1; + for (const auto & config_elem : config_elems) { if (!startsWith(config_elem, "argument")) continue; + UserDefinedExecutableFunctionArgument argument; + const auto argument_prefix = key_in_config + '.' + config_elem + '.'; - auto argument_type = DataTypeFactory::instance().get(config.getString(argument_prefix + "type")); - argument_types.emplace_back(std::move(argument_type)); + + argument.type = DataTypeFactory::instance().get(config.getString(argument_prefix + "type")); + + if (config.has(argument_prefix + "name")) + argument.name = config.getString(argument_prefix + "name"); + else + argument.name = "c" + std::to_string(argument_number); + + ++argument_number; + arguments.emplace_back(std::move(argument)); } UserDefinedExecutableFunctionConfiguration function_configuration { - .name = std::move(name), //-V1030 - .command = std::move(command_value), //-V1030 - .command_arguments = std::move(command_arguments), //-V1030 - .argument_types = std::move(argument_types), //-V1030 - .result_type = std::move(result_type), //-V1030 + .name = std::move(name), + .command = std::move(command_value), + .command_arguments = std::move(command_arguments), + .arguments = std::move(arguments), + .result_type = std::move(result_type), + .result_name = std::move(result_name), }; ShellCommandSourceCoordinator::Configuration shell_command_coordinator_configration { - .format = std::move(format), //-V1030 + .format = std::move(format), .command_termination_timeout_seconds = command_termination_timeout_seconds, .command_read_timeout_milliseconds = command_read_timeout_milliseconds, .command_write_timeout_milliseconds = command_write_timeout_milliseconds, diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index 82d8356b7c7..47bb1bfdd9f 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -232,8 +232,6 @@ HashJoin::HashJoin(std::shared_ptr table_join_, const Block & right_s : table_join(table_join_) , kind(table_join->kind()) , strictness(table_join->strictness()) - , nullable_right_side(table_join->forceNullableRight()) - , nullable_left_side(table_join->forceNullableLeft()) , any_take_last_row(any_take_last_row_) , asof_inequality(table_join->getAsofInequality()) , data(std::make_shared()) @@ -273,9 +271,6 @@ HashJoin::HashJoin(std::shared_ptr table_join_, const Block & right_s JoinCommon::createMissedColumns(sample_block_with_columns_to_add); - if (nullable_right_side) - JoinCommon::convertColumnsToNullable(sample_block_with_columns_to_add); - size_t disjuncts_num = table_join->getClauses().size(); data->maps.resize(disjuncts_num); key_sizes.reserve(disjuncts_num); @@ -306,9 +301,6 @@ HashJoin::HashJoin(std::shared_ptr table_join_, const Block & right_s if (key_columns.size() <= 1) throw Exception("ASOF join needs at least one equi-join column", ErrorCodes::SYNTAX_ERROR); - if (right_table_keys.getByName(key_names_right.back()).type->isNullable()) - throw Exception("ASOF join over right table Nullable column is not implemented", ErrorCodes::NOT_IMPLEMENTED); - size_t asof_size; asof_type = AsofRowRefs::getTypeSize(*key_columns.back(), asof_size); key_columns.pop_back(); @@ -719,12 +711,6 @@ void HashJoin::initRightBlockStructure(Block & saved_block_sample) saved_block_sample.insert(column); } } - - if (nullable_right_side) - { - JoinCommon::convertColumnsToNullable(saved_block_sample, (isFull(kind) && !multiple_disjuncts ? right_table_keys.columns() : 0)); - } - } Block HashJoin::structureRightBlock(const Block & block) const @@ -1476,9 +1462,6 @@ void HashJoin::joinBlockImpl( if constexpr (jf.right || jf.full) { materializeBlockInplace(block); - - if (nullable_left_side) - JoinCommon::convertColumnsToNullable(block); } /** For LEFT/INNER JOIN, the saved blocks do not contain keys. @@ -1527,13 +1510,13 @@ void HashJoin::joinBlockImpl( continue; const auto & col = block.getByName(left_name); - bool is_nullable = nullable_right_side || right_key.type->isNullable(); + bool is_nullable = JoinCommon::isNullable(right_key.type); auto right_col_name = getTableJoin().renamedRightColumnName(right_key.name); ColumnWithTypeAndName right_col(col.column, col.type, right_col_name); if (right_col.type->lowCardinality() != right_key.type->lowCardinality()) JoinCommon::changeLowCardinalityInplace(right_col); right_col = correctNullability(std::move(right_col), is_nullable); - block.insert(right_col); + block.insert(std::move(right_col)); } } } @@ -1559,7 +1542,7 @@ void HashJoin::joinBlockImpl( continue; const auto & col = block.getByName(left_name); - bool is_nullable = nullable_right_side || right_key.type->isNullable(); + bool is_nullable = JoinCommon::isNullable(right_key.type); ColumnPtr thin_column = filterWithBlanks(col.column, filter); @@ -1567,7 +1550,7 @@ void HashJoin::joinBlockImpl( if (right_col.type->lowCardinality() != right_key.type->lowCardinality()) JoinCommon::changeLowCardinalityInplace(right_col); right_col = correctNullability(std::move(right_col), is_nullable, null_map_filter); - block.insert(right_col); + block.insert(std::move(right_col)); if constexpr (jf.need_replication) right_keys_to_replicate.push_back(block.getPositionByName(right_key.name)); @@ -1754,8 +1737,6 @@ void HashJoin::joinBlock(Block & block, ExtraBlockPtr & not_processed) if (kind == ASTTableJoin::Kind::Right || kind == ASTTableJoin::Kind::Full) { materializeBlockInplace(block); - if (nullable_left_side) - JoinCommon::convertColumnsToNullable(block); } if (overDictionary()) @@ -2091,7 +2072,7 @@ void HashJoin::reuseJoinedData(const HashJoin & join) const ColumnWithTypeAndName & HashJoin::rightAsofKeyColumn() const { - /// It should be nullable if nullable_right_side is true + /// It should be nullable when right side is nullable return savedBlockSample().getByName(table_join->getOnlyClause().key_names_right.back()); } diff --git a/src/Interpreters/HashJoin.h b/src/Interpreters/HashJoin.h index 50c8b2c55ee..4e5ab21f08d 100644 --- a/src/Interpreters/HashJoin.h +++ b/src/Interpreters/HashJoin.h @@ -360,8 +360,6 @@ private: /// This join was created from StorageJoin and it is already filled. bool from_storage_join = false; - bool nullable_right_side; /// In case of LEFT and FULL joins, if use_nulls, convert right-side columns to Nullable. - bool nullable_left_side; /// In case of RIGHT and FULL joins, if use_nulls, convert left-side columns to Nullable. bool any_take_last_row; /// Overwrite existing values when encountering the same key again std::optional asof_type; ASOF::Inequality asof_inequality; diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index ac807f4f782..5c096a7dbac 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -286,7 +286,6 @@ InterpreterSelectQuery::InterpreterSelectQuery( query_info.ignore_projections = options.ignore_projections; query_info.is_projection_query = options.is_projection_query; - query_info.original_query = query_ptr->clone(); initSettings(); const Settings & settings = context->getSettingsRef(); @@ -310,6 +309,8 @@ InterpreterSelectQuery::InterpreterSelectQuery( ApplyWithSubqueryVisitor().visit(query_ptr); } + query_info.original_query = query_ptr->clone(); + JoinedTables joined_tables(getSubqueryContext(context), getSelectQuery(), options.with_all_cols); bool got_storage_from_query = false; @@ -1776,6 +1777,7 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc bool optimize_trivial_count = syntax_analyzer_result->optimize_trivial_count && (settings.max_parallel_replicas <= 1) + && !settings.allow_experimental_query_deduplication && storage && storage->getName() != "MaterializedMySQL" && !row_policy_filter diff --git a/src/Interpreters/MergeJoin.cpp b/src/Interpreters/MergeJoin.cpp index 7f22386f54b..c51d54c13ff 100644 --- a/src/Interpreters/MergeJoin.cpp +++ b/src/Interpreters/MergeJoin.cpp @@ -465,8 +465,6 @@ MergeJoin::MergeJoin(std::shared_ptr table_join_, const Block & right : table_join(table_join_) , size_limits(table_join->sizeLimits()) , right_sample_block(right_sample_block_) - , nullable_right_side(table_join->forceNullableRight()) - , nullable_left_side(table_join->forceNullableLeft()) , is_any_join(table_join->strictness() == ASTTableJoin::Strictness::Any) , is_all_join(table_join->strictness() == ASTTableJoin::Strictness::All) , is_semi_join(table_join->strictness() == ASTTableJoin::Strictness::Semi) @@ -545,9 +543,6 @@ MergeJoin::MergeJoin(std::shared_ptr table_join_, const Block & right JoinCommon::createMissedColumns(right_columns_to_add); - if (nullable_right_side) - JoinCommon::convertColumnsToNullable(right_columns_to_add); - makeSortAndMerge(key_names_left, left_sort_description, left_merge_description); makeSortAndMerge(key_names_right, right_sort_description, right_merge_description); @@ -710,9 +705,6 @@ void MergeJoin::joinBlock(Block & block, ExtraBlockPtr & not_processed) JoinCommon::convertToFullColumnsInplace(block, key_names_left, false); sortBlock(block, left_sort_description); - - if (nullable_left_side) - JoinCommon::convertColumnsToNullable(block); } if (!not_processed && left_blocks_buffer) diff --git a/src/Interpreters/MergeJoin.h b/src/Interpreters/MergeJoin.h index 2cf287fd2fd..89570015d0f 100644 --- a/src/Interpreters/MergeJoin.h +++ b/src/Interpreters/MergeJoin.h @@ -102,8 +102,6 @@ private: SortedBlocksWriter::SortedFiles flushed_right_blocks; Block totals; std::atomic is_in_memory{true}; - const bool nullable_right_side; - const bool nullable_left_side; const bool is_any_join; const bool is_all_join; const bool is_semi_join; diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index d17d8e4972a..26146781327 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -195,7 +195,10 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as ErrorCodes::QUERY_WITH_SAME_ID_IS_ALREADY_RUNNING); } - ProcessListForUser & user_process_list = user_to_queries[client_info.current_user]; + auto user_process_list_it = user_to_queries.find(client_info.current_user); + if (user_process_list_it == user_to_queries.end()) + user_process_list_it = user_to_queries.emplace(client_info.current_user, this).first; + ProcessListForUser & user_process_list = user_process_list_it->second; /// Actualize thread group info auto thread_group = CurrentThread::getGroup(); @@ -235,10 +238,7 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as process_it->setUserProcessList(&user_process_list); - { - BlockQueryIfMemoryLimit block_query{user_process_list.user_overcommit_tracker}; - user_process_list.queries.emplace(client_info.current_query_id, &res->get()); - } + user_process_list.queries.emplace(client_info.current_query_id, &res->get()); /// Track memory usage for all simultaneously running queries from single user. user_process_list.user_memory_tracker.setOrRaiseHardLimit(settings.max_memory_usage_for_user); @@ -289,7 +289,6 @@ ProcessListEntry::~ProcessListEntry() { if (running_query->second == process_list_element_ptr) { - BlockQueryIfMemoryLimit block_query{user_process_list.user_overcommit_tracker}; user_process_list.queries.erase(running_query->first); found = true; } @@ -502,8 +501,8 @@ ProcessList::Info ProcessList::getInfo(bool get_thread_list, bool get_profile_ev } -ProcessListForUser::ProcessListForUser() - : user_overcommit_tracker(this) +ProcessListForUser::ProcessListForUser(ProcessList * global_process_list) + : user_overcommit_tracker(global_process_list, this) { user_memory_tracker.setOvercommitTracker(&user_overcommit_tracker); } diff --git a/src/Interpreters/ProcessList.h b/src/Interpreters/ProcessList.h index 5fb4bca7e13..493b2ba81a9 100644 --- a/src/Interpreters/ProcessList.h +++ b/src/Interpreters/ProcessList.h @@ -216,7 +216,7 @@ struct ProcessListForUserInfo /// Data about queries for one user. struct ProcessListForUser { - ProcessListForUser(); + ProcessListForUser(ProcessList * global_process_list); /// query_id -> ProcessListElement(s). There can be multiple queries with the same query_id as long as all queries except one are cancelled. using QueryToElement = std::unordered_map; @@ -291,6 +291,8 @@ public: protected: friend class ProcessListEntry; + friend struct ::UserOvercommitTracker; + friend struct ::GlobalOvercommitTracker; mutable std::mutex mutex; mutable std::condition_variable have_space; /// Number of currently running queries has become less than maximum. @@ -349,10 +351,11 @@ public: max_size = max_size_; } + // Before calling this method you should be sure + // that lock is acquired. template void processEachQueryStatus(F && func) const { - std::lock_guard lk(mutex); for (auto && query : processes) func(query); } diff --git a/src/Interpreters/TableJoin.cpp b/src/Interpreters/TableJoin.cpp index 42424b81192..8fda5371f68 100644 --- a/src/Interpreters/TableJoin.cpp +++ b/src/Interpreters/TableJoin.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -25,6 +26,8 @@ #include #include +#include +#include namespace DB @@ -468,8 +471,33 @@ TableJoin::createConvertingActions(const ColumnsWithTypeAndName & left_sample_co NameToNameMap left_key_column_rename; NameToNameMap right_key_column_rename; - auto left_converting_actions = applyKeyConvertToTable(left_sample_columns, left_type_map, left_key_column_rename); - auto right_converting_actions = applyKeyConvertToTable(right_sample_columns, right_type_map, right_key_column_rename); + auto left_converting_actions = applyKeyConvertToTable(left_sample_columns, left_type_map, left_key_column_rename, forceNullableLeft()); + auto right_converting_actions = applyKeyConvertToTable(right_sample_columns, right_type_map, right_key_column_rename, forceNullableRight()); + + { + auto log_actions = [](const String & side, const ActionsDAGPtr & dag) + { + if (dag) + { + std::vector input_cols; + for (const auto & col : dag->getRequiredColumns()) + input_cols.push_back(col.name + ": " + col.type->getName()); + + std::vector output_cols; + for (const auto & col : dag->getResultColumns()) + output_cols.push_back(col.name + ": " + col.type->getName()); + + LOG_DEBUG(&Poco::Logger::get("TableJoin"), "{} JOIN converting actions: [{}] -> [{}]", + side, fmt::join(input_cols, ", "), fmt::join(output_cols, ", ")); + } + else + { + LOG_DEBUG(&Poco::Logger::get("TableJoin"), "{} JOIN converting actions: empty", side); + } + }; + log_actions("Left", left_converting_actions); + log_actions("Right", right_converting_actions); + } forAllKeys(clauses, [&](auto & left_key, auto & right_key) { @@ -482,10 +510,18 @@ TableJoin::createConvertingActions(const ColumnsWithTypeAndName & left_sample_co } template -bool TableJoin::inferJoinKeyCommonType(const LeftNamesAndTypes & left, const RightNamesAndTypes & right, bool allow_right) +void TableJoin::inferJoinKeyCommonType(const LeftNamesAndTypes & left, const RightNamesAndTypes & right, bool allow_right) { + if (strictness() == ASTTableJoin::Strictness::Asof) + { + if (clauses.size() != 1) + throw DB::Exception("ASOF join over multiple keys is not supported", ErrorCodes::NOT_IMPLEMENTED); + if (right.back().type->isNullable()) + throw DB::Exception("ASOF join over right table Nullable column is not implemented", ErrorCodes::NOT_IMPLEMENTED); + } + if (!left_type_map.empty() || !right_type_map.empty()) - return true; + return; NameToTypeMap left_types; for (const auto & col : left) @@ -544,33 +580,74 @@ bool TableJoin::inferJoinKeyCommonType(const LeftNamesAndTypes & left, const Rig formatTypeMap(left_type_map, left_types), formatTypeMap(right_type_map, right_types)); } - - return !left_type_map.empty(); } -ActionsDAGPtr TableJoin::applyKeyConvertToTable( - const ColumnsWithTypeAndName & cols_src, const NameToTypeMap & type_mapping, NameToNameMap & key_column_rename) const +static ActionsDAGPtr changeKeyTypes(const ColumnsWithTypeAndName & cols_src, + const TableJoin::NameToTypeMap & type_mapping, + bool add_new_cols, + NameToNameMap & key_column_rename) { - bool has_some_to_do = false; - ColumnsWithTypeAndName cols_dst = cols_src; + bool has_some_to_do = false; for (auto & col : cols_dst) { if (auto it = type_mapping.find(col.name); it != type_mapping.end()) { - has_some_to_do = true; col.type = it->second; col.column = nullptr; + has_some_to_do = true; } } if (!has_some_to_do) return nullptr; + return ActionsDAG::makeConvertingActions(cols_src, cols_dst, ActionsDAG::MatchColumnsMode::Name, true, add_new_cols, &key_column_rename); +} - /// Returns converting actions for tables that need to be performed before join - auto dag = ActionsDAG::makeConvertingActions( - cols_src, cols_dst, ActionsDAG::MatchColumnsMode::Name, true, !hasUsing(), &key_column_rename); +static ActionsDAGPtr changeTypesToNullable(const ColumnsWithTypeAndName & cols_src, const NameSet & exception_cols) +{ + ColumnsWithTypeAndName cols_dst = cols_src; + bool has_some_to_do = false; + for (auto & col : cols_dst) + { + if (exception_cols.contains(col.name)) + continue; + col.type = JoinCommon::convertTypeToNullable(col.type); + col.column = nullptr; + has_some_to_do = true; + } - return dag; + if (!has_some_to_do) + return nullptr; + return ActionsDAG::makeConvertingActions(cols_src, cols_dst, ActionsDAG::MatchColumnsMode::Name, true, false, nullptr); +} + +ActionsDAGPtr TableJoin::applyKeyConvertToTable( + const ColumnsWithTypeAndName & cols_src, + const NameToTypeMap & type_mapping, + NameToNameMap & key_column_rename, + bool make_nullable) const +{ + /// Create DAG to convert key columns + ActionsDAGPtr dag_stage1 = changeKeyTypes(cols_src, type_mapping, !hasUsing(), key_column_rename); + + /// Create DAG to make columns nullable if needed + if (make_nullable) + { + /// Do not need to make nullable temporary columns that would be used only as join keys, but now shown to user + NameSet cols_not_nullable; + for (const auto & t : key_column_rename) + cols_not_nullable.insert(t.second); + + ColumnsWithTypeAndName input_cols = dag_stage1 ? dag_stage1->getResultColumns() : cols_src; + ActionsDAGPtr dag_stage2 = changeTypesToNullable(input_cols, cols_not_nullable); + + /// Merge dags if we got two ones + if (dag_stage1) + return ActionsDAG::merge(std::move(*dag_stage1), std::move(*dag_stage2)); + else + return dag_stage2; + } + return dag_stage1; } void TableJoin::setStorageJoin(std::shared_ptr storage) diff --git a/src/Interpreters/TableJoin.h b/src/Interpreters/TableJoin.h index b3e5748fb2f..a3f6f7bf36b 100644 --- a/src/Interpreters/TableJoin.h +++ b/src/Interpreters/TableJoin.h @@ -152,7 +152,8 @@ private: /// Create converting actions and change key column names if required ActionsDAGPtr applyKeyConvertToTable( - const ColumnsWithTypeAndName & cols_src, const NameToTypeMap & type_mapping, NameToNameMap & key_column_rename) const; + const ColumnsWithTypeAndName & cols_src, const NameToTypeMap & type_mapping, NameToNameMap & key_column_rename, + bool make_nullable) const; void addKey(const String & left_name, const String & right_name, const ASTPtr & left_ast, const ASTPtr & right_ast = nullptr); @@ -160,7 +161,7 @@ private: /// Calculates common supertypes for corresponding join key columns. template - bool inferJoinKeyCommonType(const LeftNamesAndTypes & left, const RightNamesAndTypes & right, bool allow_right); + void inferJoinKeyCommonType(const LeftNamesAndTypes & left, const RightNamesAndTypes & right, bool allow_right); NamesAndTypesList correctedColumnsAddedByJoin() const; diff --git a/src/Interpreters/UserDefinedExecutableFunction.h b/src/Interpreters/UserDefinedExecutableFunction.h index 80d6b85ad90..434c77e9236 100644 --- a/src/Interpreters/UserDefinedExecutableFunction.h +++ b/src/Interpreters/UserDefinedExecutableFunction.h @@ -10,13 +10,20 @@ namespace DB { +struct UserDefinedExecutableFunctionArgument +{ + DataTypePtr type; + String name; +}; + struct UserDefinedExecutableFunctionConfiguration { std::string name; std::string command; std::vector command_arguments; - std::vector argument_types; + std::vector arguments; DataTypePtr result_type; + String result_name; }; class UserDefinedExecutableFunction final : public IExternalLoadable diff --git a/src/Interpreters/UserDefinedExecutableFunctionFactory.cpp b/src/Interpreters/UserDefinedExecutableFunctionFactory.cpp index 10cb806028e..8d40dc6dfc8 100644 --- a/src/Interpreters/UserDefinedExecutableFunctionFactory.cpp +++ b/src/Interpreters/UserDefinedExecutableFunctionFactory.cpp @@ -42,7 +42,7 @@ public: bool isVariadic() const override { return false; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } - size_t getNumberOfArguments() const override { return executable_function->getConfiguration().argument_types.size(); } + size_t getNumberOfArguments() const override { return executable_function->getConfiguration().arguments.size(); } bool useDefaultImplementationForConstants() const override { return true; } bool useDefaultImplementationForNulls() const override { return true; } @@ -90,7 +90,11 @@ public: auto & column_with_type = arguments_copy[i]; column_with_type.column = column_with_type.column->convertToFullColumnIfConst(); - const auto & argument_type = configuration.argument_types[i]; + const auto & argument = configuration.arguments[i]; + column_with_type.name = argument.name; + + const auto & argument_type = argument.type; + if (areTypesEqual(arguments_copy[i].type, argument_type)) continue; @@ -101,7 +105,7 @@ public: column_with_type = std::move(column_to_cast); } - ColumnWithTypeAndName result(result_type, "result"); + ColumnWithTypeAndName result(result_type, configuration.result_name); Block result_block({result}); Block arguments_block(arguments_copy); diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index f40d35e970b..c17e1173b20 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -413,9 +413,10 @@ static std::tuple executeQueryImpl( ASTPtr ast; const char * query_end; - /// Don't limit the size of internal queries. - size_t max_query_size = 0; - if (!internal) max_query_size = settings.max_query_size; + size_t max_query_size = settings.max_query_size; + /// Don't limit the size of internal queries or distributed subquery. + if (internal || client_info.query_kind == ClientInfo::QueryKind::SECONDARY_QUERY) + max_query_size = 0; String query_database; String query_table; diff --git a/src/Interpreters/join_common.cpp b/src/Interpreters/join_common.cpp index ca55fde0740..478df653f3b 100644 --- a/src/Interpreters/join_common.cpp +++ b/src/Interpreters/join_common.cpp @@ -115,10 +115,21 @@ bool canBecomeNullable(const DataTypePtr & type) return can_be_inside; } +bool isNullable(const DataTypePtr & type) +{ + bool is_nullable = type->isNullable(); + if (const auto * low_cardinality_type = typeid_cast(type.get())) + is_nullable |= low_cardinality_type->getDictionaryType()->isNullable(); + return is_nullable; +} + /// Add nullability to type. /// Note: LowCardinality(T) transformed to LowCardinality(Nullable(T)) DataTypePtr convertTypeToNullable(const DataTypePtr & type) { + if (isNullable(type)) + return type; + if (const auto * low_cardinality_type = typeid_cast(type.get())) { const auto & dict_type = low_cardinality_type->getDictionaryType(); @@ -425,10 +436,13 @@ void checkTypesOfKeys(const Block & block_left, const Names & key_names_left, DataTypePtr right_type = removeNullable(recursiveRemoveLowCardinality(block_right.getByName(key_names_right[i]).type)); if (!left_type->equals(*right_type)) - throw Exception("Type mismatch of columns to JOIN by: " - + key_names_left[i] + " " + left_type->getName() + " at left, " - + key_names_right[i] + " " + right_type->getName() + " at right", - ErrorCodes::TYPE_MISMATCH); + { + throw DB::Exception( + ErrorCodes::TYPE_MISMATCH, + "Type mismatch of columns to JOIN by: {} {} at left, {} {} at right", + key_names_left[i], left_type->getName(), + key_names_right[i], right_type->getName()); + } } } diff --git a/src/Interpreters/join_common.h b/src/Interpreters/join_common.h index 3e5a22f33bf..38b431db3e0 100644 --- a/src/Interpreters/join_common.h +++ b/src/Interpreters/join_common.h @@ -59,6 +59,7 @@ private: }; +bool isNullable(const DataTypePtr & type); bool canBecomeNullable(const DataTypePtr & type); DataTypePtr convertTypeToNullable(const DataTypePtr & type); void convertColumnToNullable(ColumnWithTypeAndName & column); diff --git a/src/Parsers/ASTSystemQuery.cpp b/src/Parsers/ASTSystemQuery.cpp index b8056862bfc..a4b0a69faaa 100644 --- a/src/Parsers/ASTSystemQuery.cpp +++ b/src/Parsers/ASTSystemQuery.cpp @@ -134,6 +134,12 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState &, << (settings.hilite ? hilite_none : ""); }; + auto print_identifier = [&](const String & identifier) + { + settings.ostr << " " << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(identifier) + << (settings.hilite ? hilite_none : ""); + }; + if (!cluster.empty()) formatOnCluster(settings); @@ -161,9 +167,19 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState &, || type == Type::RESTORE_REPLICA || type == Type::SYNC_REPLICA || type == Type::FLUSH_DISTRIBUTED - || type == Type::RELOAD_DICTIONARY) + || type == Type::RELOAD_DICTIONARY + || type == Type::RELOAD_MODEL + || type == Type::RELOAD_FUNCTION + || type == Type::RESTART_DISK) { - print_database_table(); + if (table) + print_database_table(); + else if (!target_model.empty()) + print_identifier(target_model); + else if (!target_function.empty()) + print_identifier(target_function); + else if (!disk.empty()) + print_identifier(disk); } else if (type == Type::DROP_REPLICA) { diff --git a/src/Parsers/Access/ASTCreateUserQuery.cpp b/src/Parsers/Access/ASTCreateUserQuery.cpp index 18030a5ed80..f8e1109886e 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.cpp +++ b/src/Parsers/Access/ASTCreateUserQuery.cpp @@ -34,46 +34,58 @@ namespace } String auth_type_name = AuthenticationTypeInfo::get(auth_type).name; - String by_keyword = "BY"; - std::optional by_value; + String value_prefix; + std::optional value; + const boost::container::flat_set * values = nullptr; - if ( - show_password || + if (show_password || auth_type == AuthenticationType::LDAP || - auth_type == AuthenticationType::KERBEROS - ) + auth_type == AuthenticationType::KERBEROS || + auth_type == AuthenticationType::SSL_CERTIFICATE) { switch (auth_type) { case AuthenticationType::PLAINTEXT_PASSWORD: { - by_value = auth_data.getPassword(); + value_prefix = "BY"; + value = auth_data.getPassword(); break; } case AuthenticationType::SHA256_PASSWORD: { auth_type_name = "sha256_hash"; - by_value = auth_data.getPasswordHashHex(); + value_prefix = "BY"; + value = auth_data.getPasswordHashHex(); break; } case AuthenticationType::DOUBLE_SHA1_PASSWORD: { auth_type_name = "double_sha1_hash"; - by_value = auth_data.getPasswordHashHex(); + value_prefix = "BY"; + value = auth_data.getPasswordHashHex(); break; } case AuthenticationType::LDAP: { - by_keyword = "SERVER"; - by_value = auth_data.getLDAPServerName(); + value_prefix = "SERVER"; + value = auth_data.getLDAPServerName(); break; } case AuthenticationType::KERBEROS: { - by_keyword = "REALM"; const auto & realm = auth_data.getKerberosRealm(); if (!realm.empty()) - by_value = realm; + { + value_prefix = "REALM"; + value = realm; + } + break; + } + + case AuthenticationType::SSL_CERTIFICATE: + { + value_prefix = "CN"; + values = &auth_data.getSSLCertificateCommonNames(); break; } @@ -86,10 +98,26 @@ namespace settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED WITH " << auth_type_name << (settings.hilite ? IAST::hilite_none : ""); - if (by_value) + if (!value_prefix.empty()) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " " << by_keyword << " " - << (settings.hilite ? IAST::hilite_none : "") << quoteString(*by_value); + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " " << value_prefix + << (settings.hilite ? IAST::hilite_none : ""); + } + + if (value) + { + settings.ostr << " " << quoteString(*value); + } + else if (values) + { + settings.ostr << " "; + bool need_comma = false; + for (const auto & item : *values) + { + if (std::exchange(need_comma, true)) + settings.ostr << ", "; + settings.ostr << quoteString(item); + } } } diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index c5b8c9e37b3..cde14e632dd 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -52,6 +52,7 @@ namespace bool expect_hash = false; bool expect_ldap_server_name = false; bool expect_kerberos_realm = false; + bool expect_common_names = false; if (ParserKeyword{"WITH"}.ignore(pos, expected)) { @@ -65,6 +66,8 @@ namespace expect_ldap_server_name = true; else if (check_type == AuthenticationType::KERBEROS) expect_kerberos_realm = true; + else if (check_type == AuthenticationType::SSL_CERTIFICATE) + expect_common_names = true; else if (check_type != AuthenticationType::NO_PASSWORD) expect_password = true; @@ -96,6 +99,7 @@ namespace } String value; + boost::container::flat_set common_names; if (expect_password || expect_hash) { ASTPtr ast; @@ -123,6 +127,18 @@ namespace value = ast->as().value.safeGet(); } } + else if (expect_common_names) + { + if (!ParserKeyword{"CN"}.ignore(pos, expected)) + return false; + + ASTPtr ast; + if (!ParserList{std::make_unique(), std::make_unique(TokenType::Comma), false}.parse(pos, ast, expected)) + return false; + + for (const auto & ast_child : ast->children) + common_names.insert(ast_child->as().value.safeGet()); + } auth_data = AuthenticationData{*type}; if (expect_password) @@ -133,6 +149,8 @@ namespace auth_data.setLDAPServerName(value); else if (expect_kerberos_realm) auth_data.setKerberosRealm(value); + else if (expect_common_names) + auth_data.setSSLCertificateCommonNames(std::move(common_names)); return true; }); diff --git a/src/Parsers/ParserSystemQuery.cpp b/src/Parsers/ParserSystemQuery.cpp index 915a01cc063..61e96b9c1de 100644 --- a/src/Parsers/ParserSystemQuery.cpp +++ b/src/Parsers/ParserSystemQuery.cpp @@ -64,6 +64,78 @@ static bool parseQueryWithOnClusterAndMaybeTable(std::shared_ptr return true; } +enum class SystemQueryTargetType +{ + Model, + Function, + Disk +}; + +static bool parseQueryWithOnClusterAndTarget(std::shared_ptr & res, IParser::Pos & pos, Expected & expected, SystemQueryTargetType target_type) +{ + /// Better form for user: SYSTEM target_name ON CLUSTER cluster + /// Query rewritten form + form while executing on cluster: SYSTEM ON CLUSTER cluster target_name + /// Need to support both + + String cluster; + bool parsed_on_cluster = false; + + if (ParserKeyword{"ON"}.ignore(pos, expected)) + { + if (!ASTQueryWithOnCluster::parse(pos, cluster, expected)) + return false; + parsed_on_cluster = true; + } + + String target; + ASTPtr temporary_string_literal; + + if (ParserStringLiteral{}.parse(pos, temporary_string_literal, expected)) + { + target = temporary_string_literal->as().value.safeGet(); + } + else + { + ParserIdentifier identifier_parser; + ASTPtr identifier; + + if (!identifier_parser.parse(pos, identifier, expected)) + return false; + + if (!tryGetIdentifierNameInto(identifier, target)) + return false; + } + + if (!parsed_on_cluster && ParserKeyword{"ON"}.ignore(pos, expected)) + { + if (!ASTQueryWithOnCluster::parse(pos, cluster, expected)) + return false; + } + + res->cluster = cluster; + + switch (target_type) + { + case SystemQueryTargetType::Model: + { + res->target_model = std::move(target); + break; + } + case SystemQueryTargetType::Function: + { + res->target_function = std::move(target); + break; + } + case SystemQueryTargetType::Disk: + { + res->disk = std::move(target); + break; + } + } + + return true; +} + static bool parseQueryWithOnCluster(std::shared_ptr & res, IParser::Pos & pos, Expected & expected) { @@ -112,50 +184,14 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & } case Type::RELOAD_MODEL: { - parseQueryWithOnCluster(res, pos, expected); - - ASTPtr ast; - if (ParserStringLiteral{}.parse(pos, ast, expected)) - { - res->target_model = ast->as().value.safeGet(); - } - else - { - ParserIdentifier model_parser; - ASTPtr model; - String target_model; - - if (!model_parser.parse(pos, model, expected)) - return false; - - if (!tryGetIdentifierNameInto(model, res->target_model)) - return false; - } - + if (!parseQueryWithOnClusterAndTarget(res, pos, expected, SystemQueryTargetType::Model)) + return false; break; } case Type::RELOAD_FUNCTION: { - parseQueryWithOnCluster(res, pos, expected); - - ASTPtr ast; - if (ParserStringLiteral{}.parse(pos, ast, expected)) - { - res->target_function = ast->as().value.safeGet(); - } - else - { - ParserIdentifier function_parser; - ASTPtr function; - String target_function; - - if (!function_parser.parse(pos, function, expected)) - return false; - - if (!tryGetIdentifierNameInto(function, res->target_function)) - return false; - } - + if (!parseQueryWithOnClusterAndTarget(res, pos, expected, SystemQueryTargetType::Function)) + return false; break; } case Type::DROP_REPLICA: @@ -211,14 +247,8 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & case Type::RESTART_DISK: { - parseQueryWithOnCluster(res, pos, expected); - - ASTPtr ast; - if (ParserIdentifier{}.parse(pos, ast, expected)) - res->disk = ast->as().name(); - else + if (!parseQueryWithOnClusterAndTarget(res, pos, expected, SystemQueryTargetType::Disk)) return false; - break; } diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp index c09181ed74f..102c30088c9 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -206,38 +207,19 @@ static ColumnWithTypeAndName readColumnWithDate64Data(std::shared_ptr & arrow_column, const String & column_name) { - auto internal_type = std::make_shared(); + const auto & arrow_type = static_cast(*(arrow_column->type())); + const UInt8 scale = arrow_type.unit() * 3; + auto internal_type = std::make_shared(scale, arrow_type.timezone()); auto internal_column = internal_type->createColumn(); - auto & column_data = assert_cast &>(*internal_column).getData(); + auto & column_data = assert_cast &>(*internal_column).getData(); column_data.reserve(arrow_column->length()); for (size_t chunk_i = 0, num_chunks = static_cast(arrow_column->num_chunks()); chunk_i < num_chunks; ++chunk_i) { - auto & chunk = dynamic_cast(*(arrow_column->chunk(chunk_i))); - const auto & type = static_cast(*chunk.type()); - - UInt32 divide = 1; - const auto unit = type.unit(); - switch (unit) - { - case arrow::TimeUnit::SECOND: - divide = 1; - break; - case arrow::TimeUnit::MILLI: - divide = 1000; - break; - case arrow::TimeUnit::MICRO: - divide = 1000000; - break; - case arrow::TimeUnit::NANO: - divide = 1000000000; - break; - } - + const auto & chunk = dynamic_cast(*(arrow_column->chunk(chunk_i))); for (size_t value_i = 0, length = static_cast(chunk.length()); value_i < length; ++value_i) { - auto timestamp = static_cast(chunk.Value(value_i) / divide); // ms! TODO: check other 's' 'ns' ... - column_data.emplace_back(timestamp); + column_data.emplace_back(chunk.Value(value_i)); } } return {std::move(internal_column), std::move(internal_type), column_name}; @@ -485,7 +467,6 @@ static ColumnWithTypeAndName readColumnFromArrowColumn( return readColumnWithNumericData(arrow_column, column_name); FOR_ARROW_NUMERIC_TYPES(DISPATCH) # undef DISPATCH - // TODO: support TIMESTAMP_MICROS and TIMESTAMP_MILLIS with truncated micro- and milliseconds? // TODO: read JSON as a string? // TODO: read UUID as a string? default: diff --git a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp index 014715ba9ee..b56a9c2729f 100644 --- a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp +++ b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp @@ -2,6 +2,7 @@ #if USE_ARROW || USE_PARQUET +// #include #include #include #include @@ -17,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -55,6 +57,7 @@ namespace DB extern const int UNKNOWN_EXCEPTION; extern const int UNKNOWN_TYPE; extern const int LOGICAL_ERROR; + extern const int DECIMAL_OVERFLOW; } static const std::initializer_list>> internal_type_to_arrow_type = @@ -120,6 +123,42 @@ namespace DB checkStatus(status, write_column->getName(), format_name); } + static void fillArrowArrayWithDateTime64ColumnData( + const DataTypeDateTime64 * type, + ColumnPtr write_column, + const PaddedPODArray * null_bytemap, + const String & format_name, + arrow::ArrayBuilder* array_builder, + size_t start, + size_t end) + { + const auto & column = assert_cast &>(*write_column); + arrow::TimestampBuilder & builder = assert_cast(*array_builder); + arrow::Status status; + + auto scale = type->getScale(); + bool need_rescale = scale % 3; + auto rescale_multiplier = DecimalUtils::scaleMultiplier(3 - scale % 3); + for (size_t value_i = start; value_i < end; ++value_i) + { + if (null_bytemap && (*null_bytemap)[value_i]) + { + status = builder.AppendNull(); + } + else + { + auto value = static_cast(column[value_i].get>().getValue()); + if (need_rescale) + { + if (common::mulOverflow(value, rescale_multiplier, value)) + throw Exception("Decimal math overflow", ErrorCodes::DECIMAL_OVERFLOW); + } + status = builder.Append(value); + } + checkStatus(status, write_column->getName(), format_name); + } + } + static void fillArrowArray( const String & column_name, ColumnPtr & column, @@ -480,6 +519,11 @@ namespace DB if (!callOnIndexAndDataType(column_type->getTypeId(), fill_decimal)) throw Exception{ErrorCodes::LOGICAL_ERROR, "Cannot fill arrow array with decimal data with type {}", column_type_name}; } + else if (isDateTime64(column_type)) + { + const auto * datetime64_type = assert_cast(column_type.get()); + fillArrowArrayWithDateTime64ColumnData(datetime64_type, column, null_bytemap, format_name, array_builder, start, end); + } #define DISPATCH(CPP_NUMERIC_TYPE, ARROW_BUILDER_TYPE) \ else if (#CPP_NUMERIC_TYPE == column_type_name) \ { \ @@ -546,6 +590,18 @@ namespace DB } } + static arrow::TimeUnit::type getArrowTimeUnit(const DataTypeDateTime64 * type) + { + UInt32 scale = type->getScale(); + if (scale == 0) + return arrow::TimeUnit::SECOND; + if (scale > 0 && scale <= 3) + return arrow::TimeUnit::MILLI; + if (scale > 3 && scale <= 6) + return arrow::TimeUnit::MICRO; + return arrow::TimeUnit::NANO; + } + static std::shared_ptr getArrowType( DataTypePtr column_type, ColumnPtr column, const std::string & column_name, const std::string & format_name, bool * out_is_column_nullable) { @@ -638,6 +694,12 @@ namespace DB return arrow_type_it->second; } + if (isDateTime64(column_type)) + { + const auto * datetime64_type = assert_cast(column_type.get()); + return arrow::timestamp(getArrowTimeUnit(datetime64_type), datetime64_type->getTimeZone().getTimeZone()); + } + throw Exception(ErrorCodes::UNKNOWN_TYPE, "The type '{}' of a column '{}' is not supported for conversion into {} data format.", column_type->getName(), column_name, format_name); diff --git a/src/Server/HTTP/HTTPServerRequest.cpp b/src/Server/HTTP/HTTPServerRequest.cpp index f9ef14765c9..bb72c2a4010 100644 --- a/src/Server/HTTP/HTTPServerRequest.cpp +++ b/src/Server/HTTP/HTTPServerRequest.cpp @@ -13,6 +13,12 @@ #include #include +#if USE_SSL +#include +#include +#include +#endif + namespace DB { HTTPServerRequest::HTTPServerRequest(ContextPtr context, HTTPServerResponse & response, Poco::Net::HTTPServerSession & session) @@ -69,6 +75,31 @@ bool HTTPServerRequest::checkPeerConnected() const return true; } +#if USE_SSL +bool HTTPServerRequest::havePeerCertificate() const +{ + if (!secure) + return false; + + const Poco::Net::SecureStreamSocketImpl * secure_socket = dynamic_cast(socket); + if (!secure_socket) + return false; + + return secure_socket->havePeerCertificate(); +} + +Poco::Net::X509Certificate HTTPServerRequest::peerCertificate() const +{ + if (secure) + { + const Poco::Net::SecureStreamSocketImpl * secure_socket = dynamic_cast(socket); + if (secure_socket) + return secure_socket->peerCertificate(); + } + throw Poco::Net::SSLException("No certificate available"); +} +#endif + void HTTPServerRequest::readRequest(ReadBuffer & in) { char ch; diff --git a/src/Server/HTTP/HTTPServerRequest.h b/src/Server/HTTP/HTTPServerRequest.h index 33463177743..cfaeb108095 100644 --- a/src/Server/HTTP/HTTPServerRequest.h +++ b/src/Server/HTTP/HTTPServerRequest.h @@ -3,9 +3,12 @@ #include #include #include +#include #include +namespace Poco::Net { class X509Certificate; } + namespace DB { @@ -38,6 +41,11 @@ public: /// Returns the server's address. const Poco::Net::SocketAddress & serverAddress() const { return server_address; } +#if USE_SSL + bool havePeerCertificate() const; + Poco::Net::X509Certificate peerCertificate() const; +#endif + private: /// Limits for basic sanity checks when reading a header enum Limits diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index a42df54aed7..d0f92535844 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -44,6 +44,10 @@ #include #include +#if USE_SSL +#include +#endif + namespace DB { @@ -98,6 +102,7 @@ namespace ErrorCodes extern const int INVALID_SESSION_TIMEOUT; extern const int HTTP_LENGTH_REQUIRED; + extern const int SUPPORT_IS_DISABLED; } namespace @@ -315,68 +320,93 @@ bool HTTPHandler::authenticateUser( std::string password = request.get("X-ClickHouse-Key", ""); std::string quota_key = request.get("X-ClickHouse-Quota", ""); + /// The header 'X-ClickHouse-SSL-Certificate-Auth: on' enables checking the common name + /// extracted from the SSL certificate used for this connection instead of checking password. + bool has_ssl_certificate_auth = (request.get("X-ClickHouse-SSL-Certificate-Auth", "") == "on"); + bool has_auth_headers = !user.empty() || !password.empty() || !quota_key.empty() || has_ssl_certificate_auth; + + /// User name and password can be passed using HTTP Basic auth or query parameters + /// (both methods are insecure). + bool has_http_credentials = request.hasCredentials(); + bool has_credentials_in_query_params = params.has("user") || params.has("password") || params.has("quota_key"); + std::string spnego_challenge; + std::string certificate_common_name; - if (user.empty() && password.empty() && quota_key.empty()) + if (has_auth_headers) { - /// User name and password can be passed using query parameters - /// or using HTTP Basic auth (both methods are insecure). - if (request.hasCredentials()) + /// It is prohibited to mix different authorization schemes. + if (has_http_credentials) + throw Exception("Invalid authentication: it is not allowed to use SSL certificate authentication and Authorization HTTP header simultaneously", ErrorCodes::AUTHENTICATION_FAILED); + if (has_credentials_in_query_params) + throw Exception("Invalid authentication: it is not allowed to use SSL certificate authentication and authentication via parameters simultaneously simultaneously", ErrorCodes::AUTHENTICATION_FAILED); + + if (has_ssl_certificate_auth) { - /// It is prohibited to mix different authorization schemes. - if (params.has("user") || params.has("password")) - throw Exception("Invalid authentication: it is not allowed to use Authorization HTTP header and authentication via parameters simultaneously", ErrorCodes::AUTHENTICATION_FAILED); +#if USE_SSL + if (!password.empty()) + throw Exception("Invalid authentication: it is not allowed to use SSL certificate authentication and authentication via password simultaneously", ErrorCodes::AUTHENTICATION_FAILED); - std::string scheme; - std::string auth_info; - request.getCredentials(scheme, auth_info); + if (request.havePeerCertificate()) + certificate_common_name = request.peerCertificate().commonName(); - if (Poco::icompare(scheme, "Basic") == 0) - { - HTTPBasicCredentials credentials(auth_info); - user = credentials.getUsername(); - password = credentials.getPassword(); - } - else if (Poco::icompare(scheme, "Negotiate") == 0) - { - spnego_challenge = auth_info; + if (certificate_common_name.empty()) + throw Exception("Invalid authentication: SSL certificate authentication requires nonempty certificate's Common Name", ErrorCodes::AUTHENTICATION_FAILED); +#else + throw Exception( + "SSL certificate authentication disabled because ClickHouse was built without SSL library", + ErrorCodes::SUPPORT_IS_DISABLED); +#endif + } + } + else if (has_http_credentials) + { + /// It is prohibited to mix different authorization schemes. + if (has_credentials_in_query_params) + throw Exception("Invalid authentication: it is not allowed to use Authorization HTTP header and authentication via parameters simultaneously", ErrorCodes::AUTHENTICATION_FAILED); - if (spnego_challenge.empty()) - throw Exception("Invalid authentication: SPNEGO challenge is empty", ErrorCodes::AUTHENTICATION_FAILED); - } - else - { - throw Exception("Invalid authentication: '" + scheme + "' HTTP Authorization scheme is not supported", ErrorCodes::AUTHENTICATION_FAILED); - } + std::string scheme; + std::string auth_info; + request.getCredentials(scheme, auth_info); + + if (Poco::icompare(scheme, "Basic") == 0) + { + HTTPBasicCredentials credentials(auth_info); + user = credentials.getUsername(); + password = credentials.getPassword(); + } + else if (Poco::icompare(scheme, "Negotiate") == 0) + { + spnego_challenge = auth_info; + + if (spnego_challenge.empty()) + throw Exception("Invalid authentication: SPNEGO challenge is empty", ErrorCodes::AUTHENTICATION_FAILED); } else { - user = params.get("user", "default"); - password = params.get("password", ""); + throw Exception("Invalid authentication: '" + scheme + "' HTTP Authorization scheme is not supported", ErrorCodes::AUTHENTICATION_FAILED); } quota_key = params.get("quota_key", ""); } else { - /// It is prohibited to mix different authorization schemes. - if (request.hasCredentials() || params.has("user") || params.has("password") || params.has("quota_key")) - throw Exception("Invalid authentication: it is not allowed to use X-ClickHouse HTTP headers and other authentication methods simultaneously", ErrorCodes::AUTHENTICATION_FAILED); + /// If the user name is not set we assume it's the 'default' user. + user = params.get("user", "default"); + password = params.get("password", ""); + quota_key = params.get("quota_key", ""); } - if (spnego_challenge.empty()) // I.e., now using user name and password strings ("Basic"). + if (!certificate_common_name.empty()) { if (!request_credentials) - request_credentials = std::make_unique(); + request_credentials = std::make_unique(user, certificate_common_name); - auto * basic_credentials = dynamic_cast(request_credentials.get()); - if (!basic_credentials) - throw Exception("Invalid authentication: unexpected 'Basic' HTTP Authorization scheme", ErrorCodes::AUTHENTICATION_FAILED); - - basic_credentials->setUserName(user); - basic_credentials->setPassword(password); + auto * certificate_credentials = dynamic_cast(request_credentials.get()); + if (!certificate_credentials) + throw Exception("Invalid authentication: expected SSL certificate authorization scheme", ErrorCodes::AUTHENTICATION_FAILED); } - else + else if (!spnego_challenge.empty()) { if (!request_credentials) request_credentials = server.context()->makeGSSAcceptorContext(); @@ -403,6 +433,18 @@ bool HTTPHandler::authenticateUser( return false; } } + else // I.e., now using user name and password strings ("Basic"). + { + if (!request_credentials) + request_credentials = std::make_unique(); + + auto * basic_credentials = dynamic_cast(request_credentials.get()); + if (!basic_credentials) + throw Exception("Invalid authentication: expected 'Basic' HTTP Authorization scheme", ErrorCodes::AUTHENTICATION_FAILED); + + basic_credentials->setUserName(user); + basic_credentials->setPassword(password); + } /// Set client info. It will be used for quota accounting parameters in 'setUser' method. ClientInfo & client_info = session->getClientInfo(); diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 90e10abfa92..44f208adacc 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -516,8 +516,13 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, ContextPtr context) }); if (insert_it == metadata.secondary_indices.end()) - throw Exception("Wrong index name. Cannot find index " + backQuote(after_index_name) + " to insert after.", - ErrorCodes::BAD_ARGUMENTS); + { + auto hints = metadata.secondary_indices.getHints(after_index_name); + auto hints_string = !hints.empty() ? ", may be you meant: " + toString(hints) : ""; + throw Exception( + "Wrong index name. Cannot find index " + backQuote(after_index_name) + " to insert after" + hints_string, + ErrorCodes::BAD_ARGUMENTS); + } ++insert_it; } @@ -540,7 +545,10 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, ContextPtr context) { if (if_exists) return; - throw Exception("Wrong index name. Cannot find index " + backQuote(index_name) + " to drop.", ErrorCodes::BAD_ARGUMENTS); + auto hints = metadata.secondary_indices.getHints(index_name); + auto hints_string = !hints.empty() ? ", may be you meant: " + toString(hints) : ""; + throw Exception( + "Wrong index name. Cannot find index " + backQuote(index_name) + " to drop" + hints_string, ErrorCodes::BAD_ARGUMENTS); } metadata.secondary_indices.erase(erase_it); @@ -582,7 +590,7 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, ContextPtr context) { if (if_exists) return; - throw Exception("Wrong constraint name. Cannot find constraint `" + constraint_name + "` to drop.", + throw Exception("Wrong constraint name. Cannot find constraint `" + constraint_name + "` to drop", ErrorCodes::BAD_ARGUMENTS); } constraints.erase(erase_it); @@ -763,9 +771,13 @@ bool AlterCommand::isRequireMutationStage(const StorageInMemoryMetadata & metada if (isRemovingProperty() || type == REMOVE_TTL || type == REMOVE_SAMPLE_BY) return false; - if (type == DROP_COLUMN || type == DROP_INDEX || type == DROP_PROJECTION || type == RENAME_COLUMN) + if (type == DROP_INDEX || type == DROP_PROJECTION || type == RENAME_COLUMN) return true; + /// Drop alias is metadata alter, in other case mutation is required. + if (type == DROP_COLUMN) + return metadata.columns.hasPhysical(column_name); + if (type != MODIFY_COLUMN || data_type == nullptr) return false; diff --git a/src/Storages/ConstraintsDescription.h b/src/Storages/ConstraintsDescription.h index ad8bd371f38..a5095a79ccb 100644 --- a/src/Storages/ConstraintsDescription.h +++ b/src/Storages/ConstraintsDescription.h @@ -14,7 +14,7 @@ struct ConstraintsDescription { public: ConstraintsDescription() { update(); } - ConstraintsDescription(const ASTs & constraints_); + explicit ConstraintsDescription(const ASTs & constraints_); ConstraintsDescription(const ConstraintsDescription & other); ConstraintsDescription & operator=(const ConstraintsDescription & other); diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index 7b07e929c76..13873aa21ab 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -172,23 +172,43 @@ ColumnsDescription StorageHDFS::getTableStructureFromData( const String & compression_method, ContextPtr ctx) { - auto read_buffer_creator = [&]() + const auto [path_from_uri, uri_without_path] = getPathFromUriAndUriWithoutPath(uri); + auto paths = getPathsList(path_from_uri, uri, ctx); + + std::string exception_messages; + bool read_buffer_creator_was_used = false; + for (const auto & path : paths) { - const auto [path_from_uri, uri_without_path] = getPathFromUriAndUriWithoutPath(uri); - auto paths = getPathsList(path_from_uri, uri, ctx); - if (paths.empty()) - throw Exception( - ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, - "Cannot extract table structure from {} format file, because there are no files in HDFS with provided path. You must " - "specify table structure manually", - format); + auto read_buffer_creator = [&, uri_without_path = uri_without_path]() + { + read_buffer_creator_was_used = true; - auto compression = chooseCompressionMethod(paths[0], compression_method); - return wrapReadBufferWithCompressionMethod( - std::make_unique(uri_without_path, paths[0], ctx->getGlobalContext()->getConfigRef()), compression); - }; + if (paths.empty()) + throw Exception( + ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, + "Cannot extract table structure from {} format file, because there are no files in HDFS with provided path. You must " + "specify table structure manually", + format); - return readSchemaFromFormat(format, std::nullopt, read_buffer_creator, ctx); + auto compression = chooseCompressionMethod(path, compression_method); + return wrapReadBufferWithCompressionMethod( + std::make_unique(uri_without_path, path, ctx->getGlobalContext()->getConfigRef()), compression); + }; + + try + { + return readSchemaFromFormat(format, std::nullopt, read_buffer_creator, ctx); + } + catch (...) + { + if (paths.size() == 1 || !read_buffer_creator_was_used) + throw; + + exception_messages += getCurrentExceptionMessage(false) + "\n"; + } + } + + throw Exception(ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, "All attempts to extract table structure from hdfs files failed. Errors:\n{}", exception_messages); } class HDFSSource::DisclosedGlobIterator::Impl diff --git a/src/Storages/IndicesDescription.cpp b/src/Storages/IndicesDescription.cpp index 9cdf9695172..a0a1bcbce2d 100644 --- a/src/Storages/IndicesDescription.cpp +++ b/src/Storages/IndicesDescription.cpp @@ -172,4 +172,13 @@ ExpressionActionsPtr IndicesDescription::getSingleExpressionForIndices(const Col return ExpressionAnalyzer(combined_expr_list, syntax_result, context).getActions(false); } +Names IndicesDescription::getAllRegisteredNames() const +{ + Names result; + for (const auto & index : *this) + { + result.emplace_back(index.name); + } + return result; +} } diff --git a/src/Storages/IndicesDescription.h b/src/Storages/IndicesDescription.h index 7071ec89890..72e0748778f 100644 --- a/src/Storages/IndicesDescription.h +++ b/src/Storages/IndicesDescription.h @@ -8,6 +8,7 @@ #include #include #include +#include namespace DB { @@ -61,7 +62,7 @@ struct IndexDescription }; /// All secondary indices in storage -struct IndicesDescription : public std::vector +struct IndicesDescription : public std::vector, IHints<1, IndicesDescription> { /// Index with name exists bool has(const String & name) const; @@ -72,6 +73,9 @@ struct IndicesDescription : public std::vector /// Return common expression for all stored indices ExpressionActionsPtr getSingleExpressionForIndices(const ColumnsDescription & columns, ContextPtr context) const; + +public: + Names getAllRegisteredNames() const override; }; } diff --git a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp index be3a872971e..bb4ea22a1ac 100644 --- a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp @@ -187,9 +187,7 @@ std::pair MergeFromLogEntryT merge_mutate_entry = storage.getContext()->getMergeList().insert( storage.getStorageID(), future_merged_part, - settings.memory_profiler_step, - settings.memory_profiler_sample_probability, - settings.max_untracked_memory); + settings); transaction_ptr = std::make_unique(storage); stopwatch_ptr = std::make_unique(); diff --git a/src/Storages/MergeTree/MergeList.cpp b/src/Storages/MergeTree/MergeList.cpp index a03cb053e1f..11d0bc8c565 100644 --- a/src/Storages/MergeTree/MergeList.cpp +++ b/src/Storages/MergeTree/MergeList.cpp @@ -16,23 +16,8 @@ MemoryTrackerThreadSwitcher::MemoryTrackerThreadSwitcher(MergeListEntry & merge_ { // Each merge is executed into separate background processing pool thread background_thread_memory_tracker = CurrentThread::getMemoryTracker(); - if (background_thread_memory_tracker) - { - /// From the query context it will be ("for thread") memory tracker with VariableContext::Thread level, - /// which does not have any limits and sampling settings configured. - /// And parent for this memory tracker should be ("(for query)") with VariableContext::Process level, - /// that has limits and sampling configured. - MemoryTracker * parent; - if (background_thread_memory_tracker->level == VariableContext::Thread && - (parent = background_thread_memory_tracker->getParent()) && - parent != &total_memory_tracker) - { - background_thread_memory_tracker = parent; - } - - background_thread_memory_tracker_prev_parent = background_thread_memory_tracker->getParent(); - background_thread_memory_tracker->setParent(&merge_list_entry->memory_tracker); - } + background_thread_memory_tracker_prev_parent = background_thread_memory_tracker->getParent(); + background_thread_memory_tracker->setParent(&merge_list_entry->memory_tracker); prev_untracked_memory_limit = current_thread->untracked_memory_limit; current_thread->untracked_memory_limit = merge_list_entry->max_untracked_memory; @@ -50,9 +35,7 @@ MemoryTrackerThreadSwitcher::MemoryTrackerThreadSwitcher(MergeListEntry & merge_ MemoryTrackerThreadSwitcher::~MemoryTrackerThreadSwitcher() { // Unplug memory_tracker from current background processing pool thread - - if (background_thread_memory_tracker) - background_thread_memory_tracker->setParent(background_thread_memory_tracker_prev_parent); + background_thread_memory_tracker->setParent(background_thread_memory_tracker_prev_parent); current_thread->untracked_memory_limit = prev_untracked_memory_limit; @@ -65,16 +48,14 @@ MemoryTrackerThreadSwitcher::~MemoryTrackerThreadSwitcher() MergeListElement::MergeListElement( const StorageID & table_id_, FutureMergedMutatedPartPtr future_part, - UInt64 memory_profiler_step, - UInt64 memory_profiler_sample_probability, - UInt64 max_untracked_memory_) + const Settings & settings) : table_id{table_id_} , partition_id{future_part->part_info.partition_id} , result_part_name{future_part->name} , result_part_path{future_part->path} , result_part_info{future_part->part_info} , num_parts{future_part->parts.size()} - , max_untracked_memory(max_untracked_memory_) + , max_untracked_memory(settings.max_untracked_memory) , query_id(table_id.getShortName() + "::" + result_part_name) , thread_id{getThreadId()} , merge_type{future_part->merge_type} @@ -97,8 +78,33 @@ MergeListElement::MergeListElement( } memory_tracker.setDescription("Mutate/Merge"); - memory_tracker.setProfilerStep(memory_profiler_step); - memory_tracker.setSampleProbability(memory_profiler_sample_probability); + /// MemoryTracker settings should be set here, because + /// later (see MemoryTrackerThreadSwitcher) + /// parent memory tracker will be changed, and if merge executed from the + /// query (OPTIMIZE TABLE), all settings will be lost (since + /// current_thread::memory_tracker will have Thread level MemoryTracker, + /// which does not have any settings itself, it relies on the settings of the + /// thread_group::memory_tracker, but MemoryTrackerThreadSwitcher will reset parent). + memory_tracker.setProfilerStep(settings.memory_profiler_step); + memory_tracker.setSampleProbability(settings.memory_profiler_sample_probability); + memory_tracker.setSoftLimit(settings.max_guaranteed_memory_usage); + if (settings.memory_tracker_fault_probability) + memory_tracker.setFaultProbability(settings.memory_tracker_fault_probability); + + /// Let's try to copy memory related settings from the query, + /// since settings that we have here is not from query, but global, from the table. + /// + /// NOTE: Remember, that Thread level MemoryTracker does not have any settings, + /// so it's parent is required. + MemoryTracker * query_memory_tracker = CurrentThread::getMemoryTracker(); + MemoryTracker * parent_query_memory_tracker; + if (query_memory_tracker->level == VariableContext::Thread && + (parent_query_memory_tracker = query_memory_tracker->getParent()) && + parent_query_memory_tracker != &total_memory_tracker) + { + memory_tracker.setOrRaiseHardLimit(parent_query_memory_tracker->getHardLimit()); + } + } MergeInfo MergeListElement::getInfo() const diff --git a/src/Storages/MergeTree/MergeList.h b/src/Storages/MergeTree/MergeList.h index 2df32df7f92..a944779ad44 100644 --- a/src/Storages/MergeTree/MergeList.h +++ b/src/Storages/MergeTree/MergeList.h @@ -58,6 +58,8 @@ using FutureMergedMutatedPartPtr = std::shared_ptr; struct MergeListElement; using MergeListEntry = BackgroundProcessListEntry; +struct Settings; + /** * Since merge is executed with multiple threads, this class @@ -127,9 +129,7 @@ struct MergeListElement : boost::noncopyable MergeListElement( const StorageID & table_id_, FutureMergedMutatedPartPtr future_part, - UInt64 memory_profiler_step, - UInt64 memory_profiler_sample_probability, - UInt64 max_untracked_memory_); + const Settings & settings); MergeInfo getInfo() const; diff --git a/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp b/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp index 1cbf928047c..14e43b2897e 100644 --- a/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp +++ b/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp @@ -81,9 +81,7 @@ void MergePlainMergeTreeTask::prepare() merge_list_entry = storage.getContext()->getMergeList().insert( storage.getStorageID(), future_part, - settings.memory_profiler_step, - settings.memory_profiler_sample_probability, - settings.max_untracked_memory); + settings); write_part_log = [this] (const ExecutionStatus & execution_status) { diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 5d5035d62ca..89fb27cc89c 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -241,9 +241,6 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() global_ctx->merging_column_names, global_ctx->gathering_column_names); - if (global_ctx->data->getSettings()->fsync_part_directory) - global_ctx->sync_guard = ctx->disk->getDirectorySyncGuard(local_new_part_tmp_path); - break; } default : @@ -585,12 +582,7 @@ bool MergeTask::MergeProjectionsStage::mergeMinMaxIndexAndPrepareProjections() c projection_future_part, projection.metadata, global_ctx->merge_entry, - std::make_unique( - (*global_ctx->merge_entry)->table_id, - projection_future_part, - settings.memory_profiler_step, - settings.memory_profiler_sample_probability, - settings.max_untracked_memory), + std::make_unique((*global_ctx->merge_entry)->table_id, projection_future_part, settings), global_ctx->time_of_merge, global_ctx->context, global_ctx->space_reservation, diff --git a/src/Storages/MergeTree/MergeTask.h b/src/Storages/MergeTree/MergeTask.h index aa64c4c2265..d3fc01980ea 100644 --- a/src/Storages/MergeTree/MergeTask.h +++ b/src/Storages/MergeTree/MergeTask.h @@ -155,7 +155,6 @@ private: QueryPipeline merged_pipeline; std::unique_ptr merging_executor; - SyncGuardPtr sync_guard{nullptr}; MergeTreeData::MutableDataPartPtr new_data_part{nullptr}; size_t rows_written{0}; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 23d44d177b7..7bfff3b5088 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -4720,6 +4720,10 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg if (select_query->sampleSize()) return std::nullopt; + // Currently projection don't support deduplication when moving parts between shards. + if (settings.allow_experimental_query_deduplication) + return std::nullopt; + // Currently projections don't support ARRAY JOIN yet. if (select_query->arrayJoinExpressionList().first) return std::nullopt; @@ -4730,11 +4734,16 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg if (select_query->join()) return std::nullopt; + auto query_options = SelectQueryOptions( + QueryProcessingStage::WithMergeableState, + /* depth */ 1, + /* is_subquery_= */ true + ).ignoreProjections().ignoreAlias(); InterpreterSelectQuery select( query_ptr, query_context, - SelectQueryOptions{QueryProcessingStage::WithMergeableState}.ignoreProjections().ignoreAlias(), - query_info.sets /* prepared_sets */); + query_options, + /* prepared_sets_= */ query_info.sets); const auto & analysis_result = select.getAnalysisResult(); query_info.sets = std::move(select.getQueryAnalyzer()->getPreparedSets()); diff --git a/src/Storages/MergeTree/MergeTreeIndexReader.cpp b/src/Storages/MergeTree/MergeTreeIndexReader.cpp index 03a3b811de6..95ec8acc3b9 100644 --- a/src/Storages/MergeTree/MergeTreeIndexReader.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexReader.cpp @@ -54,7 +54,7 @@ MergeTreeIndexReader::MergeTreeIndexReader( std::move(settings)); version = index_format.version; - stream->adjustForRange(MarkRange(0, getLastMark(all_mark_ranges_))); + stream->adjustRightMark(getLastMark(all_mark_ranges_)); stream->seekToStart(); } diff --git a/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/src/Storages/MergeTree/MergeTreeRangeReader.cpp index 98b8de6a0ea..e22f662960c 100644 --- a/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -695,10 +695,10 @@ MergeTreeRangeReader::ReadResult MergeTreeRangeReader::read(size_t max_rows, Mar { auto block = prev_reader->sample_block.cloneWithColumns(read_result.columns); auto block_before_prewhere = read_result.block_before_prewhere; - for (auto & ctn : block) + for (const auto & column : block) { - if (block_before_prewhere.has(ctn.name)) - block_before_prewhere.erase(ctn.name); + if (block_before_prewhere.has(column.name)) + block_before_prewhere.erase(column.name); } if (block_before_prewhere) @@ -710,8 +710,8 @@ MergeTreeRangeReader::ReadResult MergeTreeRangeReader::read(size_t max_rows, Mar block_before_prewhere.setColumns(std::move(old_columns)); } - for (auto && ctn : block_before_prewhere) - block.insert(std::move(ctn)); + for (auto & column : block_before_prewhere) + block.insert(std::move(column)); } merge_tree_reader->evaluateMissingDefaults(block, columns); } diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 6b265b9460c..726b2141ffb 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -96,6 +96,7 @@ MergeTreeReaderCompact::MergeTreeReaderCompact( cached_buffer = std::move(buffer); data_buffer = cached_buffer.get(); + compressed_data_buffer = cached_buffer.get(); } else { @@ -114,6 +115,7 @@ MergeTreeReaderCompact::MergeTreeReaderCompact( non_cached_buffer = std::move(buffer); data_buffer = non_cached_buffer.get(); + compressed_data_buffer = non_cached_buffer.get(); } } catch (...) @@ -260,10 +262,7 @@ void MergeTreeReaderCompact::seekToMark(size_t row_index, size_t column_index) MarkInCompressedFile mark = marks_loader.getMark(row_index, column_index); try { - if (cached_buffer) - cached_buffer->seek(mark.offset_in_compressed_file, mark.offset_in_decompressed_block); - if (non_cached_buffer) - non_cached_buffer->seek(mark.offset_in_compressed_file, mark.offset_in_decompressed_block); + compressed_data_buffer->seek(mark.offset_in_compressed_file, mark.offset_in_decompressed_block); } catch (Exception & e) { @@ -288,10 +287,7 @@ void MergeTreeReaderCompact::adjustUpperBound(size_t last_mark) return; last_right_offset = 0; // Zero value means the end of file. - if (cached_buffer) - cached_buffer->setReadUntilEnd(); - if (non_cached_buffer) - non_cached_buffer->setReadUntilEnd(); + data_buffer->setReadUntilEnd(); } else { @@ -299,10 +295,7 @@ void MergeTreeReaderCompact::adjustUpperBound(size_t last_mark) return; last_right_offset = right_offset; - if (cached_buffer) - cached_buffer->setReadUntilPosition(right_offset); - if (non_cached_buffer) - non_cached_buffer->setReadUntilPosition(right_offset); + data_buffer->setReadUntilPosition(right_offset); } } diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.h b/src/Storages/MergeTree/MergeTreeReaderCompact.h index 381b212df3c..aa0eb949aa1 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.h +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -41,6 +41,7 @@ private: bool isContinuousReading(size_t mark, size_t column_position); ReadBuffer * data_buffer; + CompressedReadBufferBase * compressed_data_buffer; std::unique_ptr cached_buffer; std::unique_ptr non_cached_buffer; diff --git a/src/Storages/MergeTree/MergeTreeReaderStream.cpp b/src/Storages/MergeTree/MergeTreeReaderStream.cpp index 117ef43ecb2..b337bd62dd3 100644 --- a/src/Storages/MergeTree/MergeTreeReaderStream.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderStream.cpp @@ -42,7 +42,8 @@ MergeTreeReaderStream::MergeTreeReaderStream( { size_t left_mark = mark_range.begin; size_t right_mark = mark_range.end; - auto [_, mark_range_bytes] = getRightOffsetAndBytesRange(left_mark, right_mark); + size_t left_offset = left_mark < marks_count ? marks_loader.getMark(left_mark).offset_in_compressed_file : 0; + auto mark_range_bytes = getRightOffset(right_mark) - left_offset; max_mark_range_bytes = std::max(max_mark_range_bytes, mark_range_bytes); sum_mark_range_bytes += mark_range_bytes; @@ -85,6 +86,7 @@ MergeTreeReaderStream::MergeTreeReaderStream( cached_buffer = std::move(buffer); data_buffer = cached_buffer.get(); + compressed_data_buffer = cached_buffer.get(); } else { @@ -102,22 +104,21 @@ MergeTreeReaderStream::MergeTreeReaderStream( non_cached_buffer = std::move(buffer); data_buffer = non_cached_buffer.get(); + compressed_data_buffer = non_cached_buffer.get(); } } -std::pair MergeTreeReaderStream::getRightOffsetAndBytesRange(size_t left_mark, size_t right_mark_non_included) +size_t MergeTreeReaderStream::getRightOffset(size_t right_mark_non_included) { /// NOTE: if we are reading the whole file, then right_mark == marks_count /// and we will use max_read_buffer_size for buffer size, thus avoiding the need to load marks. /// Special case, can happen in Collapsing/Replacing engines if (marks_count == 0) - return std::make_pair(0, 0); + return 0; - assert(left_mark < marks_count); assert(right_mark_non_included <= marks_count); - assert(left_mark <= right_mark_non_included); size_t result_right_offset; if (0 < right_mark_non_included && right_mark_non_included < marks_count) @@ -177,30 +178,20 @@ std::pair MergeTreeReaderStream::getRightOffsetAndBytesRange(siz } } else if (right_mark_non_included == 0) - { result_right_offset = marks_loader.getMark(right_mark_non_included).offset_in_compressed_file; - } else - { result_right_offset = file_size; - } - size_t mark_range_bytes = result_right_offset - (left_mark < marks_count ? marks_loader.getMark(left_mark).offset_in_compressed_file : 0); - - return std::make_pair(result_right_offset, mark_range_bytes); + return result_right_offset; } - void MergeTreeReaderStream::seekToMark(size_t index) { MarkInCompressedFile mark = marks_loader.getMark(index); try { - if (cached_buffer) - cached_buffer->seek(mark.offset_in_compressed_file, mark.offset_in_decompressed_block); - if (non_cached_buffer) - non_cached_buffer->seek(mark.offset_in_compressed_file, mark.offset_in_decompressed_block); + compressed_data_buffer->seek(mark.offset_in_compressed_file, mark.offset_in_decompressed_block); } catch (Exception & e) { @@ -220,10 +211,7 @@ void MergeTreeReaderStream::seekToStart() { try { - if (cached_buffer) - cached_buffer->seek(0, 0); - if (non_cached_buffer) - non_cached_buffer->seek(0, 0); + compressed_data_buffer->seek(0, 0); } catch (Exception & e) { @@ -236,24 +224,21 @@ void MergeTreeReaderStream::seekToStart() } -void MergeTreeReaderStream::adjustForRange(MarkRange range) +void MergeTreeReaderStream::adjustRightMark(size_t right_mark) { /** * Note: this method is called multiple times for the same range of marks -- each time we * read from stream, but we must update last_right_offset only if it is bigger than * the last one to avoid redundantly cancelling prefetches. */ - auto [right_offset, _] = getRightOffsetAndBytesRange(range.begin, range.end); + auto right_offset = getRightOffset(right_mark); if (!right_offset) { if (last_right_offset && *last_right_offset == 0) return; last_right_offset = 0; // Zero value means the end of file. - if (cached_buffer) - cached_buffer->setReadUntilEnd(); - if (non_cached_buffer) - non_cached_buffer->setReadUntilEnd(); + data_buffer->setReadUntilEnd(); } else { @@ -261,10 +246,7 @@ void MergeTreeReaderStream::adjustForRange(MarkRange range) return; last_right_offset = right_offset; - if (cached_buffer) - cached_buffer->setReadUntilPosition(right_offset); - if (non_cached_buffer) - non_cached_buffer->setReadUntilPosition(right_offset); + data_buffer->setReadUntilPosition(right_offset); } } diff --git a/src/Storages/MergeTree/MergeTreeReaderStream.h b/src/Storages/MergeTree/MergeTreeReaderStream.h index 96f3ea31fec..0b6b86e7f82 100644 --- a/src/Storages/MergeTree/MergeTreeReaderStream.h +++ b/src/Storages/MergeTree/MergeTreeReaderStream.h @@ -34,12 +34,13 @@ public: * Does buffer need to know something about mark ranges bounds it is going to read? * (In case of MergeTree* tables). Mostly needed for reading from remote fs. */ - void adjustForRange(MarkRange range); + void adjustRightMark(size_t right_mark); ReadBuffer * data_buffer; + CompressedReadBufferBase * compressed_data_buffer; private: - std::pair getRightOffsetAndBytesRange(size_t left_mark, size_t right_mark_non_included); + size_t getRightOffset(size_t right_mark_non_included); DiskPtr disk; std::string path_prefix; diff --git a/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/src/Storages/MergeTree/MergeTreeReaderWide.cpp index 77c0e8332d4..99a36a8e08a 100644 --- a/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -212,7 +212,7 @@ static ReadBuffer * getStream( return nullptr; MergeTreeReaderStream & stream = *it->second; - stream.adjustForRange(MarkRange(seek_to_start ? 0 : from_mark, current_task_last_mark)); + stream.adjustRightMark(current_task_last_mark); if (seek_to_start) stream.seekToStart(); diff --git a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp index 4f1a663409d..b3ee3b412b3 100644 --- a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp @@ -116,9 +116,7 @@ std::pair MutateFromLogEntry merge_mutate_entry = storage.getContext()->getMergeList().insert( storage.getStorageID(), future_mutated_part, - settings.memory_profiler_step, - settings.memory_profiler_sample_probability, - settings.max_untracked_memory); + settings); stopwatch_ptr = std::make_unique(); diff --git a/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp b/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp index 6b602484ff3..8f254b4790e 100644 --- a/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp +++ b/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp @@ -32,9 +32,7 @@ void MutatePlainMergeTreeTask::prepare() merge_list_entry = storage.getContext()->getMergeList().insert( storage.getStorageID(), future_part, - settings.memory_profiler_step, - settings.memory_profiler_sample_probability, - settings.max_untracked_memory); + settings); stopwatch = std::make_unique(); diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index cc690287ef6..c001b319c99 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -520,7 +520,6 @@ struct MutationContext DiskPtr disk; String new_part_tmp_path; - SyncGuardPtr sync_guard; IMergedBlockOutputStreamPtr out{nullptr}; String mrk_extension; @@ -633,12 +632,7 @@ public: projection_future_part, projection.metadata, ctx->mutate_entry, - std::make_unique( - (*ctx->mutate_entry)->table_id, - projection_future_part, - settings.memory_profiler_step, - settings.memory_profiler_sample_probability, - settings.max_untracked_memory), + std::make_unique((*ctx->mutate_entry)->table_id, projection_future_part, settings), *ctx->holder, ctx->time_of_mutation, ctx->context, @@ -1310,9 +1304,6 @@ bool MutateTask::prepare() ctx->disk = ctx->new_data_part->volume->getDisk(); ctx->new_part_tmp_path = ctx->new_data_part->getFullRelativePath(); - if (ctx->data->getSettings()->fsync_part_directory) - ctx->sync_guard = ctx->disk->getDirectorySyncGuard(ctx->new_part_tmp_path); - /// Don't change granularity type while mutating subset of columns ctx->mrk_extension = ctx->source_part->index_granularity_info.is_adaptive ? getAdaptiveMrkExtension(ctx->new_data_part->getType()) : getNonAdaptiveMrkExtension(); diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 9a2ec0789cd..452eb16ebc5 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -273,22 +273,38 @@ ColumnsDescription StorageFile::getTableStructureFromFile( return ColumnsDescription(source->getOutputs().front().getHeader().getNamesAndTypesList()); } - auto read_buffer_creator = [&]() + std::string exception_messages; + bool read_buffer_creator_was_used = false; + for (const auto & path : paths) { - String path; - auto it = std::find_if(paths.begin(), paths.end(), [](const String & p){ return std::filesystem::exists(p); }); - if (it == paths.end()) - throw Exception( - ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, - "Cannot extract table structure from {} format file, because there are no files with provided path. You must specify " - "table structure manually", - format); + auto read_buffer_creator = [&]() + { + read_buffer_creator_was_used = true; - path = *it; - return createReadBuffer(path, false, "File", -1, compression_method, context); - }; + if (!std::filesystem::exists(path)) + throw Exception( + ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, + "Cannot extract table structure from {} format file, because there are no files with provided path. You must specify " + "table structure manually", + format); - return readSchemaFromFormat(format, format_settings, read_buffer_creator, context); + return createReadBuffer(path, false, "File", -1, compression_method, context); + }; + + try + { + return readSchemaFromFormat(format, format_settings, read_buffer_creator, context); + } + catch (...) + { + if (paths.size() == 1 || !read_buffer_creator_was_used) + throw; + + exception_messages += getCurrentExceptionMessage(false) + "\n"; + } + } + + throw Exception(ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, "All attempts to extract table structure from files failed. Errors:\n{}", exception_messages); } bool StorageFile::isColumnOriented() const diff --git a/src/Storages/StorageJoin.cpp b/src/Storages/StorageJoin.cpp index 3a9b42f7371..9b3ec6617c0 100644 --- a/src/Storages/StorageJoin.cpp +++ b/src/Storages/StorageJoin.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -64,7 +65,7 @@ StorageJoin::StorageJoin( throw Exception{"Key column (" + key + ") does not exist in table declaration.", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE}; table_join = std::make_shared(limits, use_nulls, kind, strictness, key_names); - join = std::make_shared(table_join, metadata_snapshot->getSampleBlock().sortColumns(), overwrite); + join = std::make_shared(table_join, getRightSampleBlock(), overwrite); restore(); } @@ -82,7 +83,7 @@ SinkToStoragePtr StorageJoin::write(const ASTPtr & query, const StorageMetadataP return StorageSetOrJoinBase::write(query, metadata_snapshot, context); } -void StorageJoin::truncate(const ASTPtr &, const StorageMetadataPtr & metadata_snapshot, ContextPtr context, TableExclusiveLockHolder &) +void StorageJoin::truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr context, TableExclusiveLockHolder &) { std::lock_guard mutate_lock(mutate_mutex); TableLockHolder holder = tryLockTimedWithContext(rwlock, RWLockImpl::Write, context); @@ -92,7 +93,7 @@ void StorageJoin::truncate(const ASTPtr &, const StorageMetadataPtr & metadata_s disk->createDirectories(path + "tmp/"); increment = 0; - join = std::make_shared(table_join, metadata_snapshot->getSampleBlock().sortColumns(), overwrite); + join = std::make_shared(table_join, getRightSampleBlock(), overwrite); } void StorageJoin::checkMutationIsPossible(const MutationCommands & commands, const Settings & /* settings */) const @@ -116,7 +117,7 @@ void StorageJoin::mutate(const MutationCommands & commands, ContextPtr context) auto compressed_backup_buf = CompressedWriteBuffer(*backup_buf); auto backup_stream = NativeWriter(compressed_backup_buf, 0, metadata_snapshot->getSampleBlock()); - auto new_data = std::make_shared(table_join, metadata_snapshot->getSampleBlock().sortColumns(), overwrite); + auto new_data = std::make_shared(table_join, getRightSampleBlock(), overwrite); // New scope controls lifetime of InputStream. { @@ -167,8 +168,10 @@ HashJoinPtr StorageJoin::getJoinLocked(std::shared_ptr analyzed_join, if ((analyzed_join->forceNullableRight() && !use_nulls) || (!analyzed_join->forceNullableRight() && isLeftOrFull(analyzed_join->kind()) && use_nulls)) - throw Exception("Table " + getStorageID().getNameForLogs() + " needs the same join_use_nulls setting as present in LEFT or FULL JOIN.", - ErrorCodes::INCOMPATIBLE_TYPE_OF_JOIN); + throw Exception( + ErrorCodes::INCOMPATIBLE_TYPE_OF_JOIN, + "Table {} needs the same join_use_nulls setting as present in LEFT or FULL JOIN", + getStorageID().getNameForLogs()); /// TODO: check key columns @@ -177,7 +180,7 @@ HashJoinPtr StorageJoin::getJoinLocked(std::shared_ptr analyzed_join, /// Qualifies will be added by join implementation (HashJoin) analyzed_join->setRightKeys(key_names); - HashJoinPtr join_clone = std::make_shared(analyzed_join, metadata_snapshot->getSampleBlock().sortColumns()); + HashJoinPtr join_clone = std::make_shared(analyzed_join, getRightSampleBlock()); RWLockImpl::LockHolder holder = tryLockTimedWithContext(rwlock, RWLockImpl::Read, context); join_clone->setLock(holder); diff --git a/src/Storages/StorageJoin.h b/src/Storages/StorageJoin.h index ee685830a6c..ba59bc06334 100644 --- a/src/Storages/StorageJoin.h +++ b/src/Storages/StorageJoin.h @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB @@ -60,6 +61,22 @@ public: std::optional totalRows(const Settings & settings) const override; std::optional totalBytes(const Settings & settings) const override; + Block getRightSampleBlock() const + { + auto metadata_snapshot = getInMemoryMetadataPtr(); + Block block = metadata_snapshot->getSampleBlock().sortColumns(); + if (use_nulls && isLeftOrFull(kind)) + { + for (auto & col : block) + { + JoinCommon::convertColumnToNullable(col); + } + } + return block; + } + + bool useNulls() const { return use_nulls; } + private: Block sample_block; const Names key_names; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 451238096f4..ab42396f8da 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -3065,7 +3065,8 @@ void StorageReplicatedMergeTree::mergeSelectingTask() desired_mutation_version->second, merge_pred.getVersion()); - if (create_result == CreateMergeEntryResult::Ok) + if (create_result == CreateMergeEntryResult::Ok || + create_result == CreateMergeEntryResult::LogUpdated) break; } } diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 9a85644d825..ac462e5cb6e 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -888,23 +888,44 @@ ColumnsDescription StorageS3::getTableStructureFromDataImpl( ContextPtr ctx) { std::vector keys = {client_auth.uri.key}; - auto read_buffer_creator = [&]() + auto file_iterator = createFileIterator(client_auth, keys, is_key_with_globs, distributed_processing, ctx); + + std::string current_key; + std::string exception_messages; + bool read_buffer_creator_was_used = false; + do { - auto file_iterator = createFileIterator(client_auth, keys, is_key_with_globs, distributed_processing, ctx); - String current_key = (*file_iterator)(); - if (current_key.empty()) - throw Exception( - ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, - "Cannot extract table structure from {} format file, because there are no files with provided path in S3. You must specify " - "table structure manually", - format); + current_key = (*file_iterator)(); + auto read_buffer_creator = [&]() + { + read_buffer_creator_was_used = true; + if (current_key.empty()) + throw Exception( + ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, + "Cannot extract table structure from {} format file, because there are no files with provided path in S3. You must specify " + "table structure manually", + format); - return wrapReadBufferWithCompressionMethod( - std::make_unique(client_auth.client, client_auth.uri.bucket, current_key, max_single_read_retries, ctx->getReadSettings()), - chooseCompressionMethod(current_key, compression_method)); - }; + return wrapReadBufferWithCompressionMethod( + std::make_unique( + client_auth.client, client_auth.uri.bucket, current_key, max_single_read_retries, ctx->getReadSettings()), + chooseCompressionMethod(current_key, compression_method)); + }; - return readSchemaFromFormat(format, format_settings, read_buffer_creator, ctx); + try + { + return readSchemaFromFormat(format, format_settings, read_buffer_creator, ctx); + } + catch (...) + { + if (!is_key_with_globs || !read_buffer_creator_was_used) + throw; + + exception_messages += getCurrentExceptionMessage(false) + "\n"; + } + } while (!current_key.empty()); + + throw Exception(ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, "All attempts to extract table structure from s3 files failed. Errors:\n{}", exception_messages); } diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 508c9d8b157..e62b14224ec 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -37,6 +37,7 @@ namespace ErrorCodes extern const int NETWORK_ERROR; extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; + extern const int CANNOT_EXTRACT_TABLE_STRUCTURE; } @@ -173,41 +174,10 @@ namespace if (uri_options.empty()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Got empty url list"); - if (uri_options.size() > 1) - { - read_buf = getFirstAvailableURLReadBuffer( - uri_options, context, params, http_method, - callback, timeouts, compression_method, credentials, headers); - } - else - { - ReadSettings read_settings = context->getReadSettings(); - bool skip_url_not_found_error = glob_url && read_settings.http_skip_not_found_url_for_globs; - auto request_uri = Poco::URI(uri_options[0]); - - for (const auto & [param, value] : params) - request_uri.addQueryParameter(param, value); - - setCredentials(credentials, request_uri); - - read_buf = wrapReadBufferWithCompressionMethod( - std::make_unique( - request_uri, - http_method, - callback, - timeouts, - credentials, - context->getSettingsRef().max_http_get_redirects, - DBMS_DEFAULT_BUFFER_SIZE, - read_settings, - headers, - ReadWriteBufferFromHTTP::Range{}, - context->getRemoteHostFilter(), - /* delay_initiliazation */true, - /* use_external_buffer */false, - /* skip_url_not_found_error */skip_url_not_found_error), - chooseCompressionMethod(request_uri.getPath(), compression_method)); - } + auto first_option = uri_options.begin(); + read_buf = getFirstAvailableURLReadBuffer( + first_option, uri_options.end(), context, params, http_method, + callback, timeouts, compression_method, credentials, headers, glob_url, uri_options.size() == 1); auto input_format = FormatFactory::instance().getInput(format, *read_buf, sample_block, context, max_block_size, format_settings); QueryPipelineBuilder builder; @@ -258,7 +228,8 @@ namespace } static std::unique_ptr getFirstAvailableURLReadBuffer( - const std::vector & urls, + std::vector::const_iterator & option, + const std::vector::const_iterator & end, ContextPtr context, const URIParams & params, const String & http_method, @@ -266,14 +237,17 @@ namespace const ConnectionTimeouts & timeouts, const String & compression_method, Poco::Net::HTTPBasicCredentials & credentials, - const ReadWriteBufferFromHTTP::HTTPHeaderEntries & headers) + const ReadWriteBufferFromHTTP::HTTPHeaderEntries & headers, + bool glob_url, + bool delay_initialization) { String first_exception_message; ReadSettings read_settings = context->getReadSettings(); - for (auto option = urls.begin(); option != urls.end(); ++option) + size_t options = std::distance(option, end); + for (; option != end; ++option) { - bool skip_url_not_found_error = read_settings.http_skip_not_found_url_for_globs && option == std::prev(urls.end()); + bool skip_url_not_found_error = glob_url && read_settings.http_skip_not_found_url_for_globs && option == std::prev(end); auto request_uri = Poco::URI(*option); for (const auto & [param, value] : params) @@ -296,7 +270,7 @@ namespace headers, ReadWriteBufferFromHTTP::Range{}, context->getRemoteHostFilter(), - /* delay_initiliazation */false, + delay_initialization, /* use_external_buffer */false, /* skip_url_not_found_error */skip_url_not_found_error), chooseCompressionMethod(request_uri.getPath(), compression_method)); @@ -306,14 +280,14 @@ namespace if (first_exception_message.empty()) first_exception_message = getCurrentExceptionMessage(false); - if (urls.size() == 1) + if (options == 1) throw; tryLogCurrentException(__PRETTY_FUNCTION__); } } - throw Exception(ErrorCodes::NETWORK_ERROR, "All uri ({}) options are unreachable: {}", urls.size(), first_exception_message); + throw Exception(ErrorCodes::NETWORK_ERROR, "All uri ({}) options are unreachable: {}", options, first_exception_message); } private: @@ -449,13 +423,11 @@ ColumnsDescription IStorageURLBase::getTableStructureFromData( const std::optional & format_settings, ContextPtr context) { - ReadBufferCreator read_buffer_creator; Poco::Net::HTTPBasicCredentials credentials; + std::vector urls_to_check; if (urlWithGlobs(uri)) { - std::vector urls_to_check; - size_t max_addresses = context->getSettingsRef().glob_expansion_max_elements; auto uri_descriptions = parseRemoteDescription(uri, 0, uri.size(), ',', max_addresses); for (const auto & description : uri_descriptions) @@ -463,11 +435,24 @@ ColumnsDescription IStorageURLBase::getTableStructureFromData( auto options = parseRemoteDescription(description, 0, description.size(), '|', max_addresses); urls_to_check.insert(urls_to_check.end(), options.begin(), options.end()); } + } + else + { + urls_to_check = {uri}; + } - read_buffer_creator = [&, urls_to_check]() + String exception_messages; + bool read_buffer_creator_was_used = false; + + std::vector::const_iterator option = urls_to_check.begin(); + do + { + auto read_buffer_creator = [&]() { + read_buffer_creator_was_used = true; return StorageURLSource::getFirstAvailableURLReadBuffer( - urls_to_check, + option, + urls_to_check.end(), context, {}, Poco::Net::HTTPRequest::HTTP_GET, @@ -475,35 +460,26 @@ ColumnsDescription IStorageURLBase::getTableStructureFromData( ConnectionTimeouts::getHTTPTimeouts(context), compression_method, credentials, - headers); + headers, + false, + false); }; - } - else - { - read_buffer_creator = [&]() + + try { - auto parsed_uri = Poco::URI(uri); - StorageURLSource::setCredentials(credentials, parsed_uri); + return readSchemaFromFormat(format, format_settings, read_buffer_creator, context); + } + catch (...) + { + if (urls_to_check.size() == 1 || !read_buffer_creator_was_used) + throw; - return wrapReadBufferWithCompressionMethod( - std::make_unique( - parsed_uri, - Poco::Net::HTTPRequest::HTTP_GET, - nullptr, - ConnectionTimeouts::getHTTPTimeouts(context), - credentials, - context->getSettingsRef().max_http_get_redirects, - DBMS_DEFAULT_BUFFER_SIZE, - context->getReadSettings(), - headers, - ReadWriteBufferFromHTTP::Range{}, - context->getRemoteHostFilter(), - /* delay_initiliazation */true), - chooseCompressionMethod(parsed_uri.getPath(), compression_method)); - }; - } + exception_messages += getCurrentExceptionMessage(false) + "\n"; + } - return readSchemaFromFormat(format, format_settings, read_buffer_creator, context); + } while (++option < urls_to_check.end()); + + throw Exception(ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, "All attempts to extract table structure from urls failed. Errors:\n{}", exception_messages); } Pipe IStorageURLBase::read( diff --git a/src/Storages/System/StorageSystemUsers.cpp b/src/Storages/System/StorageSystemUsers.cpp index ca88fa688a0..d9b94f21c61 100644 --- a/src/Storages/System/StorageSystemUsers.cpp +++ b/src/Storages/System/StorageSystemUsers.cpp @@ -102,17 +102,27 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, ContextPtr conte column_storage.insertData(storage_name.data(), storage_name.length()); column_auth_type.push_back(static_cast(auth_data.getType())); - if ( - auth_data.getType() == AuthenticationType::LDAP || - auth_data.getType() == AuthenticationType::KERBEROS - ) + if (auth_data.getType() == AuthenticationType::LDAP || + auth_data.getType() == AuthenticationType::KERBEROS || + auth_data.getType() == AuthenticationType::SSL_CERTIFICATE) { Poco::JSON::Object auth_params_json; if (auth_data.getType() == AuthenticationType::LDAP) + { auth_params_json.set("server", auth_data.getLDAPServerName()); + } else if (auth_data.getType() == AuthenticationType::KERBEROS) + { auth_params_json.set("realm", auth_data.getKerberosRealm()); + } + else if (auth_data.getType() == AuthenticationType::SSL_CERTIFICATE) + { + Poco::JSON::Array::Ptr arr = new Poco::JSON::Array(); + for (const auto & common_name : auth_data.getSSLCertificateCommonNames()) + arr->add(common_name); + auth_params_json.set("common_names", arr); + } std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM oss.exceptions(std::ios::failbit); diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 0eef886625a..64e04594786 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -15,38 +15,54 @@ from get_robot_token import get_best_robot_token from upload_result_helper import upload_results from docker_pull_helper import get_image_with_version from commit_status_helper import post_commit_status -from clickhouse_helper import ClickHouseHelper, mark_flaky_tests, prepare_tests_results_for_clickhouse +from clickhouse_helper import ( + ClickHouseHelper, + mark_flaky_tests, + prepare_tests_results_for_clickhouse, +) from stopwatch import Stopwatch from rerun_helper import RerunHelper from tee_popen import TeePopen from ccache_utils import get_ccache_if_not_exists, upload_ccache -NAME = 'Fast test (actions)' +NAME = "Fast test (actions)" -def get_fasttest_cmd(workspace, output_path, ccache_path, repo_path, pr_number, commit_sha, image): - return f"docker run --cap-add=SYS_PTRACE " \ - f"-e FASTTEST_WORKSPACE=/fasttest-workspace -e FASTTEST_OUTPUT=/test_output " \ - f"-e FASTTEST_SOURCE=/ClickHouse --cap-add=SYS_PTRACE " \ - f"-e PULL_REQUEST_NUMBER={pr_number} -e COMMIT_SHA={commit_sha} -e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " \ - f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse --volume={output_path}:/test_output "\ + +def get_fasttest_cmd( + workspace, output_path, ccache_path, repo_path, pr_number, commit_sha, image +): + return ( + f"docker run --cap-add=SYS_PTRACE " + f"-e FASTTEST_WORKSPACE=/fasttest-workspace -e FASTTEST_OUTPUT=/test_output " + f"-e FASTTEST_SOURCE=/ClickHouse --cap-add=SYS_PTRACE " + f"-e PULL_REQUEST_NUMBER={pr_number} -e COMMIT_SHA={commit_sha} " + f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " + f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " + f"--volume={output_path}:/test_output " f"--volume={ccache_path}:/fasttest-workspace/ccache {image}" + ) def process_results(result_folder): test_results = [] additional_files = [] # Just upload all files from result_folder. - # If task provides processed results, then it's responsible for content of result_folder. + # If task provides processed results, then it's responsible for content of + # result_folder if os.path.exists(result_folder): - test_files = [f for f in os.listdir(result_folder) if os.path.isfile(os.path.join(result_folder, f))] + test_files = [ + f + for f in os.listdir(result_folder) + if os.path.isfile(os.path.join(result_folder, f)) + ] additional_files = [os.path.join(result_folder, f) for f in test_files] status = [] status_path = os.path.join(result_folder, "check_status.tsv") if os.path.exists(status_path): logging.info("Found test_results.tsv") - with open(status_path, 'r', encoding='utf-8') as status_file: - status = list(csv.reader(status_file, delimiter='\t')) + with open(status_path, "r", encoding="utf-8") as status_file: + status = list(csv.reader(status_file, delimiter="\t")) if len(status) != 1 or len(status[0]) != 2: logging.info("Files in result folder %s", os.listdir(result_folder)) return "error", "Invalid check_status.tsv", test_results, additional_files @@ -54,15 +70,14 @@ def process_results(result_folder): results_path = os.path.join(result_folder, "test_results.tsv") if os.path.exists(results_path): - with open(results_path, 'r', encoding='utf-8') as results_file: - test_results = list(csv.reader(results_file, delimiter='\t')) + with open(results_path, "r", encoding="utf-8") as results_file: + test_results = list(csv.reader(results_file, delimiter="\t")) if len(test_results) == 0: return "error", "Empty test_results.tsv", test_results, additional_files return state, description, test_results, additional_files - if __name__ == "__main__": logging.basicConfig(level=logging.INFO) @@ -83,9 +98,9 @@ if __name__ == "__main__": logging.info("Check is already finished according to github status, exiting") sys.exit(0) - docker_image = get_image_with_version(temp_path, 'clickhouse/fasttest') + docker_image = get_image_with_version(temp_path, "clickhouse/fasttest") - s3_helper = S3Helper('https://s3.amazonaws.com') + s3_helper = S3Helper("https://s3.amazonaws.com") workspace = os.path.join(temp_path, "fasttest-workspace") if not os.path.exists(workspace): @@ -108,15 +123,23 @@ if __name__ == "__main__": if not os.path.exists(repo_path): os.makedirs(repo_path) - run_cmd = get_fasttest_cmd(workspace, output_path, cache_path, repo_path, pr_info.number, pr_info.sha, docker_image) + run_cmd = get_fasttest_cmd( + workspace, + output_path, + cache_path, + repo_path, + pr_info.number, + pr_info.sha, + docker_image, + ) logging.info("Going to run fasttest with cmd %s", run_cmd) logs_path = os.path.join(temp_path, "fasttest-logs") if not os.path.exists(logs_path): os.makedirs(logs_path) - run_log_path = os.path.join(logs_path, 'runlog.log') - with TeePopen(run_cmd, run_log_path) as process: + run_log_path = os.path.join(logs_path, "runlog.log") + with TeePopen(run_cmd, run_log_path, timeout=40 * 60) as process: retcode = process.wait() if retcode == 0: logging.info("Run successfully") @@ -131,19 +154,21 @@ if __name__ == "__main__": for f in test_output_files: additional_logs.append(os.path.join(output_path, f)) - test_log_exists = 'test_log.txt' in test_output_files or 'test_result.txt' in test_output_files - test_result_exists = 'test_results.tsv' in test_output_files + test_log_exists = ( + "test_log.txt" in test_output_files or "test_result.txt" in test_output_files + ) + test_result_exists = "test_results.tsv" in test_output_files test_results = [] - if 'submodule_log.txt' not in test_output_files: + if "submodule_log.txt" not in test_output_files: description = "Cannot clone repository" state = "failure" - elif 'cmake_log.txt' not in test_output_files: + elif "cmake_log.txt" not in test_output_files: description = "Cannot fetch submodules" state = "failure" - elif 'build_log.txt' not in test_output_files: + elif "build_log.txt" not in test_output_files: description = "Cannot finish cmake" state = "failure" - elif 'install_log.txt' not in test_output_files: + elif "install_log.txt" not in test_output_files: description = "Cannot build ClickHouse" state = "failure" elif not test_log_exists and not test_result_exists: @@ -158,16 +183,32 @@ if __name__ == "__main__": ch_helper = ClickHouseHelper() mark_flaky_tests(ch_helper, NAME, test_results) - report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, [run_log_path] + additional_logs, NAME, True) - print("::notice ::Report url: {}".format(report_url)) + report_url = upload_results( + s3_helper, + pr_info.number, + pr_info.sha, + test_results, + [run_log_path] + additional_logs, + NAME, + True, + ) + print(f"::notice ::Report url: {report_url}") post_commit_status(gh, pr_info.sha, NAME, description, state, report_url) - prepared_events = prepare_tests_results_for_clickhouse(pr_info, test_results, state, stopwatch.duration_seconds, stopwatch.start_time_str, report_url, NAME) + prepared_events = prepare_tests_results_for_clickhouse( + pr_info, + test_results, + state, + stopwatch.duration_seconds, + stopwatch.start_time_str, + report_url, + NAME, + ) ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events) # Refuse other checks to run if fast test failed - if state != 'success': - if 'force-tests' in pr_info.labels: + if state != "success": + if "force-tests" in pr_info.labels: print("'force-tests' enabled, will report success") else: sys.exit(1) diff --git a/tests/ci/git_helper.py b/tests/ci/git_helper.py index aebbf96abe2..2d28c693087 100644 --- a/tests/ci/git_helper.py +++ b/tests/ci/git_helper.py @@ -5,8 +5,13 @@ import re import subprocess from typing import Optional -TAG_REGEXP = r"^v\d{2}[.][1-9]\d*[.][1-9]\d*[.][1-9]\d*-(testing|prestable|stable|lts)$" -SHA_REGEXP = r"^([0-9]|[a-f]){40}$" +# ^ and $ match subline in `multiple\nlines` +# \A and \Z match only start and end of the whole string +RELEASE_BRANCH_REGEXP = r"\A\d+[.]\d+\Z" +TAG_REGEXP = ( + r"\Av\d{2}[.][1-9]\d*[.][1-9]\d*[.][1-9]\d*-(testing|prestable|stable|lts)\Z" +) +SHA_REGEXP = r"\A([0-9]|[a-f]){40}\Z" # Py 3.8 removeprefix and removesuffix @@ -31,6 +36,13 @@ def commit(name: str): return name +def release_branch(name: str): + r = re.compile(RELEASE_BRANCH_REGEXP) + if not r.match(name): + raise argparse.ArgumentTypeError("release branch should be as 12.1") + return name + + class Runner: """lightweight check_output wrapper with stripping last NEW_LINE""" diff --git a/tests/ci/git_test.py b/tests/ci/git_test.py new file mode 100644 index 00000000000..785c9b62cce --- /dev/null +++ b/tests/ci/git_test.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python + +from unittest.mock import patch +import os.path as p +import unittest + +from git_helper import Git, Runner + + +class TestRunner(unittest.TestCase): + def test_init(self): + runner = Runner() + self.assertEqual(runner.cwd, p.realpath(p.dirname(__file__))) + runner = Runner("/") + self.assertEqual(runner.cwd, "/") + + def test_run(self): + runner = Runner() + output = runner.run("echo 1") + self.assertEqual(output, "1") + + +class TestGit(unittest.TestCase): + def setUp(self): + """we use dummy git object""" + run_patcher = patch("git_helper.Runner.run", return_value="") + self.run_mock = run_patcher.start() + self.addCleanup(run_patcher.stop) + update_patcher = patch("git_helper.Git.update") + update_mock = update_patcher.start() + self.addCleanup(update_patcher.stop) + self.git = Git() + update_mock.assert_called_once() + self.run_mock.assert_called_once() + self.git.new_branch = "NEW_BRANCH_NAME" + self.git.new_tag = "v21.12.333.22222-stable" + self.git.branch = "old_branch" + self.git.sha = "" + self.git.sha_short = "" + self.git.latest_tag = "" + self.git.description = "" + self.git.commits_since_tag = 0 + + def test_tags(self): + self.git.new_tag = "v21.12.333.22222-stable" + self.git.latest_tag = "v21.12.333.22222-stable" + for tag_attr in ("new_tag", "latest_tag"): + self.assertEqual(getattr(self.git, tag_attr), "v21.12.333.22222-stable") + setattr(self.git, tag_attr, "") + self.assertEqual(getattr(self.git, tag_attr), "") + for tag in ( + "v21.12.333-stable", + "v21.12.333-prestable", + "21.12.333.22222-stable", + "v21.12.333.22222-production", + ): + with self.assertRaises(Exception): + setattr(self.git, tag_attr, tag) + + def test_tweak(self): + self.git.commits_since_tag = 0 + self.assertEqual(self.git.tweak, 1) + self.git.commits_since_tag = 2 + self.assertEqual(self.git.tweak, 2) + self.git.latest_tag = "v21.12.333.22222-testing" + self.assertEqual(self.git.tweak, 22224) + self.git.commits_since_tag = 0 + self.assertEqual(self.git.tweak, 22222) diff --git a/tests/ci/performance_comparison_check.py b/tests/ci/performance_comparison_check.py index ea2f3c5196a..2700c747b12 100644 --- a/tests/ci/performance_comparison_check.py +++ b/tests/ci/performance_comparison_check.py @@ -21,8 +21,9 @@ from rerun_helper import RerunHelper IMAGE_NAME = 'clickhouse/performance-comparison' -def get_run_command(workspace, result_path, pr_to_test, sha_to_test, additional_env, image): +def get_run_command(workspace, result_path, repo_tests_path, pr_to_test, sha_to_test, additional_env, image): return f"docker run --privileged --volume={workspace}:/workspace --volume={result_path}:/output " \ + f"--volume={repo_tests_path}:/usr/share/clickhouse-test " \ f"--cap-add syslog --cap-add sys_admin --cap-add sys_rawio " \ f"-e PR_TO_TEST={pr_to_test} -e SHA_TO_TEST={sha_to_test} {additional_env} " \ f"{image}" @@ -45,6 +46,7 @@ if __name__ == "__main__": logging.basicConfig(level=logging.INFO) temp_path = os.getenv("TEMP_PATH", os.path.abspath(".")) repo_path = os.getenv("REPO_COPY", os.path.abspath("../../")) + repo_tests_path = os.path.join(repo_path, "tests") ramdrive_path = os.getenv("RAMDRIVE_PATH", os.path.join(temp_path, "ramdrive")) # currently unused, doesn't make tests more stable ramdrive_size = os.getenv("RAMDRIVE_SIZE", '0G') @@ -95,7 +97,7 @@ if __name__ == "__main__": if not os.path.exists(result_path): os.makedirs(result_path) - run_command = get_run_command(result_path, result_path, pr_info.number, pr_info.sha, docker_env, docker_image) + run_command = get_run_command(result_path, result_path, repo_tests_path, pr_info.number, pr_info.sha, docker_env, docker_image) logging.info("Going to run command %s", run_command) run_log_path = os.path.join(temp_path, "runlog.log") with TeePopen(run_command, run_log_path) as process: diff --git a/tests/ci/push_to_artifactory.py b/tests/ci/push_to_artifactory.py index 7747525df02..1f2ea139a6e 100755 --- a/tests/ci/push_to_artifactory.py +++ b/tests/ci/push_to_artifactory.py @@ -32,7 +32,6 @@ class Packages: ("clickhouse-common-static", "amd64"), ("clickhouse-common-static-dbg", "amd64"), ("clickhouse-server", "all"), - ("clickhouse-test", "all"), ) def __init__(self, version: str): diff --git a/tests/ci/pvs_check.py b/tests/ci/pvs_check.py deleted file mode 100644 index af543211c16..00000000000 --- a/tests/ci/pvs_check.py +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env python3 - -# pylint: disable=line-too-long - -import os -import json -import logging -import sys -from github import Github - -from env_helper import REPO_COPY, TEMP_PATH, GITHUB_RUN_ID, GITHUB_REPOSITORY, GITHUB_SERVER_URL -from s3_helper import S3Helper -from pr_info import PRInfo -from get_robot_token import get_best_robot_token, get_parameter_from_ssm -from upload_result_helper import upload_results -from commit_status_helper import get_commit -from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse -from stopwatch import Stopwatch -from rerun_helper import RerunHelper -from tee_popen import TeePopen - -NAME = 'PVS Studio (actions)' -LICENCE_NAME = 'Free license: ClickHouse, Yandex' -HTML_REPORT_FOLDER = 'pvs-studio-html-report' -TXT_REPORT_NAME = 'pvs-studio-task-report.txt' - - -def _process_txt_report(path): - warnings = [] - errors = [] - with open(path, 'r') as report_file: - for line in report_file: - if 'viva64' in line: - continue - - if 'warn' in line: - warnings.append(':'.join(line.split('\t')[0:2])) - elif 'err' in line: - errors.append(':'.join(line.split('\t')[0:2])) - - return warnings, errors - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - - stopwatch = Stopwatch() - - repo_path = REPO_COPY - temp_path = TEMP_PATH - - pr_info = PRInfo() - # this check modify repository so copy it to the temp directory - logging.info("Repo copy path %s", repo_path) - - gh = Github(get_best_robot_token()) - rerun_helper = RerunHelper(gh, pr_info, NAME) - if rerun_helper.is_already_finished_by_status(): - logging.info("Check is already finished according to github status, exiting") - sys.exit(0) - - images_path = os.path.join(temp_path, 'changed_images.json') - docker_image = 'clickhouse/pvs-test' - if os.path.exists(images_path): - logging.info("Images file exists") - with open(images_path, 'r') as images_fd: - images = json.load(images_fd) - logging.info("Got images %s", images) - if 'clickhouse/pvs-test' in images: - docker_image += ':' + images['clickhouse/pvs-test'] - - logging.info("Got docker image %s", docker_image) - - s3_helper = S3Helper('https://s3.amazonaws.com') - - licence_key = get_parameter_from_ssm('pvs_studio_key') - cmd = f"docker run -u $(id -u ${{USER}}):$(id -g ${{USER}}) --volume={repo_path}:/repo_folder --volume={temp_path}:/test_output -e LICENCE_NAME='{LICENCE_NAME}' -e LICENCE_KEY='{licence_key}' {docker_image}" - commit = get_commit(gh, pr_info.sha) - - run_log_path = os.path.join(temp_path, 'run_log.log') - - with TeePopen(cmd, run_log_path) as process: - retcode = process.wait() - if retcode != 0: - logging.info("Run failed") - else: - logging.info("Run Ok") - - if retcode != 0: - commit.create_status(context=NAME, description='PVS report failed to build', state='error', - target_url=f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/actions/runs/{GITHUB_RUN_ID}") - sys.exit(1) - - try: - s3_path_prefix = str(pr_info.number) + "/" + pr_info.sha + "/" + NAME.lower().replace(' ', '_') - html_urls = s3_helper.upload_test_folder_to_s3(os.path.join(temp_path, HTML_REPORT_FOLDER), s3_path_prefix) - index_html = None - - for url in html_urls: - if 'index.html' in url: - index_html = 'HTML report'.format(url) - break - - if not index_html: - commit.create_status(context=NAME, description='PVS report failed to build', state='error', - target_url=f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/actions/runs/{GITHUB_RUN_ID}") - sys.exit(1) - - txt_report = os.path.join(temp_path, TXT_REPORT_NAME) - warnings, errors = _process_txt_report(txt_report) - errors = errors + warnings - - status = 'success' - test_results = [(index_html, "Look at the report"), ("Errors count not checked", "OK")] - description = "Total errors {}".format(len(errors)) - additional_logs = [txt_report, os.path.join(temp_path, 'pvs-studio.log')] - report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, additional_logs, NAME) - - print("::notice ::Report url: {}".format(report_url)) - commit = get_commit(gh, pr_info.sha) - commit.create_status(context=NAME, description=description, state=status, target_url=report_url) - - ch_helper = ClickHouseHelper() - prepared_events = prepare_tests_results_for_clickhouse(pr_info, test_results, status, stopwatch.duration_seconds, stopwatch.start_time_str, report_url, NAME) - ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events) - except Exception as ex: - print("Got an exception", ex) - sys.exit(1) diff --git a/tests/ci/release.py b/tests/ci/release.py index f1ed27f3d8c..b1ce850d640 100755 --- a/tests/ci/release.py +++ b/tests/ci/release.py @@ -2,11 +2,11 @@ from contextlib import contextmanager -from typing import Optional +from typing import List, Optional import argparse import logging -from git_helper import commit +from git_helper import commit, release_branch from version_helper import ( FILE_WITH_VERSION_PATH, ClickHouseVersion, @@ -18,19 +18,191 @@ from version_helper import ( ) +class Repo: + VALID = ("ssh", "https", "origin") + + def __init__(self, repo: str, protocol: str): + self._repo = repo + self._url = "" + self.url = protocol + + @property + def url(self) -> str: + return self._url + + @url.setter + def url(self, protocol: str): + if protocol == "ssh": + self._url = f"git@github.com:{self}.git" + elif protocol == "https": + self._url = f"https://github.com/{self}.git" + elif protocol == "origin": + self._url = protocol + else: + raise Exception(f"protocol must be in {self.VALID}") + + def __str__(self): + return self._repo + + class Release: BIG = ("major", "minor") SMALL = ("patch",) - def __init__(self, version: ClickHouseVersion): - self._version = version - self._git = version._git + def __init__(self, repo: Repo, release_commit: str, release_type: str): + self.repo = repo self._release_commit = "" + self.release_commit = release_commit + self.release_type = release_type + self._version = get_version_from_repo() + self._git = self._version._git + self._release_branch = "" + self._rollback_stack = [] # type: List[str] def run(self, cmd: str, cwd: Optional[str] = None) -> str: - logging.info("Running in directory %s, command:\n %s", cwd or "$CWD", cmd) + cwd_text = "" + if cwd: + cwd_text = f" (CWD='{cwd}')" + logging.info("Running command%s:\n %s", cwd_text, cmd) return self._git.run(cmd, cwd) + def set_release_branch(self): + # Get the actual version for the commit before check + with self._checkout(self.release_commit, True): + self.read_version() + self.release_branch = f"{self.version.major}.{self.version.minor}" + + self.read_version() + + def read_version(self): + self._git.update() + self.version = get_version_from_repo() + + def do(self, no_check_dirty: bool, no_check_branch: bool, no_prestable: bool): + + if not no_check_dirty: + logging.info("Checking if repo is clean") + self.run("git diff HEAD --exit-code") + + self.set_release_branch() + + if not no_check_branch: + self.check_branch() + + with self._checkout(self.release_commit, True): + if self.release_type in self.BIG: + # Checkout to the commit, it will provide the correct current version + if no_prestable: + logging.info("Skipping prestable stage") + else: + with self.prestable(): + logging.info("Prestable part of the releasing is done") + + with self.testing(): + logging.info("Testing part of the releasing is done") + + elif self.release_type in self.SMALL: + with self.stable(): + logging.info("Stable part of the releasing is done") + + self.log_rollback() + + def check_no_tags_after(self): + tags_after_commit = self.run(f"git tag --contains={self.release_commit}") + if tags_after_commit: + raise Exception( + f"Commit {self.release_commit} belongs to following tags:\n" + f"{tags_after_commit}\nChoose another commit" + ) + + def check_branch(self): + if self.release_type in self.BIG: + # Commit to spin up the release must belong to a main branch + branch = "master" + output = self.run(f"git branch --contains={self.release_commit} {branch}") + if branch not in output: + raise Exception( + f"commit {self.release_commit} must belong to {branch} for " + f"{self.release_type} release" + ) + return + elif self.release_type in self.SMALL: + output = self.run( + f"git branch --contains={self.release_commit} {self.release_branch}" + ) + if self.release_branch not in output: + raise Exception( + f"commit {self.release_commit} must be in " + f"'{self.release_branch}' branch for {self.release_type} release" + ) + return + + def log_rollback(self): + if self._rollback_stack: + rollback = self._rollback_stack + rollback.reverse() + logging.info( + "To rollback the action run the following commands:\n %s", + "\n ".join(rollback), + ) + + @contextmanager + def prestable(self): + self.check_no_tags_after() + # Create release branch + self.read_version() + with self._create_branch(self.release_branch, self.release_commit): + with self._checkout(self.release_branch, True): + self.read_version() + self.version.with_description(VersionType.PRESTABLE) + with self._create_gh_release(True): + with self._bump_prestable_version(): + # At this point everything will rollback automatically + yield + + @contextmanager + def stable(self): + self.check_no_tags_after() + self.read_version() + version_type = VersionType.STABLE + if self.version.minor % 5 == 3: # our 3 and 8 are LTS + version_type = VersionType.LTS + self.version.with_description(version_type) + with self._create_gh_release(False): + self.version = self.version.update(self.release_type) + self.version.with_description(version_type) + update_cmake_version(self.version) + cmake_path = get_abs_path(FILE_WITH_VERSION_PATH) + # Checkouting the commit of the branch and not the branch itself, + # then we are able to skip rollback + with self._checkout(f"{self.release_branch}@{{0}}", False): + current_commit = self.run("git rev-parse HEAD") + self.run( + f"git commit -m " + f"'Update version to {self.version.string}' '{cmake_path}'" + ) + with self._push( + "HEAD", with_rollback_on_fail=False, remote_ref=self.release_branch + ): + # DO NOT PUT ANYTHING ELSE HERE + # The push must be the last action and mean the successful release + self._rollback_stack.append( + f"git push {self.repo.url} " + f"+{current_commit}:{self.release_branch}" + ) + yield + + @contextmanager + def testing(self): + # Create branch for a version bump + self.read_version() + self.version = self.version.update(self.release_type) + helper_branch = f"{self.version.major}.{self.version.minor}-prepare" + with self._create_branch(helper_branch, self.release_commit): + with self._checkout(helper_branch, True): + with self._bump_testing_version(helper_branch): + yield + @property def version(self) -> ClickHouseVersion: return self._version @@ -41,6 +213,14 @@ class Release: raise ValueError(f"version must be ClickHouseVersion, not {type(version)}") self._version = version + @property + def release_branch(self) -> str: + return self._release_branch + + @release_branch.setter + def release_branch(self, branch: str): + self._release_branch = release_branch(branch) + @property def release_commit(self) -> str: return self._release_commit @@ -49,49 +229,63 @@ class Release: def release_commit(self, release_commit: str): self._release_commit = commit(release_commit) - def check_no_tags_after(self): - tags_after_commit = self.run(f"git tag --contains={self.release_commit}") - if tags_after_commit: - raise Exception( - f"Commit {self.release_commit} belongs to following tags:\n" - f"{tags_after_commit}\nChoose another commit" - ) - - def check_branch(self, release_type: str): - if release_type in self.BIG: - # Commit to spin up the release must belong to a main branch - output = self.run(f"git branch --contains={self.release_commit} master") - if "master" not in output: - raise Exception( - f"commit {self.release_commit} must belong to 'master' for " - f"{release_type} release" - ) - if release_type in self.SMALL: - branch = f"{self.version.major}.{self.version.minor}" - if self._git.branch != branch: - raise Exception(f"branch must be '{branch}' for {release_type} release") - - def update(self): + @contextmanager + def _bump_prestable_version(self): + # Update only git, origal version stays the same self._git.update() - self.version = get_version_from_repo() + new_version = self.version.patch_update() + new_version.with_description("prestable") + update_cmake_version(new_version) + cmake_path = get_abs_path(FILE_WITH_VERSION_PATH) + self.run( + f"git commit -m 'Update version to {new_version.string}' '{cmake_path}'" + ) + with self._push(self.release_branch): + with self._create_gh_label( + f"v{self.release_branch}-must-backport", "10dbed" + ): + with self._create_gh_label( + f"v{self.release_branch}-affected", "c2bfff" + ): + self.run( + f"gh pr create --repo {self.repo} --title " + f"'Release pull request for branch {self.release_branch}' " + f"--head {self.release_branch} --label release " + "--body 'This PullRequest is a part of ClickHouse release " + "cycle. It is used by CI system only. Do not perform any " + "changes with it.'" + ) + # Here the prestable part is done + yield @contextmanager - def _new_branch(self, name: str, start_point: str = ""): - self.run(f"git branch {name} {start_point}") - try: + def _bump_testing_version(self, helper_branch: str): + self.read_version() + self.version = self.version.update(self.release_type) + self.version.with_description("testing") + update_cmake_version(self.version) + cmake_path = get_abs_path(FILE_WITH_VERSION_PATH) + self.run( + f"git commit -m 'Update version to {self.version.string}' '{cmake_path}'" + ) + with self._push(helper_branch): + body_file = get_abs_path(".github/PULL_REQUEST_TEMPLATE.md") + self.run( + f"gh pr create --repo {self.repo} --title 'Update version after " + f"release' --head {helper_branch} --body-file '{body_file}'" + ) + # Here the prestable part is done yield - except BaseException: - logging.warning("Rolling back created branch %s", name) - self.run(f"git branch -D {name}") - raise @contextmanager - def _checkout(self, ref: str, with_rollback: bool = False): + def _checkout(self, ref: str, with_checkout_back: bool = False): orig_ref = self._git.branch or self._git.sha need_rollback = False if ref not in (self._git.branch, self._git.sha): need_rollback = True self.run(f"git checkout {ref}") + # checkout is not put into rollback_stack intentionally + rollback_cmd = f"git checkout {orig_ref}" try: yield except BaseException: @@ -99,132 +293,90 @@ class Release: self.run(f"git reset --hard; git checkout {orig_ref}") raise else: - if with_rollback and need_rollback: - self.run(f"git checkout {orig_ref}") + if with_checkout_back and need_rollback: + self.run(rollback_cmd) @contextmanager - def prestable(self, args: argparse.Namespace): - self.check_no_tags_after() - # Create release branch - # TODO: this place is wrong. If we are in stale branch, it will produce - # a wrong version - self.update() - release_branch = f"{self.version.major}.{self.version.minor}" - with self._new_branch(release_branch, self.release_commit): - with self._checkout(release_branch, True): - self.update() - self.version.with_description(VersionType.PRESTABLE) - with self._create_gh_release(args): - with self._bump_prestable_version(release_branch, args): - # At this point everything will rollback automatically - yield - - @contextmanager - def testing(self, args: argparse.Namespace): - # Create branch for a version bump - self.update() - # TODO: this place is wrong. If we are in stale branch, it will produce - # a wrong version - self.version = self.version.update(args.release_type) - helper_branch = f"{self.version.major}.{self.version.minor}-prepare" - with self._new_branch(helper_branch, self.release_commit): - with self._checkout(helper_branch, True): - self.update() - self.version = self.version.update(args.release_type) - with self._bump_testing_version(helper_branch, args): - yield - - @contextmanager - def _bump_testing_version(self, helper_branch: str, args: argparse.Namespace): - update_cmake_version(self.version) - cmake_path = get_abs_path(FILE_WITH_VERSION_PATH) - self.run( - f"git commit -m 'Update version to {self.version.string}' '{cmake_path}'" - ) - with self._push(helper_branch, args): - body_file = get_abs_path(".github/PULL_REQUEST_TEMPLATE.md") - self.run( - f"gh pr create --repo {args.repo} --title 'Update version after " - f"release' --head {helper_branch} --body-file '{body_file}'" - ) - # Here the prestable part is done + def _create_branch(self, name: str, start_point: str = ""): + self.run(f"git branch {name} {start_point}") + rollback_cmd = f"git branch -D {name}" + self._rollback_stack.append(rollback_cmd) + try: yield + except BaseException: + logging.warning("Rolling back created branch %s", name) + self.run(rollback_cmd) + raise @contextmanager - def _bump_prestable_version(self, release_branch: str, args: argparse.Namespace): - new_version = self.version.patch_update() - update_cmake_version(new_version) - cmake_path = get_abs_path(FILE_WITH_VERSION_PATH) + def _create_gh_label(self, label: str, color_hex: str): + # API call, https://docs.github.com/en/rest/reference/issues#create-a-label self.run( - f"git commit -m 'Update version to {new_version.string}' '{cmake_path}'" + f"gh api repos/{self.repo}/labels -f name={label} -f color={color_hex}" ) - with self._push(release_branch, args): - self.run( - f"gh pr create --repo {args.repo} --title 'Release pull request for " - f"branch {release_branch}' --head {release_branch} --body 'This " - "PullRequest is a part of ClickHouse release cycle. It is used by CI " - "system only. Do not perform any changes with it.' --label release" - ) - # Here the prestable part is done + rollback_cmd = f"gh api repos/{self.repo}/labels/{label} -X DELETE" + self._rollback_stack.append(rollback_cmd) + try: yield + except BaseException: + logging.warning("Rolling back label %s", label) + self.run(rollback_cmd) + raise @contextmanager - def _create_gh_release(self, args: argparse.Namespace): - with self._create_tag(args): + def _create_gh_release(self, as_prerelease: bool): + with self._create_tag(): # Preserve tag if version is changed tag = self.version.describe + prerelease = "" + if as_prerelease: + prerelease = "--prerelease" self.run( - f"gh release create --prerelease --draft --repo {args.repo} '{tag}'" + f"gh release create {prerelease} --draft --repo {self.repo} " + f"--title 'Release {tag}' '{tag}'" ) + rollback_cmd = f"gh release delete --yes --repo {self.repo} '{tag}'" + self._rollback_stack.append(rollback_cmd) try: yield except BaseException: logging.warning("Rolling back release publishing") - self.run(f"gh release delete --yes --repo {args.repo} '{tag}'") + self.run(rollback_cmd) raise @contextmanager - def _create_tag(self, args: argparse.Namespace): + def _create_tag(self): tag = self.version.describe self.run(f"git tag -a -m 'Release {tag}' '{tag}'") + rollback_cmd = f"git tag -d '{tag}'" + self._rollback_stack.append(rollback_cmd) try: - with self._push(f"'{tag}'", args): + with self._push(f"'{tag}'"): yield except BaseException: logging.warning("Rolling back tag %s", tag) - self.run(f"git tag -d '{tag}'") + self.run(rollback_cmd) raise @contextmanager - def _push(self, ref: str, args: argparse.Namespace): - self.run(f"git push git@github.com:{args.repo}.git {ref}") + def _push(self, ref: str, with_rollback_on_fail: bool = True, remote_ref: str = ""): + if remote_ref == "": + remote_ref = ref + + self.run(f"git push {self.repo.url} {ref}:{remote_ref}") + if with_rollback_on_fail: + rollback_cmd = f"git push -d {self.repo.url} {remote_ref}" + self._rollback_stack.append(rollback_cmd) + try: yield except BaseException: - logging.warning("Rolling back pushed ref %s", ref) - self.run(f"git push -d git@github.com:{args.repo}.git {ref}") + if with_rollback_on_fail: + logging.warning("Rolling back pushed ref %s", ref) + self.run(rollback_cmd) + raise - def do(self, args: argparse.Namespace): - self.release_commit = args.commit - - if not args.no_check_dirty: - logging.info("Checking if repo is clean") - self.run("git diff HEAD --exit-code") - - if not args.no_check_branch: - self.check_branch(args.release_type) - - if args.release_type in self.BIG: - if args.no_prestable: - logging.info("Skipping prestable stage") - else: - with self.prestable(args): - logging.info("Prestable part of the releasing is done") - - with self.testing(args): - logging.info("Testing part of the releasing is done") - def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( @@ -238,10 +390,17 @@ def parse_args() -> argparse.Namespace: default="ClickHouse/ClickHouse", help="repository to create the release", ) + parser.add_argument( + "--remote-protocol", + "-p", + default="ssh", + choices=Repo.VALID, + help="repo protocol for git commands remote, 'origin' is a special case and " + "uses 'origin' as a remote", + ) parser.add_argument( "--type", default="minor", - # choices=Release.BIG+Release.SMALL, # add support later choices=Release.BIG + Release.SMALL, dest="release_type", help="a release type, new branch is created only for 'major' and 'minor'", @@ -261,67 +420,26 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--no-check-dirty", action="store_true", - help="skip check repository for uncommited changes", + help="(dangerous) skip check repository for uncommited changes", ) parser.add_argument( "--no-check-branch", action="store_true", - help="by default, 'major' and 'minor' types work only for master, and 'patch' " - "works only for a release branches, that name should be the same as " - "'$MAJOR.$MINOR' version, e.g. 22.2", - ) - parser.add_argument( - "--no-publish-release", - action="store_true", - help="by default, 'major' and 'minor' types work only for master, and 'patch' ", + help="(debug or development only) by default, 'major' and 'minor' types work " + "only for master, and 'patch' works only for a release branches, that name " + "should be the same as '$MAJOR.$MINOR' version, e.g. 22.2", ) return parser.parse_args() -def prestable(): - pass - - def main(): logging.basicConfig(level=logging.INFO) args = parse_args() - release = Release(get_version_from_repo()) + repo = Repo(args.repo, args.remote_protocol) + release = Release(repo, args.commit, args.release_type) - release.do(args) - - # if not args.no_publish_release: - # # Publish release on github for the current HEAD (master, if checked) - # git.run(f"gh release create --draft {git.new_tag} --target {git.sha}") - - ## Commit updated versions to HEAD and push to remote - # write_versions(versions_file, new_versions) - # git.run(f"git checkout -b {git.new_branch}-helper") - # git.run( - # f"git commit -m 'Auto version update to [{new_versions['VERSION_STRING']}] " - # f"[{new_versions['VERSION_REVISION']}]' {versions_file}" - # ) - # git.run(f"git push -u origin {git.new_branch}-helper") - # git.run( - # f"gh pr create --title 'Update version after release {git.new_branch}' " - # f"--body-file '{git.root}/.github/PULL_REQUEST_TEMPLATE.md'" - # ) - - ## Create a new branch from the previous commit and push there with creating - ## a PR - # git.run(f"git checkout -b {git.new_branch} HEAD~") - # write_versions(versions_file, versions) - # git.run( - # f"git commit -m 'Auto version update to [{versions['VERSION_STRING']}] " - # f"[{versions['VERSION_REVISION']}]' {versions_file}" - # ) - # git.run(f"git push -u origin {git.new_branch}") - # git.run( - # "gh pr create --title 'Release pull request for branch " - # f"{versions['VERSION_MAJOR']}.{versions['VERSION_MINOR']}' --body " - # "'This PullRequest is part of ClickHouse release cycle. It is used by CI " - # "system only. Do not perform any changes with it.' --label release" - # ) + release.do(args.no_check_dirty, args.no_check_branch, args.no_prestable) if __name__ == "__main__": diff --git a/tests/ci/report.py b/tests/ci/report.py index 156e6096605..c8ba46da106 100644 --- a/tests/ci/report.py +++ b/tests/ci/report.py @@ -120,11 +120,23 @@ def _get_status_style(status): return style -def _get_html_url(url): +def _get_html_url_name(url): if isinstance(url, str): - return '{name}'.format(url=url, name=os.path.basename(url).replace('%2B', '+').replace('%20', ' ')) + return os.path.basename(url).replace('%2B', '+').replace('%20', ' ') if isinstance(url, tuple): - return '{name}'.format(url=url[0], name=url[1].replace('%2B', '+').replace('%20', ' ')) + return url[1].replace('%2B', '+').replace('%20', ' ') + return None + + +def _get_html_url(url): + href = None + name = None + if isinstance(url, str): + href, name = url, _get_html_url_name(url) + if isinstance(url, tuple): + href, name = url[0], _get_html_url_name(url) + if href and name: + return '{name}'.format(href=href, name=_get_html_url_name(url)) return '' @@ -193,9 +205,7 @@ def create_test_html_report(header, test_result, raw_log_url, task_url, branch_u else: test_part = "" - additional_html_urls = "" - for url in additional_urls: - additional_html_urls += ' ' + _get_html_url(url) + additional_html_urls = ' '.join([_get_html_url(url) for url in sorted(additional_urls, key=_get_html_url_name)]) result = HTML_BASE_TEST_TEMPLATE.format( title=_format_header(header, branch_name), @@ -307,7 +317,7 @@ def create_build_html_report(header, build_results, build_logs_urls, artifact_ur link_separator = "
" if artifact_urls: for artifact_url in artifact_urls: - links += LINK_TEMPLATE.format(text=os.path.basename(artifact_url.replace('%2B', '+').replace('%20', ' ')), url=artifact_url) + links += LINK_TEMPLATE.format(text=_get_html_url_name(artifact_url), url=artifact_url) links += link_separator if links: links = links[:-len(link_separator)] diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py index a2403e61ac1..5b89082532d 100644 --- a/tests/ci/run_check.py +++ b/tests/ci/run_check.py @@ -69,6 +69,7 @@ TRUSTED_CONTRIBUTORS = { "s-mx", # Maxim Sabyanin, former employee, present contributor "sevirov", # technical writer, Yandex "spongedu", # Seasoned contributor + "taiyang-li", "ucasFL", # Amos Bird's friend "vdimir", # Employee "vzakaznikov", diff --git a/tests/ci/stress_check.py b/tests/ci/stress_check.py index 1f811b1e25d..bb9cec624ec 100644 --- a/tests/ci/stress_check.py +++ b/tests/ci/stress_check.py @@ -22,10 +22,11 @@ from rerun_helper import RerunHelper from tee_popen import TeePopen -def get_run_command(build_path, result_folder, server_log_folder, image): +def get_run_command(build_path, result_folder, repo_tests_path, server_log_folder, image): cmd = "docker run --cap-add=SYS_PTRACE -e S3_URL='https://clickhouse-datasets.s3.amazonaws.com' " + \ f"--volume={build_path}:/package_folder " \ f"--volume={result_folder}:/test_output " \ + f"--volume={repo_tests_path}:/usr/share/clickhouse-test " \ f"--volume={server_log_folder}:/var/log/clickhouse-server {image}" return cmd @@ -72,6 +73,7 @@ if __name__ == "__main__": stopwatch = Stopwatch() temp_path = TEMP_PATH repo_path = REPO_COPY + repo_tests_path = os.path.join(repo_path, "tests") reports_path = REPORTS_PATH check_name = sys.argv[1] @@ -106,7 +108,7 @@ if __name__ == "__main__": run_log_path = os.path.join(temp_path, "runlog.log") - run_command = get_run_command(packages_path, result_path, server_log_path, docker_image) + run_command = get_run_command(packages_path, result_path, repo_tests_path, server_log_path, docker_image) logging.info("Going to run func tests: %s", run_command) with TeePopen(run_command, run_log_path) as process: diff --git a/tests/ci/tee_popen.py b/tests/ci/tee_popen.py index 20302dacb97..7270cd6fb03 100644 --- a/tests/ci/tee_popen.py +++ b/tests/ci/tee_popen.py @@ -1,8 +1,11 @@ #!/usr/bin/env python3 from subprocess import Popen, PIPE, STDOUT -import sys +from threading import Thread +from time import sleep +import logging import os +import sys # Very simple tee logic implementation. You can specify shell command, output @@ -11,11 +14,23 @@ import os # stdout. class TeePopen: # pylint: disable=W0102 - def __init__(self, command, log_file, env=os.environ.copy()): + def __init__(self, command, log_file, env=os.environ.copy(), timeout=None): self.command = command self.log_file = log_file self.env = env self.process = None + self.timeout = timeout + + def _check_timeout(self): + sleep(self.timeout) + while self.process.poll() is None: + logging.warning( + "Killing process %s, timeout %s exceeded", + self.process.pid, + self.timeout, + ) + os.killpg(self.process.pid, 9) + sleep(10) def __enter__(self): self.process = Popen( @@ -23,11 +38,16 @@ class TeePopen: shell=True, universal_newlines=True, env=self.env, + start_new_session=True, # signall will be sent to all children stderr=STDOUT, stdout=PIPE, bufsize=1, ) self.log_file = open(self.log_file, "w", encoding="utf-8") + if self.timeout is not None and self.timeout > 0: + t = Thread(target=self._check_timeout) + t.daemon = True # does not block the program from exit + t.start() return self def __exit__(self, t, value, traceback): diff --git a/tests/ci/workflow_approve_rerun_lambda/app.py b/tests/ci/workflow_approve_rerun_lambda/app.py index 9cea6db3da2..50b9d9bfedc 100644 --- a/tests/ci/workflow_approve_rerun_lambda/app.py +++ b/tests/ci/workflow_approve_rerun_lambda/app.py @@ -114,7 +114,8 @@ TRUSTED_CONTRIBUTORS = { "s-mx", # Maxim Sabyanin, former employee, present contributor "sevirov", # technical writer, Yandex "spongedu", # Seasoned contributor - "ucasfl", # Amos Bird's friend + "taiyang-li", + "ucasFL", # Amos Bird's friend "vdimir", # Employee "vzakaznikov", "YiuRULE", diff --git a/tests/instructions/pvs-studio.txt b/tests/instructions/pvs-studio.txt deleted file mode 100644 index 6ffc266aa6f..00000000000 --- a/tests/instructions/pvs-studio.txt +++ /dev/null @@ -1,15 +0,0 @@ -https://www.viva64.com/ru/m/0036/ - -# Analyze project with 4 threads. It takes about six minutes. - -pvs-studio-analyzer analyze -o pvs-studio.log -e contrib -j 4 - -# Generate a report with "general" diagnostics of severity 1 and 2, in "tasks" format (simple text file): - -plog-converter -a GA:1,2 -t tasklist -o project.tasks pvs-studio.log - -# Generate an HTML report: - -plog-converter -a GA:1,2 -t fullhtml -o ./pvs-studio-html-report pvs-studio.log - -# Open ./pvs-studio-html-report/index.html in your browser. diff --git a/tests/integration/test_executable_user_defined_function/functions/test_function_config.xml b/tests/integration/test_executable_user_defined_function/functions/test_function_config.xml index d8f81a588a2..b2b7db83fbc 100644 --- a/tests/integration/test_executable_user_defined_function/functions/test_function_config.xml +++ b/tests/integration/test_executable_user_defined_function/functions/test_function_config.xml @@ -193,4 +193,100 @@ 0 + + executable + test_function_sum_json_unnamed_args_python + UInt64 + result_name + + UInt64 + + + UInt64 + + JSONEachRow + input_sum_json_unnamed_args.py + + + + executable_pool + test_function_sum_json_unnamed_args_pool_python + UInt64 + result_name + + UInt64 + + + UInt64 + + JSONEachRow + input_sum_json_unnamed_args.py + + + + executable + test_function_sum_json_partially_named_args_python + UInt64 + result_name + + UInt64 + argument_1 + + + UInt64 + + JSONEachRow + input_sum_json_partially_named_args.py + + + + executable_pool + test_function_sum_json_partially_named_args_pool_python + UInt64 + result_name + + UInt64 + argument_1 + + + UInt64 + + JSONEachRow + input_sum_json_partially_named_args.py + + + + executable + test_function_sum_json_named_args_python + UInt64 + result_name + + UInt64 + argument_1 + + + UInt64 + argument_2 + + JSONEachRow + input_sum_json_named_args.py + + + + executable_pool + test_function_sum_json_named_args_pool_python + UInt64 + result_name + + UInt64 + argument_1 + + + UInt64 + argument_2 + + JSONEachRow + input_sum_json_named_args.py + + diff --git a/tests/integration/test_executable_user_defined_function/test.py b/tests/integration/test_executable_user_defined_function/test.py index 94afdf8d8a9..e6542d79e4b 100644 --- a/tests/integration/test_executable_user_defined_function/test.py +++ b/tests/integration/test_executable_user_defined_function/test.py @@ -104,3 +104,30 @@ def test_executable_function_non_direct_bash(started_cluster): assert node.query("SELECT test_function_non_direct_pool_bash(toUInt64(1))") == 'Key 1\n' assert node.query("SELECT test_function_non_direct_pool_bash(1)") == 'Key 1\n' + +def test_executable_function_sum_json_python(started_cluster): + skip_test_msan(node) + + node.query("CREATE TABLE test_table (lhs UInt64, rhs UInt64) ENGINE=TinyLog;") + node.query("INSERT INTO test_table VALUES (0, 0), (1, 1), (2, 2);") + + assert node.query("SELECT test_function_sum_json_unnamed_args_python(1, 2);") == '3\n' + assert node.query("SELECT test_function_sum_json_unnamed_args_python(lhs, rhs) FROM test_table;") == '0\n2\n4\n' + + assert node.query("SELECT test_function_sum_json_partially_named_args_python(1, 2);") == '3\n' + assert node.query("SELECT test_function_sum_json_partially_named_args_python(lhs, rhs) FROM test_table;") == '0\n2\n4\n' + + assert node.query("SELECT test_function_sum_json_named_args_python(1, 2);") == '3\n' + assert node.query("SELECT test_function_sum_json_named_args_python(lhs, rhs) FROM test_table;") == '0\n2\n4\n' + + assert node.query("SELECT test_function_sum_json_unnamed_args_pool_python(1, 2);") == '3\n' + assert node.query("SELECT test_function_sum_json_unnamed_args_pool_python(lhs, rhs) FROM test_table;") == '0\n2\n4\n' + + assert node.query("SELECT test_function_sum_json_partially_named_args_python(1, 2);") == '3\n' + assert node.query("SELECT test_function_sum_json_partially_named_args_python(lhs, rhs) FROM test_table;") == '0\n2\n4\n' + + assert node.query("SELECT test_function_sum_json_named_args_pool_python(1, 2);") == '3\n' + assert node.query("SELECT test_function_sum_json_named_args_pool_python(lhs, rhs) FROM test_table;") == '0\n2\n4\n' + + node.query("DROP TABLE test_table;") + diff --git a/tests/integration/test_executable_user_defined_function/user_scripts/input_sum_json_named_args.py b/tests/integration/test_executable_user_defined_function/user_scripts/input_sum_json_named_args.py new file mode 100755 index 00000000000..6154abf665a --- /dev/null +++ b/tests/integration/test_executable_user_defined_function/user_scripts/input_sum_json_named_args.py @@ -0,0 +1,13 @@ +#!/usr/bin/python3 + +import sys +import json + +if __name__ == '__main__': + for line in sys.stdin: + value = json.loads(line) + first_arg = int(value['argument_1']) + second_arg = int(value['argument_2']) + result = {'result_name': first_arg + second_arg} + print(json.dumps(result), end='\n') + sys.stdout.flush() diff --git a/tests/integration/test_executable_user_defined_function/user_scripts/input_sum_json_partially_named_args.py b/tests/integration/test_executable_user_defined_function/user_scripts/input_sum_json_partially_named_args.py new file mode 100755 index 00000000000..1408418eb4a --- /dev/null +++ b/tests/integration/test_executable_user_defined_function/user_scripts/input_sum_json_partially_named_args.py @@ -0,0 +1,13 @@ +#!/usr/bin/python3 + +import sys +import json + +if __name__ == '__main__': + for line in sys.stdin: + value = json.loads(line) + first_arg = int(value['argument_1']) + second_arg = int(value['c2']) + result = {'result_name': first_arg + second_arg} + print(json.dumps(result), end='\n') + sys.stdout.flush() diff --git a/tests/integration/test_executable_user_defined_function/user_scripts/input_sum_json_unnamed_args.py b/tests/integration/test_executable_user_defined_function/user_scripts/input_sum_json_unnamed_args.py new file mode 100755 index 00000000000..7e0c68510c8 --- /dev/null +++ b/tests/integration/test_executable_user_defined_function/user_scripts/input_sum_json_unnamed_args.py @@ -0,0 +1,13 @@ +#!/usr/bin/python3 + +import sys +import json + +if __name__ == '__main__': + for line in sys.stdin: + value = json.loads(line) + first_arg = int(value['c1']) + second_arg = int(value['c2']) + result = {'result_name': first_arg + second_arg} + print(json.dumps(result), end='\n') + sys.stdout.flush() diff --git a/tests/integration/test_global_overcommit_tracker/__init__.py b/tests/integration/test_global_overcommit_tracker/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_global_overcommit_tracker/configs/global_overcommit_tracker.xml b/tests/integration/test_global_overcommit_tracker/configs/global_overcommit_tracker.xml new file mode 100644 index 00000000000..590759bd15d --- /dev/null +++ b/tests/integration/test_global_overcommit_tracker/configs/global_overcommit_tracker.xml @@ -0,0 +1,4 @@ + + 50000000 + 500 + \ No newline at end of file diff --git a/tests/integration/test_global_overcommit_tracker/test.py b/tests/integration/test_global_overcommit_tracker/test.py new file mode 100644 index 00000000000..4bcd9961330 --- /dev/null +++ b/tests/integration/test_global_overcommit_tracker/test.py @@ -0,0 +1,46 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +node = cluster.add_instance('node', main_configs=['configs/global_overcommit_tracker.xml']) + +@pytest.fixture(scope='module', autouse=True) +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + +TEST_QUERY_A = 'SELECT number FROM numbers(1000) GROUP BY number SETTINGS max_guaranteed_memory_usage_for_user=1' +TEST_QUERY_B = 'SELECT number FROM numbers(1000) GROUP BY number SETTINGS max_guaranteed_memory_usage_for_user=2' + +def test_overcommited_is_killed(): + node.query("CREATE USER A") + node.query("GRANT ALL ON *.* TO A") + node.query("CREATE USER B") + node.query("GRANT ALL ON *.* TO B") + + responses_A = list() + responses_B = list() + for _ in range(100): + responses_A.append(node.get_query_request(TEST_QUERY_A, user="A")) + responses_B.append(node.get_query_request(TEST_QUERY_B, user="B")) + + overcommited_killed = False + for response in responses_A: + _, err = response.get_answer_and_error() + if "MEMORY_LIMIT_EXCEEDED" in err: + overcommited_killed = True + finished = False + for response in responses_B: + _, err = response.get_answer_and_error() + if err == "": + finished = True + + assert overcommited_killed and finished, "no overcommited task was killed or all tasks are killed" + + node.query("DROP USER IF EXISTS A") + node.query("DROP USER IF EXISTS B") diff --git a/tests/integration/test_part_moves_between_shards/test.py b/tests/integration/test_part_moves_between_shards/test.py index ed7640e5f9e..6009a9d2a44 100644 --- a/tests/integration/test_part_moves_between_shards/test.py +++ b/tests/integration/test_part_moves_between_shards/test.py @@ -133,6 +133,17 @@ def test_deduplication_while_move(started_cluster): settings={"allow_experimental_query_deduplication": 1}) ) == TSV(expected) + # https://github.com/ClickHouse/ClickHouse/issues/34089 + assert TSV( + n.query("SELECT count() FROM test_deduplication_d", + settings={"allow_experimental_query_deduplication": 1}) + ) == TSV("2") + + assert TSV( + n.query("SELECT count() FROM test_deduplication_d", + settings={"allow_experimental_query_deduplication": 1, "allow_experimental_projection_optimization": 1}) + ) == TSV("2") + deduplication_invariant = ConcurrentInvariant(deduplication_invariant_test) deduplication_invariant.start() diff --git a/tests/integration/test_ssl_cert_authentication/__init__.py b/tests/integration/test_ssl_cert_authentication/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_ssl_cert_authentication/certs/ca-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/ca-cert.pem new file mode 100644 index 00000000000..293e1c7f564 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/ca-cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFhTCCA22gAwIBAgIUVRNcr0jCH3vSTxg8QYQH6CCtyF4wDQYJKoZIhvcNAQEL +BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjIwMjE4 +MDk0MzA2WhcNMzIwMjE2MDk0MzA2WjBSMQswCQYDVQQGEwJSVTETMBEGA1UECAwK +U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMQsw +CQYDVQQDDAJjYTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALwojNvu +fXQYQ4tucqNOEDHf2sNgxwxqY6QdtJ+zNfVjsK4I3Vqo8TtzxfDYGolkYem/bYJM +xQar9ehUm9ok/0kJgIo8vDXxxDJtvjz5Fd5oFWJLMxojLE9NSa0A4m18jGfbFNsF +XoU0njiInyzNaU9d4bMpaweseCZdt9Y4LR93FkuhSU/v18lWQPob8SSIij059IZP +sEUxpDOTxclAmG/Knd/6v3ecVFiQgexZM0gCtf7kcw41mxsAaP/mOexodIZDR70Y +LYjL7R2ZGhpClfQc8SO5NSpfEqsfreDX7XoaCTsy7/rqr3Nfiby6sc//awG0Ww/f +FRf2+2BU2xEwOVa3i5wU5raYY6eqFLK9q9c2IWPSqYzAmvhK2pqWQ/iaCU/Q89ow +SbKudJTLK8Y6v9LW4Q8ZLZF+CzS5cI+QEfIYqTLFdInH1BLoxx7cymEv07CDkcTo +2WtV8GdMph2P3U/9NoXQDonjCSj0lQUjgUdcrBPaIIVbIn6/5vfw8LQa8PoGDhIx +AYQkqPR+LHxCqIMzdqKZ+OXD/HPhiigpxLhF7mVRLvvoyrOZVJbcu1qmgCcQw0IE +fWzvWne+9cYC9lgt8+/k6d6B1uhYsIwwhgoj0dffFjc0sF6zfceGK+H1K2JCE0aY +zT1HlvSoZdA7lEs5xbGJnkBHqlOvQ63ynXCzAgMBAAGjUzBRMB0GA1UdDgQWBBTn +AtgFU20JF7kTZCKlY7/hi0kYRzAfBgNVHSMEGDAWgBTnAtgFU20JF7kTZCKlY7/h +i0kYRzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQCpiWgJ1XUw +a8Bdeznsa57oy+5mqQZWpRVkzTQRHEGV850OGh7WQ6u9kVAHefaHH9hsVxyggton +6/MDsu4KL5jqKmJaIAepPIOw6DTc2zs044I7W/rxRp+w1hL2TS+EahMrSPwdzCcl +NNAM0dXocGylf6qwwMqiYAR1K3UIrlyq4QTr1oEPIqJBkDg1JDYrt4T2DroPjW20 +5hlCQ/tft5ddGL0EFEaKWwAcPFm7jAwJiz2eUqmT6PcmaZ24qPn5RXVkaBAkrSga +1WgM8r3LGu2EKhdiDc5hRJKjS8RZyLvZNNzlL3+N42nGmGZkND5bV6u82OD+qn17 +LRZOt0Cr70HqszSYk/67ijjaa4n/fuuAqorV+yYB8accRXtoi00nxykT+H+yI1rD +swvcrfDvhUgY5zmunWyQUYh0q/2Hj75GbLup3Cd0B4MrBwqyCqcEugM4OSf6aRMr +e/vjeggTVPN08xE1LUkugalx0B0aoO6qFahJ2CmkAcYLLlS2N+F7TMuPavc0kVxD +I3qA5G9zvNCliSLX2+kM+LzslI8+pP/A98bvh6nW4HtZkI0jq1ks7XR0GeOhCI8E +0l/YuElxxgKhN4INKhhMoDKqPib4z8gbmkenR2CenQCpfLMIrhTXZgtw+gvEgpIE +/QK97G8XPqga6zn471wrYJnuyJli+sP7aw== +-----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/ca-cert.srl b/tests/integration/test_ssl_cert_authentication/certs/ca-cert.srl new file mode 100644 index 00000000000..c02cd0a4526 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/ca-cert.srl @@ -0,0 +1 @@ +05F10C67567FE30795D77AF2540F6AC8D4CF2461 diff --git a/tests/integration/test_ssl_cert_authentication/certs/ca-key.pem b/tests/integration/test_ssl_cert_authentication/certs/ca-key.pem new file mode 100644 index 00000000000..e85dca8553e --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/ca-key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJRAIBADANBgkqhkiG9w0BAQEFAASCCS4wggkqAgEAAoICAQC8KIzb7n10GEOL +bnKjThAx39rDYMcMamOkHbSfszX1Y7CuCN1aqPE7c8Xw2BqJZGHpv22CTMUGq/Xo +VJvaJP9JCYCKPLw18cQybb48+RXeaBViSzMaIyxPTUmtAOJtfIxn2xTbBV6FNJ44 +iJ8szWlPXeGzKWsHrHgmXbfWOC0fdxZLoUlP79fJVkD6G/EkiIo9OfSGT7BFMaQz +k8XJQJhvyp3f+r93nFRYkIHsWTNIArX+5HMONZsbAGj/5jnsaHSGQ0e9GC2Iy+0d +mRoaQpX0HPEjuTUqXxKrH63g1+16Ggk7Mu/66q9zX4m8urHP/2sBtFsP3xUX9vtg +VNsRMDlWt4ucFOa2mGOnqhSyvavXNiFj0qmMwJr4StqalkP4mglP0PPaMEmyrnSU +yyvGOr/S1uEPGS2Rfgs0uXCPkBHyGKkyxXSJx9QS6Mce3MphL9Owg5HE6NlrVfBn +TKYdj91P/TaF0A6J4wko9JUFI4FHXKwT2iCFWyJ+v+b38PC0GvD6Bg4SMQGEJKj0 +fix8QqiDM3aimfjlw/xz4YooKcS4Re5lUS776MqzmVSW3LtapoAnEMNCBH1s71p3 +vvXGAvZYLfPv5OnegdboWLCMMIYKI9HX3xY3NLBes33Hhivh9StiQhNGmM09R5b0 +qGXQO5RLOcWxiZ5AR6pTr0Ot8p1wswIDAQABAoICAQCO/c4Wccb7TFlAhD4wpumd +zX5GDq0WXV+94CldWGdARnOFvwzhkhRJ1zDtWH3KPfQ/HJBPfqIY8OQfnPUYMhej +3MnHxGJQKJyuqkHxumYJMFZX7cg3K9XHqne8NzjcddOKNa9Cx3DOkG9RjVpSRQSs +IS+d5XMGUOa6WWyVKvn3uJvD/B1n12DJDHiy2jtHRVCxOPMAg1z1KMWdwMaFrEZs +ZrHV/ow1jSN4btGd2SgkqJLA08IwYUKvoX8qQj9wzu0G/+hr5wzrsfZQEQMKQ+IL +s1b6jAzAV6IrVBbjEZXSviiXyZ0gteuCJW/acpMg+/3JPNQbWrCAFt1wluwowto/ +JAFIvlh29hfE5c+HEMpQNa0tdj7jepBn/0YEbgwpayMikKiLZXEpgheWCGypAQWp +Hm+N0Ym7HSGe82obxi8EjKRnNwFUtotWzUBKeo9aFwPZHLFlspljd+5ynDvKqXnk +txYZj6K3TtMs30HAG6fqxSPyiZ5W+5yF7nt6qLODs6m4Os+lrk1GnoqC0/uLMzIU +CRJKulrJOK4/Z2tPn9IAhcREbS4oROUeNqqo0Cfs3ssvkV7JTHF4IsKhCmElMmGa +bevOI+pvdjfECShy0Jnbtni6ece/II4/edfUp9kWN45xZLpzDjfqCVD66JS9g6ZU +i/EVll+d5zaI2TzzwZgHUQKCAQEA3d8siwXbq7x0cAB013+tvkvGMJ2EuS1TWdLk +a2P6CAnlZMWvv2cPSd2WpimHjqKxrbn6VE79mOc2l9Y1NOUUWWZATrhN7V8xMapQ +0YiYCHeaMERUAUKdzCgRN2/mRbZCBzpPBbWbb6NtKfRFJsD9zAe2JBwDVh9hvAL8 +YVBoczrEfj1ILnmtPhAJVI6s6rDsA4MgKjLs0Tt7Cc7rQxqNSpHEvwv1yLQmjp0N +L5b1TEt7fqVJ9dirykJquBYEKf55Z1qZhQzmnbu9OPnzeqGDakl5F/UsXDB5Bokp +ilcV+nFbh175Q+gTEhaSacGW8gzRw6j18PuciBjeWVEM5hhxOwKCAQEA2RnRMjv9 +46jQarJTFbIHg1SqrR87GSLnt6672M5TX9frzxMCuVDjKgdecstvLjm6X+/cPQKT +Q3javJnJXT4cx//1J7RLO6ZBVSCZf3//XntdHdFVJf5ySQtK+MJyfxjpzP6KBPfb +WPrva8p29ejbBdtsOT0M6gY5tPfadU2XEaf+BoyX9NUmu1U46Iqi+eCOjR+GVvhP +pJzGgLeOsaRVCfc9I7XPoVu3AEx5Kt55yRYm4fyGPsAd+mRDbIXMXdL0k8CfWWDr +8TT5rqKI+gFPFQCwToBW3DwHIGY+3RmoXFfQ0IJaKwOk4AB7m6HC3mv1crtjTFSM +9p74oQzNX7UG6QKCAQBEs2cygRTdH5SaXbnQRKvC4emzggLn5/4IMUIjcqioNpA+ +XOwngzz7rU6JkxBzfTMxTQYTdwYVg3qnF2AQSeK8L+o3teADYVd1PnyZ9QbGkGpB +CddNMJh17+4s0UxnR6E4Zbi0VuCTd/JEbGvBLT8pHzYqBjaOQ1dbBT2q0GAXVhoj +0Mv6ABlBv2t0MF2gqjnaeI7MIkqsGxPlHJpChAU+EtbuJUDs7cOGo2DC3KaGAlVy +CLJXGslO7rPm3oJZkn97HlWtGiqKquhTrSnUThDIJ4oEfhlHTocbG/ut53tZuiIS +T7k1arYFAtJBRv17Y7bMNBQ7k12L0s9+rpck5GqjAoIBAQCVBPSkj6tZbpII+viu +5rHjguVYyhwtx9jYK1eDnTR7kGGrlPgErjIPslkxYNSjHTsCCUnakv70jGtQlBs1 +JqJo4hesNkSB4D/uJ99VNk3a08D566uP1dUqsFa44/flp/ssG/gvKtbkf/KBwcrg +RwK4RYJG09IefUF1J8BLToQIuZBTfIP9qaXZZskWTbtK28ndsqrq3a0FaBuVVOnc +o9k/avcLoQuxTZwS12tAcs+TqOHtswGO5x5stg/V2Q2LxXbeSJTYq/+oZN2R8r0l +JmrbFsruR4fXylh189jouWjoYdrSlPdBmVG99HbkQCfbtq0XIOsrBMpxqnMtUPVT +4ZWpAoIBAQCrao4XHpRM3KsfPqdkg0vqFDBA+LzKlWu1fl8w5TGpFK8H1tv5kHNv +h0XmeU5cXwiweri3KjQz7h+qVBHZeAvyrEoxJQdImax+5OUy7lysDs+SL02gLZb3 +Z7g+u4Buwx+cv4O7hwUEDDk/5X3NBFk7/iwztKUtM+Fl8jt9K3K24s87xXp9YevI +UEawden9jVcuXQjEUrwz8cjoz/y25vK5pQ6k82PVImkMZK99/PmgmGyOl7hdRA3F +ff0Kb8pRGmV/cWRKzHaC8QchW8jdU2EGxMkrFl1DvoVKLbyDf1glRncKP9iozHAR ++s184IJCUvyMxH83uKKAiBGaDRC+Lbm7 +-----END PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client1-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/client1-cert.pem new file mode 100644 index 00000000000..bd6eea62094 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client1-cert.pem @@ -0,0 +1,30 @@ +-----BEGIN CERTIFICATE----- +MIIFMDCCAxgCFAXxDGdWf+MHldd68lQPasjUzyRfMA0GCSqGSIb3DQEBCwUAMFIx +CzAJBgNVBAYTAlJVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl +cm5ldCBXaWRnaXRzIFB0eSBMdGQxCzAJBgNVBAMMAmNhMB4XDTIyMDIxODA5NDMw +OVoXDTMyMDIxNjA5NDMwOVowVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUt +U3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UE +AwwHY2xpZW50MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMBU0fao +RrITeF4kpN81p7qirX/Gc56+Cux6u7RF1O6WU9v+V5jLw8chQZ87z4QSrFiT1ZnT +pwWYPwJ+pDk6AWEoiKuOaceOh0bjZCuxADHs+qQrye5D8GXvyFvWE2cT1pD5JNEZ +DSl2YHqNs4uTGRP9BP817iRDcuvdxpanaWxfXGfehJRMiEVgKDs+RUpoW4aVNivI +InrUWc4RXXkzaJKqhpCU3jAJBV4jSD5ZnA8PUfcoAj6z6T3I6phuDfRP5ldA3br8 +yg0hCB7Y5QrO5lRAgEoIuNnC+U6/AIwWPI36Rjiwg3EUwI/BIiL4AWjzkjSdr0mn +zyHPRk4pcn01T0GTpQi6tfZZpumDD3LkPuEy9svMpJ8ntqDnAsIJVjbg1S60hHes +yYHoQw1HxU0vrncxwcQkVaPLx0uGlioaLlvu83AVnWXbylZXsV/pLy6dE3H51GBF +DX3Zj6nkuJitk8/hNp440/Lve7SaKFPo5NdH+8ACWGdFdz3zxgPuhBDoxEeqj4c1 +FQA1ABXx2akW3lQ5VxTAg5AYORvVhJTozosr+Kn3MlRdZjl94tnVByD8MGLLE0C4 +L/qXR/IlbkOCz5LHapdC5j62ZEBwiElmMO/tMGl4ORV9tdTBrRZ9DMmKek2E8Qwz +y770PGkhp1cTzZt6UfZEympowmfjtiZfHIq1AgMBAAEwDQYJKoZIhvcNAQELBQAD +ggIBAHwRpqnpcD3EW588GSDZhZrVf3nS9M06ljQGtDUqNSI4XJp1cVT1sMaa4LjM +cWgHtayFw+jbDLwioXHjMlV+8tERH+0x+qsADG349caDYT/OF13v/jyuboUZ9AqE +KpfOQH7jCLU7rEbEl6kvT3F3xaHJg8mE7msyRFfguB2JrqZkKIj4HANxJUJo4PwB +5bq9nE3AVNAgUeQEwfu0r5SjroNpcHfm7xWqMK2mDMCsy/DvI7n97Q7vZajcTT0x +UXfgx+3CLEvLMpa2myE5OIMOeLzfZwxrxyNH7BdZsROnkGv1cX+9HZpYcue/UDxp +P2OApbTuZKaTJOyMADc17s0seE0DTAHnHAWrJwVhf8wYKKtEs+i+Sw5LNSkh5fgS +hTzGF93yClDYzWEqMSKhKPeimtpz4ZBNuGf471KbpVbUKJJvJmOxqoZ5S0kpFILL +YMALf652uf5or5d0cDNvcJTwvMi6evchIV17d/jH+MxyJQs9VCkMpJxFbMrXb3YB +b57K3Z25P6w3Qfj4zuKQFANari7Gs6qSiaUBiEhEdTQlGspkq+FLndtX818sbMk5 +LAK6JaUH0ywV2jn5XSW0irQLDXqb6Q0bSyw6pdpDjk0o4UW67JCE4kGagRDnfSqL +ZODvO/dEtVLyAsjmOx8MkqLyseI7VESVd8eiJAyL0sifh+/E +-----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client1-key.pem b/tests/integration/test_ssl_cert_authentication/certs/client1-key.pem new file mode 100644 index 00000000000..8bc1e656566 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client1-key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDAVNH2qEayE3he +JKTfNae6oq1/xnOevgrseru0RdTullPb/leYy8PHIUGfO8+EEqxYk9WZ06cFmD8C +fqQ5OgFhKIirjmnHjodG42QrsQAx7PqkK8nuQ/Bl78hb1hNnE9aQ+STRGQ0pdmB6 +jbOLkxkT/QT/Ne4kQ3Lr3caWp2lsX1xn3oSUTIhFYCg7PkVKaFuGlTYryCJ61FnO +EV15M2iSqoaQlN4wCQVeI0g+WZwPD1H3KAI+s+k9yOqYbg30T+ZXQN26/MoNIQge +2OUKzuZUQIBKCLjZwvlOvwCMFjyN+kY4sINxFMCPwSIi+AFo85I0na9Jp88hz0ZO +KXJ9NU9Bk6UIurX2Wabpgw9y5D7hMvbLzKSfJ7ag5wLCCVY24NUutIR3rMmB6EMN +R8VNL653McHEJFWjy8dLhpYqGi5b7vNwFZ1l28pWV7Ff6S8unRNx+dRgRQ192Y+p +5LiYrZPP4TaeONPy73u0mihT6OTXR/vAAlhnRXc988YD7oQQ6MRHqo+HNRUANQAV +8dmpFt5UOVcUwIOQGDkb1YSU6M6LK/ip9zJUXWY5feLZ1Qcg/DBiyxNAuC/6l0fy +JW5Dgs+Sx2qXQuY+tmRAcIhJZjDv7TBpeDkVfbXUwa0WfQzJinpNhPEMM8u+9Dxp +IadXE82belH2RMpqaMJn47YmXxyKtQIDAQABAoICAAEBsKOg19XgwjWD7ZT5e+o/ +JbdQe5RuHDKGperYnres871oBF9ZWan2I5jIwFpJmrtP8sM+V1ZxKItDzGo8QnuW +sbhsI2OW/GBDmmecIosgWWN4kzL7CgwOiDbq1OkqMmpJ04aAohAAfZrGmRT27R+s +qFUJnDh2XeicHYj2UVfu29XzVTBNgj0StsMwnT45c5ktuL3b60pHSD0K3DlhKn/y +AohJLyyDL5MBjkQ9RdLSWrR3ciOP332iSpAHq20G6ga04TQ0VH5jGN7IddJrqMry +F3nLt+Pz4EgoOcGB8Ekx8SIk0ltKJ4PZF+uk7qT0+WPrG1rAVRYxNoX8M4wyNjr4 +TcAZsV2DnGdnp+2u0SSzMczeop5hPTJKxaLaPw1JOoIk5fqW94MbEHqGnEXEIN+D +OWeUKWZ/B1YubavOeR+c3STZrh2SgmhKk6g5NMFlfnyvolPu47H8NOrewOhVG+TZ +gsQoGxSyOXwZTQ/Jd6Yg9lek8nKJBc4Res7ia/x3H+gjjRoNFI+L2HQnWztx5YMZ +H9M6hcpclZubO/w4iLq9OB2QUHn7aIT3lWRV/xS0Yh2zGCufasaMA1KSKC5zq0Fk +gCzAkYDq/ymrJs3LQQ0wegKd1akL4z5fxmXTn2v2BGoEd52uuxhL0mM/9zzRxdR2 +IsOgAym+siLXMCHTDbdVAoIBAQDuMcea66WKidS+A9frCEsabYccKzrdMEhs6Mle +orFieMC+3ZpzFIBkXPZ522I+M4nIdBKuRw9PnYTE5t30euOj60Oq905j2a+Ho4ki +kW6dC+tNDF49Hqxn9e99xbvTUi97dREcERlHA+AnRektEciyD17bi88aUy9w83Mw +G5Z+ej+9o40w8+TDopE2SIJhUAHR6LOAMq1v5y1lmTn0sbTuxZFLA0qWX9aGLi+T +4RD0MzJAtKJDbr3yPTLHAXmaMSKHhWYYgWTH9iwEhGQAm5VJy3oNJUkM7ej7Yfs7 +aTDOk61egCKhEHdWavP68MqmNOPHgnq4/edmvQnhfKtI8SMnAoIBAQDOtWDi/OnU +ZjZPnmJwwoPuXe6IjYg47bFRGv94xEpSesCAYdXNaNLPl0f/Ut9y3nXr+j+XqJWo +UqtRGFu2i9lUK3cu90GLXEaLbYWGcgL8YnJu0senLxkqxPWcGxoKmbo3xMjqk/pF +EVZ5e1qqVTlrB4q7QWmLKrS8YlcaTnChPeSBRFfryg/xvQ11Hxtq89SKkTH4ps16 +0KtiCxvfQHVASyRLIKLdyabPInB+yP3Fsn4BIx8jGtOQ/OCY01TXq9OyaRu2hJTk +qsjOLnqf6huM2so3X0Tw8AdgNoF96JJvfhwiPI5CSo9UKjhuvus1Ip5ZFFNo4Ngy +n3Zlgp1HxZzDAoIBAQC9ffqmo3sxqI8Hj3UxdIqS/rlyzm1o0+V6RwMT92gYx6nG +7fLWRGQT8+TdcotIoqWlQ7oszTlABDdAkc3XlgANQre1hkLlqqM6y/3n8zzFUVsj +E4jRJNrRZdTeAPV4mzRNCgfPhUbPuSSU+cgT48b+6L10+VeMQMtIF1T226uw+L5G +tps3a3/9pxHQ1oRquESKYo6SmT5i/M2fuvNhWBJxtdjtjTPER4AZhRqykWV0cFo1 +Ib7I2Ivh74+6w9Ciux4WJCjhq+aqMYw5F72awitU5rw1QwlHcOldO0irrfZ3EQLm +YBesfLYDmNh6NR9ydDcVXBcXnl593DvFF/IH+FYXAoIBAQCQZydLCzHy3oC8eEH+ +0fRGljooDO+IDYzcwwaLgF0HZ5eJWE97EuqKeP2kAWn2HjC07Hp2YSBDmZTyrxiK +2wG1CjRVjAeu6oShrJ4mAQnS9JdKkldFlOJ4/WUza79yflgX05IkRcIFdAo8DY+W +BLl66qbhD95CiU//dpew2fFWwx0ZrPvazar7zn1TP6rwuWvWbX5CXYyYaqP/dxE+ +khIXGyc8kI0WcWPlugJqn9CgxoO+GaIL7Ra1Z+MjACd6DyBxt3nTtKUrZZ+oYdHq +Wypp6QJxUk2gH56XeRxXMBz0ZF4VEMa0ys98FY6c1yULVqbWRhvK3aBLJRkZ6vgj +BorvAoIBAASy89mnP7d9jY7pSg/8znsUF8fQwKpRJZKS+8xgbzsZP+zT7CjxCbPL +xcNK0fl6pRBv+gyIM013R7J1uvZJ3W6rspVxlXOvofvwYSuLOjwsZA26RM8s7Do5 +e62Bg7PUHbbaD+C8HzbJlyXeQ++oddWPbIkxJMwhP1Uvy3wA6c7E7w/UACZvv20J +KriU33QmW/o0YpOX8xBVwgsCld+IfUIYm1S1mpU6k3oUfGIA5iyKx1XLTMhlaYUG +dTdExwxQp73Jk585qWSpaiQ05OrgYyzZ8OHA2kRTPK+54HSwRfn6senf3TakZHBi +zjy/DZmOU/a/EiR7MCGg+jS1x9GBxOE= +-----END PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client1-req.pem b/tests/integration/test_ssl_cert_authentication/certs/client1-req.pem new file mode 100644 index 00000000000..b821609068b --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client1-req.pem @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIEnDCCAoQCAQAwVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUx +ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UEAwwHY2xp +ZW50MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMBU0faoRrITeF4k +pN81p7qirX/Gc56+Cux6u7RF1O6WU9v+V5jLw8chQZ87z4QSrFiT1ZnTpwWYPwJ+ +pDk6AWEoiKuOaceOh0bjZCuxADHs+qQrye5D8GXvyFvWE2cT1pD5JNEZDSl2YHqN +s4uTGRP9BP817iRDcuvdxpanaWxfXGfehJRMiEVgKDs+RUpoW4aVNivIInrUWc4R +XXkzaJKqhpCU3jAJBV4jSD5ZnA8PUfcoAj6z6T3I6phuDfRP5ldA3br8yg0hCB7Y +5QrO5lRAgEoIuNnC+U6/AIwWPI36Rjiwg3EUwI/BIiL4AWjzkjSdr0mnzyHPRk4p +cn01T0GTpQi6tfZZpumDD3LkPuEy9svMpJ8ntqDnAsIJVjbg1S60hHesyYHoQw1H +xU0vrncxwcQkVaPLx0uGlioaLlvu83AVnWXbylZXsV/pLy6dE3H51GBFDX3Zj6nk +uJitk8/hNp440/Lve7SaKFPo5NdH+8ACWGdFdz3zxgPuhBDoxEeqj4c1FQA1ABXx +2akW3lQ5VxTAg5AYORvVhJTozosr+Kn3MlRdZjl94tnVByD8MGLLE0C4L/qXR/Il +bkOCz5LHapdC5j62ZEBwiElmMO/tMGl4ORV9tdTBrRZ9DMmKek2E8Qwzy770PGkh +p1cTzZt6UfZEympowmfjtiZfHIq1AgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA +fGx/D6rNeaVO/vSUGX5q1iJKd8Gnw+/8NRgbuvCDuDOSy8LyqnLmVntj8q9FHpJM +SRH3LnylMVFZdybso2ZbhR1UDReGvHCtKICG3LLP1uWwy5nS3mkGBHFm9COyFP21 +kWOit1+106gEhg2f/NXh31HFmh+myepLjPEj5KxvnQhQfaQESsDYDZAs6/qT1mqp +A7GixOXh7hIFBJ97cU7fKby0Wtv7GqKAYQkaf26ImoGijtMPIlzvwJboJWmOYzIH +zrOHqspFkJD8YvYOwLIKdahViqXU7POL9uRn0vFyaXVcyXRq83Pz+bPSW9AFYsYG +ukSZiJs1yCINZI/Mk1vlfaZWYPIbBkJZ0Ny0vw112dIEilWAkVdsmJyV95aBddQI +Md64CYWZbV5P7/0QOX+v2ZQpWVnaV0m07K6VVuTL3bw6BQ9fcj7vaql6wl8jl/9l +nEotaZiY1f1pUUko3XzXpZEFB1lGBHupuS/Plz8pfFefN/7sOZoWn1VhD9I1A8uh +b2mg6hyQ7pe2NrHOTY1+L1xxxKKHt01kvDhws09qxRXtNsLrL8tl94i1ndLjHIwD +/VRnVU04E/VoTKaEXuETLZwOZu8pLwdiejrWEAmtsbmmcKq/Bk42wa+Wrmge2Chs +V8EOAtq91AjUcQeh7s2fV6yWweMGm1J6pdkNWckCsUs= +-----END CERTIFICATE REQUEST----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client2-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/client2-cert.pem new file mode 100644 index 00000000000..886cc533fcc --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client2-cert.pem @@ -0,0 +1,30 @@ +-----BEGIN CERTIFICATE----- +MIIFMDCCAxgCFAXxDGdWf+MHldd68lQPasjUzyRgMA0GCSqGSIb3DQEBCwUAMFIx +CzAJBgNVBAYTAlJVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl +cm5ldCBXaWRnaXRzIFB0eSBMdGQxCzAJBgNVBAMMAmNhMB4XDTIyMDIxODA5NDMw +OVoXDTMyMDIxNjA5NDMwOVowVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUt +U3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UE +AwwHY2xpZW50MjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAOGIanwq +rZCqMT+ePwRkiQnD0gyVt5+kwkb8X+fdBJRF0kr70YfzMpKdZP4l4W6C0Jv/ysIH +usrI5pQxcFAIe/7DLW0JPkMLKgXsOtPNZPIkc7WYkq3cbzB0ZTsK8O3IYhwn0dAY +O49T//YqM3TLTFsG89B6uCEg7dQiP9hh6boic8M/WyAseOkJNfw+wYcTWhl1toKc +dLbo8ehESUtVhCOPVT602zBUYFkleqKPeHJ/gzl3/mTnqfeUBljGI2aXwOl7r6rI +D/or7wew2HZ81dTGDqB+yqUhBIVNseJPHOuKbke2E2qWVzAkRnX4b2ehsSaSknpC +KGWyLibaQyR0/Gt8Duu1XIsZKeFjCw27yogSTQ6xTUhLDF1anQyoJX9btSQZsTbD +3vtHbD1O07KSfiG0Z1p8LaR10RAFA7f3HLwwy6c9ExpGu5ED+co8aO5Xp5wysg8X +fYZYx4CaY3moQPJPDS6eOpUXd/6h27Fm34h9VdSj2p6j9JYsmTeEgb0x+JjAQyRS ++Koj/tbSbBqjbvO+FUaldRlHCHYCQTnjsSNBf7SxqE9lfgFitcgiHKSdD7QIfwNB +EK1o7L8OugC/SQtHGe3ngUGuNmHI9w6ItGuVqoJYP3Hwa6ClGmYlTRLoAj8NkBib +toxwGIspTlTzmmLXpqeZTPaA2K5eiq8O5DKvAgMBAAEwDQYJKoZIhvcNAQELBQAD +ggIBALp4L1aky2jfgk18tney56sUL2Us2aHqyOz9LlowWFdNMtCKo0WKpZ1qXGfQ +92QE+zc/MEdmv3V/H1MmSr7trTq1u7E5vVVI9Lq2lNbRLDQLi1+qd9E7Kdl6Oxw/ +Ecc8oxIbg86p83HhzPfJG64m3x6S6m2c4sNrHRAO/gxxJex6ZSFfQwYJZFlcvvBX +CH70RBtBG/ggasVtwqBuuIRNJ2gAtiWG2RtyGlOjPiAg7nUQiYlXLHVOjvrKDvrI +KTjzRdEUMqKtIrNUBHSbWZlxKZ2Ddavshg/0T0reAN/u5KTDxiGaQxlVEA7xfm+j +etqjzTz7LnKuRsA+Z8UUYaV6mKYfKObDoUs/12IomRCUTQi1K8MP3fGmmk+4Xiyu ++t15EqWJzhjuT2RjCAL47X6ksdOtonX9t29l6ykCvYpK1mlzG+EhqDyMIn62TNfx +OFjWwhIFgyEUWtwkihIKtv3ZVtrJVO/j+HCUfq+6IpjYHdlpdb4OaHgBtpokOtM8 +PmTHJbP2bxmNIMAU1WTfV+e/JkdTKHJclC5DTGF48yRgdKSOTq0G1eJYh4DhlEIM +vOw2rXeWR6VSkvA5vF7HANEptl1tkT3dsKR4BXkSIO16ldWBEHMM4UeXx85GGM0k +TRON4FWBMi6PXX6mrmPXcUW7AyKG2JL9gNlxRgWHVK7xmZyp +-----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client2-key.pem b/tests/integration/test_ssl_cert_authentication/certs/client2-key.pem new file mode 100644 index 00000000000..462916c0670 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client2-key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDhiGp8Kq2QqjE/ +nj8EZIkJw9IMlbefpMJG/F/n3QSURdJK+9GH8zKSnWT+JeFugtCb/8rCB7rKyOaU +MXBQCHv+wy1tCT5DCyoF7DrTzWTyJHO1mJKt3G8wdGU7CvDtyGIcJ9HQGDuPU//2 +KjN0y0xbBvPQerghIO3UIj/YYem6InPDP1sgLHjpCTX8PsGHE1oZdbaCnHS26PHo +RElLVYQjj1U+tNswVGBZJXqij3hyf4M5d/5k56n3lAZYxiNml8Dpe6+qyA/6K+8H +sNh2fNXUxg6gfsqlIQSFTbHiTxzrim5HthNqllcwJEZ1+G9nobEmkpJ6Qihlsi4m +2kMkdPxrfA7rtVyLGSnhYwsNu8qIEk0OsU1ISwxdWp0MqCV/W7UkGbE2w977R2w9 +TtOykn4htGdafC2kddEQBQO39xy8MMunPRMaRruRA/nKPGjuV6ecMrIPF32GWMeA +mmN5qEDyTw0unjqVF3f+oduxZt+IfVXUo9qeo/SWLJk3hIG9MfiYwEMkUviqI/7W +0mwao27zvhVGpXUZRwh2AkE547EjQX+0sahPZX4BYrXIIhyknQ+0CH8DQRCtaOy/ +DroAv0kLRxnt54FBrjZhyPcOiLRrlaqCWD9x8GugpRpmJU0S6AI/DZAYm7aMcBiL +KU5U85pi16anmUz2gNiuXoqvDuQyrwIDAQABAoICAHZuu3RuuOxB41DEGdWFsczV +7wS6zk1gKME8IGTS1GfEbpT/vd1FYaZKTtGDNOlieoehAGl5w6Zfb24ctBzjB7IV +7lHWy8JLJ4sqrQ2ySzM43yZac5QnMKBiTxJ9QV2sn5CnfG9pekVe2Af9yz2m0Hbw +pLIy72Q+NYXzYlGPwTwEgYPjTkgL8oZ1VssabWgwSl0aSng2DrhKhVXyHgcYZiaC +S0J9mKi9dkb5/ndFHfwKZ++Syp1UZhXjvp15lvd181DoqavmGTXHQmNog5NdJLDy +PJYdXu7t8sDJtwLfhpFOBXFU9MdBIZHfSr0CdAYYi710tMTM3wfgVIoEjcOkRzRx +36O66ehHfcyNsK52Z+DZ6uR4c+MOG0kzTiHQhyxjiu+3nYMGw1XdyE+k+eZDMPd3 +vTaR7kYOQvVvdOVAUuFZG9mK2p0mpofb9cFxFD0vJUqTYXxSdKUNIexR4mWQJw/h +rWOg/42GK4iLY2X6/CsDh6pTsM+HCzwmTGGkL54FvDsB2AhAhXPz/kGiBRTrh9/p +QBxacSPoqN+kF3u2qZRPEmjuimiW2AaXARbTABNSBQJIEmWzWOVdgUBVetGoN/ML +8mcYDmXhAc6F96eqPj0dX8cHfqYPguPhtzLj5V6XGym7hYQyOLBcE7tr2BcdjUfM +V6OFHsPNmsYWZ9F6zCv5AoIBAQD3M6gziCA0G0cG05ef0C3D9OVGWpHqr0yiR3MO +ZKsYbJJn4WOtWWvo8N5oqZBQ8VIoyGd1eiSIDuxXEWniFWjn57QN2nrDNTsEQPgk +HzomgFzuDZ7V4JsjJ9F2nAG5i2HoEwKNHdzfni6mhwGaapd+4GlET0jlC71p+h0X +CPsD6Jwabp6OUyT+xm8XW3mTWskBzKfq0OPbsdv8UB1dPt6jVrkjoe76TlTsWXWi +U9p9/h6kI984R9T10J61c21dokuL/KlHqb6TIQY3RcCgm2bfucmuawIq6vs1PBrK +VCvMX1BuTva9CYg/+hxm9Ky08jFWSCEEtzaORyN+4mmf4maFAoIBAQDpj1NoI7RP +mYqG9vHyXSDUUNbchpLOFKIaeh2DGk0sFmLi/obglsxOKu8K3r/EobNt+vpDTBxI +1EjPWdKuaXNYYjNjrVmPHdHPoHD8JmXzJDbZnXSylV9MVYSMNF+7BWUiPg3/QC7b +1a+ljJH/KEWFb0xrIfNPxVzyq8dyFOxcmLfRVLYlEW+fRYeaZ3QApxGi/BoYK8KN +vG8f/a8jpPwYCVa3JJ7/donEtsbxTkm66aacn8Vo2Y/tdo0nxyqC9PyBU+tV0u4w +aYtEZ28kpC9QheRx8D7WzhvsFc/KsshiB6jddjOVR6VgiUFCo+b/5PqpyZVTVrcs +tj8062A3KvyjAoIBAGRPn/eZS4gZcY8BmcuODKQx4j/UTNXw4KYRXE0A6LT2icqB +mZMkcDeMVpQeCqPt6SsHd4QiVmSnuZvzQwYtLe69BUGB4MMJ/LLTMl5mFZC+Efe/ +qy6bABkZ9VOuJr0GJGqqHCTrc0+CvudwbWQd0O/5XH4NtkTLqMcyaU+Jo2KIp5/K +N6kFcEO6fiX6RrFW665BP/p3XZ8u41fVorTN6EZb0LD26yTDWI64FpYSdN0fm4t7 +yv7ply9QwrZa6oxOaV2a345nASBvDDito2cI6IvstjyCy9RimiGWDEECOuup2deJ +T3KSRanAcnoM23Bpvz+F8XAacJb3ox2//qCUnIkCggEBAJHl2XllTF6pEFLs8giv +SjG26fFKE2yukPCvNb5O8MRIm68mxkSHjsqJoVeN/Act57MdI7ZkVgrcqTr15ljT +QJ2GgomSoS54tzbXB51Ls0XmamkYJezkyGobxbf7g42Fej6guwenJV5oJtfobs8Q +bhVDiF4oECDVrhFdYzKNhXT2ZWVbYIjZUnwQ5/t5Aorh0m+Ywgg1VcxKWLSIOR6w +ElZFhyjStIvqlXcPokjc2cvr5wtR9vRfa7wv4U9m59R0i0OSk6DCKc6OL9QkNNaT +xYasjR7rr6VpjSG2Il6BvhEWrdLh4qku30zlkKG7VzKk7Dyh0ykDM1u34NYC7tCn +hrcCggEBAO+Rnkk5eYYqGk/64+Qy5qA7djvvZ8AgihwJL3+ZUDSOxh0W+Er4NB6n +j0kI22N//D2j6hg93TNj9jI6lISfmY+TSikr/P+bQPGXl8wvekQxpjT5JhCYI93M +LXnSULuy7J1ujkMGdxEvfOTjvmD0ejtnuaGd+jM7hx4QNBbJj4VdV+r5BQOJAlfY +gk6n3RgAnu86szquWM6dObIz9BWtIcMVGlxA7yDmxjVDDHLwGpcwG+MTQRcHoeT6 +2+b7FtVN1NFLazfgPS3bxKs5jaUB+Ibm9BD8B7THviNikqRYqwoJMWpJgdWo/lOQ +X0ueOR40kfa077G7jNfb03qOPUR1mFw= +-----END PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client2-req.pem b/tests/integration/test_ssl_cert_authentication/certs/client2-req.pem new file mode 100644 index 00000000000..846f6db84dc --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client2-req.pem @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIEnDCCAoQCAQAwVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUx +ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UEAwwHY2xp +ZW50MjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAOGIanwqrZCqMT+e +PwRkiQnD0gyVt5+kwkb8X+fdBJRF0kr70YfzMpKdZP4l4W6C0Jv/ysIHusrI5pQx +cFAIe/7DLW0JPkMLKgXsOtPNZPIkc7WYkq3cbzB0ZTsK8O3IYhwn0dAYO49T//Yq +M3TLTFsG89B6uCEg7dQiP9hh6boic8M/WyAseOkJNfw+wYcTWhl1toKcdLbo8ehE +SUtVhCOPVT602zBUYFkleqKPeHJ/gzl3/mTnqfeUBljGI2aXwOl7r6rID/or7wew +2HZ81dTGDqB+yqUhBIVNseJPHOuKbke2E2qWVzAkRnX4b2ehsSaSknpCKGWyLiba +QyR0/Gt8Duu1XIsZKeFjCw27yogSTQ6xTUhLDF1anQyoJX9btSQZsTbD3vtHbD1O +07KSfiG0Z1p8LaR10RAFA7f3HLwwy6c9ExpGu5ED+co8aO5Xp5wysg8XfYZYx4Ca +Y3moQPJPDS6eOpUXd/6h27Fm34h9VdSj2p6j9JYsmTeEgb0x+JjAQyRS+Koj/tbS +bBqjbvO+FUaldRlHCHYCQTnjsSNBf7SxqE9lfgFitcgiHKSdD7QIfwNBEK1o7L8O +ugC/SQtHGe3ngUGuNmHI9w6ItGuVqoJYP3Hwa6ClGmYlTRLoAj8NkBibtoxwGIsp +TlTzmmLXpqeZTPaA2K5eiq8O5DKvAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA +3DJlf7AkZklzzswgm487f+y2bB7IYr55JwENASDxQEOdVcdgLC3IWu3hLiFwdqac +0Sw2OHZuETwJiIX3fD+qUT6TgbsP21W7wEQ4jfKg/bsXFMbrvw/ILkOW2JLTH4Cc +9ylCN+46dQ9heATkiF/Co+uASz9IoSDdtoycA3BuKGBZI8VGa56QmJOOsMM5NgxT +RTh2r23tV4E8AGYj3HC+b1rzK1RTlsj/m5nM9Jv0/NqoV1cprS1ONr8CBhN0ttuA +WLrG+DUZTMJYFabqTptlgejQFhiFp5HT5A+eXgZ8uEUX1I3q5jq1BEWtLdmJNZ45 +QViSJOokH/+1kfRSWiAH7pdBz4URLBcsDhAag4J7kV38t7fgdaIizY8R2Ss82iEP +xqa4A0PA065wB44zng/VrPrHoH1YnGRugXEnrqgcipC0FxUl3oQjvwOSR/E7yFU0 +GIr1MpRcyrd0z4p16783qnMpE1Aa0msED2SBKIK13WcNY+CtDF/wO47ZNywl1hBo +VkM+ohPpmonaVXNGdpdoZpeGjkBUbqkn+so4aYkX/WuZ6vY2vwdV0prD1vdAFfD2 +AeJx5ypu5aeKn6nK0eMy6W/VEJx6RLCiYVOCIcssgy31rmk4iLQJP2StYVK2mZKp +5aSR4eTv1/XlMujq+ZqcuUqA1id9wP7908Xr0DzdNdA= +-----END CERTIFICATE REQUEST----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client3-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/client3-cert.pem new file mode 100644 index 00000000000..ce9a472cb9a --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client3-cert.pem @@ -0,0 +1,30 @@ +-----BEGIN CERTIFICATE----- +MIIFMDCCAxgCFAXxDGdWf+MHldd68lQPasjUzyRhMA0GCSqGSIb3DQEBCwUAMFIx +CzAJBgNVBAYTAlJVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl +cm5ldCBXaWRnaXRzIFB0eSBMdGQxCzAJBgNVBAMMAmNhMB4XDTIyMDIxODA5NDMw +OVoXDTMyMDIxNjA5NDMwOVowVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUt +U3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UE +AwwHY2xpZW50MzCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAN8Bt8gv +50J66lQ+l/NUW+lqW4DesmSLv1BnjDd5SSA8tfczt999/l1epAGeEN/Pl4dAxXP/ +cxpx+J+xF6SKNxQ0RP+PHQMiDzCUgBq4OKs09kDQ/uvycUZlQuWPtR610TWjZR5r +VrNSwJQp3VGDdNyEbKj/yd6Yi5NC1iLuqPC20fw5/9BVTm1P2wWX7nv1AWs235s2 +yAG7pLNcgPiTfSmXyyT31YBjb9Onun7gv7exI/3K9mS+aWq6ci1xAXtykVCs551T +OQmDAUxda041YghEThO4MrZa6uSZqVwnoUcXTla+8biLYb3+9CnIjM5whAOTR+9r +jpsuuXEUOsrX9Mgb1HTS+ksmrA+Eka7MdVi60Hoon09uNvcTM8CSKNgnTzcPCM6t +J4NHDiimJM5WA/eY8i3NNCTa1HUGEeIK51UOdjIFKsvzG0TCI2FM7jQLJK5S38tI +deZ98iQbguVGhoCvRotLEAwW1M2rSOu7bxAZU4QJ93IuUfkLn2BipOuyuR55Z/6F +z5Jij/1lK2/pKWhntUHTIpG+bBHDF++0LN0aB29uIwYRkoz9JUgnNz4FDVbLvJ+z +5Ywr61t8AujZdfMZDpRYlzfWPGej8pm7/Eux5jgx/3jcLtqfqkfZLSuFjBKfkUU1 +eGsC80RupMJKIeppv541W6nQJlmJYKv7DCvrAgMBAAEwDQYJKoZIhvcNAQELBQAD +ggIBAD+YMVntBdeq7xJEL7xU4QEHzUGhDWodGMJfmswcxe7gf5Nztcq5YIug+akL +ewg0wzgCA5YGz00J92sKDF16RmYyPfkxmrCYdNGwISjNJyEEcPEVkdAzwILjv2Lq +0shFlSsf+Zp/M4XhHeirmzz/jJ9KHlzEYoCz1WOn+UGF12KgV2oQOamJSWOMCoMh +81oy90V5IlCBqnYfZCYj7cbYLBd5jZMZ+7lsVnxttzPTg1gIoP6vrLT32Ubnzx9N +IoAeiUg7az/fbnuOkJtu0cjz9aSdpjm2h2giyVAFJ8DkQ9C92tdr9DWZKn7rDO16 +TMdv0q8NFjRGhqdmqWUG6o2cUmQsJ/ZiIcHx5X1b7j7PYSS+ae9zi1tcpHAN6kCw +WHguIf5I8MIZxE741ZMBokFSIqd6Bh1EP/TUx1+g2a/nH3ZaNd4/KKADxfUU2Y58 +UwdKeX9YpcRz+NNO+1h3NoE1a/i0dhwiBf4OzBiV0WpAjQHT95IlQxTxfHFp42IH +GrbqIS3qK5DKlNFkBBk1beKxBGKmTH+Pw6fhjkuPYQzjmGo4xluivfeT8SiBT2iO +uIGLd+sitIooom0KEjHuHS9cdZ5XEPIUDAFhmIt7Y5K8J2fs+xtYzhibg3n0Q6qh +xTx7GzhTA1HSUE/467af5J3CSfpGAjZQZo/t2/A6tCumzk9F +-----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client3-key.pem b/tests/integration/test_ssl_cert_authentication/certs/client3-key.pem new file mode 100644 index 00000000000..b7464eb2866 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client3-key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDfAbfIL+dCeupU +PpfzVFvpaluA3rJki79QZ4w3eUkgPLX3M7ffff5dXqQBnhDfz5eHQMVz/3Macfif +sRekijcUNET/jx0DIg8wlIAauDirNPZA0P7r8nFGZULlj7UetdE1o2Uea1azUsCU +Kd1Rg3TchGyo/8nemIuTQtYi7qjwttH8Of/QVU5tT9sFl+579QFrNt+bNsgBu6Sz +XID4k30pl8sk99WAY2/Tp7p+4L+3sSP9yvZkvmlqunItcQF7cpFQrOedUzkJgwFM +XWtONWIIRE4TuDK2WurkmalcJ6FHF05WvvG4i2G9/vQpyIzOcIQDk0fva46bLrlx +FDrK1/TIG9R00vpLJqwPhJGuzHVYutB6KJ9Pbjb3EzPAkijYJ083DwjOrSeDRw4o +piTOVgP3mPItzTQk2tR1BhHiCudVDnYyBSrL8xtEwiNhTO40CySuUt/LSHXmffIk +G4LlRoaAr0aLSxAMFtTNq0jru28QGVOECfdyLlH5C59gYqTrsrkeeWf+hc+SYo/9 +ZStv6SloZ7VB0yKRvmwRwxfvtCzdGgdvbiMGEZKM/SVIJzc+BQ1Wy7yfs+WMK+tb +fALo2XXzGQ6UWJc31jxno/KZu/xLseY4Mf943C7an6pH2S0rhYwSn5FFNXhrAvNE +bqTCSiHqab+eNVup0CZZiWCr+wwr6wIDAQABAoIB/0I0QFst3XnfA7H+4x1Z7e9d +o8yeUFeJJUK5eub9Grh3TY4VzICM5vbRId9ZDalj95gvom7NZ15yd1zxNhOi9LcK +zXERC4vikJ/bdix4hFpPXsvfP87MKtS7OyDriNmVIIbL+zkMpLCX4JQb2ZhZblgI ++DkztrpejxEoxmmYcI8Ft1Ep5sfyi1XoXx1J/YLPOZyarcdme/oHut2EmMUzA/VV +GvnemYOEAa7UHImOL1xZOlYd6wf9f04wC7Vx1v7PBFTu/9O04TnxqnEBStns/y11 +GbjA9k0ssI8tDxpMqZRxVtBp31jqCBpflhzRbPvca1SkZLavN6baODNZzhpqAkDX +3R4lU5C7wu4jtzydUyEsCFNdtkGKlxpZRbRZk+keUC+HeCmXPED7p9egwF6Zi8VI +oaXl1KvHZO2W5x/BV9I1taEPhmOuRR49KxkU4e+IjqaWYN1qsqYqCs/od22Rah72 +KT+thr0mdxC4lb+pvteafricUQuq/dSbEY/lva7PhPQRKVX/VxOaAxBnhA1LHVgZ +imsW8W3eOQYJbxniTrz9EblWAg4dCcupsjMDUDUyACB/E6isDtYU1J2im6p4gbqw +tXg3bRh7KruIHbPSJyrFm1uqe+v97TLhpwPHKCsxE4HiJgRzaQDRckLJQebqNp3Y +e7kLLjg6uGsjAl6OwKECggEBAP5bLGVrmBmAz8RYPnG1MQWlsFg/eIhMFCqMjT3P +swPUU2VJKC3TC3OwFLxlAr0lkXol+8L8aEvxGjHksleA+1z0lav43b1/2jKgLgI6 +Ym5BxMJa+sUJpI6K7CedJ6wf2ozbpVXazvNBZ3o2l0QbC/KpX886CZH9YJgn7N0M +TfPe9er5zmETdHGTWtA0sDI8fZ8XndKmnWG9KTQCGur6gemF8SKuzGv/BnL+BZnv +bDqSvyN8Wjk35KPNeKVW78ROxRuEdB5brryGk955hX50PRRoofW8GSmLJNKNYvIj +VRkKrDKpz8gW1C2/xa9j5tQkGRFMDAptmk+yvtmDxfZz38UCggEBAOByrXLMTcwR +bz4MYcSmEdLv2VA/bZ+y0kW0frUU5il2fyQseoFbunVbTDiXYf40uueMbOONZktM +w04CXKRaTbnS/s6SGU5VW19jv+xzwrzpB2Shm08APwgFnSw40bKCpN4ZWQbOyFVq +QIMXfA0+Go3zJz37MsSgY+mzhHp4WITobVFpdlhaLvrLPCB78uInZrFsvNN6NP+K +OIbOoTA9u+BP73THHkpQdrRJaJWowpqejz8kzQ/Xu0Xe6AG1EGVp39phKpWH9TPF +8xoxjbdIGPkzCzYO3hgz6PlnWVj8iyTxklnaUblqKkY2mOlMA00ujcdF3d3IHvaM +Xolej+XeZ+8CggEBAKeZDdzaE4Oic8RtXN/xwxZ0gYj0cYhlkNgkeqCi7dL1IepY +VQg0ypP1DwTADhjx2zTAOG7XgCWh/V+o0LaFv5sVclW5iuplhzHah9ZiAB+kaHCk +IB6a5vohoc/MZqqs5oXv6LZ0ke6JRxSpSezPYYUIg5/5Hvs6GF7J1/IjPG4XmLS2 +23zto8l+jdUpEnxXjXK5zf1SWdtgF/kz9ealH9rurd/ri7kRdn9oz+oJb6f8r8ND +GfQf1yDzr65KZXxVZt1l3llukemZR2/NZN/Y2bJL64QO6AmOrLmr/emMzHLOrH5J +lCbEnBR1C14xFpTsIDRchoaMh6RCJC0Q/e0Rlv0CggEAAOIysJsBS2ZeK75cvCtz +MoNjNZ+qTNClZ0TYotncNhmTUo8iRFQaHdAoMqjV5+xJOBQjcZni5zT8J9h2iOca +GzsraaDFnLtVSsDXxpSGFbxNHSZNuDfmB6AOCFiI6sz83Sr4YMB7pWpvqpRzFpJC +BIEKjIHqpz+CZS8hvGGw54UKuSFTJ/Hi8XXPXMlgIWfKTbSB4cs/XiorIsy5cbks +fiuSY8FM6zn53afUU5KAgZ9SLQt2CzPsNtAz1Z3i3KNYEEIFquUIIBYNaPL8/dW4 +03JR/vp8AVhi+Ghhv6nu2kxhKR1k6Pf0Bqa8X16/PJSMVlZ+Extwk8Pls2C97Ee9 +3QKCAQEAgjcbHKBjd7AeyNpPSzNpv81Rry5qqOc+Cxx8LtOHBl1wc5VB5FPxfbuX +MX2skvWPnokDoXcI1a1WQwdjaZUsSoqdeyPtw8pFWiNLJZkYImiP3zMCZXYUEkzk +3EXQZryWEqBYBqxlEvTyjbBmnrAwOPOUKARFi1l9JKJ4QpdELXo9Yl+w2IQEQ5N9 +jrSY7LwS/cb25rhEc6oh/89aY83HPyABh4lC9bsciXki54YIeS+y9ijN8yCRxikr +mVGfQ0Y/qcY9spAj05yr/vnlENBB5ohxwKKsemOnH93E2GFxc1dzmWCGvISjUduB +I68TOg71OfCKgfeixNgcOvQoN+xngA== +-----END PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client3-req.pem b/tests/integration/test_ssl_cert_authentication/certs/client3-req.pem new file mode 100644 index 00000000000..7b4445b3609 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client3-req.pem @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIEnDCCAoQCAQAwVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUx +ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UEAwwHY2xp +ZW50MzCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAN8Bt8gv50J66lQ+ +l/NUW+lqW4DesmSLv1BnjDd5SSA8tfczt999/l1epAGeEN/Pl4dAxXP/cxpx+J+x +F6SKNxQ0RP+PHQMiDzCUgBq4OKs09kDQ/uvycUZlQuWPtR610TWjZR5rVrNSwJQp +3VGDdNyEbKj/yd6Yi5NC1iLuqPC20fw5/9BVTm1P2wWX7nv1AWs235s2yAG7pLNc +gPiTfSmXyyT31YBjb9Onun7gv7exI/3K9mS+aWq6ci1xAXtykVCs551TOQmDAUxd +a041YghEThO4MrZa6uSZqVwnoUcXTla+8biLYb3+9CnIjM5whAOTR+9rjpsuuXEU +OsrX9Mgb1HTS+ksmrA+Eka7MdVi60Hoon09uNvcTM8CSKNgnTzcPCM6tJ4NHDiim +JM5WA/eY8i3NNCTa1HUGEeIK51UOdjIFKsvzG0TCI2FM7jQLJK5S38tIdeZ98iQb +guVGhoCvRotLEAwW1M2rSOu7bxAZU4QJ93IuUfkLn2BipOuyuR55Z/6Fz5Jij/1l +K2/pKWhntUHTIpG+bBHDF++0LN0aB29uIwYRkoz9JUgnNz4FDVbLvJ+z5Ywr61t8 +AujZdfMZDpRYlzfWPGej8pm7/Eux5jgx/3jcLtqfqkfZLSuFjBKfkUU1eGsC80Ru +pMJKIeppv541W6nQJlmJYKv7DCvrAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA +Rggrols8hXGEcWeIEGn66kY9IVTzaTUf3oMfEbdf/2Q1QzHzmqp53yamHl5ioMgX +o5UBVxthgh1VOxkvCxIzlKDJprzVFkfwwc7h9c0HGt3No/ERobHDT6YRaGukAL5g +muIGBUseyBAOIfyqc5kbCRWfPrAOttAH4gd8XMBgO8XdfHAvyXBC8Ha55O6oriX9 +IAKL5+3nVJkBle+62OmROnstbcdKyK4UtOeki/6ptYVE0d9I+NfKjuk3eKtICW8Q +Pn3IEcNEZoFG2UQ19ENWwYEZyMZJt0aunqnm7L4RYiZT5w4meeendzXSKLKR6+Ye +ULt1sDRskgKoNRzmeCVzci05BG48jv/E7Az6aV/qhGiU2qIAPMdVXncWUhR3fj+E +CL/uLifOvfC6SnKw/7qQmgjUvEe4Duvi670a5QuImpm/mAIN22cXPc+QquSdR5xy +loz/o3JJQZemPAOM0CMIHZ+cGESxH30QCBNn5HfcOf5fRZVCss4Hl6JxHR2G4yN3 +RKEIUXR03qgSK91WHl3WvqwXgmIAiUuvPjo2i7kSuaUUHilZiXK1ngIqYfUTB5SQ +O8pG0fx3fbhVDA3RQfXeJE6FA2AyLvqOcsseRzvcQjQm4MU7p+RVaY17rI6/EkS8 +ac3E7BPwnXqSAkPSEgoiezv/Z0Hkmrcu6fIsUuf4ETU= +-----END CERTIFICATE REQUEST----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/generate_certs.sh b/tests/integration/test_ssl_cert_authentication/certs/generate_certs.sh new file mode 100755 index 00000000000..d6126d361f5 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/generate_certs.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# 1. Generate CA's private key and self-signed certificate +openssl req -newkey rsa:4096 -x509 -days 3650 -nodes -batch -keyout ca-key.pem -out ca-cert.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=ca" + +# 2. Generate server's private key and certificate signing request (CSR) +openssl req -newkey rsa:4096 -nodes -batch -keyout server-key.pem -out server-req.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=server" + +# 3. Use CA's private key to sign server's CSR and get back the signed certificate +openssl x509 -req -days 3650 -in server-req.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -extfile server-ext.cnf -out server-cert.pem + +# 4. Generate client's private key and certificate signing request (CSR) +openssl req -newkey rsa:4096 -nodes -batch -keyout client1-key.pem -out client1-req.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=client1" +openssl req -newkey rsa:4096 -nodes -batch -keyout client2-key.pem -out client2-req.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=client2" +openssl req -newkey rsa:4096 -nodes -batch -keyout client3-key.pem -out client3-req.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=client3" + +# 5. Use CA's private key to sign client's CSR and get back the signed certificate +openssl x509 -req -days 3650 -in client1-req.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out client1-cert.pem +openssl x509 -req -days 3650 -in client2-req.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out client2-cert.pem +openssl x509 -req -days 3650 -in client3-req.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out client3-cert.pem + +# 6. Generate one more self-signed certificate and private key for using as wrong certificate (because it's not signed by CA) +openssl req -newkey rsa:4096 -x509 -days 3650 -nodes -batch -keyout wrong-key.pem -out wrong-cert.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=client" diff --git a/tests/integration/test_ssl_cert_authentication/certs/server-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/server-cert.pem new file mode 100644 index 00000000000..6f8e5a3c6b1 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/server-cert.pem @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIFSTCCAzGgAwIBAgIUBfEMZ1Z/4weV13ryVA9qyNTPJF4wDQYJKoZIhvcNAQEL +BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjIwMjE4 +MDk0MzA2WhcNMzIwMjE2MDk0MzA2WjBWMQswCQYDVQQGEwJSVTETMBEGA1UECAwK +U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMQ8w +DQYDVQQDDAZzZXJ2ZXIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC8 +jV8igQGgCvu/7BJDI5VQl43VGAFjH2Na/E9P4E5uwkSlJVED1WKvIlxRWhOaQOfC +587nZVhQtHpdbCvBdKrHml4SVbTchs5SN2kZsHeqaQzcGnejnczE0SYo4xNyniSv +GiQ1M8G3fiZNflEIPM/+Ob2oI3YnVWFGy0a5rQcHZWS45KuGILMP0aRHyzyh/31c +K3i2xA7A3V2jBNuD4kHG8TLgfDeoCecTI0iU/LJnDOolX5XdpyeoJ6YyYOGg3F9e +bRmbNlJN3Iky3Vzyc4jYG7y6f5DqfebYMW6hCvLpf9lN6/gPNOb2KjL3hvJ+hbj+ +b9EkVAzpw7mW1VHEy+WbtYMPoKy08JTc7zr1tv/vQGr3XExwlC9iixZXMaVt1kP1 +TEVHv2FiUOiZsVaqtoFpS/wBvKeQdkzNy+66pRpG9bLuOnL4hlz+rwHkdBmHGk+q +cXdwglqIDqXKlCpIMSkFPH1364KLdJ2qBgWWoWCJjUmgbrA8/LU6DX+GBbEiw45T +PQKP//RMkOrHOYRD33WTU0iKP61zn5+9RD5OLxEUOtCvL7AfB+jt4vYrMTT2U3Kl +OckWxNx55bYLdLfGKtepGV2r5xzce0UMbWQrXQRuka3a/j5VJUTuUgcwgd6FoP4N +4ObW2H1YEtE5M30xpa1kcqJ1RGEWagakISgn2Z3TywIDAQABoxMwETAPBgNVHREE +CDAGhwQKBaxNMA0GCSqGSIb3DQEBCwUAA4ICAQCE2eJVcvsMmJu6xAfoE6/u6BrD +opMicCtlC2qt0BgSIzzDQ/iWjnWKGM1C+pO+2G0WTczj7ugsxjPzhkyBpuEZaWt0 +9/tJTKIrgaRZvEe0ifsJxyqL5LJgfxK7TbDPcUBKr1v+TOxPVRq0FuG16x+yka4C +rwxfBHU43FmtEFfgu13r515F3ggXcdlojkce8ZKtTAGEcN0MpbJ6XS90BHU0sy5A +APTm0fR0vM3kg1nuBLbSGF5KfASdw13gb6QsDbll0IqK8LvXYiX5CaVfkAe/pFkO +/2iIxYW74yC2gV+DcFdRPVfFxSKrdg0tDER35OYg1/vXRjV5BWr1EjE3qjrCcUZy +rlF3fms7Arr20ka2nSa8avn4ALpyJZmKasoxNAAsxivingNVZkql48OqsJ3n0qGk +LI6Yu+UM/pc78a3NHsdsCbnf8qvae4oJa1kyiochJu+gUOzHvs4Ydti9iTQn2Byo +2A2LzyVPBmSOhzdQ7SwpvHA4A2ftao+dZoA/+o4rmBtbmgxjpBPyPJTN0ZfKlpKl +Oyi57ov+cJmZctSUbP3M11gBva7aYu1Rd7/eXeCEl1FHhmKL/Ee+UrNZLiwspb2E +Sa+pOHdJX8VgsIYXku2UKaGT2QFITxO7fnxghioxgsyCKrQ+m1gL9vgXj/gJu+48 +c+5CZ9SobLdMkVOtQQ== +-----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/server-ext.cnf b/tests/integration/test_ssl_cert_authentication/certs/server-ext.cnf new file mode 100644 index 00000000000..83d9b03ccb7 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/server-ext.cnf @@ -0,0 +1 @@ +subjectAltName=IP:10.5.172.77 diff --git a/tests/integration/test_ssl_cert_authentication/certs/server-key.pem b/tests/integration/test_ssl_cert_authentication/certs/server-key.pem new file mode 100644 index 00000000000..065a2290749 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/server-key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQC8jV8igQGgCvu/ +7BJDI5VQl43VGAFjH2Na/E9P4E5uwkSlJVED1WKvIlxRWhOaQOfC587nZVhQtHpd +bCvBdKrHml4SVbTchs5SN2kZsHeqaQzcGnejnczE0SYo4xNyniSvGiQ1M8G3fiZN +flEIPM/+Ob2oI3YnVWFGy0a5rQcHZWS45KuGILMP0aRHyzyh/31cK3i2xA7A3V2j +BNuD4kHG8TLgfDeoCecTI0iU/LJnDOolX5XdpyeoJ6YyYOGg3F9ebRmbNlJN3Iky +3Vzyc4jYG7y6f5DqfebYMW6hCvLpf9lN6/gPNOb2KjL3hvJ+hbj+b9EkVAzpw7mW +1VHEy+WbtYMPoKy08JTc7zr1tv/vQGr3XExwlC9iixZXMaVt1kP1TEVHv2FiUOiZ +sVaqtoFpS/wBvKeQdkzNy+66pRpG9bLuOnL4hlz+rwHkdBmHGk+qcXdwglqIDqXK +lCpIMSkFPH1364KLdJ2qBgWWoWCJjUmgbrA8/LU6DX+GBbEiw45TPQKP//RMkOrH +OYRD33WTU0iKP61zn5+9RD5OLxEUOtCvL7AfB+jt4vYrMTT2U3KlOckWxNx55bYL +dLfGKtepGV2r5xzce0UMbWQrXQRuka3a/j5VJUTuUgcwgd6FoP4N4ObW2H1YEtE5 +M30xpa1kcqJ1RGEWagakISgn2Z3TywIDAQABAoICAQC11lTwLp/Fm7IL9fvquc9P +CMmkv2DfGi80WO2YJ8ccM8gFyEYoP0rLgYSshAUxlvSr1+iG6grQ0izMGfzctcnZ +c3rTjco9fthNG9kFCFVvh536SqAkr5MCIH3/onZn7DGOmNRgZoikkEkaJP66xgME +tuS72W8iIcoNfw63FDIaJOONGCJ+2Nw3HkOjZVIVHRLlp5rkD5H218Vs6MtWlgY/ +eO9K5SC7sskhgL6HyGe40BCjeFpMh97L4Wj7XslZ3A0xQGAYervHES9TWX5A58EK +QT2yUkIMktzklE+PicKYA08rQa1Z5Pf0YOAELSWBdS7iWi3FLjXB35sE5rbT5puH +9hZXSDWLggbefuaUJyowDEZy2aHP5pvXKBDhEANRbU8VaDyHhlNLqVNquE5Cn4HO +zPeH+KLFbbABUok7yYZmIC9Bfn+rXvNzTX6A13AdJI/HcKA5RBGtpAY/168Pt/Aq +dzuqepu42rFAvS45RNevp72IIavx/QdAA1OTgKxh3c2Mf85pIXJ51aWWLnn+EZ5/ +EsE0crfwkuKJvjubNC4oOwMTFMIBI2WsjvaAw8pQw0Kb0ksExKd0wz9mKcqR/v0I +K9oYsaHkx5je0NOZds385+zCoQHYaw1aKUd7ZLqr5G/Nf/2TEYpMWco4ETA8lzu3 +Ty/8XkNw8jd4p+7bUuz1mQKCAQEA4MNU7GWDPwUKNNSz335nGH2oBvSGbYiwLcRM +D+x2+RTfOAFSSJ+Q5tQ+327ZkAB5dK2mxmDYKB+Ln1UBIneViUflkMyh4fuutIXI +wYo+BL71r89MqhRvvMK9hWnCGtJTJedf2iQENJzVn4J76BvTPRYywBv9pofPOlj1 +MtwwMA4CZAmQpCUaF5NQr4nliYx+slkcKwlm+cOxeZGa8mkNgQdmCcTZkRz6qsiR +vQDEDiS1+5lCJ6nWW4L2tOPejNN//hVlbPGMaA0oiu7I7w4aSxnTlLhDgJzJwmN8 +NFYl+u5AcPq9iRtBnzfPmd87S9bg10zcIiMKxw898sU24Pa0jQKCAQEA1sG5hO3c +4API//k7NEWXsx5Ns2JE/AV1LtmBgqXkn1DAJ+b6V1nIUppTs0zspEWrae9KrsAk +z47qIbPaTLHuptLrvEXk2LVfzcK32a7fXXDOB5KkBhzlJM1J3PTRQFR9lr7qX6vr +EDc4p7p55IDEGnJdXa7x+z56QjpAZaHlzexQxvoWWoLBkDuoT389sdU7CbgTa4A+ +CR6D6qKd6H6tfmv5sPlvp+aje+ObacP9I4WyVjscWkzBHxS3n/fTLjY6OFv+o8PM +TdytN4+HZnu4MDJlF3vx9P6CbnnVCaScXDxPGcoSJPcoEQqoyxuvUQLDUQkzWF14 +02EvXW0dbgiPtwKCAQA0EUwFD2ceHD7HClc4+QFNDR71rYPOsBGQKJ8uOSs+fHVR +dgznwf9BWf3OqNFBqLp6KxgtcJXihZxEpt6Ca416pesqZh1CSpmoPC3LmAjR9KLZ +vX4XEHDqG3roAx3yNLMKXtU3pYxL2+Eo+INXu8ptpkzPcCyMfX2mGKGEzLllCHnJ +TuXxAJ9QwtG4OIuyF5fqHPaHicAPMCRW80If0fJM57fdn3p/QWVYVupcDGdel2aJ +CHHo2lFMFcStFvShTwWhiLdcS4CpQhMYTETEDFJO/4aiNyV8D9Y1b/J/9U0LGlJX +Wd66elPzXGx9StdjtD2V4rpENjXy8zb4nHMgHkapAoIBACvvtmTbxTSPka/M7a/k +DQU4Te1FTZfCBhdvqG9yQTPW8Xk4aD82vyUnLbihJEj3d/pUWpMl/GH6eywp/59x +R8IZpOD/67HqaY9PJw4CGPClA4HJHoWho7/DwDjUXXsrzgXpSUoJgi3vHkgyfn2h +Wn2OqEtiX19niNvDzyj71mgq0Nvkjm42EiPQEL8y6QxY85spbc+wjQCQnayDWIsY +X6ZdsNfkMFPJe+j8x+77ie6ai8HYlhRjX59cPbUcnrf1oDOnnpEincnQPCAB3VG6 +PhSeOtBzKy1UZJr1kgBHDTZRoF1GWi/14NybsazcHSIVzp/lofuSJAYa+/XBPSQl +3EECggEBALSLZPdg13906LEyznYnjgq+nMh88usegvU9qsBAFExClLLfr6Ak77og +boNoOwbaFn+xiz5M8BTJIPizJcm5GjYaqg58zotTtG51h6VgMri+fb/BUpVr7p7n +aSq3kXDZlrwZnmooCT+KcGx++w2N2SYSyZX1TELt/dpfuWJvph+E37PkONEEiHPF +ZtSA/f9lpfP5/nx1pLmv4ksKdXqpz3/kNqaf9zbhQLgOm/VoBHL4NVPYRylGpCJb +R68/7yvHBd2EskZoJB53TlJmwu+fC6ee1UiG6aqTULfEsiGidi6jIt56Gz52ox66 +BHL/JsJ0Be5xM3V4x4PtihQ3Dw546FY= +-----END PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/server-req.pem b/tests/integration/test_ssl_cert_authentication/certs/server-req.pem new file mode 100644 index 00000000000..be2f756cc7b --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/server-req.pem @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIEmzCCAoMCAQAwVjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUx +ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGc2Vy +dmVyMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAvI1fIoEBoAr7v+wS +QyOVUJeN1RgBYx9jWvxPT+BObsJEpSVRA9ViryJcUVoTmkDnwufO52VYULR6XWwr +wXSqx5peElW03IbOUjdpGbB3qmkM3Bp3o53MxNEmKOMTcp4krxokNTPBt34mTX5R +CDzP/jm9qCN2J1VhRstGua0HB2VkuOSrhiCzD9GkR8s8of99XCt4tsQOwN1dowTb +g+JBxvEy4Hw3qAnnEyNIlPyyZwzqJV+V3acnqCemMmDhoNxfXm0ZmzZSTdyJMt1c +8nOI2Bu8un+Q6n3m2DFuoQry6X/ZTev4DzTm9ioy94byfoW4/m/RJFQM6cO5ltVR +xMvlm7WDD6CstPCU3O869bb/70Bq91xMcJQvYosWVzGlbdZD9UxFR79hYlDombFW +qraBaUv8AbynkHZMzcvuuqUaRvWy7jpy+IZc/q8B5HQZhxpPqnF3cIJaiA6lypQq +SDEpBTx9d+uCi3SdqgYFlqFgiY1JoG6wPPy1Og1/hgWxIsOOUz0Cj//0TJDqxzmE +Q991k1NIij+tc5+fvUQ+Ti8RFDrQry+wHwfo7eL2KzE09lNypTnJFsTceeW2C3S3 +xirXqRldq+cc3HtFDG1kK10EbpGt2v4+VSVE7lIHMIHehaD+DeDm1th9WBLROTN9 +MaWtZHKidURhFmoGpCEoJ9md08sCAwEAAaAAMA0GCSqGSIb3DQEBCwUAA4ICAQAb +FDegAoUBz9O4JR1u68IMnGkO5nINGAPQOqf9a2BxGujnSB7Lw6SHukjkUqqgnfQ0 +x/aWOI8JVAi/ptscojgMQUDsVNsij5v+jbJE+ZAobxnTmKP0wTc2ktpf4d8UMVc8 +gyM85jLHZ8caCcuy0D97W81vgIv33dNHWtP+sfbQhX9wJ2YQTahIC8NpuQfLAOUH +EFxWil0mfN+9vRQ1C5naKtvrOPqyM0RPrWiudIJ5QjI4aSXxUCupxxnaQMoI0Y50 +MvVVT3VwWgP+hL4b+yEJFHRpE7BwCZijsLIXkXmVZoveHhiSMYen1HWIP1VMDEHP +CUtG5UQcA78CBS8qg4nyFbDU4hWClAkAt96O8Y2epJYepIoYuBBSEfrgupESMLjS +E9Hfq/H6Ac/Q3zWa320udvA+ysfS8pagkoiH9+TarrsDjhxLjg2h2bGcXKlrsP1R +mRVZwfNOl3/ZNq5HBPb9Z5WXKvcsTCQAlnHJdaSmzdyArB0guwUHg8ZZNZqCdVgL +TPsfE84yI/HlwRfuQILfGxq99p/UYFwnee5CoM/PPvaAT+9z/lykMWZA7osuBcK6 +zP8XneGmZOkmez5+YJgSC0xeaDxr2R52eQXlQEJGDbFDtQap/X+cJDGyqmGnbhSu +6XkGy0l8mAkpcurMcy3wWf6+joskZAN4Joi4ZjKsQA== +-----END CERTIFICATE REQUEST----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/wrong-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/wrong-cert.pem new file mode 100644 index 00000000000..ef95a73deba --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/wrong-cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFjTCCA3WgAwIBAgIUL2Y/QpwqqHyi43PwPeA6ygdPYK4wDQYJKoZIhvcNAQEL +BQAwVjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGY2xpZW50MB4XDTIy +MDIxODA5NDMxMFoXDTMyMDIxNjA5NDMxMFowVjELMAkGA1UEBhMCUlUxEzARBgNV +BAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0 +ZDEPMA0GA1UEAwwGY2xpZW50MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKC +AgEAxO2PSeaiNFMRRiFXpnMw07u6EIdEc1Jx3cPvZjEUg/pdEmMYkrSxr2MeqRkl +tWH8TcIIoiWDLIcM6IU0mF6a5ULu84hFb9b20qRG3wRNb5yO86HnoyzU99t98++a +9iaY1QAt03k8wq4jRjU2k/eoVSoLT5uVP5KxiNzdS2BTHFSsxrt/xcwdgkfJouHN +p+MYUekk6qaQy5fTqTpqdkgO2v/JoYCi0whBNj205d+WnS7xfeyVSJP1OJWHRZ7K +Y+LU6hz6wHIng4s/ag7VdAk0PArWs50BmH5g2zJfvt7VeTQebaJWUtSEY05odOqt +KZteUmmhxW/2M73wGVF3WAJCnaxypsjcmMZFCpMXpwyTFrqobvC3APl6SOP+Ev1M +LxhhCIDuLFu46P55KKEKjUCsYigd1VsHjjvoajGcqlPlMsVHJc9VChsQDz6agzDP +Fb/LyYbrDTTmsI57/s1jAZyemq2SEYPApJvcdZ/ucl741jI0671EZPlip9iUQgt3 +MHlc3t53/GtF2W6GF5Fogch7c+0c2BhMupAHAXwfLABvv5X8GDyjsNlwB6ea9jeC +Hw+0rEotZzCXId3daFytGNm1jI216kXLSbvz6uz1wMGS6Hrhk87whgvQ58RMNs1K +SGDFw1WFv+QZeTO7wqcn8Y/eqF7q9RBhOpPMJMX8Sx/UXuECAwEAAaNTMFEwHQYD +VR0OBBYEFCI7Iy7tY0D4HPa9BZCZxYuJ51mZMB8GA1UdIwQYMBaAFCI7Iy7tY0D4 +HPa9BZCZxYuJ51mZMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIB +AIKYtBwTp3yvUGSXorV32dnU0Hp0MOie/itgx/la6b3h2bZSoCigKmcmvMaAaNzA +pxeYSsf5wPnONpWfo9hsGrUDMT4ETnXdzA1dbidIrhJbGsY8CN217Qt3YZWNWkrz +xLwxEwAovQZqnGDvtx+tRE8i6YJO6/kca+GB7liHFvUx8zaQ6gCwfloduG8rOAeq +noeCpW/zqYQSQGK35ntQ8MTTRbi7jMOTCikvRlldS73ODQcAR7jywgBYf/i8ANtz +NoWa4KbWuqKsQKMIGOi1fMLMaNlDSzJyw6UJ2GVCcL1NxkCZi0yudfAAxWlRis9G +zLjm7YdNBiC6RVZudGhvzjlsLZpE9DgiwXqcDv3Y1dpstD5ikrNhlQo6THH1YeFy +B8vjVGZZZu4B2JEo+QWH+zFGJosD66YoaKMVuwRPwoGDQoO0Pfbpq41A4KUhB3cf +X49/rbInqwsN5MuGp4l4+T7k7Wm0Y1Qo4FXDVbFxHvvniyHUsZk9Llzf5wBLl84m +xheUGgCHSflfXuuWi76yoADHCv+Eqi4/aLJmkUewKXJlm+XYs9bdBHUI+Y10KmhA +hgcHXF56L+N4mLRwUuLxa5qwQIqNX32+piQoO9opxnVKKCptpATLE30TOMLEXBlp +J+6b1e4BIasAAEGUhTgPj/SLL0u59Bv0K5SlSn7VZ0gI +-----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/wrong-key.pem b/tests/integration/test_ssl_cert_authentication/certs/wrong-key.pem new file mode 100644 index 00000000000..b2213cd2675 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/wrong-key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDE7Y9J5qI0UxFG +IVemczDTu7oQh0RzUnHdw+9mMRSD+l0SYxiStLGvYx6pGSW1YfxNwgiiJYMshwzo +hTSYXprlQu7ziEVv1vbSpEbfBE1vnI7zoeejLNT3233z75r2JpjVAC3TeTzCriNG +NTaT96hVKgtPm5U/krGI3N1LYFMcVKzGu3/FzB2CR8mi4c2n4xhR6STqppDLl9Op +Omp2SA7a/8mhgKLTCEE2PbTl35adLvF97JVIk/U4lYdFnspj4tTqHPrAcieDiz9q +DtV0CTQ8CtaznQGYfmDbMl++3tV5NB5tolZS1IRjTmh06q0pm15SaaHFb/YzvfAZ +UXdYAkKdrHKmyNyYxkUKkxenDJMWuqhu8LcA+XpI4/4S/UwvGGEIgO4sW7jo/nko +oQqNQKxiKB3VWweOO+hqMZyqU+UyxUclz1UKGxAPPpqDMM8Vv8vJhusNNOawjnv+ +zWMBnJ6arZIRg8Ckm9x1n+5yXvjWMjTrvURk+WKn2JRCC3cweVze3nf8a0XZboYX +kWiByHtz7RzYGEy6kAcBfB8sAG+/lfwYPKOw2XAHp5r2N4IfD7SsSi1nMJch3d1o +XK0Y2bWMjbXqRctJu/Pq7PXAwZLoeuGTzvCGC9DnxEw2zUpIYMXDVYW/5Bl5M7vC +pyfxj96oXur1EGE6k8wkxfxLH9Re4QIDAQABAoICAQCjj/CAX/f/X7MsPYtQa8J1 +Sinbio42/pYmrJPNnBw/FhZxrC7/wucGFlyj9IgWZCEr8Go9SsztkeoNwn2RxJoA +q5xOV7PclX4CLIHUv/0VI8Kz5pi/NgBZMUwm7K8Xna041OI7ECqARCR2LsJ7GasN +uVMVttK6r7uXQmLnNUUydb3ffmI8xjEIQVnfWI74z60mc2+/GcOP5jXeC+/a+DSm +fudYpcAXaXbId24ls5SkTxYzEepYEtQNQFzPXXkah49yN8mpR+c74c805scxjmd9 +Kz9yhYiKwQTvaqKNpQVHmxte0gPC3lJrLPejjDtxIGOyLZw4oaqrBSpDzR9D0PTE +C+BR6VlXpVCTcAoiweuoDIxNTiJ5IbIJme3iMWxsAIJ4n10rSFFl9Cmmqbphp/6/ +XInB0X7Zyr1kBrwf+DH6DJhje5NXgGKVR9oe9jjW5v8V2tg1RrkzNU8iKBSxpvcI +x4mKhhRLYgoq/iNeYBVQrwJYktIbweVCQ5Spj7/20IrMkn3FAmMsZxGMZmLisJ9t +B0vvUkUgWxuJTsPJ2j+ytpGT0E2xIDYCpbG2EopDc8WvHcVNhagBvLC6xIjIKm7N +2zpBU2W3fPNXoToCAmaLDPYeRRpG6XaGFQAfvKUQRLBDGTfQ177qr34UBnmgvxDq +J2gA9rQm3XziLMuSlJexAQKCAQEA+yz49Ah7FFq0QffsoRb0qOJbfcmMGTRkaafb +ztto4EFSnjH2EwoSShu4DfqWw+ws1KxHlItNHHko5pVNpS4lj1OpnobW3QD7kEIV +mYKa3KowYUcCq1Gzq2RNDZqsC2BSXwx1MG0VVKYOahnu5bvzQq2Ft8W7CWBnbTbY +0Jxjs4KaOza+bH7Vfb5Yre0tlW7U5vI/YO8+YKxpxfOU9kVo8ZLQ/9r/YH8nnLa+ +Fd91+WjcUW8CTKU+Oz3lb/Vwcs6YOoAraq/wtOCqWURunBXkQtzOIn0bgBh3WEk1 +EQ+MVDHshlVVjv/rfnL571ZTT1amCJuEIwQRzLSvbso883srMQKCAQEAyLXaG3Pp +LYiRKu7Bqr5PPuqdT72UFabPpfgd5EtcFOL0xUpfRya6HyFdM25FWI8haXeg4e8N +0cIs3gMG+RRgm1xISJIZi92L0Cwj+kLFu2U5SkvAKMqZFh5q350FRi4Bp7ae4YrL +aguWLZCxhznh4D5xQGM6c8ObRfUUEMT+dnLPcj4zn9KHhoUudXjLKjPNw5v6nkbw +xtRdwANlHx/LX/d4+iwt2plDWmT+d2OLvqZcPyyghTMqV45L0p9XAXBsLnz4Zipx +7VJ8iH3jL5oaQ6YAFY+cXIrWBN0q3UYbXdkaA2ve6voioeF3KQNRmU10k7GKNRWl +pRNn62+rAR8isQKCAQAZnPVqFS9P3QwCqiCEMM4UJrkDs7jInTIcIBTnHDKuo5qk +LR4VxPImgnsbWdFj+0J7EXJfMHFVlPlZwiHf1TvZSMPEOaXRdZcxl7uSIuJd3DEA +ynf4NmWm9Zxx5bLjmhfsP1336TfCoQhZQ3m8DZV52C4Jlm1DQIRre6tSYpA8LvZB +UYzLjYeBwhZS7hu24E1vm4ZhASSQQSSsHfGzx1IzSDBt1swx7+V/MpdhrZ7fJxVI +bJSEcllNOzuZViL4Yh7d4FINGBHor/xPDA5ndkgHlXKjy7QxNM1+wEBcFATQVSL0 +c+E8qtY918Wq5VergH9/4zPvSivyfv5gwtjCT24RAoIBABP6HbJb0BqrHB/U0cvn +00Vk3rGAIgwhpUtUrcz6PzkI+enlJCSV0zKkBH3I/Pf6jw3LTWUPgSWemQ6j6H7E +K3VrMvqeKBLGw1K+Afq3yKyFP7WIYqDswV31Oxf0rgC1NY7220uBoAt3CcSRQUo/ +VZ8XN/h7p+a70mmdIhklMlqhxMoPLN48eybFfMFOe5JAw7szfDdiwjZYDti8vcTi +SkDMBeuImCvI025c3QMPEmqwbkAPdg6r8Av06tEU8PkAspPR9ntcwCgp7KE9Pm6P +fQu8qwd6WsrPOswTI2AQyUqHAFLU2sQyj13jbhPT87w5fF/y7NmpxOnwS4igfbnH +2pECggEBALO0FiJClb0GSqMXYJ+nbtRObD4AynYNVMEqYdZu5DBb6vb4T7uumTD5 +I1fKOa5SSELngUj23p2G6sVBsDyDHotGJYJwDGejHOFnEpY+J0Das0pGS40FsFC7 +qABIUaMoLKcIR9Ofcm9uu2n+koNULV2aaXj7A4OYhRCQi2PqiEx1wimdrLfGqTXn +O4rSf826ODch87vuPbfFPCaIFG28R3nByp/ZBH5QNiB3NBmc3A0tiHFnZW3cpOfW +Jm/Vu0PcNVVw32SroS2FCroR7qSWsvt61UzJtliLUiFHoUAxrXXiAxcZW1D2Hmpq +neUhT/t9hHdcMJgoxm2IITf6ip8nTnY= +-----END PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/configs/ssl_config.xml b/tests/integration/test_ssl_cert_authentication/configs/ssl_config.xml new file mode 100644 index 00000000000..163449872be --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/configs/ssl_config.xml @@ -0,0 +1,36 @@ + + + 8443 + + + + + + + + /etc/clickhouse-server/config.d/server-cert.pem + /etc/clickhouse-server/config.d/server-key.pem + /etc/clickhouse-server/config.d/ca-cert.pem + relaxed + + + + true + true + sslv2,sslv3 + true + + + + RejectCertificateHandler + + + + + \ No newline at end of file diff --git a/tests/integration/test_ssl_cert_authentication/configs/users_with_ssl_auth.xml b/tests/integration/test_ssl_cert_authentication/configs/users_with_ssl_auth.xml new file mode 100644 index 00000000000..c41776f9e78 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/configs/users_with_ssl_auth.xml @@ -0,0 +1,22 @@ + + + + + + client1 + + + + + client2 + client3 + + + + + + + qwe123 + + + diff --git a/tests/integration/test_ssl_cert_authentication/test.py b/tests/integration/test_ssl_cert_authentication/test.py new file mode 100644 index 00000000000..eceb4d10ae5 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/test.py @@ -0,0 +1,117 @@ +import pytest +from helpers.cluster import ClickHouseCluster +import urllib.request, urllib.parse +import ssl +import os.path + +HTTPS_PORT = 8443 +NODE_IP = '10.5.172.77' # It's important for the node to work at this IP because 'server-cert.pem' requires that (see server-ext.cnf). +NODE_IP_WITH_HTTPS_PORT = NODE_IP + ':' + str(HTTPS_PORT) +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) + +cluster = ClickHouseCluster(__file__) +instance = cluster.add_instance('node', ipv4_address=NODE_IP, + main_configs=['configs/ssl_config.xml', 'certs/server-key.pem', 'certs/server-cert.pem', 'certs/ca-cert.pem'], + user_configs=["configs/users_with_ssl_auth.xml"]) + + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def get_ssl_context(cert_name): + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.load_verify_locations(cafile=f'{SCRIPT_DIR}/certs/ca-cert.pem') + if cert_name: + context.load_cert_chain(f'{SCRIPT_DIR}/certs/{cert_name}-cert.pem', f'{SCRIPT_DIR}/certs/{cert_name}-key.pem') + context.verify_mode = ssl.CERT_REQUIRED + context.check_hostname = True + return context + + +def execute_query_https(query, user, enable_ssl_auth=True, cert_name=None, password=None): + url = f'https://{NODE_IP_WITH_HTTPS_PORT}/?query={urllib.parse.quote(query)}' + request = urllib.request.Request(url) + request.add_header('X-ClickHouse-User', user) + if enable_ssl_auth: + request.add_header('X-ClickHouse-SSL-Certificate-Auth', 'on') + if password: + request.add_header('X-ClickHouse-Key', password) + response = urllib.request.urlopen(request, context=get_ssl_context(cert_name)).read() + return response.decode('utf-8') + + +def test_https(): + assert execute_query_https("SELECT currentUser()", user="john", cert_name='client1') == "john\n" + assert execute_query_https("SELECT currentUser()", user="lucy", cert_name='client2') == "lucy\n" + assert execute_query_https("SELECT currentUser()", user="lucy", cert_name='client3') == "lucy\n" + + +def test_https_wrong_cert(): + # Wrong certificate: different user's certificate + with pytest.raises(Exception) as err: + execute_query_https("SELECT currentUser()", user="john", cert_name='client2') + assert "HTTP Error 403" in str(err.value) + + # Wrong certificate: self-signed certificate. + with pytest.raises(Exception) as err: + execute_query_https("SELECT currentUser()", user="john", cert_name='wrong') + assert "unknown ca" in str(err.value) + + # No certificate. + with pytest.raises(Exception) as err: + execute_query_https("SELECT currentUser()", user="john") + assert "HTTP Error 403" in str(err.value) + + # No header enabling SSL authentication. + with pytest.raises(Exception) as err: + execute_query_https("SELECT currentUser()", user="john", enable_ssl_auth=False, cert_name='client1') + + +def test_https_non_ssl_auth(): + # Users with non-SSL authentication are allowed, in this case we can skip sending a client certificate at all (because "verificationMode" is set to "relaxed"). + #assert execute_query_https("SELECT currentUser()", user="peter", enable_ssl_auth=False) == "peter\n" + assert execute_query_https("SELECT currentUser()", user="jane", enable_ssl_auth=False, password='qwe123') == "jane\n" + + # But we still can send a certificate if we want. + assert execute_query_https("SELECT currentUser()", user="peter", enable_ssl_auth=False, cert_name='client1') == "peter\n" + assert execute_query_https("SELECT currentUser()", user="peter", enable_ssl_auth=False, cert_name='client2') == "peter\n" + assert execute_query_https("SELECT currentUser()", user="peter", enable_ssl_auth=False, cert_name='client3') == "peter\n" + + assert execute_query_https("SELECT currentUser()", user="jane", enable_ssl_auth=False, password='qwe123', cert_name='client1') == "jane\n" + assert execute_query_https("SELECT currentUser()", user="jane", enable_ssl_auth=False, password='qwe123', cert_name='client2') == "jane\n" + assert execute_query_https("SELECT currentUser()", user="jane", enable_ssl_auth=False, password='qwe123', cert_name='client3') == "jane\n" + + # However if we send a certificate it must not be wrong. + with pytest.raises(Exception) as err: + execute_query_https("SELECT currentUser()", user="peter", enable_ssl_auth=False, cert_name='wrong') + assert "unknown ca" in str(err.value) + with pytest.raises(Exception) as err: + execute_query_https("SELECT currentUser()", user="jane", enable_ssl_auth=False, password='qwe123', cert_name='wrong') + assert "unknown ca" in str(err.value) + + +def test_create_user(): + instance.query("CREATE USER emma IDENTIFIED WITH ssl_certificate CN 'client3'") + assert execute_query_https("SELECT currentUser()", user="emma", cert_name='client3') == "emma\n" + assert instance.query("SHOW CREATE USER emma") == "CREATE USER emma IDENTIFIED WITH ssl_certificate CN \\'client3\\'\n" + + instance.query("ALTER USER emma IDENTIFIED WITH ssl_certificate CN 'client2'") + assert execute_query_https("SELECT currentUser()", user="emma", cert_name='client2') == "emma\n" + assert instance.query("SHOW CREATE USER emma") == "CREATE USER emma IDENTIFIED WITH ssl_certificate CN \\'client2\\'\n" + + with pytest.raises(Exception) as err: + execute_query_https("SELECT currentUser()", user="emma", cert_name='client3') + assert "HTTP Error 403" in str(err.value) + + assert instance.query("SHOW CREATE USER lucy") == "CREATE USER lucy IDENTIFIED WITH ssl_certificate CN \\'client2\\', \\'client3\\'\n" + + assert instance.query("SELECT name, auth_type, auth_params FROM system.users WHERE name IN ['emma', 'lucy'] ORDER BY name") ==\ + "emma\tssl_certificate\t{\"common_names\":[\"client2\"]}\n"\ + "lucy\tssl_certificate\t{\"common_names\":[\"client2\",\"client3\"]}\n" diff --git a/tests/integration/test_storage_hdfs/test.py b/tests/integration/test_storage_hdfs/test.py index b0836a38c9e..82b06ade6be 100644 --- a/tests/integration/test_storage_hdfs/test.py +++ b/tests/integration/test_storage_hdfs/test.py @@ -410,6 +410,17 @@ def test_format_detection(started_cluster): assert(int(result) == 1) +def test_schema_inference_with_globs(started_cluster): + node1.query(f"insert into table function hdfs('hdfs://hdfs1:9000/data1.jsoncompacteachrow', 'JSONCompactEachRow', 'x Nullable(UInt32)') select NULL") + node1.query(f"insert into table function hdfs('hdfs://hdfs1:9000/data2.jsoncompacteachrow', 'JSONCompactEachRow', 'x Nullable(UInt32)') select 0") + + result = node1.query(f"desc hdfs('hdfs://hdfs1:9000/data*.jsoncompacteachrow')") + assert(result.strip() == 'c1\tNullable(Float64)') + + result = node1.query(f"select * from hdfs('hdfs://hdfs1:9000/data*.jsoncompacteachrow')") + assert(sorted(result.split()) == ['0', '\\N']) + + if __name__ == '__main__': cluster.start() input("Cluster created, press any key to destroy...") diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index 3fc44469998..7fb880119a7 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -994,6 +994,27 @@ def test_format_detection(started_cluster): assert(int(result) == 1) +def test_schema_inference_from_globs(started_cluster): + bucket = started_cluster.minio_bucket + instance = started_cluster.instances["dummy"] + + instance.query(f"insert into table function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test1.jsoncompacteachrow', 'JSONCompactEachRow', 'x Nullable(UInt32)') select NULL") + instance.query(f"insert into table function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test2.jsoncompacteachrow', 'JSONCompactEachRow', 'x Nullable(UInt32)') select 0") + + url_filename = "test{1,2}.jsoncompacteachrow" + result = instance.query(f"desc url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{url_filename}')") + assert(result.strip() == 'c1\tNullable(Float64)') + + result = instance.query(f"select * from url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{url_filename}')") + assert(sorted(result.split()) == ['0', '\\N']) + + result = instance.query(f"desc s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test*.jsoncompacteachrow')") + assert(result.strip() == 'c1\tNullable(Float64)') + + result = instance.query(f"select * from s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test*.jsoncompacteachrow')") + assert(sorted(result.split()) == ['0', '\\N']) + + def test_signatures(started_cluster): bucket = started_cluster.minio_bucket instance = started_cluster.instances["dummy"] diff --git a/tests/performance/README.md b/tests/performance/README.md index 69d97e5c3d2..2391a1a2ece 100644 --- a/tests/performance/README.md +++ b/tests/performance/README.md @@ -21,6 +21,6 @@ TODO @akuzm ### How to validate single test ``` -pip3 install clickhouse_driver +pip3 install clickhouse_driver scipy ../../docker/test/performance-comparison/perf.py --runs 1 insert_parallel.xml ``` diff --git a/tests/performance/h3.xml b/tests/performance/h3.xml index ce00ebbc9ec..5b697884c42 100644 --- a/tests/performance/h3.xml +++ b/tests/performance/h3.xml @@ -1,6 +1,4 @@ - - - SELECT count() FROM zeros(100000) WHERE NOT ignore(geoToH3(37.62 + rand(1) / 0x100000000, 55.75 + rand(2) / 0x100000000, 15)) + SELECT count() FROM zeros(100000) WHERE NOT ignore(geoToH3(37.62 + rand(1) / 0x100000000, 55.75 + rand(2) / 0x100000000, toUInt8(15))) diff --git a/tests/queries/0_stateless/00398_url_functions.reference b/tests/queries/0_stateless/00398_url_functions.reference index e5f89fbea78..feba95fb1b3 100644 --- a/tests/queries/0_stateless/00398_url_functions.reference +++ b/tests/queries/0_stateless/00398_url_functions.reference @@ -59,6 +59,14 @@ query=hello world+foo+bar query=hello world+foo+bar query=hello world+foo+bar query=hello world foo+bar + + +\N +\N +hello%20world%20foo%2Bbar +hello+world+foo%2Bbar +http://paul@127.0.0.1/?query=hello world foo+bar +http://paul@127.0.0.1/?query=hello world foo+bar ====FRAGMENT==== @@ -74,6 +82,10 @@ query=hello world+foo+bar#a=b query=hello world+foo+bar#a=b #a=b query=hello world foo+bar#a=b +hello%20world%20foo%2Bbar%23a%3Db +hello+world+foo%2Bbar%23a%3Db +http://paul@127.0.0.1/?query=hello world foo+bar#a=b +http://paul@127.0.0.1/?query=hello world foo+bar#a=b ====CUT TO FIRST SIGNIFICANT SUBDOMAIN==== example.com example.com diff --git a/tests/queries/0_stateless/00398_url_functions.sql b/tests/queries/0_stateless/00398_url_functions.sql index ea71ed226d7..66fe591bb58 100644 --- a/tests/queries/0_stateless/00398_url_functions.sql +++ b/tests/queries/0_stateless/00398_url_functions.sql @@ -64,6 +64,14 @@ SELECT decodeURLComponent(queryString('http://127.0.0.1:443/?query=hello%20world SELECT decodeURLComponent(queryString('http://paul@127.0.0.1:443/?query=hello%20world+foo%2Bbar')); SELECT decodeURLComponent(queryString('//paul@127.0.0.1:443/?query=hello%20world+foo%2Bbar')); SELECT decodeURLFormComponent(queryString('//paul@127.0.0.1:443/?query=hello%20world+foo%2Bbar')); +SELECT encodeURLComponent(''); +SELECT encodeURLFormComponent(''); +SELECT encodeURLComponent(NULL); +SELECT encodeURLFormComponent(NULL); +SELECT encodeURLComponent('hello world foo+bar'); +SELECT encodeURLFormComponent('hello world foo+bar'); +SELECT decodeURLComponent(encodeURLComponent('http://paul@127.0.0.1/?query=hello world foo+bar')); +SELECT decodeURLFormComponent(encodeURLFormComponent('http://paul@127.0.0.1/?query=hello world foo+bar')); SELECT '====FRAGMENT===='; SELECT decodeURLComponent(fragment('http://127.0.0.1/?query=hello%20world+foo%2Bbar')); @@ -81,6 +89,10 @@ SELECT decodeURLComponent(queryStringAndFragment('http://paul@127.0.0.1/?query=h SELECT decodeURLComponent(queryStringAndFragment('//paul@127.0.0.1/?query=hello%20world+foo%2Bbar#a=b')); SELECT decodeURLComponent(queryStringAndFragment('//paul@127.0.0.1/#a=b')); SELECT decodeURLFormComponent(queryStringAndFragment('//paul@127.0.0.1/?query=hello%20world+foo%2Bbar#a=b')); +SELECT encodeURLComponent('hello world foo+bar#a=b'); +SELECT encodeURLFormComponent('hello world foo+bar#a=b'); +SELECT decodeURLComponent(encodeURLComponent('http://paul@127.0.0.1/?query=hello world foo+bar#a=b')); +SELECT decodeURLFormComponent(encodeURLFormComponent('http://paul@127.0.0.1/?query=hello world foo+bar#a=b')); SELECT '====CUT TO FIRST SIGNIFICANT SUBDOMAIN===='; SELECT cutToFirstSignificantSubdomain('http://www.example.com'); diff --git a/tests/queries/0_stateless/00502_custom_partitioning_local.sql b/tests/queries/0_stateless/00502_custom_partitioning_local.sql index 55c9b559ed0..b7eb08c919e 100644 --- a/tests/queries/0_stateless/00502_custom_partitioning_local.sql +++ b/tests/queries/0_stateless/00502_custom_partitioning_local.sql @@ -19,7 +19,7 @@ ALTER TABLE not_partitioned DETACH PARTITION ID 'all'; SELECT 'Sum after DETACH PARTITION:'; SELECT sum(x) FROM not_partitioned; SELECT 'system.detached_parts after DETACH PARTITION:'; -SELECT * FROM system.detached_parts WHERE table = 'not_partitioned'; +SELECT * FROM system.detached_parts WHERE database = currentDatabase() AND table = 'not_partitioned'; DROP TABLE not_partitioned; diff --git a/tests/queries/0_stateless/00612_http_max_query_size_for_distributed.reference b/tests/queries/0_stateless/00612_http_max_query_size_for_distributed.reference new file mode 100644 index 00000000000..8a1218a1024 --- /dev/null +++ b/tests/queries/0_stateless/00612_http_max_query_size_for_distributed.reference @@ -0,0 +1,5 @@ +1 +2 +3 +4 +5 diff --git a/tests/queries/0_stateless/00612_http_max_query_size_for_distributed.sql b/tests/queries/0_stateless/00612_http_max_query_size_for_distributed.sql new file mode 100644 index 00000000000..1802fadc57b --- /dev/null +++ b/tests/queries/0_stateless/00612_http_max_query_size_for_distributed.sql @@ -0,0 +1,19 @@ +-- Tags: no-parallel + +DROP TABLE IF EXISTS data_00612; +DROP TABLE IF EXISTS dist_00612; + +CREATE TABLE data_00612 (key UInt64, val UInt64) ENGINE = MergeTree ORDER BY key; +CREATE TABLE dist_00612 AS data_00612 ENGINE = Distributed(test_shard_localhost, currentDatabase(), data_00612, rand()); + +SET insert_distributed_sync=1; +SET prefer_localhost_replica=0; +SET max_query_size=29; +INSERT INTO dist_00612 VALUES(1, 1), (2, 2), (3, 3), (4, 4), (5, 5); +SELECT key FROM dist_00612; + +SET max_query_size=262144; +SET insert_distributed_sync=0; +SET prefer_localhost_replica=1; +DROP TABLE dist_00612; +DROP TABLE data_00612; diff --git a/tests/queries/0_stateless/00900_long_parquet.reference b/tests/queries/0_stateless/00900_long_parquet.reference index 9ee4fc11a55..bbdad7243bd 100644 --- a/tests/queries/0_stateless/00900_long_parquet.reference +++ b/tests/queries/0_stateless/00900_long_parquet.reference @@ -34,32 +34,32 @@ ContextLock Number of times the lock of Context was acquired or tried to acquire. This is global lock. Query Number of queries to be interpreted and potentially executed. Does not include queries that failed to parse or were rejected due to AST size limits, quota limits or limits on the number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries. original: --128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1.032 -1.064 string-1 fixedstring-1\0\0 2003-04-05 2003-02-03 04:05:06 --108 108 -1016 1116 -1032 1132 -1064 1164 -1.032 -1.064 string-0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 04:05:06 -127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1.032 -1.064 string-2 fixedstring-2\0\0 2004-06-07 2004-02-03 04:05:06 +-128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1.032 -1.064 string-1 fixedstring-1\0\0 2003-04-05 2003-02-03 04:05:06 2003-02-03 04:05:06.789000000 +-108 108 -1016 1116 -1032 1132 -1064 1164 -1.032 -1.064 string-0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 04:05:06 2002-02-03 04:05:06.789000000 +127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1.032 -1.064 string-2 fixedstring-2\0\0 2004-06-07 2004-02-03 04:05:06 2004-02-03 04:05:06.789000000 converted: --128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1.032 -1.064 string-1 fixedstring-1\0\0 2003-04-05 2003-02-03 04:05:06 --108 108 -1016 1116 -1032 1132 -1064 1164 -1.032 -1.064 string-0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 04:05:06 -127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1.032 -1.064 string-2 fixedstring-2\0\0 2004-06-07 2004-02-03 04:05:06 +-128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1.032 -1.064 string-1 fixedstring-1\0\0 2003-04-05 2003-02-03 04:05:06 2003-02-03 04:05:06.789000000 +-108 108 -1016 1116 -1032 1132 -1064 1164 -1.032 -1.064 string-0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 04:05:06 2002-02-03 04:05:06.789000000 +127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1.032 -1.064 string-2 fixedstring-2\0\0 2004-06-07 2004-02-03 04:05:06 2004-02-03 04:05:06.789000000 diff: dest: -79 81 82 83 84 85 86 87 88 89 str01\0\0\0\0\0\0\0\0\0\0 fstr1\0\0\0\0\0\0\0\0\0\0 2003-03-04 2004-05-06 00:00:00 -80 81 82 83 84 85 86 87 88 89 str02 fstr2\0\0\0\0\0\0\0\0\0\0 2005-03-04 2006-08-09 10:11:12 +79 81 82 83 84 85 86 87 88 89 str01\0\0\0\0\0\0\0\0\0\0 fstr1\0\0\0\0\0\0\0\0\0\0 2003-03-04 2004-05-06 00:00:00 2004-05-06 07:08:09.012000000 +80 81 82 83 84 85 86 87 88 89 str02 fstr2\0\0\0\0\0\0\0\0\0\0 2005-03-04 2006-08-09 10:11:12 2006-08-09 10:11:12.345000000 min: --128 0 0 0 0 0 0 0 -1 -1 string-1\0\0\0\0\0\0\0 fixedstring-1\0\0 2003-04-05 2003-02-03 --108 108 8 92 -8 108 -40 -116 -1 -1 string-0\0\0\0\0\0\0\0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 -79 81 82 83 84 85 86 87 88 89 str01\0\0\0\0\0\0\0\0\0\0 fstr1\0\0\0\0\0\0\0\0\0\0 2003-03-04 2004-05-06 -127 -1 -1 -1 -1 -1 -1 -1 -1 -1 string-2\0\0\0\0\0\0\0 fixedstring-2\0\0 2004-06-07 2004-02-03 +-128 0 0 0 0 0 0 0 -1 -1 string-1\0\0\0\0\0\0\0 fixedstring-1\0\0 2003-04-05 2003-02-03 2003-02-03 04:05:06.789000000 +-108 108 8 92 -8 108 -40 -116 -1 -1 string-0\0\0\0\0\0\0\0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 2002-02-03 04:05:06.789000000 +79 81 82 83 84 85 86 87 88 89 str01\0\0\0\0\0\0\0\0\0\0 fstr1\0\0\0\0\0\0\0\0\0\0 2003-03-04 2004-05-06 2004-05-06 07:08:09.012000000 +127 -1 -1 -1 -1 -1 -1 -1 -1 -1 string-2\0\0\0\0\0\0\0 fixedstring-2\0\0 2004-06-07 2004-02-03 2004-02-03 04:05:06.789000000 max: --128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1 -1 string-1 fixedstring-1\0\0 2003-04-05 00:00:00 2003-02-03 04:05:06 --108 108 -1016 1116 -1032 1132 -1064 1164 -1 -1 string-0 fixedstring\0\0\0\0 2001-02-03 00:00:00 2002-02-03 04:05:06 -80 81 82 83 84 85 86 87 88 89 str02 fstr2 2005-03-04 05:06:07 2006-08-09 10:11:12 -127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1 -1 string-2 fixedstring-2\0\0 2004-06-07 00:00:00 2004-02-03 04:05:06 +-128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1 -1 string-1 fixedstring-1\0\0 2003-04-05 00:00:00 2003-02-03 04:05:06 2003-02-03 04:05:06.789000000 +-108 108 -1016 1116 -1032 1132 -1064 1164 -1 -1 string-0 fixedstring\0\0\0\0 2001-02-03 00:00:00 2002-02-03 04:05:06 2002-02-03 04:05:06.789000000 +80 81 82 83 84 85 86 87 88 89 str02 fstr2 2005-03-04 05:06:07 2006-08-09 10:11:12 2006-08-09 10:11:12.345000000 +127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1 -1 string-2 fixedstring-2\0\0 2004-06-07 00:00:00 2004-02-03 04:05:06 2004-02-03 04:05:06.789000000 dest from null: --128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1.032 -1.064 string-1 fixedstring-1\0\0 2003-04-05 2003-02-03 04:05:06 --108 108 -1016 1116 -1032 1132 -1064 1164 -1.032 -1.064 string-0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 04:05:06 -127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1.032 -1.064 string-2 fixedstring-2\0\0 2004-06-07 2004-02-03 04:05:06 -\N \N \N \N \N \N \N \N \N \N \N \N \N \N +-128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1.032 -1.064 string-1 fixedstring-1\0\0 2003-04-05 2003-02-03 04:05:06 2003-02-03 04:05:06.789000000 +-108 108 -1016 1116 -1032 1132 -1064 1164 -1.032 -1.064 string-0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 04:05:06 2002-02-03 04:05:06.789000000 +127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1.032 -1.064 string-2 fixedstring-2\0\0 2004-06-07 2004-02-03 04:05:06 2004-02-03 04:05:06.789000000 +\N \N \N \N \N \N \N \N \N \N \N \N \N \N \N 1 [1,-2,3] [1,2,3] [100,-200,300] [100,200,300] [10000000,-20000000,30000000] [10000000,2000000,3000000] [100000000000000,-200000000000,3000000000000] [100000000000000,20000000000000,3000000000000] ['Some string','Some string','Some string'] ['0000','1111','2222'] [42.42,424.2,0.4242] [424242.424242,4242042420.242424,42] ['2000-01-01','2001-01-01','2002-01-01'] ['2000-01-01 00:00:00','2001-01-01 00:00:00','2002-01-01 00:00:00'] [0.2,10,4] [4,10000.1,10000.1] [1000000000,90,101001.01] 1 [1,-2,3] [1,2,3] [100,-200,300] [100,200,300] [10000000,-20000000,30000000] [10000000,2000000,3000000] [100000000000000,-200000000000,3000000000000] [100000000000000,20000000000000,3000000000000] ['Some string','Some string','Some string'] ['0000','1111','2222'] [42.42,424.2,0.4242] [424242.424242,4242042420.242424,42] ['2000-01-01','2001-01-01','2002-01-01'] ['2000-01-01 00:00:00','2001-01-01 00:00:00','2002-01-01 00:00:00'] [0.2,10,4] [4,10000.1,10000.1] [1000000000,90,101001.01] 2 [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] diff --git a/tests/queries/0_stateless/00900_long_parquet.sh b/tests/queries/0_stateless/00900_long_parquet.sh index 6d2d57b5a6e..5d6317b2787 100755 --- a/tests/queries/0_stateless/00900_long_parquet.sh +++ b/tests/queries/0_stateless/00900_long_parquet.sh @@ -46,20 +46,20 @@ ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS parquet_types1" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS parquet_types2" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS parquet_types3" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS parquet_types4" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_types1 (int8 Int8, uint8 UInt8, int16 Int16, uint16 UInt16, int32 Int32, uint32 UInt32, int64 Int64, uint64 UInt64, float32 Float32, float64 Float64, string String, fixedstring FixedString(15), date Date, datetime DateTime) ENGINE = Memory" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_types2 (int8 Int8, uint8 UInt8, int16 Int16, uint16 UInt16, int32 Int32, uint32 UInt32, int64 Int64, uint64 UInt64, float32 Float32, float64 Float64, string String, fixedstring FixedString(15), date Date, datetime DateTime) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_types1 (int8 Int8, uint8 UInt8, int16 Int16, uint16 UInt16, int32 Int32, uint32 UInt32, int64 Int64, uint64 UInt64, float32 Float32, float64 Float64, string String, fixedstring FixedString(15), date Date, datetime DateTime, datetime64 DateTime64(9)) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_types2 (int8 Int8, uint8 UInt8, int16 Int16, uint16 UInt16, int32 Int32, uint32 UInt32, int64 Int64, uint64 UInt64, float32 Float32, float64 Float64, string String, fixedstring FixedString(15), date Date, datetime DateTime, datetime64 DateTime64(9)) ENGINE = Memory" # convert min type -${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_types3 (int8 Int8, uint8 Int8, int16 Int8, uint16 Int8, int32 Int8, uint32 Int8, int64 Int8, uint64 Int8, float32 Int8, float64 Int8, string FixedString(15), fixedstring FixedString(15), date Date, datetime Date) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_types3 (int8 Int8, uint8 Int8, int16 Int8, uint16 Int8, int32 Int8, uint32 Int8, int64 Int8, uint64 Int8, float32 Int8, float64 Int8, string FixedString(15), fixedstring FixedString(15), date Date, datetime Date, datetime64 DateTime64(9)) ENGINE = Memory" # convert max type -${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_types4 (int8 Int64, uint8 Int64, int16 Int64, uint16 Int64, int32 Int64, uint32 Int64, int64 Int64, uint64 Int64, float32 Int64, float64 Int64, string String, fixedstring String, date DateTime, datetime DateTime) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_types4 (int8 Int64, uint8 Int64, int16 Int64, uint16 Int64, int32 Int64, uint32 Int64, int64 Int64, uint64 Int64, float32 Int64, float64 Int64, string String, fixedstring String, date DateTime, datetime DateTime, datetime64 DateTime64(9)) ENGINE = Memory" -${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types1 values ( -108, 108, -1016, 1116, -1032, 1132, -1064, 1164, -1.032, -1.064, 'string-0', 'fixedstring', '2001-02-03', '2002-02-03 04:05:06')" +${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types1 values ( -108, 108, -1016, 1116, -1032, 1132, -1064, 1164, -1.032, -1.064, 'string-0', 'fixedstring', '2001-02-03', '2002-02-03 04:05:06', toDateTime64('2002-02-03 04:05:06.789', 9))" # min -${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types1 values ( -128, 0, -32768, 0, -2147483648, 0, -9223372036854775808, 0, -1.032, -1.064, 'string-1', 'fixedstring-1', '2003-04-05', '2003-02-03 04:05:06')" +${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types1 values ( -128, 0, -32768, 0, -2147483648, 0, -9223372036854775808, 0, -1.032, -1.064, 'string-1', 'fixedstring-1', '2003-04-05', '2003-02-03 04:05:06', toDateTime64('2003-02-03 04:05:06.789', 9))" # max -${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types1 values ( 127, 255, 32767, 65535, 2147483647, 4294967295, 9223372036854775807, 9223372036854775807, -1.032, -1.064, 'string-2', 'fixedstring-2', '2004-06-07', '2004-02-03 04:05:06')" +${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types1 values ( 127, 255, 32767, 65535, 2147483647, 4294967295, 9223372036854775807, 9223372036854775807, -1.032, -1.064, 'string-2', 'fixedstring-2', '2004-06-07', '2004-02-03 04:05:06', toDateTime64('2004-02-03 04:05:06.789', 9))" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_types1 FORMAT Parquet" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types2 FORMAT Parquet" @@ -73,11 +73,11 @@ echo diff: diff "${CLICKHOUSE_TMP}"/parquet_all_types_1.dump "${CLICKHOUSE_TMP}"/parquet_all_types_2.dump ${CLICKHOUSE_CLIENT} --query="TRUNCATE TABLE parquet_types2" -${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types3 values ( 79, 81, 82, 83, 84, 85, 86, 87, 88, 89, 'str01', 'fstr1', '2003-03-04', '2004-05-06')" +${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types3 values ( 79, 81, 82, 83, 84, 85, 86, 87, 88, 89, 'str01', 'fstr1', '2003-03-04', '2004-05-06', toDateTime64('2004-05-06 07:08:09.012', 9))" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_types3 ORDER BY int8 FORMAT Parquet" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types2 FORMAT Parquet" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_types1 ORDER BY int8 FORMAT Parquet" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types3 FORMAT Parquet" -${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types4 values ( 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 'str02', 'fstr2', '2005-03-04 05:06:07', '2006-08-09 10:11:12')" +${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types4 values ( 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 'str02', 'fstr2', '2005-03-04 05:06:07', '2006-08-09 10:11:12', toDateTime64('2006-08-09 10:11:12.345', 9))" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_types4 ORDER BY int8 FORMAT Parquet" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types2 FORMAT Parquet" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_types1 ORDER BY int8 FORMAT Parquet" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types4 FORMAT Parquet" @@ -92,9 +92,9 @@ ${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_types4 ORDER BY int8" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS parquet_types5" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS parquet_types6" ${CLICKHOUSE_CLIENT} --query="TRUNCATE TABLE parquet_types2" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_types5 (int8 Nullable(Int8), uint8 Nullable(UInt8), int16 Nullable(Int16), uint16 Nullable(UInt16), int32 Nullable(Int32), uint32 Nullable(UInt32), int64 Nullable(Int64), uint64 Nullable(UInt64), float32 Nullable(Float32), float64 Nullable(Float64), string Nullable(String), fixedstring Nullable(FixedString(15)), date Nullable(Date), datetime Nullable(DateTime)) ENGINE = Memory" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_types6 (int8 Nullable(Int8), uint8 Nullable(UInt8), int16 Nullable(Int16), uint16 Nullable(UInt16), int32 Nullable(Int32), uint32 Nullable(UInt32), int64 Nullable(Int64), uint64 Nullable(UInt64), float32 Nullable(Float32), float64 Nullable(Float64), string Nullable(String), fixedstring Nullable(FixedString(15)), date Nullable(Date), datetime Nullable(DateTime)) ENGINE = Memory" -${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types5 values ( NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_types5 (int8 Nullable(Int8), uint8 Nullable(UInt8), int16 Nullable(Int16), uint16 Nullable(UInt16), int32 Nullable(Int32), uint32 Nullable(UInt32), int64 Nullable(Int64), uint64 Nullable(UInt64), float32 Nullable(Float32), float64 Nullable(Float64), string Nullable(String), fixedstring Nullable(FixedString(15)), date Nullable(Date), datetime Nullable(DateTime), datetime64 Nullable(DateTime64(9))) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_types6 (int8 Nullable(Int8), uint8 Nullable(UInt8), int16 Nullable(Int16), uint16 Nullable(UInt16), int32 Nullable(Int32), uint32 Nullable(UInt32), int64 Nullable(Int64), uint64 Nullable(UInt64), float32 Nullable(Float32), float64 Nullable(Float64), string Nullable(String), fixedstring Nullable(FixedString(15)), date Nullable(Date), datetime Nullable(DateTime), datetime64 Nullable(DateTime64(9))) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types5 values ( NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_types5 ORDER BY int8 FORMAT Parquet" > "${CLICKHOUSE_TMP}"/parquet_all_types_5.parquet ${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_types5 ORDER BY int8 FORMAT Parquet" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types6 FORMAT Parquet" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_types1 ORDER BY int8 FORMAT Parquet" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types6 FORMAT Parquet" diff --git a/tests/queries/0_stateless/00900_orc_load.reference b/tests/queries/0_stateless/00900_orc_load.reference index 04c72eccb17..d4590fe5790 100644 --- a/tests/queries/0_stateless/00900_orc_load.reference +++ b/tests/queries/0_stateless/00900_orc_load.reference @@ -1,4 +1,4 @@ -0 0 0 0 0 2019-01-01 test1 -2147483647 -1 9223372036854775806 123.345345 345345.3453451212 2019-01-01 test2 -0 0 0 0 0 2019-01-01 test1 -2147483647 -1 9223372036854775806 123.345345 345345.3453451212 2019-01-01 test2 +0 0 0 0 0 2019-01-01 test1 2019-01-01 02:03:04.567 +2147483647 -1 9223372036854775806 123.345345 345345.3453451212 2019-01-01 test2 2019-01-01 02:03:04.567 +0 0 0 0 0 2019-01-01 test1 2019-01-01 02:03:04.567 +2147483647 -1 9223372036854775806 123.345345 345345.3453451212 2019-01-01 test2 2019-01-01 02:03:04.567 diff --git a/tests/queries/0_stateless/00900_orc_load.sh b/tests/queries/0_stateless/00900_orc_load.sh index 4503c8eb8a5..b3f2c39e5d2 100755 --- a/tests/queries/0_stateless/00900_orc_load.sh +++ b/tests/queries/0_stateless/00900_orc_load.sh @@ -8,9 +8,13 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) DATA_FILE=$CUR_DIR/data_orc/test.orc ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS orc_load" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE orc_load (int Int32, smallint Int8, bigint Int64, float Float32, double Float64, date Date, y String) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE orc_load (int Int32, smallint Int8, bigint Int64, float Float32, double Float64, date Date, y String, datetime64 DateTime64(3)) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="insert into orc_load values (0, 0, 0, 0, 0, '2019-01-01', 'test1', toDateTime64('2019-01-01 02:03:04.567', 3)), (2147483647, -1, 9223372036854775806, 123.345345, 345345.3453451212, '2019-01-01', 'test2', toDateTime64('2019-01-01 02:03:04.567', 3))" +${CLICKHOUSE_CLIENT} --query="select * from orc_load FORMAT ORC" > $DATA_FILE +${CLICKHOUSE_CLIENT} --query="truncate table orc_load" + cat "$DATA_FILE" | ${CLICKHOUSE_CLIENT} -q "insert into orc_load format ORC" timeout 3 ${CLICKHOUSE_CLIENT} -q "insert into orc_load format ORC" < $DATA_FILE ${CLICKHOUSE_CLIENT} --query="select * from orc_load" - ${CLICKHOUSE_CLIENT} --query="drop table orc_load" +rm -rf "$DATA_FILE" diff --git a/tests/queries/0_stateless/00926_multimatch.sql b/tests/queries/0_stateless/00926_multimatch.sql index 6a90bfec914..90cc289b5a5 100644 --- a/tests/queries/0_stateless/00926_multimatch.sql +++ b/tests/queries/0_stateless/00926_multimatch.sql @@ -1,5 +1,4 @@ --- Tags: no-fasttest --- Tag no-fasttest: Hyperscan +-- Tags: no-fasttest, use-hyperscan select 0 = multiMatchAny(materialize('mpnsguhwsitzvuleiwebwjfitmsg'), ['wbirxqoabpblrnvvmjizj', 'cfcxhuvrexyzyjsh', 'oldhtubemyuqlqbwvwwkwin', 'bumoozxdkjglzu', 'intxlfohlxmajjomw', 'dxkeghohv', 'arsvmwwkjeopnlwnan', 'ouugllgowpqtaxslcopkytbfhifaxbgt', 'hkedmjlbcrzvryaopjqdjjc', 'tbqkljywstuahzh', 'o', 'wowoclosyfcuwotmvjygzuzhrery', 'vpefjiffkhlggntcu', 'ytdixvasrorhripzfhjdmlhqksmctyycwp']) from system.numbers limit 10; select 0 = multiMatchAny(materialize('qjjzqexjpgkglgxpzrbqbnskq'), ['vaiatcjacmlffdzsejpdareqzy', 'xspcfzdufkmecud', 'bcvtbuqtctq', 'nkcopwbfytgemkqcfnnno', 'dylxnzuyhq', 'tno', 'scukuhufly', 'cdyquzuqlptv', 'ohluyfeksyxepezdhqmtfmgkvzsyph', 'ualzwtahvqvtijwp', 'jg', 'gwbawqlngzcknzgtmlj', 'qimvjcgbkkp', 'eaedbcgyrdvv', 'qcwrncjoewwedyyewcdkh', 'uqcvhngoqngmitjfxpznqomertqnqcveoqk', 'ydrgjiankgygpm', 'axepgap']) from system.numbers limit 10; diff --git a/tests/queries/0_stateless/00929_multi_match_edit_distance.sql b/tests/queries/0_stateless/00929_multi_match_edit_distance.sql index 176e2a24b28..5f13477aa28 100644 --- a/tests/queries/0_stateless/00929_multi_match_edit_distance.sql +++ b/tests/queries/0_stateless/00929_multi_match_edit_distance.sql @@ -1,5 +1,4 @@ --- Tags: no-fasttest --- Tag no-fasttest: Hyperscan +-- Tags: no-fasttest, use-hyperscan SET send_logs_level = 'fatal'; diff --git a/tests/queries/0_stateless/01273_arrow.reference b/tests/queries/0_stateless/01273_arrow.reference index 89eca82f8ef..b4d011754dc 100644 --- a/tests/queries/0_stateless/01273_arrow.reference +++ b/tests/queries/0_stateless/01273_arrow.reference @@ -32,29 +32,29 @@ 991 990 original: --128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1.032 -1.064 string-1 fixedstring-1\0\0 2003-04-05 2003-02-03 04:05:06 --108 108 -1016 1116 -1032 1132 -1064 1164 -1.032 -1.064 string-0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 04:05:06 -127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1.032 -1.064 string-2 fixedstring-2\0\0 2004-06-07 2004-02-03 04:05:06 +-128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1.032 -1.064 string-1 fixedstring-1\0\0 2003-04-05 2003-02-03 04:05:06 2003-02-03 04:05:06.789012 +-108 108 -1016 1116 -1032 1132 -1064 1164 -1.032 -1.064 string-0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 04:05:06 2002-02-03 04:05:06.789012 +127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1.032 -1.064 string-2 fixedstring-2\0\0 2004-06-07 2004-02-03 04:05:06 2004-02-03 04:05:06.789012 converted: --128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1.032 -1.064 string-1 fixedstring-1\0\0 2003-04-05 2003-02-03 04:05:06 --108 108 -1016 1116 -1032 1132 -1064 1164 -1.032 -1.064 string-0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 04:05:06 -127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1.032 -1.064 string-2 fixedstring-2\0\0 2004-06-07 2004-02-03 04:05:06 +-128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1.032 -1.064 string-1 fixedstring-1\0\0 2003-04-05 2003-02-03 04:05:06 2003-02-03 04:05:06.789012 +-108 108 -1016 1116 -1032 1132 -1064 1164 -1.032 -1.064 string-0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 04:05:06 2002-02-03 04:05:06.789012 +127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1.032 -1.064 string-2 fixedstring-2\0\0 2004-06-07 2004-02-03 04:05:06 2004-02-03 04:05:06.789012 diff: dest: -79 81 82 83 84 85 86 87 88 89 str01\0\0\0\0\0\0\0\0\0\0 fstr1\0\0\0\0\0\0\0\0\0\0 2003-03-04 2004-05-06 00:00:00 -80 81 82 83 84 85 86 87 88 89 str02 fstr2\0\0\0\0\0\0\0\0\0\0 2005-03-04 2006-08-09 10:11:12 +79 81 82 83 84 85 86 87 88 89 str01\0\0\0\0\0\0\0\0\0\0 fstr1\0\0\0\0\0\0\0\0\0\0 2003-03-04 2004-05-06 00:00:00 2005-02-03 04:05:06.789012 +80 81 82 83 84 85 86 87 88 89 str02 fstr2\0\0\0\0\0\0\0\0\0\0 2005-03-04 2006-08-09 10:11:12 2007-02-03 04:05:06.789012 min: --128 0 0 0 0 0 0 0 -1 -1 string-1\0\0\0\0\0\0\0 fixedstring-1\0\0 2003-04-05 2003-02-03 --108 108 8 92 -8 108 -40 -116 -1 -1 string-0\0\0\0\0\0\0\0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 -79 81 82 83 84 85 86 87 88 89 str01\0\0\0\0\0\0\0\0\0\0 fstr1\0\0\0\0\0\0\0\0\0\0 2003-03-04 2004-05-06 -127 -1 -1 -1 -1 -1 -1 -1 -1 -1 string-2\0\0\0\0\0\0\0 fixedstring-2\0\0 2004-06-07 2004-02-03 +-128 0 0 0 0 0 0 0 -1 -1 string-1\0\0\0\0\0\0\0 fixedstring-1\0\0 2003-04-05 2003-02-03 2003-02-03 04:05:06.789012 +-108 108 8 92 -8 108 -40 -116 -1 -1 string-0\0\0\0\0\0\0\0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 2002-02-03 04:05:06.789012 +79 81 82 83 84 85 86 87 88 89 str01\0\0\0\0\0\0\0\0\0\0 fstr1\0\0\0\0\0\0\0\0\0\0 2003-03-04 2004-05-06 2005-02-03 04:05:06.789012 +127 -1 -1 -1 -1 -1 -1 -1 -1 -1 string-2\0\0\0\0\0\0\0 fixedstring-2\0\0 2004-06-07 2004-02-03 2004-02-03 04:05:06.789012 max: --128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1 -1 string-1 fixedstring-1\0\0 2003-04-05 00:00:00 2003-02-03 04:05:06 --108 108 -1016 1116 -1032 1132 -1064 1164 -1 -1 string-0 fixedstring\0\0\0\0 2001-02-03 00:00:00 2002-02-03 04:05:06 -80 81 82 83 84 85 86 87 88 89 str02 fstr2 2005-03-04 05:06:07 2006-08-09 10:11:12 -127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1 -1 string-2 fixedstring-2\0\0 2004-06-07 00:00:00 2004-02-03 04:05:06 +-128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1 -1 string-1 fixedstring-1\0\0 2003-04-05 00:00:00 2003-02-03 04:05:06 2003-02-03 04:05:06.789012 +-108 108 -1016 1116 -1032 1132 -1064 1164 -1 -1 string-0 fixedstring\0\0\0\0 2001-02-03 00:00:00 2002-02-03 04:05:06 2002-02-03 04:05:06.789012 +80 81 82 83 84 85 86 87 88 89 str02 fstr2 2005-03-04 05:06:07 2006-08-09 10:11:12 2007-02-03 04:05:06.789012 +127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1 -1 string-2 fixedstring-2\0\0 2004-06-07 00:00:00 2004-02-03 04:05:06 2004-02-03 04:05:06.789012 dest from null: --128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1.032 -1.064 string-1 fixedstring-1\0\0 2003-04-05 2003-02-03 04:05:06 --108 108 -1016 1116 -1032 1132 -1064 1164 -1.032 -1.064 string-0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 04:05:06 -127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1.032 -1.064 string-2 fixedstring-2\0\0 2004-06-07 2004-02-03 04:05:06 -\N \N \N \N \N \N \N \N \N \N \N \N \N \N +-128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1.032 -1.064 string-1 fixedstring-1\0\0 2003-04-05 2003-02-03 04:05:06 2003-02-03 04:05:06.789 +-108 108 -1016 1116 -1032 1132 -1064 1164 -1.032 -1.064 string-0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 04:05:06 2002-02-03 04:05:06.789 +127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1.032 -1.064 string-2 fixedstring-2\0\0 2004-06-07 2004-02-03 04:05:06 2004-02-03 04:05:06.789 +\N \N \N \N \N \N \N \N \N \N \N \N \N \N \N diff --git a/tests/queries/0_stateless/01273_arrow.sh b/tests/queries/0_stateless/01273_arrow.sh index 01a27ac12d1..90638b37c3e 100755 --- a/tests/queries/0_stateless/01273_arrow.sh +++ b/tests/queries/0_stateless/01273_arrow.sh @@ -40,20 +40,20 @@ ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types1" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types2" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types3" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types4" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types1 (int8 Int8, uint8 UInt8, int16 Int16, uint16 UInt16, int32 Int32, uint32 UInt32, int64 Int64, uint64 UInt64, float32 Float32, float64 Float64, string String, fixedstring FixedString(15), date Date, datetime DateTime) ENGINE = Memory" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types2 (int8 Int8, uint8 UInt8, int16 Int16, uint16 UInt16, int32 Int32, uint32 UInt32, int64 Int64, uint64 UInt64, float32 Float32, float64 Float64, string String, fixedstring FixedString(15), date Date, datetime DateTime) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types1 (int8 Int8, uint8 UInt8, int16 Int16, uint16 UInt16, int32 Int32, uint32 UInt32, int64 Int64, uint64 UInt64, float32 Float32, float64 Float64, string String, fixedstring FixedString(15), date Date, datetime DateTime, datetime64 DateTime64(6)) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types2 (int8 Int8, uint8 UInt8, int16 Int16, uint16 UInt16, int32 Int32, uint32 UInt32, int64 Int64, uint64 UInt64, float32 Float32, float64 Float64, string String, fixedstring FixedString(15), date Date, datetime DateTime, datetime64 DateTime64(6)) ENGINE = Memory" # convert min type -${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types3 (int8 Int8, uint8 Int8, int16 Int8, uint16 Int8, int32 Int8, uint32 Int8, int64 Int8, uint64 Int8, float32 Int8, float64 Int8, string FixedString(15), fixedstring FixedString(15), date Date, datetime Date) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types3 (int8 Int8, uint8 Int8, int16 Int8, uint16 Int8, int32 Int8, uint32 Int8, int64 Int8, uint64 Int8, float32 Int8, float64 Int8, string FixedString(15), fixedstring FixedString(15), date Date, datetime Date, datetime64 DateTime64(6)) ENGINE = Memory" # convert max type -${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types4 (int8 Int64, uint8 Int64, int16 Int64, uint16 Int64, int32 Int64, uint32 Int64, int64 Int64, uint64 Int64, float32 Int64, float64 Int64, string String, fixedstring String, date DateTime, datetime DateTime) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types4 (int8 Int64, uint8 Int64, int16 Int64, uint16 Int64, int32 Int64, uint32 Int64, int64 Int64, uint64 Int64, float32 Int64, float64 Int64, string String, fixedstring String, date DateTime, datetime DateTime, datetime64 DateTime64(6)) ENGINE = Memory" -${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types1 values ( -108, 108, -1016, 1116, -1032, 1132, -1064, 1164, -1.032, -1.064, 'string-0', 'fixedstring', '2001-02-03', '2002-02-03 04:05:06')" +${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types1 values ( -108, 108, -1016, 1116, -1032, 1132, -1064, 1164, -1.032, -1.064, 'string-0', 'fixedstring', '2001-02-03', '2002-02-03 04:05:06', toDateTime64('2002-02-03 04:05:06.789012', 6))" # min -${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types1 values ( -128, 0, -32768, 0, -2147483648, 0, -9223372036854775808, 0, -1.032, -1.064, 'string-1', 'fixedstring-1', '2003-04-05', '2003-02-03 04:05:06')" +${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types1 values ( -128, 0, -32768, 0, -2147483648, 0, -9223372036854775808, 0, -1.032, -1.064, 'string-1', 'fixedstring-1', '2003-04-05', '2003-02-03 04:05:06', toDateTime64('2003-02-03 04:05:06.789012', 6))" # max -${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types1 values ( 127, 255, 32767, 65535, 2147483647, 4294967295, 9223372036854775807, 9223372036854775807, -1.032, -1.064, 'string-2', 'fixedstring-2', '2004-06-07', '2004-02-03 04:05:06')" +${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types1 values ( 127, 255, 32767, 65535, 2147483647, 4294967295, 9223372036854775807, 9223372036854775807, -1.032, -1.064, 'string-2', 'fixedstring-2', '2004-06-07', '2004-02-03 04:05:06', toDateTime64('2004-02-03 04:05:06.789012', 6))" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types1 FORMAT Arrow" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types2 FORMAT Arrow" @@ -67,11 +67,11 @@ echo diff: diff "${CLICKHOUSE_TMP}"/arrow_all_types_1.dump "${CLICKHOUSE_TMP}"/arrow_all_types_2.dump ${CLICKHOUSE_CLIENT} --query="TRUNCATE TABLE arrow_types2" -${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types3 values ( 79, 81, 82, 83, 84, 85, 86, 87, 88, 89, 'str01', 'fstr1', '2003-03-04', '2004-05-06')" +${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types3 values ( 79, 81, 82, 83, 84, 85, 86, 87, 88, 89, 'str01', 'fstr1', '2003-03-04', '2004-05-06', toDateTime64('2005-02-03 04:05:06.789012', 6))" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types3 ORDER BY int8 FORMAT Arrow" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types2 FORMAT Arrow" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types1 ORDER BY int8 FORMAT Arrow" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types3 FORMAT Arrow" -${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types4 values ( 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 'str02', 'fstr2', '2005-03-04 05:06:07', '2006-08-09 10:11:12')" +${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types4 values ( 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 'str02', 'fstr2', '2005-03-04 05:06:07', '2006-08-09 10:11:12', toDateTime64('2007-02-03 04:05:06.789012', 6))" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types4 ORDER BY int8 FORMAT Arrow" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types2 FORMAT Arrow" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types1 ORDER BY int8 FORMAT Arrow" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types4 FORMAT Arrow" @@ -86,9 +86,9 @@ ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types4 ORDER BY int8" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types5" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types6" ${CLICKHOUSE_CLIENT} --query="TRUNCATE TABLE arrow_types2" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types5 (int8 Nullable(Int8), uint8 Nullable(UInt8), int16 Nullable(Int16), uint16 Nullable(UInt16), int32 Nullable(Int32), uint32 Nullable(UInt32), int64 Nullable(Int64), uint64 Nullable(UInt64), float32 Nullable(Float32), float64 Nullable(Float64), string Nullable(String), fixedstring Nullable(FixedString(15)), date Nullable(Date), datetime Nullable(DateTime)) ENGINE = Memory" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types6 (int8 Nullable(Int8), uint8 Nullable(UInt8), int16 Nullable(Int16), uint16 Nullable(UInt16), int32 Nullable(Int32), uint32 Nullable(UInt32), int64 Nullable(Int64), uint64 Nullable(UInt64), float32 Nullable(Float32), float64 Nullable(Float64), string Nullable(String), fixedstring Nullable(FixedString(15)), date Nullable(Date), datetime Nullable(DateTime)) ENGINE = Memory" -${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types5 values ( NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types5 (int8 Nullable(Int8), uint8 Nullable(UInt8), int16 Nullable(Int16), uint16 Nullable(UInt16), int32 Nullable(Int32), uint32 Nullable(UInt32), int64 Nullable(Int64), uint64 Nullable(UInt64), float32 Nullable(Float32), float64 Nullable(Float64), string Nullable(String), fixedstring Nullable(FixedString(15)), date Nullable(Date), datetime Nullable(DateTime), datetime64 Nullable(DateTime64)) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types6 (int8 Nullable(Int8), uint8 Nullable(UInt8), int16 Nullable(Int16), uint16 Nullable(UInt16), int32 Nullable(Int32), uint32 Nullable(UInt32), int64 Nullable(Int64), uint64 Nullable(UInt64), float32 Nullable(Float32), float64 Nullable(Float64), string Nullable(String), fixedstring Nullable(FixedString(15)), date Nullable(Date), datetime Nullable(DateTime), datetime64 Nullable(DateTime64)) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types5 values ( NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types5 ORDER BY int8 FORMAT Arrow" > "${CLICKHOUSE_TMP}"/arrow_all_types_5.arrow ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types5 ORDER BY int8 FORMAT Arrow" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types6 FORMAT Arrow" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types1 ORDER BY int8 FORMAT Arrow" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types6 FORMAT Arrow" diff --git a/tests/queries/0_stateless/01316_create_user_syntax_hilite.reference b/tests/queries/0_stateless/01316_create_user_syntax_hilite.reference index ed7daeb3609..d1e2cba5663 100644 --- a/tests/queries/0_stateless/01316_create_user_syntax_hilite.reference +++ b/tests/queries/0_stateless/01316_create_user_syntax_hilite.reference @@ -1 +1 @@ -CREATE USER user IDENTIFIED WITH plaintext_password BY 'hello' +CREATE USER user IDENTIFIED WITH plaintext_password BY 'hello' diff --git a/tests/queries/0_stateless/01428_nullable_asof_join.sql b/tests/queries/0_stateless/01428_nullable_asof_join.sql index c812e6ecfab..30e5c51eb1c 100644 --- a/tests/queries/0_stateless/01428_nullable_asof_join.sql +++ b/tests/queries/0_stateless/01428_nullable_asof_join.sql @@ -18,13 +18,13 @@ SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(ma FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a ASOF LEFT JOIN (SELECT 1 as pk, toNullable(0) as dt) b USING(pk, dt) -ORDER BY a.dt;-- { serverError 48 } +ORDER BY a.dt; -- { serverError 48 } SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt)) FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a ASOF LEFT JOIN (SELECT 1 as pk, toNullable(0) as dt) b USING(pk, dt) -ORDER BY a.dt;-- { serverError 48 } +ORDER BY a.dt; -- { serverError 48 } select 'left asof on'; @@ -44,13 +44,13 @@ SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(ma FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a ASOF LEFT JOIN (SELECT 1 as pk, toNullable(0) as dt) b ON a.pk = b.pk AND a.dt >= b.dt -ORDER BY a.dt;-- { serverError 48 } +ORDER BY a.dt; -- { serverError 48 } SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt)) FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a ASOF LEFT JOIN (SELECT 1 as pk, toNullable(0) as dt) b -ON a.pk = b.pk AND a.dt >= b.dt -ORDER BY a.dt;-- { serverError 48 } +ON a.dt >= b.dt AND a.pk = b.pk +ORDER BY a.dt; -- { serverError 48 } select 'asof using'; @@ -70,13 +70,13 @@ SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(ma FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a ASOF JOIN (SELECT 1 as pk, toNullable(0) as dt) b USING(pk, dt) -ORDER BY a.dt;-- { serverError 48 } +ORDER BY a.dt; -- { serverError 48 } SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt)) FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a ASOF JOIN (SELECT 1 as pk, toNullable(0) as dt) b USING(pk, dt) -ORDER BY a.dt;-- { serverError 48 } +ORDER BY a.dt; -- { serverError 48 } select 'asof on'; @@ -96,10 +96,16 @@ SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(ma FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a ASOF JOIN (SELECT 1 as pk, toNullable(0) as dt) b ON a.pk = b.pk AND a.dt >= b.dt -ORDER BY a.dt;-- { serverError 48 } +ORDER BY a.dt; -- { serverError 48 } SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt)) FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a ASOF JOIN (SELECT 1 as pk, toNullable(0) as dt) b ON a.pk = b.pk AND a.dt >= b.dt -ORDER BY a.dt;-- { serverError 48 } +ORDER BY a.dt; -- { serverError 48 } + +SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt)) +FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a +ASOF JOIN (SELECT 1 as pk, toNullable(0) as dt) b +ON a.dt >= b.dt AND a.pk = b.pk +ORDER BY a.dt; -- { serverError 48 } diff --git a/tests/queries/0_stateless/01502_jemalloc_percpu_arena.reference b/tests/queries/0_stateless/01502_jemalloc_percpu_arena.reference index 1eb57a24638..fe093e39a56 100644 --- a/tests/queries/0_stateless/01502_jemalloc_percpu_arena.reference +++ b/tests/queries/0_stateless/01502_jemalloc_percpu_arena.reference @@ -1,4 +1,5 @@ +: Number of CPUs detected is not deterministic. Per-CPU arena disabled. 1 -: Number of CPUs is not deterministic -: Number of CPUs is not deterministic, but narenas is set. Hope you not what you are doing and you have set narenas to largest possible CPU ID. +: Number of CPUs detected is not deterministic. Per-CPU arena disabled. +100000000 1 diff --git a/tests/queries/0_stateless/01502_jemalloc_percpu_arena.sh b/tests/queries/0_stateless/01502_jemalloc_percpu_arena.sh index 265ca4a6763..b3ea6eca3f4 100755 --- a/tests/queries/0_stateless/01502_jemalloc_percpu_arena.sh +++ b/tests/queries/0_stateless/01502_jemalloc_percpu_arena.sh @@ -9,10 +9,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) ncpus="$(getconf _NPROCESSORS_ONLN)" -# In debug build the following settings enabled by default: -# - abort_conf -# - abort -# Disable them explicitly (will enable when required). +# Disable them explicitly, to avoid failing on "Number of CPUs is not deterministic". export MALLOC_CONF=abort_conf:false,abort:false # Regression for: @@ -20,17 +17,10 @@ export MALLOC_CONF=abort_conf:false,abort:false # $ taskset --cpu-list 8 ./clickhouse local -q 'select 1' # : ../contrib/jemalloc/src/jemalloc.c:321: Failed assertion: "ind <= narenas_total_get()" # Aborted (core dumped) -taskset --cpu-list $((ncpus-1)) ${CLICKHOUSE_LOCAL} -q 'select 1' -# just in case something more complicated -taskset --cpu-list $((ncpus-1)) ${CLICKHOUSE_LOCAL} -q 'select * from numbers_mt(100000000) settings max_threads=100 FORMAT Null' +taskset --cpu-list $((ncpus-1)) ${CLICKHOUSE_LOCAL} -q 'select 1' 2>&1 -# this command should fail because percpu arena will be disabled, -# and with abort_conf:true it is not allowed -( - # subshell is required to suppress "Aborted" message from the shell. - MALLOC_CONF=abort_conf:true,abort:true - taskset --cpu-list $((ncpus-1)) ${CLICKHOUSE_LOCAL} -q 'select 1' -) |& grep -F 'Number of CPUs is not deterministic' +# just in case something more complicated +taskset --cpu-list $((ncpus-1)) ${CLICKHOUSE_LOCAL} -q 'select count() from numbers_mt(100000000) settings max_threads=100' 2>&1 # this command should not fail because we specify narenas explicitly # (even with abort_conf:true) diff --git a/tests/queries/0_stateless/01505_pipeline_executor_UAF.sh b/tests/queries/0_stateless/01505_pipeline_executor_UAF.sh index ff22597c620..c2750ad31b2 100755 --- a/tests/queries/0_stateless/01505_pipeline_executor_UAF.sh +++ b/tests/queries/0_stateless/01505_pipeline_executor_UAF.sh @@ -5,14 +5,19 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -# In debug build abort_conf:true is set by default, disable it explicitly -# to avoid "Number of CPUs is not deterministic" error from jemalloc. -export MALLOC_CONF=abort_conf:false +# Avoid failures due to "Number of CPUs is not deterministic" error from jemalloc. +export MALLOC_CONF=abort_conf:false,abort:false # Regression for UAF in ThreadPool. # (Triggered under TSAN) for _ in {1..10}; do ${CLICKHOUSE_LOCAL} -q 'select * from numbers_mt(100000000) settings max_threads=100 FORMAT Null' # Binding to specific CPU is not required, but this makes the test more reliable. - taskset --cpu-list 0 ${CLICKHOUSE_LOCAL} -q 'select * from numbers_mt(100000000) settings max_threads=100 FORMAT Null' + taskset --cpu-list 0 ${CLICKHOUSE_LOCAL} -q 'select * from numbers_mt(100000000) settings max_threads=100 FORMAT Null' 2>&1 | { + # build with santiziers does not have jemalloc + # and for jemalloc we have separate test + # 01502_jemalloc_percpu_arena + grep -v ': Number of CPUs detected is not deterministic. Per-CPU arena disabled.' + } done +exit 0 diff --git a/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.reference b/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.reference index f57d5df6efd..613c455fc59 100644 --- a/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.reference +++ b/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.reference @@ -12,3 +12,4 @@ memory in_memory_parts_insert_sync 1 wide fsync_part_directory,vertical 1 +2 diff --git a/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.sql b/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.sql index 644cf063a33..598e1ef3c34 100644 --- a/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.sql +++ b/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.sql @@ -43,8 +43,10 @@ optimize table data_01643 final; drop table data_01643; select 'wide fsync_part_directory,vertical'; -create table data_01643 (key Int) engine=MergeTree() order by key settings min_bytes_for_wide_part=0, fsync_part_directory=1, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1; +create table data_01643 (key Int) engine=MergeTree() order by key settings min_bytes_for_wide_part=0, fsync_part_directory=1, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, vertical_merge_algorithm_min_columns_to_activate=0; insert into data_01643 values (1); -select * from data_01643; -optimize table data_01643 final; +insert into data_01643 values (2); +select * from data_01643 order by key; +-- vertical merge does not supports deduplicate, hence no FINAL +optimize table data_01643; drop table data_01643; diff --git a/tests/queries/0_stateless/01662_date_ubsan.reference b/tests/queries/0_stateless/01662_date_ubsan.reference index 3c90ee8960e..b9bff160f6b 100644 --- a/tests/queries/0_stateless/01662_date_ubsan.reference +++ b/tests/queries/0_stateless/01662_date_ubsan.reference @@ -1,34 +1,5 @@ --- { echo } --- tests with INT64_MIN (via overflow) -SELECT addMinutes(toDateTime('2021-01-01 00:00:00', 'GMT'), 9223372036854775808); -2021-01-01 00:00:00 -SELECT addHours(toDateTime('2021-01-01 00:00:00', 'GMT'), 9223372036854775808); -2021-01-01 00:00:00 -SELECT addWeeks(toDateTime('2021-01-01 00:00:00', 'GMT'), 9223372036854775808); -2021-01-01 00:00:00 -SELECT addDays(toDateTime('2021-01-01 00:00:00', 'GMT'), 9223372036854775808); -2021-01-01 00:00:00 -SELECT addYears(toDateTime('2021-01-01 00:00:00', 'GMT'), 9223372036854775808); -2021-01-01 00:00:00 --- tests with INT64_MAX -SELECT addMinutes(toDateTime('2020-01-01 00:00:00', 'GMT'), 9223372036854775807); 2019-12-31 23:59:00 -SELECT addHours(toDateTime('2020-01-01 00:00:00', 'GMT'), 9223372036854775807); 2019-12-31 23:00:00 -SELECT addWeeks(toDateTime('2020-01-01 00:00:00', 'GMT'), 9223372036854775807); 2019-12-25 00:00:00 -SELECT addDays(toDateTime('2020-01-01 00:00:00', 'GMT'), 9223372036854775807); 2019-12-31 00:00:00 -SELECT addYears(toDateTime('2020-01-01 00:00:00', 'GMT'), 9223372036854775807); 2019-01-01 00:00:00 --- tests with inf -SELECT addMinutes(toDateTime('2021-01-01 00:00:00', 'GMT'), inf); -2021-01-01 00:00:00 -SELECT addHours(toDateTime('2021-01-01 00:00:00', 'GMT'), inf); -2021-01-01 00:00:00 -SELECT addWeeks(toDateTime('2021-01-01 00:00:00', 'GMT'), inf); -2021-01-01 00:00:00 -SELECT addDays(toDateTime('2021-01-01 00:00:00', 'GMT'), inf); -2021-01-01 00:00:00 -SELECT addYears(toDateTime('2021-01-01 00:00:00', 'GMT'), inf); -2021-01-01 00:00:00 diff --git a/tests/queries/0_stateless/01662_date_ubsan.sql b/tests/queries/0_stateless/01662_date_ubsan.sql index 784853bef3d..fd197044fad 100644 --- a/tests/queries/0_stateless/01662_date_ubsan.sql +++ b/tests/queries/0_stateless/01662_date_ubsan.sql @@ -1,19 +1,18 @@ --- { echo } --- tests with INT64_MIN (via overflow) -SELECT addMinutes(toDateTime('2021-01-01 00:00:00', 'GMT'), 9223372036854775808); -SELECT addHours(toDateTime('2021-01-01 00:00:00', 'GMT'), 9223372036854775808); -SELECT addWeeks(toDateTime('2021-01-01 00:00:00', 'GMT'), 9223372036854775808); -SELECT addDays(toDateTime('2021-01-01 00:00:00', 'GMT'), 9223372036854775808); -SELECT addYears(toDateTime('2021-01-01 00:00:00', 'GMT'), 9223372036854775808); -- tests with INT64_MAX SELECT addMinutes(toDateTime('2020-01-01 00:00:00', 'GMT'), 9223372036854775807); SELECT addHours(toDateTime('2020-01-01 00:00:00', 'GMT'), 9223372036854775807); SELECT addWeeks(toDateTime('2020-01-01 00:00:00', 'GMT'), 9223372036854775807); SELECT addDays(toDateTime('2020-01-01 00:00:00', 'GMT'), 9223372036854775807); SELECT addYears(toDateTime('2020-01-01 00:00:00', 'GMT'), 9223372036854775807); +-- tests with INT64_MIN (via overflow) +SELECT addMinutes(toDateTime('2021-01-01 00:00:00', 'GMT'), 9223372036854775808); -- { serverError DECIMAL_OVERFLOW } +SELECT addHours(toDateTime('2021-01-01 00:00:00', 'GMT'), 9223372036854775808); -- { serverError DECIMAL_OVERFLOW } +SELECT addWeeks(toDateTime('2021-01-01 00:00:00', 'GMT'), 9223372036854775808); -- { serverError DECIMAL_OVERFLOW } +SELECT addDays(toDateTime('2021-01-01 00:00:00', 'GMT'), 9223372036854775808); -- { serverError DECIMAL_OVERFLOW } +SELECT addYears(toDateTime('2021-01-01 00:00:00', 'GMT'), 9223372036854775808); -- { serverError DECIMAL_OVERFLOW } -- tests with inf -SELECT addMinutes(toDateTime('2021-01-01 00:00:00', 'GMT'), inf); -SELECT addHours(toDateTime('2021-01-01 00:00:00', 'GMT'), inf); -SELECT addWeeks(toDateTime('2021-01-01 00:00:00', 'GMT'), inf); -SELECT addDays(toDateTime('2021-01-01 00:00:00', 'GMT'), inf); -SELECT addYears(toDateTime('2021-01-01 00:00:00', 'GMT'), inf); +SELECT addMinutes(toDateTime('2021-01-01 00:00:00', 'GMT'), inf); -- { serverError DECIMAL_OVERFLOW } +SELECT addHours(toDateTime('2021-01-01 00:00:00', 'GMT'), inf); -- { serverError DECIMAL_OVERFLOW } +SELECT addWeeks(toDateTime('2021-01-01 00:00:00', 'GMT'), inf); -- { serverError DECIMAL_OVERFLOW } +SELECT addDays(toDateTime('2021-01-01 00:00:00', 'GMT'), inf); -- { serverError DECIMAL_OVERFLOW } +SELECT addYears(toDateTime('2021-01-01 00:00:00', 'GMT'), inf); -- { serverError DECIMAL_OVERFLOW } diff --git a/tests/queries/0_stateless/01670_dictionary_create_key_expression.reference b/tests/queries/0_stateless/01670_dictionary_create_key_expression.reference index eb246761f9c..49c1542f109 100644 --- a/tests/queries/0_stateless/01670_dictionary_create_key_expression.reference +++ b/tests/queries/0_stateless/01670_dictionary_create_key_expression.reference @@ -1,8 +1,8 @@ Simple -5791441145865411458 Test2 -3450587330153346914 Test1 3111929972906540512 Test3 +3450587330153346914 Test1 +5791441145865411458 Test2 Complex 3111929972906540512 5 Test3 -5791441145865411458 5 Test2 3450587330153346914 5 Test1 +5791441145865411458 5 Test2 diff --git a/tests/queries/0_stateless/01670_dictionary_create_key_expression.sql b/tests/queries/0_stateless/01670_dictionary_create_key_expression.sql index 32e7dc17479..97c04ce445f 100644 --- a/tests/queries/0_stateless/01670_dictionary_create_key_expression.sql +++ b/tests/queries/0_stateless/01670_dictionary_create_key_expression.sql @@ -17,7 +17,7 @@ SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'test_for LIFETIME(MIN 1 MAX 10) LAYOUT(HASHED()); -SELECT * FROM database_dictionary_test_key_expression.test_query_log_dictionary_simple; +SELECT * FROM database_dictionary_test_key_expression.test_query_log_dictionary_simple ORDER BY value_id; DROP DICTIONARY IF EXISTS database_dictionary_test_key_expression.test_query_log_dictionary_simple; @@ -34,7 +34,7 @@ SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'test_for LIFETIME(MIN 1 MAX 10) LAYOUT(COMPLEX_KEY_HASHED()); -SELECT * FROM database_dictionary_test_key_expression.test_query_log_dictionary_complex; +SELECT * FROM database_dictionary_test_key_expression.test_query_log_dictionary_complex ORDER BY value_id; DROP DICTIONARY IF EXISTS database_dictionary_test_key_expression.test_query_log_dictionary_complex; diff --git a/tests/queries/0_stateless/01680_date_time_add_ubsan.sql b/tests/queries/0_stateless/01680_date_time_add_ubsan.sql index f4690116e1a..d2c443bddf9 100644 --- a/tests/queries/0_stateless/01680_date_time_add_ubsan.sql +++ b/tests/queries/0_stateless/01680_date_time_add_ubsan.sql @@ -1,2 +1,3 @@ -SELECT DISTINCT result FROM (SELECT toStartOfFifteenMinutes(toDateTime(toStartOfFifteenMinutes(toDateTime(1000.0001220703125) + (number * 65536))) + (number * 9223372036854775807)) AS result FROM system.numbers LIMIT 1048576) ORDER BY result DESC NULLS FIRST FORMAT Null; +SELECT DISTINCT result FROM (SELECT toStartOfFifteenMinutes(toDateTime(toStartOfFifteenMinutes(toDateTime(1000.0001220703125) + (number * 65536))) + (number * 9223372036854775807)) AS result FROM system.numbers LIMIT 1048576) ORDER BY result DESC NULLS FIRST FORMAT Null; -- { serverError 407 } +SELECT DISTINCT result FROM (SELECT toStartOfFifteenMinutes(toDateTime(toStartOfFifteenMinutes(toDateTime(1000.0001220703125) + (number * 65536))) + toInt64(number * 9223372036854775807)) AS result FROM system.numbers LIMIT 1048576) ORDER BY result DESC NULLS FIRST FORMAT Null; SELECT round(round(round(round(round(100)), round(round(round(round(NULL), round(65535)), toTypeName(now() + 9223372036854775807) LIKE 'DateTime%DateTime%DateTime%DateTime%', round(-2)), 255), round(NULL)))); diff --git a/tests/queries/0_stateless/01681_arg_min_max_if_fix.reference b/tests/queries/0_stateless/01681_arg_min_max_if_fix.reference index 75a0b4104b3..e69de29bb2d 100644 --- a/tests/queries/0_stateless/01681_arg_min_max_if_fix.reference +++ b/tests/queries/0_stateless/01681_arg_min_max_if_fix.reference @@ -1 +0,0 @@ -0 0 2 diff --git a/tests/queries/0_stateless/01681_arg_min_max_if_fix.sql b/tests/queries/0_stateless/01681_arg_min_max_if_fix.sql index b0aab898536..5edd52e0841 100644 --- a/tests/queries/0_stateless/01681_arg_min_max_if_fix.sql +++ b/tests/queries/0_stateless/01681_arg_min_max_if_fix.sql @@ -1 +1 @@ -SELECT bitAnd(number, toUInt64(pow(257, 20) - 1048576)) AS k, argMaxIf(k, if((number % 255) = 256, toInt256(65535), number), number > 42), uniq(number) AS u FROM numbers(2) GROUP BY toInt256(-2, NULL), k; +SELECT bitAnd(number, toUInt64(pow(257, 20) - 1048576)) AS k, argMaxIf(k, if((number % 255) = 256, toInt256(65535), number), number > 42), uniq(number) AS u FROM numbers(2) GROUP BY toInt256(-2, NULL), k FORMAT Null diff --git a/tests/queries/0_stateless/01773_datetime64_add_ubsan.reference b/tests/queries/0_stateless/01773_datetime64_add_ubsan.reference index aa47d0d46d4..e69de29bb2d 100644 --- a/tests/queries/0_stateless/01773_datetime64_add_ubsan.reference +++ b/tests/queries/0_stateless/01773_datetime64_add_ubsan.reference @@ -1,2 +0,0 @@ -0 -0 diff --git a/tests/queries/0_stateless/01773_datetime64_add_ubsan.sql b/tests/queries/0_stateless/01773_datetime64_add_ubsan.sql index f7267f2b6b4..70dcd6a133f 100644 --- a/tests/queries/0_stateless/01773_datetime64_add_ubsan.sql +++ b/tests/queries/0_stateless/01773_datetime64_add_ubsan.sql @@ -1,2 +1,2 @@ -- The result is unspecified but UBSan should not argue. -SELECT ignore(addHours(now64(3), inf)) FROM numbers(2); +SELECT ignore(addHours(now64(3), inf)) FROM numbers(2); -- { serverError 407 } diff --git a/tests/queries/0_stateless/02100_multiple_hosts_command_line_set.reference b/tests/queries/0_stateless/02100_multiple_hosts_command_line_set.reference index da7b788b157..40d20c08279 100644 --- a/tests/queries/0_stateless/02100_multiple_hosts_command_line_set.reference +++ b/tests/queries/0_stateless/02100_multiple_hosts_command_line_set.reference @@ -1,14 +1,29 @@ -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 +1 +=== Backward compatibility test +1 +=== Cannot resolve host +1 +1 +=== Bad arguments +1 +1 +=== Not alive host +1 +1 +1 +1 +1 +=== Code 210 with ipv6 +1 +1 +1 +1 +1 +1 +=== Values form config +1 +1 +=== 1 1 1 diff --git a/tests/queries/0_stateless/02100_multiple_hosts_command_line_set.sh b/tests/queries/0_stateless/02100_multiple_hosts_command_line_set.sh index 0e7a9f17225..168b9be00c6 100755 --- a/tests/queries/0_stateless/02100_multiple_hosts_command_line_set.sh +++ b/tests/queries/0_stateless/02100_multiple_hosts_command_line_set.sh @@ -7,55 +7,75 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # default values test ${CLICKHOUSE_CLIENT} --query "SELECT 1" -# backward compatibility test +echo '=== Backward compatibility test' ${CLICKHOUSE_CLIENT} --host "${CLICKHOUSE_HOST}" --port "${CLICKHOUSE_PORT_TCP}" --query "SELECT 1"; +echo '=== Cannot resolve host' not_resolvable_host="notlocalhost" -exception_msg="Cannot resolve host (${not_resolvable_host}), error 0: ${not_resolvable_host}. -Code: 198. DB::Exception: Not found address of host: ${not_resolvable_host}. (DNS_ERROR) -" -error="$(${CLICKHOUSE_CLIENT} --host "${CLICKHOUSE_HOST}" --host "${not_resolvable_host}" --query "SELECT 1" 2>&1 > /dev/null)"; -[ "${error}" == "${exception_msg}" ]; echo "$?" +error="$(${CLICKHOUSE_CLIENT} --host "${not_resolvable_host}" --query "SELECT 1" 2>&1 > /dev/null)"; +echo "${error}" | grep -Fc "DNS_ERROR" +echo "${error}" | grep -Fq "${not_resolvable_host}" && echo 1 || echo 0 +echo '=== Bad arguments' not_number_port="abc" -exception_msg="Bad arguments: the argument ('${CLICKHOUSE_HOST}:${not_number_port}') for option '--host' is invalid." -error="$(${CLICKHOUSE_CLIENT} --host "${CLICKHOUSE_HOST}" --port "${not_number_port}" --query "SELECT 1" 2>&1 > /dev/null)"; -[ "${error}" == "${exception_msg}" ]; echo "$?" +error="$(${CLICKHOUSE_CLIENT} --host "${CLICKHOUSE_HOST}" --port "${not_number_port}" --query "SELECT 1" 2>&1 > /dev/null)"; +echo "${error}" | grep -Fc "Bad arguments" +echo "${error}" | grep -Fc "${not_number_port}" + +echo '=== Not alive host' not_alive_host="10.100.0.0" ${CLICKHOUSE_CLIENT} --host "${not_alive_host}" --host "${CLICKHOUSE_HOST}" --query "SELECT 1"; not_alive_port="1" -exception_msg="Code: 210. DB::NetException: Connection refused (${CLICKHOUSE_HOST}:${not_alive_port}). (NETWORK_ERROR) -" error="$(${CLICKHOUSE_CLIENT} --host "${CLICKHOUSE_HOST}" --port "${not_alive_port}" --query "SELECT 1" 2>&1 > /dev/null)" -[ "${error}" == "${exception_msg}" ]; echo "$?" +echo "${error}" | grep -Fc "Code: 210" +echo "${error}" | grep -Fc "${CLICKHOUSE_HOST}:${not_alive_port}" + ${CLICKHOUSE_CLIENT} --host "${CLICKHOUSE_HOST}" --port "${not_alive_port}" --host "${CLICKHOUSE_HOST}" --query "SELECT 1"; ${CLICKHOUSE_CLIENT} --host "${CLICKHOUSE_HOST}" --port "${CLICKHOUSE_PORT_TCP}" --port "${not_alive_port}" --query "SELECT 1"; +echo '=== Code 210 with ipv6' + ipv6_host_without_brackets="2001:3984:3989::1:1000" -exception_msg="Code: 210. DB::NetException: Connection refused (${ipv6_host_without_brackets}). (NETWORK_ERROR) -" error="$(${CLICKHOUSE_CLIENT} --host "${ipv6_host_without_brackets}" --query "SELECT 1" 2>&1 > /dev/null)" -[ "${error}" == "${exception_msg}" ]; echo "$?" +echo "${error}" | grep -Fc "Code: 210" +echo "${error}" | grep -Fc "${ipv6_host_without_brackets}" ipv6_host_with_brackets="[2001:3984:3989::1:1000]" -exception_msg="Code: 210. DB::NetException: Connection refused (${ipv6_host_with_brackets}). (NETWORK_ERROR) -" + error="$(${CLICKHOUSE_CLIENT} --host "${ipv6_host_with_brackets}" --query "SELECT 1" 2>&1 > /dev/null)" -[ "${error}" == "${exception_msg}" ]; echo "$?" +echo "${error}" | grep -Fc "Code: 210" +echo "${error}" | grep -Fc "${ipv6_host_with_brackets}" -exception_msg="Code: 210. DB::NetException: Connection refused (${ipv6_host_with_brackets}:${not_alive_port}). (NETWORK_ERROR) -" error="$(${CLICKHOUSE_CLIENT} --host "${ipv6_host_with_brackets}" --port "${not_alive_port}" --query "SELECT 1" 2>&1 > /dev/null)" -[ "${error}" == "${exception_msg}" ]; echo "$?" +echo "${error}" | grep -Fc "Code: 210" +echo "${error}" | grep -Fc "${ipv6_host_with_brackets}:${not_alive_port}" +echo '=== Values form config' -${CLICKHOUSE_CLIENT} --query "SELECT 1"; -${CLICKHOUSE_CLIENT} --port "${CLICKHOUSE_PORT_TCP}" --query "SELECT 1"; -${CLICKHOUSE_CLIENT} --host "${CLICKHOUSE_HOST}" --query "SELECT 1"; -${CLICKHOUSE_CLIENT} --port "${CLICKHOUSE_PORT_TCP}" --host "${CLICKHOUSE_HOST}" --query "SELECT 1"; -${CLICKHOUSE_CLIENT} --port "${CLICKHOUSE_PORT_TCP}" --host "${CLICKHOUSE_HOST}" --host "{$not_alive_host}" --port "${CLICKHOUSE_PORT_TCP}" --query "SELECT 1"; -${CLICKHOUSE_CLIENT} --port "${CLICKHOUSE_PORT_TCP}" --host "{$not_alive_host}" --host "${CLICKHOUSE_HOST}" --query "SELECT 1" 2> /dev/null; -${CLICKHOUSE_CLIENT} --port "${CLICKHOUSE_PORT_TCP}" --port "${CLICKHOUSE_PORT_TCP}" --port "${CLICKHOUSE_PORT_TCP}" --host "{$not_alive_host}" --host "${CLICKHOUSE_HOST}" --query "SELECT 1"; +CUSTOM_CONFIG="$CURDIR/02100_config.xml" +rm -f ${CUSTOM_CONFIG} + +cat << EOF > ${CUSTOM_CONFIG} + + ${not_alive_host} + ${not_alive_port} + +EOF + +error="$(${CLICKHOUSE_CLIENT} --config ${CUSTOM_CONFIG} --query "SELECT 1" 2>&1 > /dev/null)" +echo "${error}" | grep -Fc "DB::NetException" +echo "${error}" | grep -Fc "${not_alive_host}:${not_alive_port}" +rm -f ${CUSTOM_CONFIG} + +echo '===' + +${CLICKHOUSE_CLIENT} --query "SELECT 1"; +${CLICKHOUSE_CLIENT} --port "${CLICKHOUSE_PORT_TCP}" --query "SELECT 1"; +${CLICKHOUSE_CLIENT} --host "${CLICKHOUSE_HOST}" --query "SELECT 1"; +${CLICKHOUSE_CLIENT} --port "${CLICKHOUSE_PORT_TCP}" --host "${CLICKHOUSE_HOST}" --query "SELECT 1"; +${CLICKHOUSE_CLIENT} --port "${CLICKHOUSE_PORT_TCP}" --host "${CLICKHOUSE_HOST}" --host "{$not_alive_host}" --port "${CLICKHOUSE_PORT_TCP}" --query "SELECT 1"; +${CLICKHOUSE_CLIENT} --port "${CLICKHOUSE_PORT_TCP}" --host "{$not_alive_host}" --host "${CLICKHOUSE_HOST}" --query "SELECT 1" 2> /dev/null; +${CLICKHOUSE_CLIENT} --port "${CLICKHOUSE_PORT_TCP}" --port "${CLICKHOUSE_PORT_TCP}" --port "${CLICKHOUSE_PORT_TCP}" --host "{$not_alive_host}" --host "${CLICKHOUSE_HOST}" --query "SELECT 1"; diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index 678fe35fd96..f97eaace72a 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -60,7 +60,7 @@ CREATE TABLE system.table_functions\n(\n `name` String\n)\nENGINE = SystemTab CREATE TABLE system.tables\n(\n `database` String,\n `name` String,\n `uuid` UUID,\n `engine` String,\n `is_temporary` UInt8,\n `data_paths` Array(String),\n `metadata_path` String,\n `metadata_modification_time` DateTime,\n `dependencies_database` Array(String),\n `dependencies_table` Array(String),\n `create_table_query` String,\n `engine_full` String,\n `as_select` String,\n `partition_key` String,\n `sorting_key` String,\n `primary_key` String,\n `sampling_key` String,\n `storage_policy` String,\n `total_rows` Nullable(UInt64),\n `total_bytes` Nullable(UInt64),\n `lifetime_rows` Nullable(UInt64),\n `lifetime_bytes` Nullable(UInt64),\n `comment` String,\n `has_own_data` UInt8,\n `loading_dependencies_database` Array(String),\n `loading_dependencies_table` Array(String),\n `loading_dependent_database` Array(String),\n `loading_dependent_table` Array(String),\n `table` String\n)\nENGINE = SystemTables()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' CREATE TABLE system.time_zones\n(\n `time_zone` String\n)\nENGINE = SystemTimeZones()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' CREATE TABLE system.user_directories\n(\n `name` String,\n `type` String,\n `params` String,\n `precedence` UInt64\n)\nENGINE = SystemUserDirectories()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' -CREATE TABLE system.users\n(\n `name` String,\n `id` UUID,\n `storage` String,\n `auth_type` Enum8(\'no_password\' = 0, \'plaintext_password\' = 1, \'sha256_password\' = 2, \'double_sha1_password\' = 3, \'ldap\' = 4, \'kerberos\' = 5),\n `auth_params` String,\n `host_ip` Array(String),\n `host_names` Array(String),\n `host_names_regexp` Array(String),\n `host_names_like` Array(String),\n `default_roles_all` UInt8,\n `default_roles_list` Array(String),\n `default_roles_except` Array(String),\n `grantees_any` UInt8,\n `grantees_list` Array(String),\n `grantees_except` Array(String),\n `default_database` String\n)\nENGINE = SystemUsers()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' +CREATE TABLE system.users\n(\n `name` String,\n `id` UUID,\n `storage` String,\n `auth_type` Enum8(\'no_password\' = 0, \'plaintext_password\' = 1, \'sha256_password\' = 2, \'double_sha1_password\' = 3, \'ldap\' = 4, \'kerberos\' = 5, \'ssl_certificate\' = 6),\n `auth_params` String,\n `host_ip` Array(String),\n `host_names` Array(String),\n `host_names_regexp` Array(String),\n `host_names_like` Array(String),\n `default_roles_all` UInt8,\n `default_roles_list` Array(String),\n `default_roles_except` Array(String),\n `grantees_any` UInt8,\n `grantees_list` Array(String),\n `grantees_except` Array(String),\n `default_database` String\n)\nENGINE = SystemUsers()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' CREATE TABLE system.warnings\n(\n `message` String\n)\nENGINE = SystemWarnings()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' CREATE TABLE system.zeros\n(\n `zero` UInt8\n)\nENGINE = SystemZeros()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' CREATE TABLE system.zeros_mt\n(\n `zero` UInt8\n)\nENGINE = SystemZeros()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' diff --git a/tests/queries/0_stateless/02206_array_starts_ends_with.reference b/tests/queries/0_stateless/02206_array_starts_ends_with.reference new file mode 100644 index 00000000000..e0dacfc06e0 --- /dev/null +++ b/tests/queries/0_stateless/02206_array_starts_ends_with.reference @@ -0,0 +1,30 @@ +1 +1 +0 +- +1 +1 +0 +1 +0 +- +1 +0 +1 +0 +- +1 +1 +0 +- +1 +1 +0 +1 +0 +- +1 +0 +- +1 +1 diff --git a/tests/queries/0_stateless/02206_array_starts_ends_with.sql b/tests/queries/0_stateless/02206_array_starts_ends_with.sql new file mode 100644 index 00000000000..39b02c29dc0 --- /dev/null +++ b/tests/queries/0_stateless/02206_array_starts_ends_with.sql @@ -0,0 +1,36 @@ +select startsWith([], []); +select startsWith([1], []); +select startsWith([], [1]); +select '-'; + +select startsWith([NULL], [NULL]); +select startsWith([NULL], []); +select startsWith([], [NULL]); +select startsWith([NULL, 1], [NULL]); +select startsWith([NULL, 1], [1]); +select '-'; + +select startsWith([1, 2, 3, 4], [1, 2, 3]); +select startsWith([1, 2, 3, 4], [1, 2, 4]); +select startsWith(['a', 'b', 'c'], ['a', 'b']); +select startsWith(['a', 'b', 'c'], ['b']); +select '-'; + +select endsWith([], []); +select endsWith([1], []); +select endsWith([], [1]); +select '-'; + +select endsWith([NULL], [NULL]); +select endsWith([NULL], []); +select endsWith([], [NULL]); +select endsWith([1, NULL], [NULL]); +select endsWith([NULL, 1], [NULL]); +select '-'; + +select endsWith([1, 2, 3, 4], [3, 4]); +select endsWith([1, 2, 3, 4], [3]); +select '-'; + +select startsWith([1], emptyArrayUInt8()); +select endsWith([1], emptyArrayUInt8()); diff --git a/tests/queries/0_stateless/02207_s3_content_type.reference b/tests/queries/0_stateless/02207_s3_content_type.reference new file mode 100644 index 00000000000..2b0a5bcadc2 --- /dev/null +++ b/tests/queries/0_stateless/02207_s3_content_type.reference @@ -0,0 +1,2 @@ +ContentLength:2144451 +ContentType:binary/octet-stream diff --git a/tests/queries/0_stateless/02207_s3_content_type.sh b/tests/queries/0_stateless/02207_s3_content_type.sh new file mode 100755 index 00000000000..5ede30e867c --- /dev/null +++ b/tests/queries/0_stateless/02207_s3_content_type.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: needs s3 + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT --query " +INSERT INTO TABLE FUNCTION s3('http://localhost:11111/test/content-type.csv.gz', 'test', 'testtest', 'CSV', 'number UInt64') SELECT number FROM numbers(1000000) SETTINGS s3_max_single_part_upload_size = 10000, s3_truncate_on_insert = 1; +" + +aws --endpoint-url http://localhost:11111 s3api head-object --bucket test --key content-type.csv.gz | grep Content | sed 's/[ \t,"]*//g' diff --git a/tests/queries/0_stateless/02222_allow_experimental_projection_optimization__enable_global_with_statement.reference b/tests/queries/0_stateless/02222_allow_experimental_projection_optimization__enable_global_with_statement.reference new file mode 100644 index 00000000000..4ace962aa0f --- /dev/null +++ b/tests/queries/0_stateless/02222_allow_experimental_projection_optimization__enable_global_with_statement.reference @@ -0,0 +1,14 @@ +-- { echoOn } +WITH + (SELECT * FROM data_02222) AS bm1, + (SELECT * FROM data_02222) AS bm2, + (SELECT * FROM data_02222) AS bm3, + (SELECT * FROM data_02222) AS bm4, + (SELECT * FROM data_02222) AS bm5, + (SELECT * FROM data_02222) AS bm6, + (SELECT * FROM data_02222) AS bm7, + (SELECT * FROM data_02222) AS bm8, + (SELECT * FROM data_02222) AS bm9, + (SELECT * FROM data_02222) AS bm10 +SELECT bm1, bm2, bm3, bm4, bm5, bm6, bm7, bm8, bm9, bm10 FROM data_02222; +0 0 0 0 0 0 0 0 0 0 diff --git a/tests/queries/0_stateless/02222_allow_experimental_projection_optimization__enable_global_with_statement.sql b/tests/queries/0_stateless/02222_allow_experimental_projection_optimization__enable_global_with_statement.sql new file mode 100644 index 00000000000..f870b985d81 --- /dev/null +++ b/tests/queries/0_stateless/02222_allow_experimental_projection_optimization__enable_global_with_statement.sql @@ -0,0 +1,17 @@ +DROP TABLE IF EXISTS data_02222; +CREATE TABLE data_02222 engine=MergeTree() ORDER BY dummy AS SELECT * FROM system.one; +-- { echoOn } +WITH + (SELECT * FROM data_02222) AS bm1, + (SELECT * FROM data_02222) AS bm2, + (SELECT * FROM data_02222) AS bm3, + (SELECT * FROM data_02222) AS bm4, + (SELECT * FROM data_02222) AS bm5, + (SELECT * FROM data_02222) AS bm6, + (SELECT * FROM data_02222) AS bm7, + (SELECT * FROM data_02222) AS bm8, + (SELECT * FROM data_02222) AS bm9, + (SELECT * FROM data_02222) AS bm10 +SELECT bm1, bm2, bm3, bm4, bm5, bm6, bm7, bm8, bm9, bm10 FROM data_02222; +-- { echoOff } +DROP TABLE data_02222; diff --git a/tests/queries/0_stateless/02223_h3_test_const_columns.reference b/tests/queries/0_stateless/02223_h3_test_const_columns.reference new file mode 100644 index 00000000000..5102199ef58 --- /dev/null +++ b/tests/queries/0_stateless/02223_h3_test_const_columns.reference @@ -0,0 +1,80 @@ +583031433791012863 +587531185127686143 +583031433791012863 +587531185127686143 +607221000000 +86745854035 +607220.9782 +86745.85403 +4106166334463.9233 +85294486110.07852 +12781831077.715292 +0.10116268528089567 +0.002101380838405832 +0.00031490306268786255 +0 +2 +3 +9.961887434044831 +3.7652395323603707 +1.4231267757782213 +1107712.591 +418676.0055 +158244.6558 +1107.712591 +418.6760055 +158.2446558 +(-173.6412167681162,-14.130272474941535) +(172.68095885060296,-83.6576608516349) +(-94.46556851304558,-69.1999982492279) +[(-25.60370257696877,-170.61932339479839),(-16.505947603561054,-161.6348206171839),(-5.762860491436932,-165.41674992858833),(-3.968796976609588,-176.05696384421356),(-11.54529597541476,175.98600155652952),(-22.19754138630238,177.51613498805204)] +[(-82.24829137508873,167.18203088800593),(-83.41761096812805,158.00531624510785),(-84.94207431820979,162.09183616506846),(-85.00324300064887,-178.60454506450245),(-83.46691212211444,-172.41232929697492),(-82.25118471750908,179.4928586395771)] +[(-69.70201806837188,-95.63006768303532),(-69.78121889088241,-93.8329499937899),(-69.26603652285242,-92.70414199751751),(-68.6908704290193,-93.35479180342097),(-68.62037380778602,-95.0614247833063),(-69.11663254992226,-96.20491957306085)] +[579275502070530047,579768083279773695,578888473977552895,579662530163507199,579205133326352383,578466261512486911,578712552117108735] +[578888473977552895,580225480116928511,579979189512306687,578114417791598591,578079233419509759,578712552117108735,579310686442618879,578606999000842239,578571814628753407,579205133326352383,579275502070530047,578466261512486911,579240317698441215,577727389698621439,579838452023951359,579662530163507199,579768083279773695,580331033233195007] +[577375545977733119,578431077140398079,579099580210085887,579732898907684863,580612508209905663,579275502070530047,580190295744839679,577094071001022463,578606999000842239,579029211465908223,577727389698621439,579240317698441215,579662530163507199,578571814628753407,580331033233195007,580295848861106175,579205133326352383,577903311559065599,578114417791598591,579838452023951359,577445914721910783,577868127186976767,578079233419509759,579592161419329535,578501445884575743,578712552117108735,580225480116928511,580471770721550335,580647692581994495,578466261512486911,579768083279773695,578888473977552895,579979189512306687,579310686442618879] +77 +121 +116 +1 +0 +1 +[603909588852408319,603909588986626047,603909589120843775,603909589255061503,603909589389279231,603909589523496959,603909589657714687] +[599405990164561919] +576918149140578303 +581395360488882175 +809bfffffffffff +82f39ffffffffff +83e9abfffffffff +579205133326352383 +589753847883235327 +594082350283882495 +0 +0 +1 +0 +1 +0 +1 +[7] +[1,6,11,7,2] +[7] +[1,6,11,7,2] +581496515558637567 +585996266895310847 +590499385486344191 +1263609.6633631135 +1263609.663363112 +1263609.6633631124 +1263609.6633631117 +1263.6096633631134 +1263.6096633631118 +1263.6096633631123 +1263.6096633631116 +0.19833750417794152 +0.19833750417794127 +0.19833750417794133 +0.19833750417794122 +842 +5882 +41162 diff --git a/tests/queries/0_stateless/02223_h3_test_const_columns.sql b/tests/queries/0_stateless/02223_h3_test_const_columns.sql new file mode 100644 index 00000000000..5bae85edb23 --- /dev/null +++ b/tests/queries/0_stateless/02223_h3_test_const_columns.sql @@ -0,0 +1,31 @@ +-- Tags: no-fasttest + +select geoToH3(toFloat64(0),toFloat64(1),arrayJoin([1,2])); +select h3ToParent(641573946153969375, arrayJoin([1,2])); +SELECT h3HexAreaM2(arrayJoin([1,2])); +SELECT h3HexAreaKm2(arrayJoin([1,2])); +SELECT h3CellAreaM2(arrayJoin([579205133326352383,589753847883235327,594082350283882495])); +SELECT NULL, toFloat64('-1'), -2147483648, h3CellAreaM2(arrayJoin([9223372036854775807, 65535, NULL])); -- { serverError 117 } +SELECT h3CellAreaRads2(arrayJoin([579205133326352383,589753847883235327,594082350283882495])); +SELECT NULL, toFloat64('-1'), -2147483648, h3CellAreaRads2(arrayJoin([9223372036854775807, 65535, NULL])); -- { serverError 117 } +SELECT h3GetResolution(arrayJoin([579205133326352383,589753847883235327,594082350283882495])); +SELECT h3EdgeAngle(arrayJoin([0,1,2])); +SELECT h3EdgeLengthM(arrayJoin([0,1,2])); +SELECT h3EdgeLengthKm(arrayJoin([0,1,2])); +SELECT h3ToGeo(arrayJoin([579205133326352383,589753847883235327,594082350283882495])); +SELECT h3ToGeoBoundary(arrayJoin([579205133326352383,589753847883235327,594082350283882495])); +SELECT h3kRing(arrayJoin([579205133326352383]), arrayJoin([toUInt16(1),toUInt16(2),toUInt16(3)])); +SELECT h3GetBaseCell(arrayJoin([579205133326352383,589753847883235327,594082350283882495])); +SELECT h3IndexesAreNeighbors(617420388351344639, arrayJoin([617420388352655359, 617420388351344639, 617420388352917503])); +SELECT h3ToChildren(599405990164561919, arrayJoin([6,5])); +SELECT h3ToParent(599405990164561919, arrayJoin([0,1])); +SELECT h3ToString(arrayJoin([579205133326352383,589753847883235327,594082350283882495])); +SELECT stringToH3(h3ToString(arrayJoin([579205133326352383,589753847883235327,594082350283882495]))); +SELECT h3IsResClassIII(arrayJoin([579205133326352383,589753847883235327,594082350283882495])); +SELECT h3IsPentagon(arrayJoin([stringToH3('8f28308280f18f2'),stringToH3('821c07fffffffff'),stringToH3('0x8f28308280f18f2L'),stringToH3('0x821c07fffffffffL')])); +SELECT h3GetFaces(arrayJoin([stringToH3('8f28308280f18f2'),stringToH3('821c07fffffffff'),stringToH3('0x8f28308280f18f2L'),stringToH3('0x821c07fffffffffL')])); +SELECT h3ToCenterChild(577023702256844799, arrayJoin([1,2,3])); +SELECT h3ExactEdgeLengthM(arrayJoin([1298057039473278975,1370114633511206911,1442172227549134847,1514229821587062783])); +SELECT h3ExactEdgeLengthKm(arrayJoin([1298057039473278975,1370114633511206911,1442172227549134847,1514229821587062783])); +SELECT h3ExactEdgeLengthRads(arrayJoin([1298057039473278975,1370114633511206911,1442172227549134847,1514229821587062783])); +SELECT h3NumHexagons(arrayJoin([1,2,3])); diff --git a/tests/queries/0_stateless/02224_s2_test_const_columns.reference b/tests/queries/0_stateless/02224_s2_test_const_columns.reference new file mode 100644 index 00000000000..9982596f097 --- /dev/null +++ b/tests/queries/0_stateless/02224_s2_test_const_columns.reference @@ -0,0 +1,19 @@ +4704772434919038107 +1527555102286129111 +(37.79506681471008,55.7129059052841) +(-73.98300293448631,40.755097559353004) +[1157339245694594831,1157339245694594835,1157339245694594931,1157339245694594827] +[5074766987100422144,5074766712222515200,5074767536856236032,5074767261978329088] +1 +0 +1 +1 +(4534655147792050737,60.2088283994957) +(5179062030687166815,5177056748191934217) +(5178914411069187297,5177057445452335297) +0 +0 +(5179062030687166815,5177056748191934217) +(5179062030687166815,5177057445452335297) +(5178914411069187297,5177056748191934217) +(6304347505408739331,8070450532247928833) diff --git a/tests/queries/0_stateless/02224_s2_test_const_columns.sql b/tests/queries/0_stateless/02224_s2_test_const_columns.sql new file mode 100644 index 00000000000..f33a7f2b696 --- /dev/null +++ b/tests/queries/0_stateless/02224_s2_test_const_columns.sql @@ -0,0 +1,12 @@ +-- Tags: no-fasttest + +SELECT geoToS2(37.79506683, arrayJoin([55.71290588,37.79506683])); +SELECT s2ToGeo(arrayJoin([4704772434919038107,9926594385212866560])); +SELECT s2GetNeighbors(arrayJoin([1157339245694594829, 5074766849661468672])); +SELECT s2CellsIntersect(9926595209846587392, arrayJoin([9926594385212866560, 5074766849661468672])); +SELECT s2CapContains(1157339245694594829, toFloat64(1), arrayJoin([1157347770437378819,1157347770437378389])); +SELECT s2CapUnion(3814912406305146967, toFloat64(1), 1157347770437378819, toFloat64(1)); +SELECT s2RectAdd(5178914411069187297, 5177056748191934217, arrayJoin([5179056748191934217,5177914411069187297])); +SELECT s2RectContains(5179062030687166815, 5177056748191934217, arrayJoin([5177914411069187297, 5177914411069187297])); +SELECT s2RectUnion(5178914411069187297, 5177056748191934217, 5179062030687166815, arrayJoin([5177056748191934217, 5177914411069187297])); +SELECT s2RectIntersection(5178914411069187297, 5177056748191934217, 5179062030687166815, arrayJoin([5177056748191934217,1157347770437378819])); diff --git a/tests/queries/0_stateless/02225_hints_for_indeices.reference b/tests/queries/0_stateless/02225_hints_for_indeices.reference new file mode 100644 index 00000000000..2c94e483710 --- /dev/null +++ b/tests/queries/0_stateless/02225_hints_for_indeices.reference @@ -0,0 +1,2 @@ +OK +OK diff --git a/tests/queries/0_stateless/02225_hints_for_indeices.sh b/tests/queries/0_stateless/02225_hints_for_indeices.sh new file mode 100755 index 00000000000..f4cfa17f8db --- /dev/null +++ b/tests/queries/0_stateless/02225_hints_for_indeices.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS t" + +$CLICKHOUSE_CLIENT --query="CREATE TABLE t ENGINE=MergeTree ORDER BY n AS SELECT number AS n FROM numbers(10)" + +$CLICKHOUSE_CLIENT --query="ALTER TABLE t ADD INDEX test_index n TYPE minmax GRANULARITY 32" + +$CLICKHOUSE_CLIENT --query="ALTER TABLE t DROP INDEX test_indes" 2>&1 | grep -q "may be you meant: \['test_index'\]" && echo 'OK' || echo 'FAIL' + +$CLICKHOUSE_CLIENT --query="ALTER TABLE t ADD INDEX test_index1 n TYPE minmax GRANULARITY 4 AFTER test_indes" 2>&1 | grep -q "may be you meant: \['test_index'\]" && echo 'OK' || echo 'FAIL' + +$CLICKHOUSE_CLIENT --query="DROP TABLE t" diff --git a/tests/queries/0_stateless/02225_unwinder_dwarf_version.reference b/tests/queries/0_stateless/02225_unwinder_dwarf_version.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/02225_unwinder_dwarf_version.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/02225_unwinder_dwarf_version.sh b/tests/queries/0_stateless/02225_unwinder_dwarf_version.sh new file mode 100755 index 00000000000..59dcba401f2 --- /dev/null +++ b/tests/queries/0_stateless/02225_unwinder_dwarf_version.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +# In case if DWARF-5 debug info is generated during build, it cannot parse it and correctly show this exception. This is why we limit DWARF version to 4 max. +$CLICKHOUSE_LOCAL --query "SELECT throwIf(1)" 2>&1 | grep -c 'FUNCTION_THROW_IF_VALUE_IS_NON_ZERO' diff --git a/tests/queries/0_stateless/data_orc/test.orc b/tests/queries/0_stateless/data_orc/test.orc deleted file mode 100644 index 1b2c9aa4922..00000000000 Binary files a/tests/queries/0_stateless/data_orc/test.orc and /dev/null differ diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index cff412d9fd8..664dbf837fd 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,3 +1,4 @@ +v22.2.2.1-stable 2022-02-17 v22.1.3.7-stable 2022-01-23 v22.1.2.2-stable 2022-01-19 v21.12.4.1-stable 2022-01-23 diff --git a/utils/release/release_lib.sh b/utils/release/release_lib.sh index 9f6c2285d93..538d596d263 100644 --- a/utils/release/release_lib.sh +++ b/utils/release/release_lib.sh @@ -243,16 +243,6 @@ function make_rpm { cat ${PACKAGE}-$VERSION_FULL-2.spec_tmp >> ${PACKAGE}-$VERSION_FULL-2.spec rpm_pack - PACKAGE=clickhouse-test - ARCH=all - TARGET=noarch - deb_unpack - mv ${PACKAGE}-$VERSION_FULL-2.spec ${PACKAGE}-$VERSION_FULL-2.spec_tmp - echo "Requires: python3" >> ${PACKAGE}-$VERSION_FULL-2.spec - #echo "Requires: python3-termcolor" >> ${PACKAGE}-$VERSION-2.spec - cat ${PACKAGE}-$VERSION_FULL-2.spec_tmp >> ${PACKAGE}-$VERSION_FULL-2.spec - rpm_pack - PACKAGE=clickhouse-common-static ARCH=amd64 TARGET=x86_64 @@ -271,7 +261,7 @@ function make_tgz { VERSION_FULL="${VERSION_STRING}" PACKAGE_DIR=${PACKAGE_DIR=../} - for PACKAGE in clickhouse-server clickhouse-client clickhouse-test clickhouse-common-static clickhouse-common-static-dbg; do + for PACKAGE in clickhouse-server clickhouse-client clickhouse-common-static clickhouse-common-static-dbg; do alien --verbose --scripts --generate --to-tgz ${PACKAGE_DIR}${PACKAGE}_${VERSION_FULL}_*.deb PKGDIR="./${PACKAGE}-${VERSION_FULL}" if [ ! -d "$PKGDIR/install" ]; then diff --git a/utils/zero_copy/zero_copy_schema_converter.py b/utils/zero_copy/zero_copy_schema_converter.py index 6fdd03add5a..c5edef78cce 100755 --- a/utils/zero_copy/zero_copy_schema_converter.py +++ b/utils/zero_copy/zero_copy_schema_converter.py @@ -4,6 +4,7 @@ import socket import uuid from kazoo.client import KazooClient + def parse_args(): """ Parse command-line arguments. @@ -17,7 +18,7 @@ def parse_args(): parser.add_argument('-u', '--user', default='', help='ZooKeeper ACL user') parser.add_argument('-p', '--password', default='', help='ZooKeeper ACL password') parser.add_argument('-r', '--root', default='/clickhouse', help='ZooKeeper root path for ClickHouse') - parser.add_argument('-z', '--zcroot', default='zero_copy', help='ZooKeeper node for new zero-copy data') + parser.add_argument('-z', '--zcroot', default='clickhouse/zero_copy', help='ZooKeeper node for new zero-copy data') parser.add_argument('--dryrun', default=False, action='store_true', help='Do not perform any actions') parser.add_argument('--cleanup', default=False, action='store_true', help='Clean old nodes') parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Verbose mode') @@ -34,7 +35,7 @@ zc_nodes = ['zero_copy_s3', 'zero_copy_hdfs'] def convert_node(client, args, path, zc_node): base_path = f'{path}/{zc_node}/shared' parts = client.get_children(base_path) - table_id_path = f'{path}/table_id' + table_id_path = f'{path}/table_shared_id' table_id = '' if client.exists(table_id_path): table_id = client.get(table_id_path)[0].decode('UTF-8') diff --git a/website/images/backgrounds/bg-card-pattern-red.png b/website/images/backgrounds/bg-card-pattern-red.png index a944ec45bd0..84f781590e6 100644 Binary files a/website/images/backgrounds/bg-card-pattern-red.png and b/website/images/backgrounds/bg-card-pattern-red.png differ diff --git a/website/images/backgrounds/bg-quotes.svg b/website/images/backgrounds/bg-quotes.svg index 651351a0812..2bab82c0d75 100644 --- a/website/images/backgrounds/bg-quotes.svg +++ b/website/images/backgrounds/bg-quotes.svg @@ -1,12 +1 @@ - - - Artboard - - - - - - - - - \ No newline at end of file +Artboard \ No newline at end of file diff --git a/website/images/icons/icon-arrow.svg b/website/images/icons/icon-arrow.svg index 3a03b2da23b..22f67781c6e 100644 --- a/website/images/icons/icon-arrow.svg +++ b/website/images/icons/icon-arrow.svg @@ -1,17 +1 @@ - - - icon-arrow - - - - - - - - - - - - - - \ No newline at end of file +icon-arrow \ No newline at end of file diff --git a/website/images/icons/icon-blog-black.svg b/website/images/icons/icon-blog-black.svg index 2e448c5c0da..42d5c1a3010 100644 --- a/website/images/icons/icon-blog-black.svg +++ b/website/images/icons/icon-blog-black.svg @@ -1,14 +1 @@ - - - icon-blog - - - - - - - - - - - \ No newline at end of file +icon-blog \ No newline at end of file diff --git a/website/images/icons/icon-facebook-gray.svg b/website/images/icons/icon-facebook-gray.svg index fc42770a749..6c0e3190e0a 100644 --- a/website/images/icons/icon-facebook-gray.svg +++ b/website/images/icons/icon-facebook-gray.svg @@ -1,15 +1 @@ - - - icon-facebook-gray - - - - - - - \ No newline at end of file +icon-facebook-gray \ No newline at end of file diff --git a/website/images/icons/icon-facebook.svg b/website/images/icons/icon-facebook.svg index 910235c7ed8..c7c66a5cb5d 100644 --- a/website/images/icons/icon-facebook.svg +++ b/website/images/icons/icon-facebook.svg @@ -1,19 +1 @@ - - - Group - - - - - - - - - - - - - - - - \ No newline at end of file +Group \ No newline at end of file diff --git a/website/images/icons/icon-github.svg b/website/images/icons/icon-github.svg index c22c8563b21..79f936ad51b 100644 --- a/website/images/icons/icon-github.svg +++ b/website/images/icons/icon-github.svg @@ -1,9 +1 @@ - - - icon-github - - - - - - \ No newline at end of file +icon-github \ No newline at end of file diff --git a/website/images/icons/icon-google.svg b/website/images/icons/icon-google.svg index 4fa5d1eb118..732c960a052 100644 --- a/website/images/icons/icon-google.svg +++ b/website/images/icons/icon-google.svg @@ -1,12 +1 @@ - - - icon-google - - - - - - - - - \ No newline at end of file +icon-google \ No newline at end of file diff --git a/website/images/icons/icon-linkedin-alt-gray.svg b/website/images/icons/icon-linkedin-alt-gray.svg index e286486ab9b..eb5931c3a65 100644 --- a/website/images/icons/icon-linkedin-alt-gray.svg +++ b/website/images/icons/icon-linkedin-alt-gray.svg @@ -1,13 +1 @@ - - - icon-linkedin-gray - - - - - - - - - - \ No newline at end of file +icon-linkedin-gray \ No newline at end of file diff --git a/website/images/icons/icon-linkedin-gray.svg b/website/images/icons/icon-linkedin-gray.svg index ff19bbbe686..684f35e8f47 100644 --- a/website/images/icons/icon-linkedin-gray.svg +++ b/website/images/icons/icon-linkedin-gray.svg @@ -1,15 +1 @@ - - - icon-linkedin-gray - - - - - - - \ No newline at end of file +icon-linkedin-gray \ No newline at end of file diff --git a/website/images/icons/icon-linkedin.png b/website/images/icons/icon-linkedin.png index e50cf591487..1facc9e76df 100644 Binary files a/website/images/icons/icon-linkedin.png and b/website/images/icons/icon-linkedin.png differ diff --git a/website/images/icons/icon-menu.svg b/website/images/icons/icon-menu.svg index 49aa5b52d37..08d3edb0dfa 100644 --- a/website/images/icons/icon-menu.svg +++ b/website/images/icons/icon-menu.svg @@ -1,10 +1 @@ - - - - - - - - - - + \ No newline at end of file diff --git a/website/images/icons/icon-performance.svg b/website/images/icons/icon-performance.svg index 758149fc872..9fcd2a6558a 100644 --- a/website/images/icons/icon-performance.svg +++ b/website/images/icons/icon-performance.svg @@ -1,16 +1 @@ - - - icon-performance - - - - - - - - - - - - - \ No newline at end of file +icon-performance \ No newline at end of file diff --git a/website/images/icons/icon-reliability.svg b/website/images/icons/icon-reliability.svg index 0b18d1451c0..add18fe0e76 100644 --- a/website/images/icons/icon-reliability.svg +++ b/website/images/icons/icon-reliability.svg @@ -1,16 +1 @@ - - - icon-reliability - - - - - - - - - - - - - \ No newline at end of file +icon-reliability \ No newline at end of file diff --git a/website/images/icons/icon-scalability.svg b/website/images/icons/icon-scalability.svg index 4b0a99dc233..9b015300f99 100644 --- a/website/images/icons/icon-scalability.svg +++ b/website/images/icons/icon-scalability.svg @@ -1,16 +1 @@ - - - icon-scalability - - - - - - - - - - - - - \ No newline at end of file +icon-scalability \ No newline at end of file diff --git a/website/images/icons/icon-security.svg b/website/images/icons/icon-security.svg index 716f7cc5789..da341224ed3 100644 --- a/website/images/icons/icon-security.svg +++ b/website/images/icons/icon-security.svg @@ -1,16 +1 @@ - - - icon-security - - - - - - - - - - - - - \ No newline at end of file +icon-security \ No newline at end of file diff --git a/website/images/icons/icon-slack-black.svg b/website/images/icons/icon-slack-black.svg index 89b6ddcbada..03420c85534 100644 --- a/website/images/icons/icon-slack-black.svg +++ b/website/images/icons/icon-slack-black.svg @@ -1,14 +1 @@ - - - icon-slack - - - - - - - - - - - \ No newline at end of file +icon-slack \ No newline at end of file diff --git a/website/images/icons/icon-slack.svg b/website/images/icons/icon-slack.svg index cb74d47cb04..42e33c4d65f 100644 --- a/website/images/icons/icon-slack.svg +++ b/website/images/icons/icon-slack.svg @@ -1,12 +1 @@ - - - icon-slack - - - - - - - - - \ No newline at end of file +icon-slack \ No newline at end of file diff --git a/website/images/icons/icon-stack-overflow.svg b/website/images/icons/icon-stack-overflow.svg index 5c9a2d57d2b..c8244ac9e4c 100644 --- a/website/images/icons/icon-stack-overflow.svg +++ b/website/images/icons/icon-stack-overflow.svg @@ -1,16 +1 @@ - - - icon-stack-overflow - - - - - - - - - - - - - \ No newline at end of file +icon-stack-overflow \ No newline at end of file diff --git a/website/images/icons/icon-telegram.svg b/website/images/icons/icon-telegram.svg index 5ab06341383..3bf90eea1a4 100644 --- a/website/images/icons/icon-telegram.svg +++ b/website/images/icons/icon-telegram.svg @@ -1,12 +1 @@ - - - icon-telegram - - - - - - - - - \ No newline at end of file +icon-telegram \ No newline at end of file diff --git a/website/images/icons/icon-twitter-gray.svg b/website/images/icons/icon-twitter-gray.svg index 115fe5ff390..b209f7b07cb 100644 --- a/website/images/icons/icon-twitter-gray.svg +++ b/website/images/icons/icon-twitter-gray.svg @@ -1,11 +1 @@ - - - icon-twitter-gray - - - - - - - - \ No newline at end of file +icon-twitter-gray \ No newline at end of file diff --git a/website/images/icons/icon-twitter.svg b/website/images/icons/icon-twitter.svg index b25ac9353ac..ef11bb2f0ba 100644 --- a/website/images/icons/icon-twitter.svg +++ b/website/images/icons/icon-twitter.svg @@ -1,16 +1 @@ - - - + \ No newline at end of file diff --git a/website/images/icons/icon-youtube-black.svg b/website/images/icons/icon-youtube-black.svg index 0cb017d64a4..bb20f823a06 100644 --- a/website/images/icons/icon-youtube-black.svg +++ b/website/images/icons/icon-youtube-black.svg @@ -1,15 +1 @@ - - - icon-youtube - - - - - - - - - - - - +icon-youtube \ No newline at end of file diff --git a/website/images/icons/icon-youtube.svg b/website/images/icons/icon-youtube.svg index ee676649770..224fc1c49cf 100644 --- a/website/images/icons/icon-youtube.svg +++ b/website/images/icons/icon-youtube.svg @@ -1,10 +1 @@ - - - icon-youtube - - - - - - - \ No newline at end of file +icon-youtube \ No newline at end of file diff --git a/website/images/logo-clickhouse.svg b/website/images/logo-clickhouse.svg index 560e15572d9..b6bb0cd2d07 100644 --- a/website/images/logo-clickhouse.svg +++ b/website/images/logo-clickhouse.svg @@ -1,28 +1 @@ - - - ClickHouse Logo - - - - \ No newline at end of file +ClickHouse Logo \ No newline at end of file diff --git a/website/images/logos/logo-almaz-capital.svg b/website/images/logos/logo-almaz-capital.svg index ed2357083ed..20cf10bd205 100644 --- a/website/images/logos/logo-almaz-capital.svg +++ b/website/images/logos/logo-almaz-capital.svg @@ -1 +1 @@ -AC Logo \ No newline at end of file +AC Logo \ No newline at end of file diff --git a/website/images/logos/logo-cloudflare.svg b/website/images/logos/logo-cloudflare.svg index e4bf4f2f803..9c6b408c388 100644 --- a/website/images/logos/logo-cloudflare.svg +++ b/website/images/logos/logo-cloudflare.svg @@ -1,25 +1 @@ - - - cf-logo-h - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file +cf-logo-h \ No newline at end of file diff --git a/website/images/logos/logo-firstmark.svg b/website/images/logos/logo-firstmark.svg index e529766bae4..9f372958295 100644 --- a/website/images/logos/logo-firstmark.svg +++ b/website/images/logos/logo-firstmark.svg @@ -1,579 +1 @@ - - - logo-firstmark - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file +logo-firstmark \ No newline at end of file diff --git a/website/images/logos/logo-yandex.png b/website/images/logos/logo-yandex.png index 92349036b3f..02ea52f2c7f 100644 Binary files a/website/images/logos/logo-yandex.png and b/website/images/logos/logo-yandex.png differ diff --git a/website/images/photos/aaron-katz.jpg b/website/images/photos/aaron-katz.jpg index 02aff426b33..3e03047a247 100644 Binary files a/website/images/photos/aaron-katz.jpg and b/website/images/photos/aaron-katz.jpg differ diff --git a/website/images/photos/alexander-sapin.jpg b/website/images/photos/alexander-sapin.jpg index 83121a980e7..8f132c6f406 100644 Binary files a/website/images/photos/alexander-sapin.jpg and b/website/images/photos/alexander-sapin.jpg differ diff --git a/website/images/photos/alexander-tokmakov.jpg b/website/images/photos/alexander-tokmakov.jpg index 3215788443f..dcee6b9d551 100644 Binary files a/website/images/photos/alexander-tokmakov.jpg and b/website/images/photos/alexander-tokmakov.jpg differ diff --git a/website/images/photos/alexey-milovidov.jpg b/website/images/photos/alexey-milovidov.jpg index 31a4ab5311d..9b5b5a12e88 100644 Binary files a/website/images/photos/alexey-milovidov.jpg and b/website/images/photos/alexey-milovidov.jpg differ diff --git a/website/images/photos/andy-james.jpg b/website/images/photos/andy-james.jpg new file mode 100644 index 00000000000..d1f5b47f39c Binary files /dev/null and b/website/images/photos/andy-james.jpg differ diff --git a/website/images/photos/anton-popov.jpg b/website/images/photos/anton-popov.jpg index 663fc9255e9..86fefbba27a 100644 Binary files a/website/images/photos/anton-popov.jpg and b/website/images/photos/anton-popov.jpg differ diff --git a/website/images/photos/arno-van-driel.jpg b/website/images/photos/arno-van-driel.jpg index 9a7855491aa..878ab2c336d 100644 Binary files a/website/images/photos/arno-van-driel.jpg and b/website/images/photos/arno-van-driel.jpg differ diff --git a/website/images/photos/caryn-marooney.jpg b/website/images/photos/caryn-marooney.jpg index 446e890ebe1..fa0dd29ad1f 100644 Binary files a/website/images/photos/caryn-marooney.jpg and b/website/images/photos/caryn-marooney.jpg differ diff --git a/website/images/photos/dmitry-novik.jpg b/website/images/photos/dmitry-novik.jpg index c2ce965e48d..d688407e5e4 100644 Binary files a/website/images/photos/dmitry-novik.jpg and b/website/images/photos/dmitry-novik.jpg differ diff --git a/website/images/photos/dorota-szeremeta.jpg b/website/images/photos/dorota-szeremeta.jpg index 70e427a1154..1e7c0aa2471 100644 Binary files a/website/images/photos/dorota-szeremeta.jpg and b/website/images/photos/dorota-szeremeta.jpg differ diff --git a/website/images/photos/ilya-yatsishin.jpg b/website/images/photos/ilya-yatsishin.jpg index b8d9d0866a1..23d10a86c7b 100644 Binary files a/website/images/photos/ilya-yatsishin.jpg and b/website/images/photos/ilya-yatsishin.jpg differ diff --git a/website/images/photos/ivan-blinkov.jpg b/website/images/photos/ivan-blinkov.jpg index 0e70ff6f773..b11d4504bde 100644 Binary files a/website/images/photos/ivan-blinkov.jpg and b/website/images/photos/ivan-blinkov.jpg differ diff --git a/website/images/photos/jason-chan.jpg b/website/images/photos/jason-chan.jpg index 6ce0b524cb2..2f69200a0b4 100644 Binary files a/website/images/photos/jason-chan.jpg and b/website/images/photos/jason-chan.jpg differ diff --git a/website/images/photos/kseniia-sumarokova.jpg b/website/images/photos/kseniia-sumarokova.jpg index 0fd4efeee25..8125d39869a 100644 Binary files a/website/images/photos/kseniia-sumarokova.jpg and b/website/images/photos/kseniia-sumarokova.jpg differ diff --git a/website/images/photos/maksim-kita.jpg b/website/images/photos/maksim-kita.jpg index e6aac5737cb..d740059b71a 100644 Binary files a/website/images/photos/maksim-kita.jpg and b/website/images/photos/maksim-kita.jpg differ diff --git a/website/images/photos/manas-alekar.jpg b/website/images/photos/manas-alekar.jpg index 307a860455f..a5463a527a9 100644 Binary files a/website/images/photos/manas-alekar.jpg and b/website/images/photos/manas-alekar.jpg differ diff --git a/website/images/photos/mark-zitnik.jpg b/website/images/photos/mark-zitnik.jpg new file mode 100644 index 00000000000..7ffc5105258 Binary files /dev/null and b/website/images/photos/mark-zitnik.jpg differ diff --git a/website/images/photos/melvyn-peignon.jpg b/website/images/photos/melvyn-peignon.jpg index 532c1759c65..3c4d2a212a0 100644 Binary files a/website/images/photos/melvyn-peignon.jpg and b/website/images/photos/melvyn-peignon.jpg differ diff --git a/website/images/photos/miel-donkers.jpg b/website/images/photos/miel-donkers.jpg new file mode 100644 index 00000000000..8d67c2440b1 Binary files /dev/null and b/website/images/photos/miel-donkers.jpg differ diff --git a/website/images/photos/mike-hayes.jpg b/website/images/photos/mike-hayes.jpg index 13dfbe4cefd..df0380063fa 100644 Binary files a/website/images/photos/mike-hayes.jpg and b/website/images/photos/mike-hayes.jpg differ diff --git a/website/images/photos/nihat-hosgur.jpg b/website/images/photos/nihat-hosgur.jpg index ad47b4aba50..fb3dd451713 100644 Binary files a/website/images/photos/nihat-hosgur.jpg and b/website/images/photos/nihat-hosgur.jpg differ diff --git a/website/images/photos/nikita-mikhailov.jpg b/website/images/photos/nikita-mikhailov.jpg index 549e5bc06e8..d956d334798 100644 Binary files a/website/images/photos/nikita-mikhailov.jpg and b/website/images/photos/nikita-mikhailov.jpg differ diff --git a/website/images/photos/nikolai-kochetov.jpg b/website/images/photos/nikolai-kochetov.jpg index 924df4e59cb..135852aa078 100644 Binary files a/website/images/photos/nikolai-kochetov.jpg and b/website/images/photos/nikolai-kochetov.jpg differ diff --git a/website/images/photos/pavel-kruglov.jpg b/website/images/photos/pavel-kruglov.jpg index 6287ef1bfe3..e1699785c9b 100644 Binary files a/website/images/photos/pavel-kruglov.jpg and b/website/images/photos/pavel-kruglov.jpg differ diff --git a/website/images/photos/rich-raposa.jpg b/website/images/photos/rich-raposa.jpg index f8338d6e3df..2d155356e1f 100644 Binary files a/website/images/photos/rich-raposa.jpg and b/website/images/photos/rich-raposa.jpg differ diff --git a/website/images/photos/thom-oconnor.jpg b/website/images/photos/thom-oconnor.jpg index 23d2d10382b..e44ece64548 100644 Binary files a/website/images/photos/thom-oconnor.jpg and b/website/images/photos/thom-oconnor.jpg differ diff --git a/website/images/photos/tyler-hannan.jpg b/website/images/photos/tyler-hannan.jpg new file mode 100644 index 00000000000..e14d9544bfe Binary files /dev/null and b/website/images/photos/tyler-hannan.jpg differ diff --git a/website/images/photos/vitaly-baranov.jpg b/website/images/photos/vitaly-baranov.jpg index 516b73073c4..2f0df629d5b 100644 Binary files a/website/images/photos/vitaly-baranov.jpg and b/website/images/photos/vitaly-baranov.jpg differ diff --git a/website/images/photos/vladimir-cherkasov.jpg b/website/images/photos/vladimir-cherkasov.jpg index 9d8246be2ae..31c2cb42eee 100644 Binary files a/website/images/photos/vladimir-cherkasov.jpg and b/website/images/photos/vladimir-cherkasov.jpg differ diff --git a/website/images/photos/yossi-kahlon.jpg b/website/images/photos/yossi-kahlon.jpg index dbcff5185c4..4a9672f99bc 100644 Binary files a/website/images/photos/yossi-kahlon.jpg and b/website/images/photos/yossi-kahlon.jpg differ diff --git a/website/images/photos/yury-izrailevsky.jpg b/website/images/photos/yury-izrailevsky.jpg index 785479d4ae3..0115a248e1e 100644 Binary files a/website/images/photos/yury-izrailevsky.jpg and b/website/images/photos/yury-izrailevsky.jpg differ diff --git a/website/templates/company/team.html b/website/templates/company/team.html index c2e6bfe496d..01341fb4a5f 100644 --- a/website/templates/company/team.html +++ b/website/templates/company/team.html @@ -154,6 +154,19 @@
+ + + +

+ {{ _('Miel Donkers') }} +

+

+ {{ _('Senior Cloud Software Engineer') }} +

+ +
+
+ @@ -229,6 +242,19 @@ {{ _('Associate, Business Strategy & Ops') }}

+
+
+ + + + +

+ {{ _('Tyler Hannan') }} +

+

+ {{ _('Senior Director, Developer Advocacy') }} +

+
@@ -268,6 +294,19 @@ {{ _('Account Executive, AMER') }}

+
+
+ + + + +

+ {{ _('Andy James') }} +

+

+ {{ _('Business Technology Full Stack Lead') }} +

+
@@ -697,6 +736,19 @@ {{ _('Senior Technical Project Manager') }}

+
+
+ + + + +

+ {{ _('Mark Zitnik') }} +

+

+ {{ _('Senior Cloud Software Engineer') }} +

+
diff --git a/website/templates/footer.html b/website/templates/footer.html index ec2afc2aa6a..efb6b1a0299 100644 --- a/website/templates/footer.html +++ b/website/templates/footer.html @@ -3,16 +3,16 @@
- © 2016–2021 {{ _('ClickHouse, Inc.') }} + © 2016–2022 {{ _('ClickHouse, Inc.') }}
{{ _('ClickHouse source code is published under the Apache 2.0 License.') }} {{ _('Software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.') }}
-
- Trademark Policy -
+
+ Trademark Policy +